From ae6f8c50a43e89511c2d322d6fb5fbcdb470379e Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Thu, 4 Sep 2025 13:02:24 -0400 Subject: [PATCH 1/9] test --- dispatcher.go | 30 ++ go.mod | 18 ++ go.sum | 30 ++ internal/batch/batch.go | 40 +++ internal/batch/batch_submitter.go | 8 + internal/batch/batch_test.go | 115 +++++++ internal/kinesis/kinesis.go | 318 +++++++++++++++++++ internal/kinesis/kinesis_test.go | 159 ++++++++++ outboundSender.go | 511 +++++++++++++++++------------- outboundSender_test.go | 30 +- senderWrapper.go | 4 +- webhook_dispatcher.go | 250 +++++++++++++++ webhook_outbound_sender.go | 161 ++++++++++ 13 files changed, 1447 insertions(+), 227 deletions(-) create mode 100644 dispatcher.go create mode 100644 internal/batch/batch.go create mode 100644 internal/batch/batch_submitter.go create mode 100644 internal/batch/batch_test.go create mode 100644 internal/kinesis/kinesis.go create mode 100644 internal/kinesis/kinesis_test.go create mode 100644 webhook_dispatcher.go create mode 100644 webhook_outbound_sender.go diff --git a/dispatcher.go b/dispatcher.go new file mode 100644 index 00000000..d199c827 --- /dev/null +++ b/dispatcher.go @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2021 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "container/ring" + + "github.com/xmidt-org/wrp-go/v3" +) + +type Dispatcher interface { + QueueOverflow() + Send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) +} + +func DispatcherFactory(webhook bool, obs *CaduceusOutboundSender) Dispatcher { + + if webhook { + return &WebhookDispatcher{ + obs: obs, + } + } + + // TODO + return &WebhookDispatcher{ + obs: obs, + } + +} diff --git a/go.mod b/go.mod index 3c5f2a68..47e74af0 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.24 require ( emperror.dev/emperror v0.33.0 + github.com/aws/aws-sdk-go-v2/credentials v1.18.10 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/go-kit/kit v0.13.0 github.com/gorilla/mux v1.8.1 @@ -28,11 +29,28 @@ require ( go.uber.org/zap v1.27.0 ) +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect + github.com/aws/smithy-go v1.23.0 // indirect +) + require ( emperror.dev/errors v0.8.1 // indirect github.com/SermoDigital/jose v0.9.2-0.20161205224733-f6df55f235c2 // indirect github.com/VividCortex/gohistogram v1.0.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.38.3 + github.com/aws/aws-sdk-go-v2/config v1.31.6 + github.com/aws/aws-sdk-go-v2/service/kinesis v1.40.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 github.com/beorn7/perks v1.0.1 // indirect github.com/billhathaway/consistentHash v0.0.0-20140718022140-addea16d2229 // indirect github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 // indirect diff --git a/go.sum b/go.sum index aac3be40..e47cb4b6 100644 --- a/go.sum +++ b/go.sum @@ -679,8 +679,38 @@ github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2 v1.38.3 h1:B6cV4oxnMs45fql4yRH+/Po/YU+597zgWqvDpYMturk= +github.com/aws/aws-sdk-go-v2 v1.38.3/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= +github.com/aws/aws-sdk-go-v2/config v1.31.6 h1:a1t8fXY4GT4xjyJExz4knbuoxSCacB5hT/WgtfPyLjo= +github.com/aws/aws-sdk-go-v2/config v1.31.6/go.mod h1:5ByscNi7R+ztvOGzeUaIu49vkMk2soq5NaH5PYe33MQ= +github.com/aws/aws-sdk-go-v2/credentials v1.18.10 h1:xdJnXCouCx8Y0NncgoptztUocIYLKeQxrCgN6x9sdhg= +github.com/aws/aws-sdk-go-v2/credentials v1.18.10/go.mod h1:7tQk08ntj914F/5i9jC4+2HQTAuJirq7m1vZVIhEkWs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 h1:wbjnrrMnKew78/juW7I2BtKQwa1qlf6EjQgS69uYY14= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6/go.mod h1:AtiqqNrDioJXuUgz3+3T0mBWN7Hro2n9wll2zRUc0ww= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 h1:uF68eJA6+S9iVr9WgX1NaRGyQ/6MdIyc4JNUo6TN1FA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6/go.mod h1:qlPeVZCGPiobx8wb1ft0GHT5l+dc6ldnwInDFaMvC7Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 h1:pa1DEC6JoI0zduhZePp3zmhWvk/xxm4NB8Hy/Tlsgos= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6/go.mod h1:gxEjPebnhWGJoaDdtDkA0JX46VRg1wcTHYe63OfX5pE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6 h1:LHS1YAIJXJ4K9zS+1d/xa9JAA9sL2QyXIQCQFQW/X08= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6/go.mod h1:c9PCiTEuh0wQID5/KqA32J+HAgZxN9tOGXKCiYJjTZI= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.40.1 h1:9QC0AF6gakV1TZuGp3NEUNl/6gXt3rfIifnxd+dWwbw= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.40.1/go.mod h1:UpSQbmXxFiDGDrvqsTgEm3YijDf9cg/Ti+s2W0SeFEU= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 h1:8OLZnVJPvjnrxEwHFg9hVUof/P4sibH+Ea4KKuqAGSg= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.1/go.mod h1:27M3BpVi0C02UiQh1w9nsBEit6pLhlaH3NHna6WUbDE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 h1:gKWSTnqudpo8dAxqBqZnDoDWCiEh/40FziUjr/mo6uA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2/go.mod h1:x7+rkNmRoEN1U13A6JE2fXne9EWyJy54o3n6d4mGaXQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 h1:YZPjhyaGzhDQEvsffDEcpycq49nl7fiGcfJTIo8BszI= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.2/go.mod h1:2dIN8qhQfv37BdUYGgEC8Q3tteM3zFxTI1MLO2O3J3c= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= +github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= diff --git a/internal/batch/batch.go b/internal/batch/batch.go new file mode 100644 index 00000000..e24f939a --- /dev/null +++ b/internal/batch/batch.go @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: LicenseRef-COMCAST + +package batch + +import () + +func GetBatch[T any](start int, batchSize int, items []T) ([]T, bool) { + var batch []T + more := true + if (len(items) - start) <= batchSize { + batch = items[start:] + more = false + } else { + batch = items[start : start+batchSize] + } + + return batch, more +} + +func GetBatches[T any](batchSize int, items []T) [][]T { + batches := [][]T{} + + if len(items) == 0 { + return batches + } + + notDone := true + start := 0 + for notDone { + items, more := GetBatch(start, batchSize, items) + notDone = more + start += batchSize + + batches = append(batches, items) + } + + return batches + +} diff --git a/internal/batch/batch_submitter.go b/internal/batch/batch_submitter.go new file mode 100644 index 00000000..143e582f --- /dev/null +++ b/internal/batch/batch_submitter.go @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: LicenseRef-COMCAST + +package batch + +type BatchSubmitter[T any] interface { + SubmitBatch(events []T) error +} diff --git a/internal/batch/batch_test.go b/internal/batch/batch_test.go new file mode 100644 index 00000000..e5c1f7de --- /dev/null +++ b/internal/batch/batch_test.go @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: LicenseRef-COMCAST + +package batch + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetBatch(t *testing.T) { + batchSize := 5 + + items := []string{} + batchOfItems, more := GetBatch(0, batchSize, items) + assert.False(t, more) + assert.Equal(t, 0, len(batchOfItems)) + + items = []string{"a", "b", "c", "d"} + batchOfItems, more = GetBatch(0, batchSize, items) + assert.False(t, more) + assert.Equal(t, 4, len(batchOfItems)) + + items = []string{"a", "b", "c", "d", "e"} + batchOfItems, more = GetBatch(0, batchSize, items) + assert.False(t, more) + assert.Equal(t, 5, len(batchOfItems)) + + items = []string{"a", "b", "c", "d", "e", "f"} + batchOfItems, more = GetBatch(0, batchSize, items) + assert.True(t, more) + assert.Equal(t, 5, len(batchOfItems)) + batchOfItems, more = GetBatch(0+batchSize, batchSize, items) + assert.False(t, more) + assert.Equal(t, 1, len(batchOfItems)) +} + +func TestGetLargeBatch(t *testing.T) { + batchSize := 5 + + start := 0 + items := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"} + batchOfItems, more := GetBatch(start, batchSize, items) + assert.True(t, more) + assert.Equal(t, 5, len(batchOfItems)) + + start += batchSize + batchOfItems, more = GetBatch(start, batchSize, items) + assert.True(t, more) + assert.Equal(t, 5, len(batchOfItems)) + + start += batchSize + batchOfItems, more = GetBatch(start, batchSize, items) + assert.False(t, more) + assert.Equal(t, 1, len(batchOfItems)) +} + +func TestGetBatches100Items(t *testing.T) { + batchSize := 25 + + items := []string{} + + for i := 0; i < 100; i++ { + items = append(items, fmt.Sprintf("%d", i)) + } + + batches := GetBatches(batchSize, items) + assert.Equal(t, 4, len(batches)) + assert.Equal(t, 25, len(batches[0])) + assert.Equal(t, 25, len(batches[1])) + assert.Equal(t, 25, len(batches[2])) + assert.Equal(t, 25, len(batches[3])) +} + +func TestGetBatchesEmptyItems(t *testing.T) { + batchSize := 25 + + items := []string{} + + batches := GetBatches(batchSize, items) + assert.Equal(t, 0, len(batches)) +} + +func TestGetBatchesLessThanOne(t *testing.T) { + batchSize := 5 + + items := []string{"a", "b", "c", "d"} + + batches := GetBatches(batchSize, items) + assert.Equal(t, 1, len(batches)) + assert.Equal(t, 4, len(batches[0])) +} + +func TestGetBatchesExactlyOne(t *testing.T) { + batchSize := 5 + + items := []string{"a", "b", "c", "d", "e"} + + batches := GetBatches(batchSize, items) + assert.Equal(t, 1, len(batches)) + assert.Equal(t, 5, len(batches[0])) +} + +func TestGetBatchesOneOver(t *testing.T) { + batchSize := 5 + + items := []string{"a", "b", "c", "d", "e", "f"} + + batches := GetBatches(batchSize, items) + assert.Equal(t, 2, len(batches)) + assert.Equal(t, 5, len(batches[0])) + assert.Equal(t, 1, len(batches[1])) +} diff --git a/internal/kinesis/kinesis.go b/internal/kinesis/kinesis.go new file mode 100644 index 00000000..d4e034c1 --- /dev/null +++ b/internal/kinesis/kinesis.go @@ -0,0 +1,318 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: LicenseRef-COMCAST + +package kinesis + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/kinesis" + "github.com/aws/aws-sdk-go-v2/service/kinesis/types" + + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/xmidt-org/caduceus/internal/batch" + "go.uber.org/zap" +) + +const MaxPutRecordsBatchSize = 500 + +type KinesisClientAPI interface { + PutRecord(event []byte, stream string, partitionKey string) (*kinesis.PutRecordOutput, error) + PutRecords(items []Item, stream string) (int, error) + GetRecords(stream string) (*kinesis.GetRecordsOutput, error) +} + +type KinesisAPI interface { + PutRecord(context.Context, *kinesis.PutRecordInput, ...func(*kinesis.Options)) (*kinesis.PutRecordOutput, error) + PutRecords(context.Context, *kinesis.PutRecordsInput, ...func(*kinesis.Options)) (*kinesis.PutRecordsOutput, error) + GetRecords(context.Context, *kinesis.GetRecordsInput, ...func(*kinesis.Options)) (*kinesis.GetRecordsOutput, error) + GetShardIterator(context.Context, *kinesis.GetShardIteratorInput, ...func(*kinesis.Options)) (*kinesis.GetShardIteratorOutput, error) + DescribeStream(context.Context, *kinesis.DescribeStreamInput, ...func(*kinesis.Options)) (*kinesis.DescribeStreamOutput, error) +} + +type KinesisClient struct { + svc KinesisAPI + logger *zap.Logger + config Config + credsExpireAt time.Time +} + +type Config struct { + Region string `validate:"empty=false"` + Endpoint string `validate:"empty=false"` + Role string + Credentials Credentials +} + +type Credentials struct { + AccessKey string + SecretKey string + SessionToken string +} + +type Item struct { + PartitionKey string + Item []byte +} + +func New(cfg Config, logger *zap.Logger) (KinesisClientAPI, error) { + k, credsExpireAt, err := getClient(cfg, logger) + if err != nil { + return nil, err + } + + return &KinesisClient{ + svc: k, + logger: logger, + config: cfg, + credsExpireAt: credsExpireAt, + }, nil +} + +func getClient(cfg Config, logger *zap.Logger) (KinesisAPI, time.Time, error) { + ctx := context.Background() + + // try removing this, not sure why we need it + if cfg.Role != "" && + (cfg.Credentials.AccessKey != "" || + cfg.Credentials.SecretKey != "" || + cfg.Credentials.SessionToken != "") { + return nil, time.Time{}, fmt.Errorf("cannot specify both role and credentials") + } + + var credsExpireAt time.Time + var awsConfig aws.Config + var err error + + logger.Debug("kinesis role is", zap.String("role", cfg.Role)) + + if cfg.Role != "" { // we need to authenticate with aws and assume a cross-account role + + // config to retrieve sts credentials + awsConfig, err = config.LoadDefaultConfig( + ctx, + config.WithRegion(cfg.Region), + config.WithRetryMaxAttempts(3), + //config.WithCredentialsChainVerboseErrors(true), + config.WithHTTPClient( + &http.Client{Timeout: 10 * time.Second}, + ), + ) + if err != nil { + logger.Error("unable to load aws config for sts credentials", zap.String("role", cfg.Role), zap.Error(err)) + return nil, time.Time{}, err + } + + // retrieve credentials for cross-account role + client := sts.NewFromConfig(awsConfig) + creds := stscreds.NewAssumeRoleProvider(client, cfg.Role) + stsResponse, err := creds.Retrieve(ctx) + if err != nil { + logger.Error("unable to authenticate kinesis assumed role", zap.String("role", cfg.Role), zap.Error(err)) + return nil, time.Time{}, err + } + + // create config for kinesis client with cross account credentials + awsConfig, err = config.LoadDefaultConfig( + ctx, + config.WithRegion(cfg.Region), + config.WithBaseEndpoint(cfg.Endpoint), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + stsResponse.AccessKeyID, + stsResponse.SecretAccessKey, + stsResponse.SessionToken), + ), + ) + if err != nil { + logger.Error("unable to create kinesis client config", zap.String("role", cfg.Role), zap.Error(err)) + return nil, time.Time{}, err + } + + credsExpireAt = stsResponse.Expires + } else { + // create config to talk to local kinesis + var err error + awsConfig, err = config.LoadDefaultConfig( + ctx, + config.WithRegion(cfg.Region), + config.WithBaseEndpoint(cfg.Endpoint), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + cfg.Credentials.AccessKey, + cfg.Credentials.SecretKey, + cfg.Credentials.SessionToken, + ), + ), + ) + if err != nil { + logger.Error("unable to create local kinesis client config", zap.String("role", cfg.Role), zap.Error(err)) + return nil, time.Time{}, err + } + + // if cfg.Credentials.AccessKey != "" || + // cfg.Credentials.SecretKey != "" || + // cfg.Credentials.SessionToken != "" { + + // awsConfig = awsConfig.WithCredentials( + // credentials.NewStaticCredentials( + // cfg.Credentials.AccessKey, + // cfg.Credentials.SecretKey, + // cfg.Credentials.SessionToken, + // )) + // } + } + + k := kinesis.NewFromConfig(awsConfig) + + return k, credsExpireAt, nil +} + +func (kc *KinesisClient) PutRecord(event []byte, stream string, partitionKey string) (*kinesis.PutRecordOutput, error) { + kc.logger.Debug("putting event to kinesis", zap.String("event", string(event))) + + kc.refreshClient() + + putOutput, err := kc.svc.PutRecord(context.Background(), &kinesis.PutRecordInput{ + Data: event, + StreamName: &stream, + PartitionKey: aws.String(partitionKey), + }) + + if err != nil { + kc.logger.Error("PutRecord failed", zap.Error(err)) + return putOutput, err + } + + kc.logger.Debug("Published Event", zap.Any("result", putOutput), zap.String("endpoint", kc.config.Endpoint)) + return putOutput, nil +} + +func (kc *KinesisClient) PutRecords(items []Item, stream string) (int, error) { + notDone := true + start := 0 + failedRecordCount := int32(0) + for notDone { + batch, more := batch.GetBatch(start, MaxPutRecordsBatchSize, items) + notDone = more + start += MaxPutRecordsBatchSize + + kc.logger.Debug("putting events to kinesis", zap.Int("size", len(items))) + + records := []types.PutRecordsRequestEntry{} + for _, item := range batch { + entry := types.PutRecordsRequestEntry{ + Data: item.Item, + PartitionKey: aws.String(item.PartitionKey), + } + records = append(records, entry) + } + + kc.refreshClient() + putOutput, err := kc.svc.PutRecords(context.Background(), &kinesis.PutRecordsInput{ + Records: records, + StreamName: &stream, + }) + + if putOutput != nil { + failedRecordCount += *putOutput.FailedRecordCount + } + + if err != nil { + kc.logger.Error("PutRecords failed", zap.Error(err)) + return int(failedRecordCount), err + } + kc.logger.Debug("Published Records", zap.Any("result", putOutput), zap.String("endpoint", kc.config.Endpoint)) + } + + return int(failedRecordCount), nil +} + +// get all records - for integration testing, only, not used in production + +func (kc *KinesisClient) GetRecords(stream string) (*kinesis.GetRecordsOutput, error) { + kc.logger.Debug("getting records", zap.String("endpoint", kc.config.Endpoint)) + + // get starting sequence number first, then shard iterator for that + // retrieve iterator + shardId := "shardId-000000000000" + + description, err := kc.svc.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{ + StreamName: aws.String(stream), + }) + + if err != nil { + kc.logger.Error("error describing stream", zap.Error(err)) + return nil, err + } + if description == nil { + return nil, fmt.Errorf("description is nil") + } + if description.StreamDescription == nil { + return nil, fmt.Errorf("stream description is nil") + } + if len(description.StreamDescription.Shards) == 0 { + return nil, fmt.Errorf("no shards found") + } + if description.StreamDescription.Shards[0].SequenceNumberRange == nil { + return nil, fmt.Errorf("sequence number range is nil") + } + if description.StreamDescription.Shards[0].SequenceNumberRange.StartingSequenceNumber == nil { + return nil, fmt.Errorf("starting sequence number is nil") + } + sequenceNo := description.StreamDescription.Shards[0].SequenceNumberRange.StartingSequenceNumber + + iteratorOutput, err := kc.svc.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + // Shard Id is provided when making put record(s) request. + ShardId: &shardId, + //ShardIteratorType: aws.String("TRIM_HORIZON"), + ShardIteratorType: types.ShardIteratorTypeAtSequenceNumber, + StartingSequenceNumber: sequenceNo, + // ShardIteratorType: aws.String("LATEST"), + StreamName: &stream, + }) + + if err != nil { + kc.logger.Error("error getting iterator output", zap.Error(err)) + return nil, err + } + + // get records use shard iterator for making request + records, err := kc.svc.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iteratorOutput.ShardIterator, + //StreamARN: &streamARN, + }) + + if err != nil { + kc.logger.Error("error getting records", zap.Error(err)) + panic(err) + } + + if err != nil { + kc.logger.Error(fmt.Sprintf("Error: %v", err)) + return records, err + } + + return records, nil +} + +func (kc *KinesisClient) refreshClient() { + if kc.config.Role != "" { + if kc.credsExpireAt.Unix() <= time.Now().Add(3*time.Minute).Unix() { + k, credsExpireAt, err := getClient(kc.config, kc.logger) + if err != nil { + kc.logger.Error("error refreshing kinesis client", zap.Error(err)) + return + } + + // save expiration locally because asking for it is a drain on the metadata server + kc.credsExpireAt = credsExpireAt + kc.svc = k + } + } +} diff --git a/internal/kinesis/kinesis_test.go b/internal/kinesis/kinesis_test.go new file mode 100644 index 00000000..399f674c --- /dev/null +++ b/internal/kinesis/kinesis_test.go @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: LicenseRef-COMCAST + +package kinesis + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/kinesis" + "github.com/aws/aws-sdk-go-v2/service/kinesis/types" + "go.uber.org/zap" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +const SomeStream = "some-stream" + +var logger, _ = zap.NewProduction() + +var shardId = "shardId" +var sequenceNumber = "123" + +var putRecordOutput = kinesis.PutRecordOutput{ + EncryptionType: types.EncryptionTypeNone, + ShardId: &shardId, + SequenceNumber: &sequenceNumber, +} + +type mockKinesis struct{ mock.Mock } + +func newMockKinesis() *mockKinesis { return &mockKinesis{} } + +func (m *mockKinesis) PutRecords(ctx context.Context, input *kinesis.PutRecordsInput, optFuns ...func(*kinesis.Options)) (*kinesis.PutRecordsOutput, error) { + args := m.Called(ctx, input, optFuns) + return args.Get(0).(*kinesis.PutRecordsOutput), args.Error(1) +} + +func (m *mockKinesis) PutRecord(ctx context.Context, input *kinesis.PutRecordInput, optFuns ...func(*kinesis.Options)) (*kinesis.PutRecordOutput, error) { + m.Called(ctx, input, optFuns) + return &putRecordOutput, nil +} + +func (m *mockKinesis) GetRecords(ctx context.Context, input *kinesis.GetRecordsInput, optFuns ...func(*kinesis.Options)) (*kinesis.GetRecordsOutput, error) { + m.Called(ctx, input, optFuns) + + return nil, nil +} + +func (m *mockKinesis) DescribeStream(ctx context.Context, input *kinesis.DescribeStreamInput, optFuns ...func(*kinesis.Options)) (*kinesis.DescribeStreamOutput, error) { + m.Called(ctx, input, optFuns) + + return nil, nil +} + +func (m *mockKinesis) GetShardIterator(ctx context.Context, input *kinesis.GetShardIteratorInput, optFuns ...func(*kinesis.Options)) (*kinesis.GetShardIteratorOutput, error) { + m.Called(ctx, input, optFuns) + + return nil, nil +} + +func TestNewNoAwsRole(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "accessKey") + t.Setenv("AWS_SECRET_ACCESS_KEY", "secretKey") + + cfg := Config{ + Region: "na", + Endpoint: "http://localhost", + } + + kc, err := New(cfg, logger) + kclient := kc.(*KinesisClient) + + assert.Nil(t, err) + assert.NotNil(t, kclient.svc) +} + +func TestPutRecords(t *testing.T) { + items := []Item{{PartitionKey: "mac1", Item: []byte("test1")}, {PartitionKey: "mac2", Item: []byte("test2")}} + stream := SomeStream + + failedRecordInputCount := int32(10) + putRecordsOutput := &kinesis.PutRecordsOutput{ + EncryptionType: types.EncryptionTypeNone, + FailedRecordCount: &failedRecordInputCount, + } + + m := newMockKinesis() + m.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, nil) + kc := KinesisClient{ + logger: logger, + svc: m, + } + + failedRecordCount, err := kc.PutRecords(items, stream) + t.Log(err) + assert.Nil(t, err) + assert.Equal(t, 10, failedRecordCount) +} + +func TestPutRecordsError(t *testing.T) { + items := []Item{{PartitionKey: "mac1", Item: []byte("test1")}, {PartitionKey: "mac2", Item: []byte("test2")}} + stream := SomeStream + + failedRecordInputCount := int32(0) + putRecordsOutput := &kinesis.PutRecordsOutput{ + EncryptionType: types.EncryptionTypeNone, + FailedRecordCount: &failedRecordInputCount, + } + + m := newMockKinesis() + m.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, errors.New("some db error")) + kc := KinesisClient{ + logger: logger, + svc: m, + } + + failedRecordCount, err := kc.PutRecords(items, stream) + t.Log(err) + assert.NotNil(t, err) + assert.Equal(t, 0, failedRecordCount) +} + +func TestPutRecordsErrorNilOutput(t *testing.T) { + items := []Item{{PartitionKey: "mac1", Item: []byte("test1")}, {PartitionKey: "mac2", Item: []byte("test2")}} + stream := SomeStream + + putRecordsOutput := (*kinesis.PutRecordsOutput)(nil) + m := newMockKinesis() + m.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, errors.New("some db error")) + kc := KinesisClient{ + logger: logger, + svc: m, + } + + failedRecordCount, err := kc.PutRecords(items, stream) + t.Log(err) + assert.NotNil(t, err) + assert.Equal(t, 0, failedRecordCount) +} + +func TestPutRecord(t *testing.T) { + event := []byte("test") + stream := SomeStream + partitionKey := "some-key" + + m := newMockKinesis() + m.On("PutRecord", mock.Anything, mock.Anything, mock.Anything).Return(&putRecordOutput) + kc := KinesisClient{ + logger: logger, + svc: m, + } + + _, err := kc.PutRecord(event, stream, partitionKey) + t.Log(err) + assert.Nil(t, err) +} diff --git a/outboundSender.go b/outboundSender.go index 8a222ec8..6b009cfa 100644 --- a/outboundSender.go +++ b/outboundSender.go @@ -3,36 +3,27 @@ package main import ( - "bytes" "container/ring" - "crypto/hmac" - "crypto/sha1" - "encoding/hex" - "encoding/json" + "strings" + "errors" "fmt" - "io" "math/rand" - "net/http" + "net/url" "regexp" - "strconv" - "strings" "sync" "sync/atomic" "time" "go.uber.org/zap" - gokitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/xmidt-org/ancla" - "github.com/xmidt-org/webpa-common/v2/device" "github.com/xmidt-org/webpa-common/v2/semaphore" - "github.com/xmidt-org/webpa-common/v2/xhttp" + "github.com/xmidt-org/wrp-go/v3" - "github.com/xmidt-org/wrp-go/v3/wrphttp" ) // failureText is human readable text for the failure message @@ -90,6 +81,9 @@ type OutboundSenderFactory struct { // DisablePartnerIDs dictates whether or not to enforce the partner ID check. DisablePartnerIDs bool + + // Dispatcher sends the events + Dispatcher Dispatcher } type OutboundSender interface { @@ -106,7 +100,7 @@ type CaduceusOutboundSender struct { listener ancla.InternalWebhook deliverUntil time.Time dropUntil time.Time - sender httpClient + httpSender httpClient events []*regexp.Regexp matcher []*regexp.Regexp queueSize int @@ -124,6 +118,7 @@ type CaduceusOutboundSender struct { customPIDs []string disablePartnerIDs bool clientMiddleware func(httpClient) httpClient + dispatcher Dispatcher } // New creates a new OutboundSender object from the factory, or returns an error. @@ -155,7 +150,7 @@ func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) { // since id is only used to enrich metrics and logging, remove invalid UTF-8 characters from the URL id: strings.ToValidUTF8(osf.Listener.Webhook.Config.URL, ""), listener: osf.Listener, - sender: osf.Sender, + httpSender: osf.Sender, queueSize: osf.QueueSize, cutOffPeriod: osf.CutOffPeriod, deliverUntil: osf.Listener.Webhook.Until, @@ -191,11 +186,15 @@ func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) { caduceusOutboundSender.workers = semaphore.New(caduceusOutboundSender.maxWorkers) caduceusOutboundSender.wg.Add(1) - go caduceusOutboundSender.dispatcher() return caduceusOutboundSender, nil } +func (obs *CaduceusOutboundSender) Dispatch(d Dispatcher) { + obs.dispatcher = d + go obs.dispatch() +} + // Update applies user configurable values for the outbound sender when a // webhook is registered func (obs *CaduceusOutboundSender) Update(wh ancla.InternalWebhook) (err error) { @@ -408,7 +407,7 @@ func (obs *CaduceusOutboundSender) Queue(msg *wrp.Message) { obs.logger.Debug("event added to outbound queue", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) default: obs.logger.Debug("queue full. event dropped", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) - obs.queueOverflow() + obs.dispatcher.QueueOverflow() obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "queue_full"}).Add(1.0) } } @@ -440,7 +439,7 @@ func (obs *CaduceusOutboundSender) Empty(droppedCounter prometheus.Counter) { obs.metrics.queueDepthGauge.With(prometheus.Labels{urlLabel: obs.id}).Set(0.0) } -func (obs *CaduceusOutboundSender) dispatcher() { +func (obs *CaduceusOutboundSender) dispatch() { defer obs.wg.Done() var ( urls *ring.Ring @@ -501,215 +500,283 @@ Loop: obs.workers.Acquire() obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(1.0) - go obs.send(urls, secret, accept, msg) + go obs.dispatcher.Send(urls, secret, accept, msg) } for i := 0; i < obs.maxWorkers; i++ { obs.workers.Acquire() } } +// func (obs *CaduceusOutboundSender) dispatcher() { +// defer obs.wg.Done() +// var ( +// urls *ring.Ring +// secret, accept string +// ) + +// Loop: +// for { +// // Always pull a new queue in case we have been cutoff or are shutting +// // down. +// msg, ok := <-obs.queue.Load().(chan *wrp.Message) +// // The dispatcher cannot get stuck blocking here forever (caused by an +// // empty queue that is replaced and then Queue() starts adding to the +// // new queue) because: +// // - queue is only replaced on cutoff and shutdown +// // - on cutoff, the first queue is always full so we will definitely +// // get a message, drop it because we're cut off, then get the new +// // queue and block until the cut off ends and Queue() starts queueing +// // messages again. +// // - on graceful shutdown, the queue is closed and then the dispatcher +// // will send all messages, then break the loop, gather workers, and +// // exit. +// // - on non graceful shutdown, the queue is closed and then replaced +// // with a new, empty queue that is also closed. +// // - If the first queue is empty, we immediately break the loop, +// // gather workers, and exit. +// // - If the first queue has messages, we drop a message as expired +// // pull in the new queue which is empty and closed, break the +// // loop, gather workers, and exit. +// // This is only true when a queue is empty and closed, which for us +// // only happens on Shutdown(). +// if !ok { +// break Loop +// } +// obs.metrics.queueDepthGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(-1.0) +// obs.mutex.RLock() +// urls = obs.urls +// // Move to the next URL to try 1st the next time. +// // This is okay because we run a single dispatcher and it's the +// // only one updating this field. +// obs.urls = obs.urls.Next() +// deliverUntil := obs.deliverUntil +// dropUntil := obs.dropUntil +// secret = obs.listener.Webhook.Config.Secret +// accept = obs.listener.Webhook.Config.ContentType +// obs.mutex.RUnlock() + +// now := time.Now() + +// if now.Before(dropUntil) { +// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "cut_off"}).Add(1.0) +// continue +// } +// if now.After(deliverUntil) { +// obs.Empty(obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "expired"})) +// continue +// } +// obs.workers.Acquire() +// obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(1.0) + +// go obs.send(urls, secret, accept, msg) +// } +// for i := 0; i < obs.maxWorkers; i++ { +// obs.workers.Acquire() +// } +// } + // worker is the routine that actually takes the queued messages and delivers // them to the listeners outside webpa -func (obs *CaduceusOutboundSender) send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) { - defer func() { - if r := recover(); nil != r { - obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: DropsDueToPanic}).Add(1.0) - obs.logger.Error("goroutine send() panicked", zap.String("id", obs.id), zap.Any("panic", r)) - // don't silence the panic - panic(r) - } - - obs.workers.Release() - obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(-1.0) - }() - - payload := msg.Payload - body := payload - var payloadReader *bytes.Reader - - // Use the internal content type unless the accept type is wrp - contentType := msg.ContentType - switch acceptType { - case "wrp", wrp.MimeTypeMsgpack, wrp.MimeTypeWrp: - // WTS - We should pass the original, raw WRP event instead of - // re-encoding it. - contentType = wrp.MimeTypeMsgpack - buffer := bytes.NewBuffer([]byte{}) - encoder := wrp.NewEncoder(buffer, wrp.Msgpack) - encoder.Encode(msg) - body = buffer.Bytes() - } - payloadReader = bytes.NewReader(body) - - req, err := http.NewRequest("POST", urls.Value.(string), payloadReader) - if err != nil { - // Report drop - obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "invalid_config"}).Add(1.0) - obs.logger.Error("Invalid URL", zap.String("url", urls.Value.(string)), zap.String("id", obs.id), zap.Error(err)) - return - } - - req.Header.Set("Content-Type", contentType) - - // Add x-Midt-* headers - wrphttp.AddMessageHeaders(req.Header, msg) - - // Provide the old headers for now - req.Header.Set("X-Webpa-Event", strings.TrimPrefix(msg.Destination, "event:")) - req.Header.Set("X-Webpa-Transaction-Id", msg.TransactionUUID) - - // Add the device id without the trailing service - id, _ := device.ParseID(msg.Source) - // Deprecated: X-Webpa-Device-Id should only be used for backwards compatibility. - // Use X-Webpa-Source instead. - req.Header.Set("X-Webpa-Device-Id", string(id)) - // Deprecated: X-Webpa-Device-Name should only be used for backwards compatibility. - // Use X-Webpa-Source instead. - req.Header.Set("X-Webpa-Device-Name", string(id)) - req.Header.Set("X-Webpa-Source", msg.Source) - req.Header.Set("X-Webpa-Destination", msg.Destination) - - // Apply the secret - - if secret != "" { - s := hmac.New(sha1.New, []byte(secret)) - s.Write(body) - sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(s.Sum(nil))) - req.Header.Set("X-Webpa-Signature", sig) - } - - // since eventType is only used to enrich metrics and logging, remove invalid UTF-8 characters from the URL - eventType := strings.ToValidUTF8(msg.FindEventStringSubMatch(), "") - - retryOptions := xhttp.RetryOptions{ - Logger: obs.logger, - Retries: obs.deliveryRetries, - Interval: obs.deliveryInterval, - Counter: gokitprometheus.NewCounter(obs.metrics.deliveryRetryCounter.MustCurryWith(prometheus.Labels{urlLabel: obs.id, eventLabel: eventType})), - // Always retry on failures up to the max count. - ShouldRetry: xhttp.ShouldRetry, - ShouldRetryStatus: xhttp.RetryCodes, - } - - // update subsequent requests with the next url in the list upon failure - retryOptions.UpdateRequest = func(request *http.Request) { - urls = urls.Next() - tmp, err := url.Parse(urls.Value.(string)) - if err != nil { - obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: updateRequestURLFailedReason}).Add(1) - obs.logger.Error("failed to update url", zap.String("url", urls.Value.(string)), zap.Error(err)) - return - } - request.URL = tmp - } - - // Send it - obs.logger.Debug("attempting to send event", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) - - retryer := xhttp.RetryTransactor(retryOptions, obs.sender.Do) - client := obs.clientMiddleware(doerFunc(retryer)) - resp, err := client.Do(req) - - var deliveryCounterLabels prometheus.Labels - code := messageDroppedCode - reason := noErrReason - l := obs.logger - if err != nil { - // Report failure - reason = getDoErrReason(err) - if resp != nil { - code = strconv.Itoa(resp.StatusCode) - } - - l = obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) - deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} - obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason}).Add(1) - } else { - // Report Result - code = strconv.Itoa(resp.StatusCode) - // read until the response is complete before closing to allow - // connection reuse - if resp.Body != nil { - io.Copy(io.Discard, resp.Body) - resp.Body.Close() - } - - deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} - } - - obs.metrics.deliveryCounter.With(deliveryCounterLabels).Add(1.0) - l.Debug("event sent-ish", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination), zap.String("code", code), zap.String("url", req.URL.String())) -} - -// queueOverflow handles the logic of what to do when a queue overflows: -// cutting off the webhook for a time and sending a cut off notification -// to the failure URL. -func (obs *CaduceusOutboundSender) queueOverflow() { - obs.mutex.Lock() - if time.Now().Before(obs.dropUntil) { - obs.mutex.Unlock() - return - } - obs.dropUntil = time.Now().Add(obs.cutOffPeriod) - obs.metrics.dropUntilGauge.With(prometheus.Labels{urlLabel: obs.id}).Set(float64(obs.dropUntil.Unix())) - secret := obs.listener.Webhook.Config.Secret - failureMsg := obs.failureMsg - failureURL := obs.listener.Webhook.FailureURL - obs.mutex.Unlock() - - obs.metrics.cutOffCounter.With(prometheus.Labels{urlLabel: obs.id}).Add(1.0) - - // We empty the queue but don't close the channel, because we're not - // shutting down. - obs.Empty(obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "cut_off"})) - - msg, err := json.Marshal(failureMsg) - if err != nil { - obs.logger.Error("Cut-off notification json.Marshal failed", zap.Any("failureMessage", obs.failureMsg), zap.String("for", obs.id), zap.Error(err)) - return - } - - // if no URL to send cut off notification to, do nothing - if failureURL == "" { - return - } - - // Send a "you've been cut off" warning message - payload := bytes.NewReader(msg) - req, err := http.NewRequest("POST", failureURL, payload) - if err != nil { - // Failure - obs.logger.Error("Unable to send cut-off notification", zap.String("notification", - failureURL), zap.String("for", obs.id), zap.Error(err)) - return - } - req.Header.Set("Content-Type", wrp.MimeTypeJson) - - if secret != "" { - h := hmac.New(sha1.New, []byte(secret)) - h.Write(msg) - sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil))) - req.Header.Set("X-Webpa-Signature", sig) - } - - resp, err := obs.sender.Do(req) - if err != nil { - // Failure - obs.logger.Error("Unable to send cut-off notification", zap.String("notification", failureURL), zap.String("for", obs.id), zap.Error(err)) - return - } - - if resp == nil { - // Failure - obs.logger.Error("Unable to send cut-off notification, nil response", zap.String("notification", failureURL)) - return - } - - // Success - - if resp.Body != nil { - io.Copy(io.Discard, resp.Body) - resp.Body.Close() - - } -} +// func (obs *CaduceusOutboundSender) send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) { +// defer func() { +// if r := recover(); nil != r { +// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: DropsDueToPanic}).Add(1.0) +// obs.logger.Error("goroutine send() panicked", zap.String("id", obs.id), zap.Any("panic", r)) +// // don't silence the panic +// panic(r) +// } + +// obs.workers.Release() +// obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(-1.0) +// }() + +// payload := msg.Payload +// body := payload +// var payloadReader *bytes.Reader + +// // Use the internal content type unless the accept type is wrp +// contentType := msg.ContentType +// switch acceptType { +// case "wrp", wrp.MimeTypeMsgpack, wrp.MimeTypeWrp: +// // WTS - We should pass the original, raw WRP event instead of +// // re-encoding it. +// contentType = wrp.MimeTypeMsgpack +// buffer := bytes.NewBuffer([]byte{}) +// encoder := wrp.NewEncoder(buffer, wrp.Msgpack) +// encoder.Encode(msg) +// body = buffer.Bytes() +// } +// payloadReader = bytes.NewReader(body) + +// req, err := http.NewRequest("POST", urls.Value.(string), payloadReader) +// if err != nil { +// // Report drop +// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "invalid_config"}).Add(1.0) +// obs.logger.Error("Invalid URL", zap.String("url", urls.Value.(string)), zap.String("id", obs.id), zap.Error(err)) +// return +// } + +// req.Header.Set("Content-Type", contentType) + +// // Add x-Midt-* headers +// wrphttp.AddMessageHeaders(req.Header, msg) + +// // Provide the old headers for now +// req.Header.Set("X-Webpa-Event", strings.TrimPrefix(msg.Destination, "event:")) +// req.Header.Set("X-Webpa-Transaction-Id", msg.TransactionUUID) + +// // Add the device id without the trailing service +// id, _ := device.ParseID(msg.Source) +// // Deprecated: X-Webpa-Device-Id should only be used for backwards compatibility. +// // Use X-Webpa-Source instead. +// req.Header.Set("X-Webpa-Device-Id", string(id)) +// // Deprecated: X-Webpa-Device-Name should only be used for backwards compatibility. +// // Use X-Webpa-Source instead. +// req.Header.Set("X-Webpa-Device-Name", string(id)) +// req.Header.Set("X-Webpa-Source", msg.Source) +// req.Header.Set("X-Webpa-Destination", msg.Destination) + +// // Apply the secret + +// if secret != "" { +// s := hmac.New(sha1.New, []byte(secret)) +// s.Write(body) +// sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(s.Sum(nil))) +// req.Header.Set("X-Webpa-Signature", sig) +// } + +// // since eventType is only used to enrich metrics and logging, remove invalid UTF-8 characters from the URL +// eventType := strings.ToValidUTF8(msg.FindEventStringSubMatch(), "") + +// retryOptions := xhttp.RetryOptions{ +// Logger: obs.logger, +// Retries: obs.deliveryRetries, +// Interval: obs.deliveryInterval, +// Counter: gokitprometheus.NewCounter(obs.metrics.deliveryRetryCounter.MustCurryWith(prometheus.Labels{urlLabel: obs.id, eventLabel: eventType})), +// // Always retry on failures up to the max count. +// ShouldRetry: xhttp.ShouldRetry, +// ShouldRetryStatus: xhttp.RetryCodes, +// } + +// // update subsequent requests with the next url in the list upon failure +// retryOptions.UpdateRequest = func(request *http.Request) { +// urls = urls.Next() +// tmp, err := url.Parse(urls.Value.(string)) +// if err != nil { +// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: updateRequestURLFailedReason}).Add(1) +// obs.logger.Error("failed to update url", zap.String("url", urls.Value.(string)), zap.Error(err)) +// return +// } +// request.URL = tmp +// } + +// // Send it +// obs.logger.Debug("attempting to send event", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) + +// retryer := xhttp.RetryTransactor(retryOptions, obs.sender.Do) +// client := obs.clientMiddleware(doerFunc(retryer)) +// resp, err := client.Do(req) + +// var deliveryCounterLabels prometheus.Labels +// code := messageDroppedCode +// reason := noErrReason +// l := obs.logger +// if err != nil { +// // Report failure +// reason = getDoErrReason(err) +// if resp != nil { +// code = strconv.Itoa(resp.StatusCode) +// } + +// l = obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) +// deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} +// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason}).Add(1) +// } else { +// // Report Result +// code = strconv.Itoa(resp.StatusCode) +// // read until the response is complete before closing to allow +// // connection reuse +// if resp.Body != nil { +// io.Copy(io.Discard, resp.Body) +// resp.Body.Close() +// } + +// deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} +// } + +// obs.metrics.deliveryCounter.With(deliveryCounterLabels).Add(1.0) +// l.Debug("event sent-ish", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination), zap.String("code", code), zap.String("url", req.URL.String())) +// } + +// // queueOverflow handles the logic of what to do when a queue overflows: +// // cutting off the webhook for a time and sending a cut off notification +// // to the failure URL. +// func (obs *CaduceusOutboundSender) queueOverflow() { +// obs.mutex.Lock() +// if time.Now().Before(obs.dropUntil) { +// obs.mutex.Unlock() +// return +// } +// obs.dropUntil = time.Now().Add(obs.cutOffPeriod) +// obs.metrics.dropUntilGauge.With(prometheus.Labels{urlLabel: obs.id}).Set(float64(obs.dropUntil.Unix())) +// secret := obs.listener.Webhook.Config.Secret +// failureMsg := obs.failureMsg +// failureURL := obs.listener.Webhook.FailureURL +// obs.mutex.Unlock() + +// obs.metrics.cutOffCounter.With(prometheus.Labels{urlLabel: obs.id}).Add(1.0) + +// // We empty the queue but don't close the channel, because we're not +// // shutting down. +// obs.Empty(obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "cut_off"})) + +// msg, err := json.Marshal(failureMsg) +// if err != nil { +// obs.logger.Error("Cut-off notification json.Marshal failed", zap.Any("failureMessage", obs.failureMsg), zap.String("for", obs.id), zap.Error(err)) +// return +// } + +// // if no URL to send cut off notification to, do nothing +// if failureURL == "" { +// return +// } + +// // Send a "you've been cut off" warning message +// payload := bytes.NewReader(msg) +// req, err := http.NewRequest("POST", failureURL, payload) +// if err != nil { +// // Failure +// obs.logger.Error("Unable to send cut-off notification", zap.String("notification", +// failureURL), zap.String("for", obs.id), zap.Error(err)) +// return +// } +// req.Header.Set("Content-Type", wrp.MimeTypeJson) + +// if secret != "" { +// h := hmac.New(sha1.New, []byte(secret)) +// h.Write(msg) +// sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil))) +// req.Header.Set("X-Webpa-Signature", sig) +// } + +// resp, err := obs.sender.Do(req) +// if err != nil { +// // Failure +// obs.logger.Error("Unable to send cut-off notification", zap.String("notification", failureURL), zap.String("for", obs.id), zap.Error(err)) +// return +// } + +// if resp == nil { +// // Failure +// obs.logger.Error("Unable to send cut-off notification, nil response", zap.String("notification", failureURL)) +// return +// } + +// // Success + +// if resp.Body != nil { +// io.Copy(io.Discard, resp.Body) +// resp.Body.Close() + +// } +// } diff --git a/outboundSender_test.go b/outboundSender_test.go index 80f18f82..e381a887 100644 --- a/outboundSender_test.go +++ b/outboundSender_test.go @@ -830,7 +830,12 @@ func TestOverflowNoFailureURL(t *testing.T) { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - obs.(*CaduceusOutboundSender).queueOverflow() + whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) + if err != nil { + assert.Fail("Error returned by NewWebhookOutboundSender") + } + + whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } @@ -876,7 +881,13 @@ func TestOverflowValidFailureURL(t *testing.T) { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - obs.(*CaduceusOutboundSender).queueOverflow() + whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) + if err != nil { + assert.Fail("Error returned by NewWebhookOutboundSender") + } + + whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + assert.NotNil(output.String()) } @@ -923,7 +934,12 @@ func TestOverflowValidFailureURLWithSecret(t *testing.T) { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - obs.(*CaduceusOutboundSender).queueOverflow() + whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) + if err != nil { + assert.Fail("Error returned by NewWebhookOutboundSender") + } + + whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } @@ -961,7 +977,13 @@ func TestOverflowValidFailureURLError(t *testing.T) { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - obs.(*CaduceusOutboundSender).queueOverflow() + whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) + if err != nil { + assert.Fail("Error returned by NewWebhookOutboundSender") + } + + whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + assert.NotNil(output.String()) } diff --git a/senderWrapper.go b/senderWrapper.go index 1a658c12..0f3715af 100644 --- a/senderWrapper.go +++ b/senderWrapper.go @@ -114,7 +114,7 @@ func (swf SenderWrapperFactory) New() (SenderWrapper, error) { func (sw *CaduceusSenderWrapper) Update(list []ancla.InternalWebhook) { // We'll like need this, so let's get one ready osf := OutboundSenderFactory{ - Sender: sw.sender, + Sender: sw.sender, // ** REMOVE *** Dispatcher goes here - dispatcher factory? CutOffPeriod: sw.cutOffPeriod, NumWorkers: sw.numWorkersPerSender, QueueSize: sw.queueSizePerSender, @@ -153,6 +153,7 @@ func (sw *CaduceusSenderWrapper) Update(list []ancla.InternalWebhook) { } } +// *** REMOVE - wrp.Message entrypoint ****// // Queue is used to send all the possible outbound senders a request. This // function performs the fan-out and filtering to multiple possible endpoints. func (sw *CaduceusSenderWrapper) Queue(msg *wrp.Message) { @@ -161,6 +162,7 @@ func (sw *CaduceusSenderWrapper) Queue(msg *wrp.Message) { sw.metrics.eventType.With(prometheus.Labels{eventLabel: msg.FindEventStringSubMatch()}).Add(1) + // easiest thing to do might be to add a method to CaduceusSenderWrapper to add the pre-defined outbound senders? for _, v := range sw.senders { v.Queue(msg) } diff --git a/webhook_dispatcher.go b/webhook_dispatcher.go new file mode 100644 index 00000000..4aa07699 --- /dev/null +++ b/webhook_dispatcher.go @@ -0,0 +1,250 @@ +// SPDX-FileCopyrightText: 2021 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "container/ring" + "crypto/hmac" + "crypto/sha1" //nolint:gosec + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + gokitprometheus "github.com/go-kit/kit/metrics/prometheus" + + "github.com/prometheus/client_golang/prometheus" + "github.com/xmidt-org/webpa-common/v2/device" + "github.com/xmidt-org/webpa-common/v2/xhttp" + "github.com/xmidt-org/wrp-go/v3" + "github.com/xmidt-org/wrp-go/v3/wrphttp" + "go.uber.org/zap" +) + +type WebhookDispatcher struct { + obs *CaduceusOutboundSender +} + +// needed - wg, queue, httpclient, urls, metrics +func NewWebhookDispatcher(obs *CaduceusOutboundSender) Dispatcher { + return &WebhookDispatcher{ + obs: obs, + } + +} + +// worker is the routine that actually takes the queued messages and delivers +// them to the listeners outside webpa +func (d *WebhookDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) { + defer func() { + if r := recover(); nil != r { + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.obs.id, reasonLabel: DropsDueToPanic}).Add(1.0) + d.obs.logger.Error("goroutine send() panicked", zap.String("id", d.obs.id), zap.Any("panic", r)) + // don't silence the panic + panic(r) + } + + d.obs.workers.Release() + d.obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: d.obs.id}).Add(-1.0) + }() + + payload := msg.Payload + body := payload + var payloadReader *bytes.Reader + + // Use the internal content type unless the accept type is wrp + contentType := msg.ContentType + switch acceptType { + // Not sure if we will break clients here if we lint this + case "wrp", wrp.MimeTypeMsgpack, wrp.MimeTypeWrp: //nolint:staticcheck + // WTS - We should pass the original, raw WRP event instead of + // re-encoding it. + contentType = wrp.MimeTypeMsgpack + buffer := bytes.NewBuffer([]byte{}) + encoder := wrp.NewEncoder(buffer, wrp.Msgpack) + encoder.Encode(msg) + body = buffer.Bytes() + } + payloadReader = bytes.NewReader(body) + + req, err := http.NewRequest("POST", urls.Value.(string), payloadReader) + if err != nil { + // Report drop + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.obs.id, reasonLabel: "invalid_config"}).Add(1.0) + d.obs.logger.Error("Invalid URL", zap.String("url", urls.Value.(string)), zap.String("id", d.obs.id), zap.Error(err)) + return + } + + req.Header.Set("Content-Type", contentType) + + // Add x-Midt-* headers + wrphttp.AddMessageHeaders(req.Header, msg) + + // Provide the old headers for now + req.Header.Set("X-Webpa-Event", strings.TrimPrefix(msg.Destination, "event:")) + req.Header.Set("X-Webpa-Transaction-Id", msg.TransactionUUID) + + // Add the device id without the trailing service + id, _ := device.ParseID(msg.Source) + // Deprecated: X-Webpa-Device-Id should only be used for backwards compatibility. + // Use X-Webpa-Source instead. + req.Header.Set("X-Webpa-Device-Id", string(id)) + // Deprecated: X-Webpa-Device-Name should only be used for backwards compatibility. + // Use X-Webpa-Source instead. + req.Header.Set("X-Webpa-Device-Name", string(id)) + req.Header.Set("X-Webpa-Source", msg.Source) + req.Header.Set("X-Webpa-Destination", msg.Destination) + + // Apply the secret + + if secret != "" { + s := hmac.New(sha1.New, []byte(secret)) + s.Write(body) + sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(s.Sum(nil))) + req.Header.Set("X-Webpa-Signature", sig) + } + + // since eventType is only used to enrich metrics and logging, remove invalid UTF-8 characters from the URL + eventType := strings.ToValidUTF8(msg.FindEventStringSubMatch(), "") + + retryOptions := xhttp.RetryOptions{ + Logger: d.obs.logger, + Retries: d.obs.deliveryRetries, + Interval: d.obs.deliveryInterval, + Counter: gokitprometheus.NewCounter(d.obs.metrics.deliveryRetryCounter.MustCurryWith(prometheus.Labels{urlLabel: d.obs.id, eventLabel: eventType})), + // Always retry on failures up to the max count. + ShouldRetry: xhttp.ShouldRetry, + ShouldRetryStatus: xhttp.RetryCodes, + } + + // update subsequent requests with the next url in the list upon failure + retryOptions.UpdateRequest = func(request *http.Request) { + urls = urls.Next() + tmp, err := url.Parse(urls.Value.(string)) + if err != nil { + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: updateRequestURLFailedReason}).Add(1) + d.obs.logger.Error("failed to update url", zap.String("url", urls.Value.(string)), zap.Error(err)) + return + } + request.URL = tmp + } + + // Send it + d.obs.logger.Debug("attempting to send event", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) + + // do not know how to address bodyclose on the below line + retryer := xhttp.RetryTransactor(retryOptions, d.obs.httpSender.Do) //nolint:bodyclose + client := d.obs.clientMiddleware(doerFunc(retryer)) + resp, err := client.Do(req) + defer func() { + if resp.Body != nil { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } + }() + + var deliveryCounterLabels prometheus.Labels + code := messageDroppedCode + reason := noErrReason + l := d.obs.logger + if err != nil { + // Report failure + reason = getDoErrReason(err) + if resp != nil { + code = strconv.Itoa(resp.StatusCode) + } + + l = d.obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) + deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason}).Add(1) + } else { + // Report Result + code = strconv.Itoa(resp.StatusCode) + + deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} + } + + d.obs.metrics.deliveryCounter.With(deliveryCounterLabels).Add(1.0) + l.Debug("event sent-ish", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination), zap.String("code", code), zap.String("url", req.URL.String())) +} + +// queueOverflow handles the logic of what to do when a queue overflows: +// cutting off the webhook for a time and sending a cut off notification +// to the failure URL. +func (d *WebhookDispatcher) QueueOverflow() { + d.obs.mutex.Lock() + if time.Now().Before(d.obs.dropUntil) { + d.obs.mutex.Unlock() + return + } + d.obs.dropUntil = time.Now().Add(d.obs.cutOffPeriod) + d.obs.metrics.dropUntilGauge.With(prometheus.Labels{urlLabel: d.obs.id}).Set(float64(d.obs.dropUntil.Unix())) + secret := d.obs.listener.Webhook.Config.Secret + failureMsg := d.obs.failureMsg + failureURL := d.obs.listener.Webhook.FailureURL + d.obs.mutex.Unlock() + + d.obs.metrics.cutOffCounter.With(prometheus.Labels{urlLabel: d.obs.id}).Add(1.0) + + // We empty the queue but don't close the channel, because we're not + // shutting down. + d.obs.Empty(d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.obs.id, reasonLabel: "cut_off"})) + + msg, err := json.Marshal(failureMsg) + if err != nil { + d.obs.logger.Error("Cut-off notification json.Marshal failed", zap.Any("failureMessage", d.obs.failureMsg), zap.String("for", d.obs.id), zap.Error(err)) + return + } + + // if no URL to send cut off notification to, do nothing + if failureURL == "" { + return + } + + // Send a "you've been cut off" warning message + payload := bytes.NewReader(msg) + req, err := http.NewRequest("POST", failureURL, payload) + if err != nil { + // Failure + d.obs.logger.Error("Unable to send cut-off notification", zap.String("notification", + failureURL), zap.String("for", d.obs.id), zap.Error(err)) + return + } + req.Header.Set("Content-Type", wrp.MimeTypeJson) + + if secret != "" { + // we can't break the existing clients + h := hmac.New(sha1.New, []byte(secret)) //nolint:gosec + h.Write(msg) + sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil))) + req.Header.Set("X-Webpa-Signature", sig) + } + + resp, err := d.obs.httpSender.Do(req) + if err != nil { + // Failure + d.obs.logger.Error("Unable to send cut-off notification", zap.String("notification", failureURL), zap.String("for", d.obs.id), zap.Error(err)) + return + } + + if resp == nil { + // Failure + d.obs.logger.Error("Unable to send cut-off notification, nil response", zap.String("notification", failureURL)) + return + } + + // Success + + if resp.Body != nil { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + + } +} diff --git a/webhook_outbound_sender.go b/webhook_outbound_sender.go new file mode 100644 index 00000000..d7aa5a23 --- /dev/null +++ b/webhook_outbound_sender.go @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: 2021 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "container/ring" + "errors" + "fmt" + "math/rand" + "net/url" + "regexp" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/xmidt-org/ancla" + "github.com/xmidt-org/wrp-go/v3" + "go.uber.org/zap" +) + +const Secret = "XxxxxX" + +type WebhookOutboundSender struct { + obs *CaduceusOutboundSender +} + +func NewWebhookOutboundSender(obs *CaduceusOutboundSender) (OutboundSender, error) { + dispatcher := NewWebhookDispatcher(obs) + + whSender := &WebhookOutboundSender{ + obs: obs, + } + + whSender.Dispatch(dispatcher) + + return whSender, nil +} + +func (s *WebhookOutboundSender) RetiredSince() time.Time { + s.obs.mutex.RLock() + deliverUntil := s.obs.deliverUntil + s.obs.mutex.RUnlock() + return deliverUntil +} + +func (s *WebhookOutboundSender) Dispatch(d Dispatcher) { + s.obs.Dispatch(d) +} + +func (s *WebhookOutboundSender) Queue(msg *wrp.Message) { + s.obs.Queue(msg) +} + +func (s *WebhookOutboundSender) Shutdown(gentle bool) { + s.obs.Shutdown(gentle) +} + +func (s *WebhookOutboundSender) Update(wh ancla.InternalWebhook) (err error) { + // Validate the failure URL, if present + if wh.Webhook.FailureURL != "" { + if _, err = url.ParseRequestURI(wh.Webhook.FailureURL); err != nil { + return + } + } + + // Create and validate the event regex objects + // nolint:prealloc + var events []*regexp.Regexp + for _, event := range wh.Webhook.Events { + var re *regexp.Regexp + if re, err = regexp.Compile(event); err != nil { + return + } + + events = append(events, re) + } + if len(events) < 1 { + err = errors.New("events must not be empty") + return + } + + // Create the matcher regex objects + matcher := []*regexp.Regexp{} + for _, item := range wh.Webhook.Matcher.DeviceID { + if item == ".*" { + // Match everything - skip the filtering + matcher = []*regexp.Regexp{} + break + } + + var re *regexp.Regexp + if re, err = regexp.Compile(item); err != nil { + err = fmt.Errorf("invalid matcher item: '%s'", item) + return + } + matcher = append(matcher, re) + } + + // Validate the various urls + urlCount := len(wh.Webhook.Config.AlternativeURLs) + for i := 0; i < urlCount; i++ { + _, err = url.Parse(wh.Webhook.Config.AlternativeURLs[i]) + if err != nil { + s.obs.logger.Error("failed to update url", zap.Any("url", wh.Webhook.Config.AlternativeURLs[i]), zap.Error(err)) + return + } + } + + s.obs.metrics.renewalTimeGauge.With(prometheus.Labels{urlLabel: s.obs.id}).Set(float64(time.Now().Unix())) + + // write/update obs + s.obs.mutex.Lock() + + s.obs.listener = wh + + s.obs.failureMsg.Original = wh + // Don't share the secret with others when there is an error. + s.obs.failureMsg.Original.Webhook.Config.Secret = Secret + + s.obs.listener.Webhook.FailureURL = wh.Webhook.FailureURL + s.obs.deliverUntil = wh.Webhook.Until + s.obs.metrics.deliverUntilGauge.With(prometheus.Labels{urlLabel: s.obs.id}).Set(float64(s.obs.deliverUntil.Unix())) + + s.obs.events = events + + s.obs.metrics.deliveryRetryMaxGauge.With(prometheus.Labels{urlLabel: s.obs.id}).Set(float64(s.obs.deliveryRetries)) + + // if matcher list is empty set it nil for Queue() logic + s.obs.matcher = nil + if 0 < len(matcher) { + s.obs.matcher = matcher + } + + if urlCount == 0 { + s.obs.urls = ring.New(1) + s.obs.urls.Value = s.obs.id + } else { + r := ring.New(urlCount) + for i := 0; i < urlCount; i++ { + r.Value = wh.Webhook.Config.AlternativeURLs[i] + r = r.Next() + } + s.obs.urls = r + } + + // Randomize where we start so all the instances don't synchronize + // not sure why lint is complaining about below line, it is not using crypto/rand + r := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec + offset := r.Intn(s.obs.urls.Len()) + for 0 < offset { + s.obs.urls = s.obs.urls.Next() + offset-- + } + + // Update this here in case we make this configurable later + s.obs.metrics.maxWorkersGauge.With(prometheus.Labels{urlLabel: s.obs.id}).Set(float64(s.obs.maxWorkers)) + + s.obs.mutex.Unlock() + + return +} From 552e98080e4d2f4c2b578f832ea305784868655e Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Thu, 4 Sep 2025 13:17:13 -0400 Subject: [PATCH 2/9] switch internal license out --- internal/batch/batch.go | 2 +- internal/batch/batch_submitter.go | 2 +- internal/batch/batch_test.go | 2 +- internal/kinesis/kinesis.go | 2 +- internal/kinesis/kinesis_test.go | 2 +- webhook_dispatcher.go | 2 +- webhook_outbound_sender.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/batch/batch.go b/internal/batch/batch.go index e24f939a..1c3cedf9 100644 --- a/internal/batch/batch.go +++ b/internal/batch/batch.go @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC -// SPDX-License-Identifier: LicenseRef-COMCAST +// SPDX-License-Identifier: Apache-2.0 package batch diff --git a/internal/batch/batch_submitter.go b/internal/batch/batch_submitter.go index 143e582f..76359f21 100644 --- a/internal/batch/batch_submitter.go +++ b/internal/batch/batch_submitter.go @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC -// SPDX-License-Identifier: LicenseRef-COMCAST +// SPDX-License-Identifier: Apache-2.0 package batch diff --git a/internal/batch/batch_test.go b/internal/batch/batch_test.go index e5c1f7de..e2571cfd 100644 --- a/internal/batch/batch_test.go +++ b/internal/batch/batch_test.go @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC -// SPDX-License-Identifier: LicenseRef-COMCAST +// SPDX-License-Identifier: Apache-2.0 package batch diff --git a/internal/kinesis/kinesis.go b/internal/kinesis/kinesis.go index d4e034c1..3544674a 100644 --- a/internal/kinesis/kinesis.go +++ b/internal/kinesis/kinesis.go @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC -// SPDX-License-Identifier: LicenseRef-COMCAST +// SPDX-License-Identifier: Apache-2.0 package kinesis diff --git a/internal/kinesis/kinesis_test.go b/internal/kinesis/kinesis_test.go index 399f674c..10c431c7 100644 --- a/internal/kinesis/kinesis_test.go +++ b/internal/kinesis/kinesis_test.go @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC -// SPDX-License-Identifier: LicenseRef-COMCAST +// SPDX-License-Identifier: Apache-2.0 package kinesis diff --git a/webhook_dispatcher.go b/webhook_dispatcher.go index 4aa07699..ad7a7239 100644 --- a/webhook_dispatcher.go +++ b/webhook_dispatcher.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2021 Comcast Cable Communications Management, LLC +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC // SPDX-License-Identifier: Apache-2.0 package main diff --git a/webhook_outbound_sender.go b/webhook_outbound_sender.go index d7aa5a23..bb4d88fb 100644 --- a/webhook_outbound_sender.go +++ b/webhook_outbound_sender.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2021 Comcast Cable Communications Management, LLC +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC // SPDX-License-Identifier: Apache-2.0 package main From bfdc24e3fe40627551e079eb8fcfe34f5330c73c Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Fri, 5 Sep 2025 09:30:06 -0400 Subject: [PATCH 3/9] fix factory and unit tests --- outboundSender.go | 2 +- outboundSender_test.go | 35 ++++++++--------------------------- webhook_dispatcher.go | 2 +- webhook_outbound_sender.go | 1 + 4 files changed, 11 insertions(+), 29 deletions(-) diff --git a/outboundSender.go b/outboundSender.go index 6b009cfa..eed54e89 100644 --- a/outboundSender.go +++ b/outboundSender.go @@ -187,7 +187,7 @@ func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) { caduceusOutboundSender.workers = semaphore.New(caduceusOutboundSender.maxWorkers) caduceusOutboundSender.wg.Add(1) - return caduceusOutboundSender, nil + return NewWebhookOutboundSender(caduceusOutboundSender) } func (obs *CaduceusOutboundSender) Dispatch(d Dispatcher) { diff --git a/outboundSender_test.go b/outboundSender_test.go index e381a887..3234ecdc 100644 --- a/outboundSender_test.go +++ b/outboundSender_test.go @@ -197,6 +197,7 @@ func simpleFactorySetup(trans *transport, cutOffPeriod time.Duration, matcher [] DeliveryRetries: 1, Logger: zap.NewNop(), } + } func simpleRequest() *wrp.Message { @@ -830,12 +831,7 @@ func TestOverflowNoFailureURL(t *testing.T) { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) - if err != nil { - assert.Fail("Error returned by NewWebhookOutboundSender") - } - - whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + //obs.QueueOverflow() assert.NotNil(output.String()) } @@ -877,16 +873,11 @@ func TestOverflowValidFailureURL(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*CaduceusOutboundSender); !ok { + if _, ok := obs.(*WebhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) - if err != nil { - assert.Fail("Error returned by NewWebhookOutboundSender") - } - - whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } @@ -930,16 +921,11 @@ func TestOverflowValidFailureURLWithSecret(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*CaduceusOutboundSender); !ok { + if _, ok := obs.(*WebhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) - if err != nil { - assert.Fail("Error returned by NewWebhookOutboundSender") - } - - whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } @@ -973,16 +959,11 @@ func TestOverflowValidFailureURLError(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*CaduceusOutboundSender); !ok { + if _, ok := obs.(*WebhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") } - whs, err := NewWebhookOutboundSender(obs.(*CaduceusOutboundSender)) - if err != nil { - assert.Fail("Error returned by NewWebhookOutboundSender") - } - - whs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } diff --git a/webhook_dispatcher.go b/webhook_dispatcher.go index ad7a7239..9f0d4c8e 100644 --- a/webhook_dispatcher.go +++ b/webhook_dispatcher.go @@ -144,7 +144,7 @@ func (d *WebhookDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg client := d.obs.clientMiddleware(doerFunc(retryer)) resp, err := client.Do(req) defer func() { - if resp.Body != nil { + if resp != nil { io.Copy(io.Discard, resp.Body) resp.Body.Close() } diff --git a/webhook_outbound_sender.go b/webhook_outbound_sender.go index bb4d88fb..ab2ed04f 100644 --- a/webhook_outbound_sender.go +++ b/webhook_outbound_sender.go @@ -24,6 +24,7 @@ type WebhookOutboundSender struct { obs *CaduceusOutboundSender } +// this needs to be a factory and you should not be able to create it any other way func NewWebhookOutboundSender(obs *CaduceusOutboundSender) (OutboundSender, error) { dispatcher := NewWebhookDispatcher(obs) From a0d55709565ec49fa8741fbc8695277986029e0a Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Fri, 5 Sep 2025 09:38:13 -0400 Subject: [PATCH 4/9] fix test --- outboundSender_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/outboundSender_test.go b/outboundSender_test.go index 3234ecdc..ff3480e9 100644 --- a/outboundSender_test.go +++ b/outboundSender_test.go @@ -791,13 +791,13 @@ func TestUpdate(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*CaduceusOutboundSender); !ok { - assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") + if _, ok := obs.(*WebhookOutboundSender); !ok { + assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } - assert.Equal(now, obs.(*CaduceusOutboundSender).deliverUntil, "Delivery should match original value.") + assert.Equal(now, obs.(*WebhookOutboundSender).obs.deliverUntil, "Delivery should match original value.") obs.Update(w2) - assert.Equal(later, obs.(*CaduceusOutboundSender).deliverUntil, "Delivery should match new value.") + assert.Equal(later, obs.(*WebhookOutboundSender).obs.deliverUntil, "Delivery should match new value.") obs.Shutdown(true) } @@ -827,8 +827,8 @@ func TestOverflowNoFailureURL(t *testing.T) { assert.Nil(err) - if _, ok := obs.(*CaduceusOutboundSender); !ok { - assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") + if _, ok := obs.(*WebhookOutboundSender); !ok { + assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } //obs.QueueOverflow() @@ -874,7 +874,7 @@ func TestOverflowValidFailureURL(t *testing.T) { assert.Nil(err) if _, ok := obs.(*WebhookOutboundSender); !ok { - assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") + assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() @@ -922,7 +922,7 @@ func TestOverflowValidFailureURLWithSecret(t *testing.T) { assert.Nil(err) if _, ok := obs.(*WebhookOutboundSender); !ok { - assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") + assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() @@ -960,7 +960,7 @@ func TestOverflowValidFailureURLError(t *testing.T) { assert.Nil(err) if _, ok := obs.(*WebhookOutboundSender); !ok { - assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a CaduceusOutboundSender.") + assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() From 54c9fbbde3f44284535978d6f33919dcc6713b4d Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Wed, 10 Sep 2025 11:22:08 -0400 Subject: [PATCH 5/9] add stream sender --- dispatcher.go | 15 -- go.mod | 1 + go.sum | 2 + internal/kinesis/kinesis.go | 6 +- internal/kinesis/kinesis_test.go | 2 +- internal/stream/sender.go | 82 +++++++++ internal/stream/sender_test.go | 71 ++++++++ main.go | 15 ++ outboundSender.go | 292 ++----------------------------- outboundSender_test.go | 20 +-- senderWrapper.go | 18 +- stream_dispatcher.go | 102 +++++++++++ stream_outbound_sender.go | 57 ++++++ webhook_dispatcher.go | 1 - webhook_outbound_sender.go | 16 +- 15 files changed, 387 insertions(+), 313 deletions(-) create mode 100644 internal/stream/sender.go create mode 100644 internal/stream/sender_test.go create mode 100644 stream_dispatcher.go create mode 100644 stream_outbound_sender.go diff --git a/dispatcher.go b/dispatcher.go index d199c827..830b8481 100644 --- a/dispatcher.go +++ b/dispatcher.go @@ -13,18 +13,3 @@ type Dispatcher interface { QueueOverflow() Send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) } - -func DispatcherFactory(webhook bool, obs *CaduceusOutboundSender) Dispatcher { - - if webhook { - return &WebhookDispatcher{ - obs: obs, - } - } - - // TODO - return &WebhookDispatcher{ - obs: obs, - } - -} diff --git a/go.mod b/go.mod index 47e74af0..c2f7a5d8 100644 --- a/go.mod +++ b/go.mod @@ -115,6 +115,7 @@ require ( github.com/xmidt-org/argus v0.10.18 // indirect github.com/xmidt-org/arrange v0.4.0 // indirect github.com/xmidt-org/chronon v0.1.4 // indirect + github.com/xmidt-org/retry v0.0.4 go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect diff --git a/go.sum b/go.sum index e47cb4b6..aae7f4f0 100644 --- a/go.sum +++ b/go.sum @@ -1793,6 +1793,8 @@ github.com/xmidt-org/httpaux v0.3.2/go.mod h1:qmlPisXf80FTi3y4gX43eYbCVruSQyvu+F github.com/xmidt-org/httpaux v0.4.0/go.mod h1:UypqZwuZV1nn8D6+K1JDb+im9IZrLNg/2oO/Bgiybxc= github.com/xmidt-org/httpaux v0.4.2 h1:O6KTPy9Dtx6m5Ezb0nCQeAZWvBDgZvx82P16FSNauIE= github.com/xmidt-org/httpaux v0.4.2/go.mod h1:tZJ+SBoGNCxDOLopuSqrxaCkIVAQ+aPjNRf2XfMVwJA= +github.com/xmidt-org/retry v0.0.4 h1:GUnMqjNUm2W0qfjmXS99keQ4qvyFsjh+HC66NXqLQfU= +github.com/xmidt-org/retry v0.0.4/go.mod h1:Btl7o0Ts6iNEkF2liNiQepkkpHrK4rdGmqDVQ6KRHxo= github.com/xmidt-org/sallust v0.1.5/go.mod h1:azcKBypudADIeZ3Em8zGjVq3yQ7n4ueSvM/degHMIxo= github.com/xmidt-org/sallust v0.2.0/go.mod h1:HCQQn7po8czynjxtNVyZ5vzWuTqJyJwnPWkQoqBX67s= github.com/xmidt-org/sallust v0.2.1/go.mod h1:68C0DLwD5xlhRznXTWmfUhx0etyrFpSOzYGU7jzmpzs= diff --git a/internal/kinesis/kinesis.go b/internal/kinesis/kinesis.go index 3544674a..fa7ce9fc 100644 --- a/internal/kinesis/kinesis.go +++ b/internal/kinesis/kinesis.go @@ -40,7 +40,7 @@ type KinesisAPI interface { type KinesisClient struct { svc KinesisAPI logger *zap.Logger - config Config + config *Config credsExpireAt time.Time } @@ -62,7 +62,7 @@ type Item struct { Item []byte } -func New(cfg Config, logger *zap.Logger) (KinesisClientAPI, error) { +func New(cfg *Config, logger *zap.Logger) (KinesisClientAPI, error) { k, credsExpireAt, err := getClient(cfg, logger) if err != nil { return nil, err @@ -76,7 +76,7 @@ func New(cfg Config, logger *zap.Logger) (KinesisClientAPI, error) { }, nil } -func getClient(cfg Config, logger *zap.Logger) (KinesisAPI, time.Time, error) { +func getClient(cfg *Config, logger *zap.Logger) (KinesisAPI, time.Time, error) { ctx := context.Background() // try removing this, not sure why we need it diff --git a/internal/kinesis/kinesis_test.go b/internal/kinesis/kinesis_test.go index 10c431c7..32794d76 100644 --- a/internal/kinesis/kinesis_test.go +++ b/internal/kinesis/kinesis_test.go @@ -70,7 +70,7 @@ func TestNewNoAwsRole(t *testing.T) { Endpoint: "http://localhost", } - kc, err := New(cfg, logger) + kc, err := New(&cfg, logger) kclient := kc.(*KinesisClient) assert.Nil(t, err) diff --git a/internal/stream/sender.go b/internal/stream/sender.go new file mode 100644 index 00000000..7b125ac7 --- /dev/null +++ b/internal/stream/sender.go @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: LicenseRef-COMCAST + +package stream + +import ( + "context" + "encoding/json" + "time" + + "github.com/xmidt-org/caduceus/internal/kinesis" + "github.com/xmidt-org/retry" + "github.com/xmidt-org/wrp-go/v3" + "go.uber.org/zap" +) + +const schemaVersion = "1.1" +const retries = 3 + +type StreamEventSender struct { + kc kinesis.KinesisClientAPI + url string + schemaVersion string + logger *zap.Logger +} + +type EventSender interface { + OnEvent(event []*wrp.Message) (int, error) + GetUrl() string +} + +var kPutRunner, _ = retry.NewRunner[int]( + retry.WithPolicyFactory[int](retry.Config{ + Interval: 10 * time.Millisecond, + MaxRetries: retries, + }), +) + +func New(url string, version string, kc kinesis.KinesisClientAPI, logger *zap.Logger) (EventSender, error) { + if schemaVersion == "" { + version = schemaVersion + } + return &StreamEventSender{ + kc: kc, + url: url, + schemaVersion: version, + logger: logger, + }, nil +} + +func (s *StreamEventSender) GetUrl() string { + return s.url +} + +// TODO - add a queue and a channel instead +func (s *StreamEventSender) OnEvent(msgs []*wrp.Message) (int, error) { + items := []kinesis.Item{} + for _, m := range msgs { + data, err := json.Marshal(m) + if err != nil { + s.logger.Error("error marshaling statusEvent", zap.Any("event", m), zap.Error(err)) + continue + } + items = append(items, kinesis.Item{Item: data, PartitionKey: m.TransactionUUID}) + } + + attempts := 0 + failedRecordCount, err := kPutRunner.Run( + context.Background(), + func(_ context.Context) (int, error) { + attempts++ + failedRecordCount, err := s.kc.PutRecords(items, s.url) + if err != nil { + s.logger.Error("kinesis.PutRecords error", zap.Int("attempt", attempts), zap.Error(err)) + } + + return failedRecordCount, err + }, + ) + + return failedRecordCount, err +} diff --git a/internal/stream/sender_test.go b/internal/stream/sender_test.go new file mode 100644 index 00000000..d115a193 --- /dev/null +++ b/internal/stream/sender_test.go @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: LicenseRef-COMCAST + +package stream + +import ( + "encoding/json" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/kinesis" + "go.uber.org/zap" + + "github.com/segmentio/ksuid" + "github.com/xmidt-org/wrp-go/v3" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + mykinesis "github.com/xmidt-org/caduceus/internal/kinesis" +) + +type mockKinesisClient struct{ mock.Mock } + +func newMockKinesisClient() *mockKinesisClient { return &mockKinesisClient{} } + +func (m *mockKinesisClient) PutRecord(event []byte, stream string, partitionKey string) (*kinesis.PutRecordOutput, error) { + args := m.Called(event, stream, partitionKey) + pio, _ := args.Get(0).(*kinesis.PutRecordOutput) + return pio, args.Error(1) +} + +func (m *mockKinesisClient) PutRecords(items []mykinesis.Item, stream string) (int, error) { + args := m.Called(items, stream) + return args.Get(0).(int), args.Error(1) +} + +func (m *mockKinesisClient) GetRecords(stream string) (*kinesis.GetRecordsOutput, error) { + args := m.Called(stream) + pio, _ := args.Get(0).(*kinesis.GetRecordsOutput) + return pio, args.Error(1) +} + +func TestOnEvent(t *testing.T) { + assert := assert.New(t) + + uuid := ksuid.New() + + e := &wrp.Message{ + TransactionUUID: uuid.String(), + } + + var kc = newMockKinesisClient() + + sender, err := New("", "", kc, zap.NewExample()) + assert.Nil(err) + + mockCall := kc.On("PutRecords", mock.Anything, mock.Anything).Return(0, nil) + + events := []*wrp.Message{e} + failedRecordCount, err := sender.OnEvent(events) + assert.Equal(0, failedRecordCount) + assert.Nil(err) + + items := mockCall.Parent.Calls[0].Arguments.Get(0).([]mykinesis.Item) + var msg = wrp.Message{} + err = json.Unmarshal(items[0].Item, &msg) + assert.NoError(err) + + assert.Equal("112233445566", items[0].PartitionKey) + // TODO - assert message content + assert.Equal("TestStream", mockCall.Parent.Calls[0].Arguments.Get(1).(string)) +} diff --git a/main.go b/main.go index be22bd38..3779b5fc 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "github.com/spf13/viper" "github.com/xmidt-org/ancla" "github.com/xmidt-org/bascule/basculehelper" + "github.com/xmidt-org/caduceus/internal/kinesis" "github.com/xmidt-org/candlelight" "github.com/xmidt-org/httpaux/recovery" "github.com/xmidt-org/sallust" @@ -114,6 +115,19 @@ func caduceus(arguments []string) int { return 1 } + kinesisConfig := new(kinesis.Config) + err = v.Unmarshal(kinesisConfig) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to unmarshal kinesis configuration data into struct: %s\n", err) + return 1 + } + + streamSender, err := kinesis.New(kinesisConfig, logger) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to kinesis client: %s\n", err) + return 1 + } + tracing, err := loadTracing(v, applicationName) if err != nil { fmt.Fprintf(os.Stderr, "Unable to build tracing component: %v \n", err) @@ -147,6 +161,7 @@ func caduceus(arguments []string) int { Transport: tr, Timeout: caduceusConfig.Sender.ClientTimeout, }).Do), + StreamSender: streamSender, CustomPIDs: caduceusConfig.Sender.CustomPIDs, DisablePartnerIDs: caduceusConfig.Sender.DisablePartnerIDs, }.New() diff --git a/outboundSender.go b/outboundSender.go index eed54e89..dee98e01 100644 --- a/outboundSender.go +++ b/outboundSender.go @@ -20,6 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/xmidt-org/ancla" + "github.com/xmidt-org/caduceus/internal/kinesis" "github.com/xmidt-org/webpa-common/v2/semaphore" @@ -52,6 +53,12 @@ type OutboundSenderFactory struct { // Sender func(*http.Request) (*http.Response, error) Sender httpClient + // The kinesis client to use for outbound requests. + StreamSender kinesis.KinesisClientAPI + + // version + StreamVersion string + // The number of delivery workers to create and use. NumWorkers int @@ -84,6 +91,9 @@ type OutboundSenderFactory struct { // Dispatcher sends the events Dispatcher Dispatcher + + // whether or not this is a webhook or a stream + IsStream bool } type OutboundSender interface { @@ -101,6 +111,8 @@ type CaduceusOutboundSender struct { deliverUntil time.Time dropUntil time.Time httpSender httpClient + streamSender kinesis.KinesisClientAPI + streamVersion string events []*regexp.Regexp matcher []*regexp.Regexp queueSize int @@ -151,6 +163,8 @@ func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) { id: strings.ToValidUTF8(osf.Listener.Webhook.Config.URL, ""), listener: osf.Listener, httpSender: osf.Sender, + streamSender: osf.StreamSender, + streamVersion: osf.StreamVersion, queueSize: osf.QueueSize, cutOffPeriod: osf.CutOffPeriod, deliverUntil: osf.Listener.Webhook.Until, @@ -187,6 +201,10 @@ func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) { caduceusOutboundSender.workers = semaphore.New(caduceusOutboundSender.maxWorkers) caduceusOutboundSender.wg.Add(1) + if osf.IsStream { + return NewStreamOutboundSender(caduceusOutboundSender) + } + return NewWebhookOutboundSender(caduceusOutboundSender) } @@ -506,277 +524,3 @@ Loop: obs.workers.Acquire() } } - -// func (obs *CaduceusOutboundSender) dispatcher() { -// defer obs.wg.Done() -// var ( -// urls *ring.Ring -// secret, accept string -// ) - -// Loop: -// for { -// // Always pull a new queue in case we have been cutoff or are shutting -// // down. -// msg, ok := <-obs.queue.Load().(chan *wrp.Message) -// // The dispatcher cannot get stuck blocking here forever (caused by an -// // empty queue that is replaced and then Queue() starts adding to the -// // new queue) because: -// // - queue is only replaced on cutoff and shutdown -// // - on cutoff, the first queue is always full so we will definitely -// // get a message, drop it because we're cut off, then get the new -// // queue and block until the cut off ends and Queue() starts queueing -// // messages again. -// // - on graceful shutdown, the queue is closed and then the dispatcher -// // will send all messages, then break the loop, gather workers, and -// // exit. -// // - on non graceful shutdown, the queue is closed and then replaced -// // with a new, empty queue that is also closed. -// // - If the first queue is empty, we immediately break the loop, -// // gather workers, and exit. -// // - If the first queue has messages, we drop a message as expired -// // pull in the new queue which is empty and closed, break the -// // loop, gather workers, and exit. -// // This is only true when a queue is empty and closed, which for us -// // only happens on Shutdown(). -// if !ok { -// break Loop -// } -// obs.metrics.queueDepthGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(-1.0) -// obs.mutex.RLock() -// urls = obs.urls -// // Move to the next URL to try 1st the next time. -// // This is okay because we run a single dispatcher and it's the -// // only one updating this field. -// obs.urls = obs.urls.Next() -// deliverUntil := obs.deliverUntil -// dropUntil := obs.dropUntil -// secret = obs.listener.Webhook.Config.Secret -// accept = obs.listener.Webhook.Config.ContentType -// obs.mutex.RUnlock() - -// now := time.Now() - -// if now.Before(dropUntil) { -// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "cut_off"}).Add(1.0) -// continue -// } -// if now.After(deliverUntil) { -// obs.Empty(obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "expired"})) -// continue -// } -// obs.workers.Acquire() -// obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(1.0) - -// go obs.send(urls, secret, accept, msg) -// } -// for i := 0; i < obs.maxWorkers; i++ { -// obs.workers.Acquire() -// } -// } - -// worker is the routine that actually takes the queued messages and delivers -// them to the listeners outside webpa -// func (obs *CaduceusOutboundSender) send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) { -// defer func() { -// if r := recover(); nil != r { -// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: DropsDueToPanic}).Add(1.0) -// obs.logger.Error("goroutine send() panicked", zap.String("id", obs.id), zap.Any("panic", r)) -// // don't silence the panic -// panic(r) -// } - -// obs.workers.Release() -// obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: obs.id}).Add(-1.0) -// }() - -// payload := msg.Payload -// body := payload -// var payloadReader *bytes.Reader - -// // Use the internal content type unless the accept type is wrp -// contentType := msg.ContentType -// switch acceptType { -// case "wrp", wrp.MimeTypeMsgpack, wrp.MimeTypeWrp: -// // WTS - We should pass the original, raw WRP event instead of -// // re-encoding it. -// contentType = wrp.MimeTypeMsgpack -// buffer := bytes.NewBuffer([]byte{}) -// encoder := wrp.NewEncoder(buffer, wrp.Msgpack) -// encoder.Encode(msg) -// body = buffer.Bytes() -// } -// payloadReader = bytes.NewReader(body) - -// req, err := http.NewRequest("POST", urls.Value.(string), payloadReader) -// if err != nil { -// // Report drop -// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "invalid_config"}).Add(1.0) -// obs.logger.Error("Invalid URL", zap.String("url", urls.Value.(string)), zap.String("id", obs.id), zap.Error(err)) -// return -// } - -// req.Header.Set("Content-Type", contentType) - -// // Add x-Midt-* headers -// wrphttp.AddMessageHeaders(req.Header, msg) - -// // Provide the old headers for now -// req.Header.Set("X-Webpa-Event", strings.TrimPrefix(msg.Destination, "event:")) -// req.Header.Set("X-Webpa-Transaction-Id", msg.TransactionUUID) - -// // Add the device id without the trailing service -// id, _ := device.ParseID(msg.Source) -// // Deprecated: X-Webpa-Device-Id should only be used for backwards compatibility. -// // Use X-Webpa-Source instead. -// req.Header.Set("X-Webpa-Device-Id", string(id)) -// // Deprecated: X-Webpa-Device-Name should only be used for backwards compatibility. -// // Use X-Webpa-Source instead. -// req.Header.Set("X-Webpa-Device-Name", string(id)) -// req.Header.Set("X-Webpa-Source", msg.Source) -// req.Header.Set("X-Webpa-Destination", msg.Destination) - -// // Apply the secret - -// if secret != "" { -// s := hmac.New(sha1.New, []byte(secret)) -// s.Write(body) -// sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(s.Sum(nil))) -// req.Header.Set("X-Webpa-Signature", sig) -// } - -// // since eventType is only used to enrich metrics and logging, remove invalid UTF-8 characters from the URL -// eventType := strings.ToValidUTF8(msg.FindEventStringSubMatch(), "") - -// retryOptions := xhttp.RetryOptions{ -// Logger: obs.logger, -// Retries: obs.deliveryRetries, -// Interval: obs.deliveryInterval, -// Counter: gokitprometheus.NewCounter(obs.metrics.deliveryRetryCounter.MustCurryWith(prometheus.Labels{urlLabel: obs.id, eventLabel: eventType})), -// // Always retry on failures up to the max count. -// ShouldRetry: xhttp.ShouldRetry, -// ShouldRetryStatus: xhttp.RetryCodes, -// } - -// // update subsequent requests with the next url in the list upon failure -// retryOptions.UpdateRequest = func(request *http.Request) { -// urls = urls.Next() -// tmp, err := url.Parse(urls.Value.(string)) -// if err != nil { -// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: updateRequestURLFailedReason}).Add(1) -// obs.logger.Error("failed to update url", zap.String("url", urls.Value.(string)), zap.Error(err)) -// return -// } -// request.URL = tmp -// } - -// // Send it -// obs.logger.Debug("attempting to send event", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) - -// retryer := xhttp.RetryTransactor(retryOptions, obs.sender.Do) -// client := obs.clientMiddleware(doerFunc(retryer)) -// resp, err := client.Do(req) - -// var deliveryCounterLabels prometheus.Labels -// code := messageDroppedCode -// reason := noErrReason -// l := obs.logger -// if err != nil { -// // Report failure -// reason = getDoErrReason(err) -// if resp != nil { -// code = strconv.Itoa(resp.StatusCode) -// } - -// l = obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) -// deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} -// obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason}).Add(1) -// } else { -// // Report Result -// code = strconv.Itoa(resp.StatusCode) -// // read until the response is complete before closing to allow -// // connection reuse -// if resp.Body != nil { -// io.Copy(io.Discard, resp.Body) -// resp.Body.Close() -// } - -// deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} -// } - -// obs.metrics.deliveryCounter.With(deliveryCounterLabels).Add(1.0) -// l.Debug("event sent-ish", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination), zap.String("code", code), zap.String("url", req.URL.String())) -// } - -// // queueOverflow handles the logic of what to do when a queue overflows: -// // cutting off the webhook for a time and sending a cut off notification -// // to the failure URL. -// func (obs *CaduceusOutboundSender) queueOverflow() { -// obs.mutex.Lock() -// if time.Now().Before(obs.dropUntil) { -// obs.mutex.Unlock() -// return -// } -// obs.dropUntil = time.Now().Add(obs.cutOffPeriod) -// obs.metrics.dropUntilGauge.With(prometheus.Labels{urlLabel: obs.id}).Set(float64(obs.dropUntil.Unix())) -// secret := obs.listener.Webhook.Config.Secret -// failureMsg := obs.failureMsg -// failureURL := obs.listener.Webhook.FailureURL -// obs.mutex.Unlock() - -// obs.metrics.cutOffCounter.With(prometheus.Labels{urlLabel: obs.id}).Add(1.0) - -// // We empty the queue but don't close the channel, because we're not -// // shutting down. -// obs.Empty(obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: obs.id, reasonLabel: "cut_off"})) - -// msg, err := json.Marshal(failureMsg) -// if err != nil { -// obs.logger.Error("Cut-off notification json.Marshal failed", zap.Any("failureMessage", obs.failureMsg), zap.String("for", obs.id), zap.Error(err)) -// return -// } - -// // if no URL to send cut off notification to, do nothing -// if failureURL == "" { -// return -// } - -// // Send a "you've been cut off" warning message -// payload := bytes.NewReader(msg) -// req, err := http.NewRequest("POST", failureURL, payload) -// if err != nil { -// // Failure -// obs.logger.Error("Unable to send cut-off notification", zap.String("notification", -// failureURL), zap.String("for", obs.id), zap.Error(err)) -// return -// } -// req.Header.Set("Content-Type", wrp.MimeTypeJson) - -// if secret != "" { -// h := hmac.New(sha1.New, []byte(secret)) -// h.Write(msg) -// sig := fmt.Sprintf("sha1=%s", hex.EncodeToString(h.Sum(nil))) -// req.Header.Set("X-Webpa-Signature", sig) -// } - -// resp, err := obs.sender.Do(req) -// if err != nil { -// // Failure -// obs.logger.Error("Unable to send cut-off notification", zap.String("notification", failureURL), zap.String("for", obs.id), zap.Error(err)) -// return -// } - -// if resp == nil { -// // Failure -// obs.logger.Error("Unable to send cut-off notification, nil response", zap.String("notification", failureURL)) -// return -// } - -// // Success - -// if resp.Body != nil { -// io.Copy(io.Discard, resp.Body) -// resp.Body.Close() - -// } -// } diff --git a/outboundSender_test.go b/outboundSender_test.go index ff3480e9..dfe3a36d 100644 --- a/outboundSender_test.go +++ b/outboundSender_test.go @@ -791,13 +791,13 @@ func TestUpdate(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*WebhookOutboundSender); !ok { + if _, ok := obs.(*webhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } - assert.Equal(now, obs.(*WebhookOutboundSender).obs.deliverUntil, "Delivery should match original value.") + assert.Equal(now, obs.(*webhookOutboundSender).obs.deliverUntil, "Delivery should match original value.") obs.Update(w2) - assert.Equal(later, obs.(*WebhookOutboundSender).obs.deliverUntil, "Delivery should match new value.") + assert.Equal(later, obs.(*webhookOutboundSender).obs.deliverUntil, "Delivery should match new value.") obs.Shutdown(true) } @@ -827,7 +827,7 @@ func TestOverflowNoFailureURL(t *testing.T) { assert.Nil(err) - if _, ok := obs.(*WebhookOutboundSender); !ok { + if _, ok := obs.(*webhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } @@ -873,11 +873,11 @@ func TestOverflowValidFailureURL(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*WebhookOutboundSender); !ok { + if _, ok := obs.(*webhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } - obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + obs.(*webhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } @@ -921,11 +921,11 @@ func TestOverflowValidFailureURLWithSecret(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*WebhookOutboundSender); !ok { + if _, ok := obs.(*webhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } - obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + obs.(*webhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } @@ -959,11 +959,11 @@ func TestOverflowValidFailureURLError(t *testing.T) { obs, err := obsf.New() assert.Nil(err) - if _, ok := obs.(*WebhookOutboundSender); !ok { + if _, ok := obs.(*webhookOutboundSender); !ok { assert.Fail("Interface returned by OutboundSenderFactory.New() must be implemented by a WebhookOutboundSender.") } - obs.(*WebhookOutboundSender).obs.dispatcher.QueueOverflow() + obs.(*webhookOutboundSender).obs.dispatcher.QueueOverflow() assert.NotNil(output.String()) } diff --git a/senderWrapper.go b/senderWrapper.go index 0f3715af..3ef4e59d 100644 --- a/senderWrapper.go +++ b/senderWrapper.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/xmidt-org/ancla" + "github.com/xmidt-org/caduceus/internal/kinesis" "github.com/xmidt-org/wrp-go/v3" "go.uber.org/zap" ) @@ -41,9 +42,15 @@ type SenderWrapperFactory struct { // The logger implementation to share with OutboundSenders. Logger *zap.Logger - // The http client Do() function to share with OutboundSenders. + // The http client Do() function to share with WebhookOutboundSenders. Sender httpClient + // The kinesis client to share with StreamOutboundSenders. + StreamSender kinesis.KinesisClientAPI + + // kinesis stream format version + StreamVersion string + // CustomPIDs is a custom list of allowed PartnerIDs that will be used if a message // has no partner IDs. CustomPIDs []string @@ -61,6 +68,8 @@ type SenderWrapper interface { // CaduceusSenderWrapper contains no external parameters. type CaduceusSenderWrapper struct { sender httpClient + streamSender kinesis.KinesisClientAPI + streamVersion string numWorkersPerSender int queueSizePerSender int deliveryRetries int @@ -87,6 +96,8 @@ func (swf SenderWrapperFactory) New() (SenderWrapper, error) { sw := &CaduceusSenderWrapper{ sender: swf.Sender, + streamSender: swf.StreamSender, + streamVersion: swf.StreamVersion, numWorkersPerSender: swf.NumWorkersPerSender, queueSizePerSender: swf.QueueSizePerSender, deliveryRetries: swf.DeliveryRetries, @@ -124,6 +135,7 @@ func (sw *CaduceusSenderWrapper) Update(list []ancla.InternalWebhook) { Logger: sw.logger, CustomPIDs: sw.customPIDs, DisablePartnerIDs: sw.disablePartnerIDs, + IsStream: false, } ids := make([]struct { @@ -181,6 +193,10 @@ func (sw *CaduceusSenderWrapper) Shutdown(gentle bool) { close(sw.shutdown) } +func (sw *CaduceusSenderWrapper) loadStreams() { + +} + // undertaker looks at the OutboundSenders periodically and prunes the ones // that have been retired for too long, freeing up resources. func undertaker(sw *CaduceusSenderWrapper) { diff --git a/stream_dispatcher.go b/stream_dispatcher.go new file mode 100644 index 00000000..97a540e4 --- /dev/null +++ b/stream_dispatcher.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "container/ring" + "strings" + + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/xmidt-org/caduceus/internal/stream" + + "github.com/xmidt-org/wrp-go/v3" + "go.uber.org/zap" +) + +type StreamDispatcher struct { + obs *CaduceusOutboundSender + sender stream.EventSender +} + +func NewStreamDispatcher(obs *CaduceusOutboundSender) (Dispatcher, error) { + url := obs.urls.Value.(string) + sender, err := stream.New(url, obs.streamVersion, obs.streamSender, obs.logger) + if err != nil { + obs.logger.Error("error creating stream sender", zap.Error(err)) + return nil, err + } + + return &StreamDispatcher{ + obs: obs, + sender: sender, + }, nil + +} + +// Note - first go around we will not batch the records +// worker is the routine that actually takes the queued messages and delivers +// them to the listeners outside webpa +func (d *StreamDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) { + defer func() { + if r := recover(); nil != r { + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.obs.id, reasonLabel: DropsDueToPanic}).Add(1.0) + d.obs.logger.Error("stream goroutine send() panicked", zap.String("id", d.obs.id), zap.Any("panic", r)) + // don't silence the panic + panic(r) + } + + d.obs.workers.Release() + d.obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: d.obs.id}).Add(-1.0) + }() + + // Send it + d.obs.logger.Debug("attempting to send event", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) + + msgs := []*wrp.Message{msg} + failedRecordCount, err := d.sender.OnEvent(msgs) + + eventType := strings.ToValidUTF8(msg.FindEventStringSubMatch(), "") + var deliveryCounterLabels prometheus.Labels + code := messageDroppedCode + reason := "no_err" + l := d.obs.logger + if err != nil { + reason = "send_error" + l = d.obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) + deliveryCounterLabels = prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason}).Add(1) + } else if failedRecordCount > 0 { + reason = "some_records_failed" + l = d.obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) + deliveryCounterLabels = prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason}).Add(1) + } else { + deliveryCounterLabels = prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason, codeLabel: "success", eventLabel: eventType} + } + + d.obs.metrics.deliveryCounter.With(deliveryCounterLabels).Add(1.0) + l.Debug("event sent-ish", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination), zap.String("code", code), zap.String("url", d.sender.GetUrl())) +} + +// queueOverflow handles the logic of what to do when a queue overflows: +// cutting off the stream for a time (TODO - should we send cutoff message to the stream?) +func (d *StreamDispatcher) QueueOverflow() { + d.obs.mutex.Lock() + if time.Now().Before(d.obs.dropUntil) { + d.obs.mutex.Unlock() + return + } + d.obs.dropUntil = time.Now().Add(d.obs.cutOffPeriod) + d.obs.metrics.dropUntilGauge.With(prometheus.Labels{urlLabel: d.obs.id}).Set(float64(d.obs.dropUntil.Unix())) + d.obs.mutex.Unlock() + + d.obs.metrics.cutOffCounter.With(prometheus.Labels{urlLabel: d.obs.id}).Add(1.0) + + // We empty the queue but don't close the channel, because we're not + // shutting down. + d.obs.Empty(d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.obs.id, reasonLabel: "cut_off"})) +} diff --git a/stream_outbound_sender.go b/stream_outbound_sender.go new file mode 100644 index 00000000..52915433 --- /dev/null +++ b/stream_outbound_sender.go @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "github.com/xmidt-org/ancla" + "github.com/xmidt-org/wrp-go/v3" + "time" +) + +type streamOutboundSender struct { + obs *CaduceusOutboundSender +} + +// TODO - you should not be able to create it any other way +func NewStreamOutboundSender(obs *CaduceusOutboundSender) (OutboundSender, error) { + dispatcher, err := NewStreamDispatcher(obs) + if err != nil { + return nil, err + } + + sender := &streamOutboundSender{ + obs: obs, + } + + sender.Dispatch(dispatcher) + + return sender, nil +} + +func (s *streamOutboundSender) RetiredSince() time.Time { + // stream senders never expire + deliverUntil := time.Now().UTC().Add(time.Hour * 24) + s.obs.mutex.RLock() + s.obs.deliverUntil = deliverUntil + s.obs.mutex.RUnlock() + return deliverUntil +} + +func (s *streamOutboundSender) Dispatch(d Dispatcher) { + s.obs.Dispatch(d) +} + +func (s *streamOutboundSender) Queue(msg *wrp.Message) { + s.obs.Queue(msg) +} + +func (s *streamOutboundSender) Shutdown(gentle bool) { + s.obs.Shutdown(gentle) +} + +func (s *streamOutboundSender) Update(wh ancla.InternalWebhook) error { + // this is not truly a webhook and is built statically from config at startup + s.obs.logger.Info("Update is NOOP for stream senders") + return nil +} diff --git a/webhook_dispatcher.go b/webhook_dispatcher.go index 9f0d4c8e..9ee367bb 100644 --- a/webhook_dispatcher.go +++ b/webhook_dispatcher.go @@ -167,7 +167,6 @@ func (d *WebhookDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg } else { // Report Result code = strconv.Itoa(resp.StatusCode) - deliveryCounterLabels = prometheus.Labels{urlLabel: req.URL.String(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} } diff --git a/webhook_outbound_sender.go b/webhook_outbound_sender.go index ab2ed04f..839c6fc0 100644 --- a/webhook_outbound_sender.go +++ b/webhook_outbound_sender.go @@ -20,15 +20,15 @@ import ( const Secret = "XxxxxX" -type WebhookOutboundSender struct { +type webhookOutboundSender struct { obs *CaduceusOutboundSender } -// this needs to be a factory and you should not be able to create it any other way +// TODO - you should not be able to create it any other way func NewWebhookOutboundSender(obs *CaduceusOutboundSender) (OutboundSender, error) { dispatcher := NewWebhookDispatcher(obs) - whSender := &WebhookOutboundSender{ + whSender := &webhookOutboundSender{ obs: obs, } @@ -37,26 +37,26 @@ func NewWebhookOutboundSender(obs *CaduceusOutboundSender) (OutboundSender, erro return whSender, nil } -func (s *WebhookOutboundSender) RetiredSince() time.Time { +func (s *webhookOutboundSender) RetiredSince() time.Time { s.obs.mutex.RLock() deliverUntil := s.obs.deliverUntil s.obs.mutex.RUnlock() return deliverUntil } -func (s *WebhookOutboundSender) Dispatch(d Dispatcher) { +func (s *webhookOutboundSender) Dispatch(d Dispatcher) { s.obs.Dispatch(d) } -func (s *WebhookOutboundSender) Queue(msg *wrp.Message) { +func (s *webhookOutboundSender) Queue(msg *wrp.Message) { s.obs.Queue(msg) } -func (s *WebhookOutboundSender) Shutdown(gentle bool) { +func (s *webhookOutboundSender) Shutdown(gentle bool) { s.obs.Shutdown(gentle) } -func (s *WebhookOutboundSender) Update(wh ancla.InternalWebhook) (err error) { +func (s *webhookOutboundSender) Update(wh ancla.InternalWebhook) (err error) { // Validate the failure URL, if present if wh.Webhook.FailureURL != "" { if _, err = url.ParseRequestURI(wh.Webhook.FailureURL); err != nil { From ef66d97d174beec164ea61bed51d14aab559deae Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Wed, 10 Sep 2025 15:59:41 -0400 Subject: [PATCH 6/9] finish wiring for stream senders --- caduceus.yaml | 15 +++++++++++++++ main.go | 16 ++++++++++++---- senderWrapper.go | 26 +++++++++++++++++++++----- stream_outbound_sender.go | 8 +++++++- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/caduceus.yaml b/caduceus.yaml index 04badfa2..726def4d 100644 --- a/caduceus.yaml +++ b/caduceus.yaml @@ -421,6 +421,21 @@ sender: # Defaults to 'false'. disablePartnerIDs: false +# subcategory of senders that are predefined, listen for events and send to streams +# instead of webhook callbacks +outbound_stream_senders: + - PartnerIDs: + - comcast + - sky-italia + - cox + - rogers + Webhook: + Address: http://some-address + Events: + - ".*" + + + # (Deprecated) # profilerFrequency: 15 # profilerDuration: 15 diff --git a/main.go b/main.go index 3779b5fc..eb78e9ea 100644 --- a/main.go +++ b/main.go @@ -122,7 +122,14 @@ func caduceus(arguments []string) int { return 1 } - streamSender, err := kinesis.New(kinesisConfig, logger) + streamSenderConfig := new(StreamSenderConfig) + err = v.Unmarshal(streamSenderConfig) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to unmarshal stream sender config data into struct: %s\n", err) + return 1 + } + + streamClient, err := kinesis.New(kinesisConfig, logger) if err != nil { fmt.Fprintf(os.Stderr, "Unable to kinesis client: %s\n", err) return 1 @@ -161,9 +168,10 @@ func caduceus(arguments []string) int { Transport: tr, Timeout: caduceusConfig.Sender.ClientTimeout, }).Do), - StreamSender: streamSender, - CustomPIDs: caduceusConfig.Sender.CustomPIDs, - DisablePartnerIDs: caduceusConfig.Sender.DisablePartnerIDs, + StreamClient: streamClient, + StreamSenderConfig: streamSenderConfig, + CustomPIDs: caduceusConfig.Sender.CustomPIDs, + DisablePartnerIDs: caduceusConfig.Sender.DisablePartnerIDs, }.New() if err != nil { diff --git a/senderWrapper.go b/senderWrapper.go index 3ef4e59d..330324be 100644 --- a/senderWrapper.go +++ b/senderWrapper.go @@ -46,11 +46,14 @@ type SenderWrapperFactory struct { Sender httpClient // The kinesis client to share with StreamOutboundSenders. - StreamSender kinesis.KinesisClientAPI + StreamClient kinesis.KinesisClientAPI // kinesis stream format version StreamVersion string + // config for predefined stream senders + StreamSenderConfig *StreamSenderConfig + // CustomPIDs is a custom list of allowed PartnerIDs that will be used if a message // has no partner IDs. CustomPIDs []string @@ -70,6 +73,7 @@ type CaduceusSenderWrapper struct { sender httpClient streamSender kinesis.KinesisClientAPI streamVersion string + streamSenderConfig *StreamSenderConfig numWorkersPerSender int queueSizePerSender int deliveryRetries int @@ -96,7 +100,7 @@ func (swf SenderWrapperFactory) New() (SenderWrapper, error) { sw := &CaduceusSenderWrapper{ sender: swf.Sender, - streamSender: swf.StreamSender, + streamSender: swf.StreamClient, streamVersion: swf.StreamVersion, numWorkersPerSender: swf.NumWorkersPerSender, queueSizePerSender: swf.QueueSizePerSender, @@ -114,6 +118,9 @@ func (swf SenderWrapperFactory) New() (SenderWrapper, error) { } sw.wg.Add(1) + + sw.loadStreamSenders() + go undertaker(sw) return sw, nil @@ -123,6 +130,13 @@ func (swf SenderWrapperFactory) New() (SenderWrapper, error) { // additions, or updates. This code takes care of building new OutboundSenders // and maintaining the existing OutboundSenders. func (sw *CaduceusSenderWrapper) Update(list []ancla.InternalWebhook) { + sw.updateSenders(list, false) +} + +// Update is called when we get changes to our webhook listeners with either +// additions, or updates. This code takes care of building new OutboundSenders +// and maintaining the existing OutboundSenders. +func (sw *CaduceusSenderWrapper) updateSenders(list []ancla.InternalWebhook, stream bool) { // We'll like need this, so let's get one ready osf := OutboundSenderFactory{ Sender: sw.sender, // ** REMOVE *** Dispatcher goes here - dispatcher factory? @@ -135,7 +149,7 @@ func (sw *CaduceusSenderWrapper) Update(list []ancla.InternalWebhook) { Logger: sw.logger, CustomPIDs: sw.customPIDs, DisablePartnerIDs: sw.disablePartnerIDs, - IsStream: false, + IsStream: stream, } ids := make([]struct { @@ -193,8 +207,10 @@ func (sw *CaduceusSenderWrapper) Shutdown(gentle bool) { close(sw.shutdown) } -func (sw *CaduceusSenderWrapper) loadStreams() { - +// load predefined event listeners which write output to streams instead of +// webhook callbacks +func (sw *CaduceusSenderWrapper) loadStreamSenders() { + sw.updateSenders(sw.streamSenderConfig.OutboundStreamSenders, true) } // undertaker looks at the OutboundSenders periodically and prunes the ones diff --git a/stream_outbound_sender.go b/stream_outbound_sender.go index 52915433..89973f6f 100644 --- a/stream_outbound_sender.go +++ b/stream_outbound_sender.go @@ -4,11 +4,17 @@ package main import ( + "time" + "github.com/xmidt-org/ancla" "github.com/xmidt-org/wrp-go/v3" - "time" ) +type StreamSenderConfig struct { + // these senders are not really webhooks, but we can treat them as such + OutboundStreamSenders []ancla.InternalWebhook +} + type streamOutboundSender struct { obs *CaduceusOutboundSender } From 30883a5c74724921cd4b4cfadb3b1878df7c9805 Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Thu, 11 Sep 2025 11:50:22 -0400 Subject: [PATCH 7/9] fix tests --- internal/kinesis/kinesis_test.go | 94 +++++++++++++++++--------------- internal/stream/sender_test.go | 4 +- outboundSender_test.go | 2 + senderWrapper.go | 2 +- senderWrapper_test.go | 14 +++++ 5 files changed, 68 insertions(+), 48 deletions(-) diff --git a/internal/kinesis/kinesis_test.go b/internal/kinesis/kinesis_test.go index 32794d76..60a44236 100644 --- a/internal/kinesis/kinesis_test.go +++ b/internal/kinesis/kinesis_test.go @@ -14,11 +14,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" ) const SomeStream = "some-stream" -var logger, _ = zap.NewProduction() +var logger = zap.NewExample() var shardId = "shardId" var sequenceNumber = "123" @@ -77,7 +78,31 @@ func TestNewNoAwsRole(t *testing.T) { assert.NotNil(t, kclient.svc) } -func TestPutRecords(t *testing.T) { +type KinesisSuite struct { + suite.Suite + kc KinesisClient + mockKinesis *mockKinesis +} + +func TestKinesisSuite(t *testing.T) { + suite.Run(t, new(KinesisSuite)) +} + +func (suite *KinesisSuite) SetupTest() { + m := newMockKinesis() + kc := KinesisClient{ + logger: logger, + svc: m, + config: &Config{ + Region: "na", + Endpoint: "http://localhost", + }, + } + suite.kc = kc + suite.mockKinesis = m +} + +func (suite *KinesisSuite) TestPutRecords() { items := []Item{{PartitionKey: "mac1", Item: []byte("test1")}, {PartitionKey: "mac2", Item: []byte("test2")}} stream := SomeStream @@ -87,20 +112,15 @@ func TestPutRecords(t *testing.T) { FailedRecordCount: &failedRecordInputCount, } - m := newMockKinesis() - m.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, nil) - kc := KinesisClient{ - logger: logger, - svc: m, - } + suite.mockKinesis.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, nil) - failedRecordCount, err := kc.PutRecords(items, stream) - t.Log(err) - assert.Nil(t, err) - assert.Equal(t, 10, failedRecordCount) + failedRecordCount, err := suite.kc.PutRecords(items, stream) + suite.T().Log(err) + suite.Nil(err) + suite.Equal(10, failedRecordCount) } -func TestPutRecordsError(t *testing.T) { +func (suite *KinesisSuite) TestPutRecordsError() { items := []Item{{PartitionKey: "mac1", Item: []byte("test1")}, {PartitionKey: "mac2", Item: []byte("test2")}} stream := SomeStream @@ -110,50 +130,36 @@ func TestPutRecordsError(t *testing.T) { FailedRecordCount: &failedRecordInputCount, } - m := newMockKinesis() - m.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, errors.New("some db error")) - kc := KinesisClient{ - logger: logger, - svc: m, - } + suite.mockKinesis.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, errors.New("some db error")) - failedRecordCount, err := kc.PutRecords(items, stream) - t.Log(err) - assert.NotNil(t, err) - assert.Equal(t, 0, failedRecordCount) + failedRecordCount, err := suite.kc.PutRecords(items, stream) + suite.T().Log(err) + suite.NotNil(err) + suite.Equal(0, failedRecordCount) } -func TestPutRecordsErrorNilOutput(t *testing.T) { +func (suite *KinesisSuite) TestPutRecordsErrorNilOutput() { items := []Item{{PartitionKey: "mac1", Item: []byte("test1")}, {PartitionKey: "mac2", Item: []byte("test2")}} stream := SomeStream putRecordsOutput := (*kinesis.PutRecordsOutput)(nil) - m := newMockKinesis() - m.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, errors.New("some db error")) - kc := KinesisClient{ - logger: logger, - svc: m, - } - failedRecordCount, err := kc.PutRecords(items, stream) - t.Log(err) - assert.NotNil(t, err) - assert.Equal(t, 0, failedRecordCount) + suite.mockKinesis.On("PutRecords", mock.Anything, mock.Anything, mock.Anything).Return(putRecordsOutput, errors.New("some db error")) + + failedRecordCount, err := suite.kc.PutRecords(items, stream) + suite.T().Log(err) + suite.NotNil(err) + suite.Equal(0, failedRecordCount) } -func TestPutRecord(t *testing.T) { +func (suite *KinesisSuite) TestPutRecord() { event := []byte("test") stream := SomeStream partitionKey := "some-key" - m := newMockKinesis() - m.On("PutRecord", mock.Anything, mock.Anything, mock.Anything).Return(&putRecordOutput) - kc := KinesisClient{ - logger: logger, - svc: m, - } + suite.mockKinesis.On("PutRecord", mock.Anything, mock.Anything, mock.Anything).Return(&putRecordOutput) - _, err := kc.PutRecord(event, stream, partitionKey) - t.Log(err) - assert.Nil(t, err) + _, err := suite.kc.PutRecord(event, stream, partitionKey) + suite.T().Log(err) + suite.Nil(err) } diff --git a/internal/stream/sender_test.go b/internal/stream/sender_test.go index d115a193..1adf2744 100644 --- a/internal/stream/sender_test.go +++ b/internal/stream/sender_test.go @@ -65,7 +65,5 @@ func TestOnEvent(t *testing.T) { err = json.Unmarshal(items[0].Item, &msg) assert.NoError(err) - assert.Equal("112233445566", items[0].PartitionKey) - // TODO - assert message content - assert.Equal("TestStream", mockCall.Parent.Calls[0].Arguments.Get(1).(string)) + assert.Equal(uuid.String(), items[0].PartitionKey) } diff --git a/outboundSender_test.go b/outboundSender_test.go index dfe3a36d..82fa8530 100644 --- a/outboundSender_test.go +++ b/outboundSender_test.go @@ -26,6 +26,8 @@ import ( const testLocalhostURL = "http://localhost:9999/foo" +//const testLocalhostStreamURL = "http://localhost:8888/someStream" + // TODO Improve all of these tests // Make a simple RoundTrip implementation that let's me short-circuit the network diff --git a/senderWrapper.go b/senderWrapper.go index 330324be..1d874f1b 100644 --- a/senderWrapper.go +++ b/senderWrapper.go @@ -101,6 +101,7 @@ func (swf SenderWrapperFactory) New() (SenderWrapper, error) { sw := &CaduceusSenderWrapper{ sender: swf.Sender, streamSender: swf.StreamClient, + streamSenderConfig: swf.StreamSenderConfig, streamVersion: swf.StreamVersion, numWorkersPerSender: swf.NumWorkersPerSender, queueSizePerSender: swf.QueueSizePerSender, @@ -179,7 +180,6 @@ func (sw *CaduceusSenderWrapper) updateSenders(list []ancla.InternalWebhook, str } } -// *** REMOVE - wrp.Message entrypoint ****// // Queue is used to send all the possible outbound senders a request. This // function performs the fan-out and filtering to multiple possible endpoints. func (sw *CaduceusSenderWrapper) Queue(msg *wrp.Message) { diff --git a/senderWrapper_test.go b/senderWrapper_test.go index f66eef9a..d0c642fb 100644 --- a/senderWrapper_test.go +++ b/senderWrapper_test.go @@ -99,7 +99,21 @@ func getFakeFactory() *SenderWrapperFactory { On("With", prometheus.Labels{"content_type": "http"}).Return(fakeIgnore). On("With", prometheus.Labels{"content_type": "other"}).Return(fakeIgnore) + streamSenderConfig := &StreamSenderConfig{ + // OutboundStreamSenders: []ancla.InternalWebhook{ + // { + // Webhook: ancla.Webhook{ + // Config: ancla.DeliveryConfig{ + // URL: testLocalhostStreamURL, + // }, + // Events: []string{"iot"}, + // }, + // }, + // }, + } + return &SenderWrapperFactory{ + StreamSenderConfig: streamSenderConfig, NumWorkersPerSender: 10, QueueSizePerSender: 10, CutOffPeriod: 30 * time.Second, From 72250f8d386777469084f0f45f3e8b605bcb17a3 Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Thu, 11 Sep 2025 15:28:22 -0400 Subject: [PATCH 8/9] more tests --- mocks_test.go | 15 ++++++++ outboundSender.go | 7 +--- outboundSender_test.go | 27 ++++++++------- stream_dispatcher.go | 1 + stream_outbound_sender.go | 7 ++-- stream_outbound_sender_test.go | 62 ++++++++++++++++++++++++++++++++++ webhook_outbound_sender.go | 7 ++-- 7 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 stream_outbound_sender_test.go diff --git a/mocks_test.go b/mocks_test.go index 9c8e6b52..2ec9b3bd 100644 --- a/mocks_test.go +++ b/mocks_test.go @@ -3,6 +3,7 @@ package main import ( + "container/ring" "time" "unicode/utf8" @@ -259,3 +260,17 @@ func mockTime(one, two time.Time) func() time.Time { return one } } + +// mock dispatcher + +type mockDispatcher struct { + mock.Mock +} + +func (m *mockDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg *wrp.Message) { + m.Called(urls, secret, acceptType, msg) +} + +func (m *mockDispatcher) QueueOverflow() { + m.Called() +} diff --git a/outboundSender.go b/outboundSender.go index dee98e01..b4e946a8 100644 --- a/outboundSender.go +++ b/outboundSender.go @@ -205,12 +205,7 @@ func (osf OutboundSenderFactory) New() (obs OutboundSender, err error) { return NewStreamOutboundSender(caduceusOutboundSender) } - return NewWebhookOutboundSender(caduceusOutboundSender) -} - -func (obs *CaduceusOutboundSender) Dispatch(d Dispatcher) { - obs.dispatcher = d - go obs.dispatch() + return NewWebhookOutboundSender(caduceusOutboundSender) // TODO } // Update applies user configurable values for the outbound sender when a diff --git a/outboundSender_test.go b/outboundSender_test.go index 82fa8530..0b29bf70 100644 --- a/outboundSender_test.go +++ b/outboundSender_test.go @@ -53,7 +53,7 @@ func getNewTestOutputLogger(out io.Writer) *zap.Logger { } func simpleSetup(trans *transport, cutOffPeriod time.Duration, matcher []string) (OutboundSender, error) { - return simpleFactorySetup(trans, cutOffPeriod, matcher).New() + return simpleFactorySetup(trans, cutOffPeriod, matcher, false).New() } // simpleFactorySetup sets up a outboundSender with metrics. @@ -71,7 +71,7 @@ func simpleSetup(trans *transport, cutOffPeriod time.Duration, matcher []string) // case 2: On("With", []string{eventLabel, unknown} // 4. Mimic the metric behavior using On: // fakeSlow.On("Add", 1.0).Return() -func simpleFactorySetup(trans *transport, cutOffPeriod time.Duration, matcher []string) *OutboundSenderFactory { +func simpleFactorySetup(trans *transport, cutOffPeriod time.Duration, matcher []string, stream bool) *OutboundSenderFactory { if nil == trans.fn { trans.fn = func(req *http.Request, count int) (resp *http.Response, err error) { resp = &http.Response{Status: "200 OK", @@ -198,6 +198,7 @@ func simpleFactorySetup(trans *transport, cutOffPeriod time.Duration, matcher [] QueueSize: 10, DeliveryRetries: 1, Logger: zap.NewNop(), + IsStream: stream, } } @@ -668,7 +669,7 @@ func TestInvalidSender(t *testing.T) { assert := assert.New(t) trans := &transport{} - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Sender = nil obs, err := obsf.New() assert.Nil(obs) @@ -689,7 +690,7 @@ func TestInvalidLogger(t *testing.T) { w.Webhook.Config.ContentType = wrp.MimeTypeJson trans := &transport{} - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w obsf.Sender = doerFunc((&http.Client{}).Do) obsf.Logger = nil @@ -714,7 +715,7 @@ func TestFailureURL(t *testing.T) { w.Webhook.Config.ContentType = wrp.MimeTypeJson trans := &transport{} - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w obsf.Sender = doerFunc((&http.Client{}).Do) obs, err := obsf.New() @@ -735,7 +736,7 @@ func TestInvalidEvents(t *testing.T) { w.Webhook.Config.ContentType = wrp.MimeTypeJson trans := &transport{} - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w obsf.Sender = doerFunc((&http.Client{}).Do) obs, err := obsf.New() @@ -752,7 +753,7 @@ func TestInvalidEvents(t *testing.T) { w2.Webhook.Config.URL = testLocalhostURL w2.Webhook.Config.ContentType = wrp.MimeTypeJson - obsf = simpleFactorySetup(trans, time.Second, nil) + obsf = simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w2 obsf.Sender = doerFunc((&http.Client{}).Do) obs, err = obsf.New() @@ -787,7 +788,7 @@ func TestUpdate(t *testing.T) { w2.Webhook.Config.ContentType = wrp.MimeTypeMsgpack trans := &transport{} - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w1 obsf.Sender = doerFunc((&http.Client{}).Do) obs, err := obsf.New() @@ -821,7 +822,7 @@ func TestOverflowNoFailureURL(t *testing.T) { w.Webhook.Config.ContentType = wrp.MimeTypeJson trans := &transport{} - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w obsf.Logger = logger obsf.Sender = doerFunc((&http.Client{}).Do) @@ -869,7 +870,7 @@ func TestOverflowValidFailureURL(t *testing.T) { w.Webhook.Config.URL = testLocalhostURL w.Webhook.Config.ContentType = wrp.MimeTypeJson - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w obsf.Logger = logger obs, err := obsf.New() @@ -917,7 +918,7 @@ func TestOverflowValidFailureURLWithSecret(t *testing.T) { w.Webhook.Config.ContentType = wrp.MimeTypeJson w.Webhook.Config.Secret = "123456" - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w obsf.Logger = logger obs, err := obsf.New() @@ -955,7 +956,7 @@ func TestOverflowValidFailureURLError(t *testing.T) { w.Webhook.Config.URL = testLocalhostURL w.Webhook.Config.ContentType = wrp.MimeTypeJson - obsf := simpleFactorySetup(trans, time.Second, nil) + obsf := simpleFactorySetup(trans, time.Second, nil, false) obsf.Listener = w obsf.Logger = logger obs, err := obsf.New() @@ -1003,7 +1004,7 @@ func TestOverflow(t *testing.T) { w.Config.URL = testLocalhostURL w.Config.ContentType = wrp.MimeTypeJson - obsf := simpleFactorySetup(trans, 4*time.Second, nil) + obsf := simpleFactorySetup(trans, 4*time.Second, nil, false) obsf.NumWorkers = 1 obsf.QueueSize = 2 obsf.Logger = logger diff --git a/stream_dispatcher.go b/stream_dispatcher.go index 97a540e4..2ab1f8ce 100644 --- a/stream_dispatcher.go +++ b/stream_dispatcher.go @@ -24,6 +24,7 @@ type StreamDispatcher struct { func NewStreamDispatcher(obs *CaduceusOutboundSender) (Dispatcher, error) { url := obs.urls.Value.(string) + // TODO - sender should hit alternatives (east vs west) if kinesis is down sender, err := stream.New(url, obs.streamVersion, obs.streamSender, obs.logger) if err != nil { obs.logger.Error("error creating stream sender", zap.Error(err)) diff --git a/stream_outbound_sender.go b/stream_outbound_sender.go index 89973f6f..1fd5a30b 100644 --- a/stream_outbound_sender.go +++ b/stream_outbound_sender.go @@ -30,7 +30,8 @@ func NewStreamOutboundSender(obs *CaduceusOutboundSender) (OutboundSender, error obs: obs, } - sender.Dispatch(dispatcher) + obs.dispatcher = dispatcher + go obs.dispatch() return sender, nil } @@ -44,10 +45,6 @@ func (s *streamOutboundSender) RetiredSince() time.Time { return deliverUntil } -func (s *streamOutboundSender) Dispatch(d Dispatcher) { - s.obs.Dispatch(d) -} - func (s *streamOutboundSender) Queue(msg *wrp.Message) { s.obs.Queue(msg) } diff --git a/stream_outbound_sender_test.go b/stream_outbound_sender_test.go new file mode 100644 index 00000000..775b7e2b --- /dev/null +++ b/stream_outbound_sender_test.go @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "github.com/xmidt-org/ancla" + "github.com/xmidt-org/wrp-go/v3" +) + +type StreamOutboundSenderSuite struct { + suite.Suite + streamOutboundSender *streamOutboundSender + mockDispatcher *mockDispatcher +} + +func TestStreamOutboundSenderSuite(t *testing.T) { + suite.Run(t, new(StreamOutboundSenderSuite)) +} + +func (suite *StreamOutboundSenderSuite) SetupTest() { + trans := &transport{} + obs, err := simpleFactorySetup(trans, time.Second, nil, true).New() + mockDispatcher := new(mockDispatcher) + + suite.NotNil(obs) + suite.Nil(err) + suite.streamOutboundSender = obs.(*streamOutboundSender) + suite.streamOutboundSender.obs.dispatcher = mockDispatcher + suite.mockDispatcher = mockDispatcher +} + +func (suite *StreamOutboundSenderSuite) TestRetiredSince() { + suite.Greater(suite.streamOutboundSender.RetiredSince(), time.Now().UTC()) +} + +func (suite *StreamOutboundSenderSuite) TestShutdown() { + suite.streamOutboundSender.Shutdown(true) + + time.Sleep(100 * time.Millisecond) + + _, ok := <-suite.streamOutboundSender.obs.queue.Load().(chan *wrp.Message) + suite.False(ok) +} + +func (suite *StreamOutboundSenderSuite) TestQueue() { + suite.mockDispatcher.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + req := simpleRequestWithPartnerIDs() + req.Destination = "event:iot" + suite.streamOutboundSender.Queue(req) + time.Sleep(100 * time.Millisecond) + suite.mockDispatcher.AssertCalled(suite.T(), "Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +func (suite *StreamOutboundSenderSuite) TestUpdate() { + suite.streamOutboundSender.Update(ancla.InternalWebhook{}) +} diff --git a/webhook_outbound_sender.go b/webhook_outbound_sender.go index 839c6fc0..a54536f1 100644 --- a/webhook_outbound_sender.go +++ b/webhook_outbound_sender.go @@ -32,7 +32,8 @@ func NewWebhookOutboundSender(obs *CaduceusOutboundSender) (OutboundSender, erro obs: obs, } - whSender.Dispatch(dispatcher) + obs.dispatcher = dispatcher + go obs.dispatch() return whSender, nil } @@ -44,10 +45,6 @@ func (s *webhookOutboundSender) RetiredSince() time.Time { return deliverUntil } -func (s *webhookOutboundSender) Dispatch(d Dispatcher) { - s.obs.Dispatch(d) -} - func (s *webhookOutboundSender) Queue(msg *wrp.Message) { s.obs.Queue(msg) } From 47cbc87515cc991caa6235785af7a4dc0c9ec85b Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Wed, 17 Sep 2025 12:23:08 -0400 Subject: [PATCH 9/9] unit tests --- internal/stream/sender.go | 21 ++--- internal/stream/sender_test.go | 6 +- mocks_test.go | 16 ++++ outboundSender_test.go | 1 - stream_dispatcher.go | 30 ++++--- stream_dispatcher_test.go | 145 +++++++++++++++++++++++++++++++++ stream_outbound_sender_test.go | 4 +- 7 files changed, 193 insertions(+), 30 deletions(-) create mode 100644 stream_dispatcher_test.go diff --git a/internal/stream/sender.go b/internal/stream/sender.go index 7b125ac7..ba592210 100644 --- a/internal/stream/sender.go +++ b/internal/stream/sender.go @@ -17,16 +17,14 @@ import ( const schemaVersion = "1.1" const retries = 3 -type StreamEventSender struct { +type EventStreamSender struct { kc kinesis.KinesisClientAPI - url string schemaVersion string logger *zap.Logger } -type EventSender interface { - OnEvent(event []*wrp.Message) (int, error) - GetUrl() string +type StreamSender interface { + OnEvent(event []*wrp.Message, url string) (int, error) } var kPutRunner, _ = retry.NewRunner[int]( @@ -36,24 +34,19 @@ var kPutRunner, _ = retry.NewRunner[int]( }), ) -func New(url string, version string, kc kinesis.KinesisClientAPI, logger *zap.Logger) (EventSender, error) { +func New(version string, kc kinesis.KinesisClientAPI, logger *zap.Logger) (StreamSender, error) { if schemaVersion == "" { version = schemaVersion } - return &StreamEventSender{ + return &EventStreamSender{ kc: kc, - url: url, schemaVersion: version, logger: logger, }, nil } -func (s *StreamEventSender) GetUrl() string { - return s.url -} - // TODO - add a queue and a channel instead -func (s *StreamEventSender) OnEvent(msgs []*wrp.Message) (int, error) { +func (s *EventStreamSender) OnEvent(msgs []*wrp.Message, url string) (int, error) { items := []kinesis.Item{} for _, m := range msgs { data, err := json.Marshal(m) @@ -69,7 +62,7 @@ func (s *StreamEventSender) OnEvent(msgs []*wrp.Message) (int, error) { context.Background(), func(_ context.Context) (int, error) { attempts++ - failedRecordCount, err := s.kc.PutRecords(items, s.url) + failedRecordCount, err := s.kc.PutRecords(items, url) if err != nil { s.logger.Error("kinesis.PutRecords error", zap.Int("attempt", attempts), zap.Error(err)) } diff --git a/internal/stream/sender_test.go b/internal/stream/sender_test.go index 1adf2744..d9e303c8 100644 --- a/internal/stream/sender_test.go +++ b/internal/stream/sender_test.go @@ -18,6 +18,8 @@ import ( mykinesis "github.com/xmidt-org/caduceus/internal/kinesis" ) +const testUrl = "http://localhost:9999" + type mockKinesisClient struct{ mock.Mock } func newMockKinesisClient() *mockKinesisClient { return &mockKinesisClient{} } @@ -50,13 +52,13 @@ func TestOnEvent(t *testing.T) { var kc = newMockKinesisClient() - sender, err := New("", "", kc, zap.NewExample()) + sender, err := New("", kc, zap.NewExample()) assert.Nil(err) mockCall := kc.On("PutRecords", mock.Anything, mock.Anything).Return(0, nil) events := []*wrp.Message{e} - failedRecordCount, err := sender.OnEvent(events) + failedRecordCount, err := sender.OnEvent(events, testUrl) assert.Equal(0, failedRecordCount) assert.Nil(err) diff --git a/mocks_test.go b/mocks_test.go index 2ec9b3bd..8aeaf142 100644 --- a/mocks_test.go +++ b/mocks_test.go @@ -274,3 +274,19 @@ func (m *mockDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg *w func (m *mockDispatcher) QueueOverflow() { m.Called() } + +// mock event sender + +type mockStreamSender struct { + mock.Mock +} + +func (m *mockStreamSender) OnEvent(msg []*wrp.Message, url string) (int, error) { + args := m.Called(msg, url) + return args.Get(0).(int), args.Error(1) +} + +func (m *mockStreamSender) GetUrl() string { + args := m.Called() + return args.Get(0).(string) +} diff --git a/outboundSender_test.go b/outboundSender_test.go index 0b29bf70..21d17e6a 100644 --- a/outboundSender_test.go +++ b/outboundSender_test.go @@ -15,7 +15,6 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" - //"github.com/stretchr/testify/mock" "io" "net" "net/http" diff --git a/stream_dispatcher.go b/stream_dispatcher.go index 2ab1f8ce..4e9ef5d5 100644 --- a/stream_dispatcher.go +++ b/stream_dispatcher.go @@ -19,13 +19,13 @@ import ( type StreamDispatcher struct { obs *CaduceusOutboundSender - sender stream.EventSender + sender stream.StreamSender } func NewStreamDispatcher(obs *CaduceusOutboundSender) (Dispatcher, error) { - url := obs.urls.Value.(string) + // TODO - sender should hit alternatives (east vs west) if kinesis is down - sender, err := stream.New(url, obs.streamVersion, obs.streamSender, obs.logger) + sender, err := stream.New(obs.streamVersion, obs.streamSender, obs.logger) if err != nil { obs.logger.Error("error creating stream sender", zap.Error(err)) return nil, err @@ -54,11 +54,17 @@ func (d *StreamDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg d.obs.metrics.currentWorkersGauge.With(prometheus.Labels{urlLabel: d.obs.id}).Add(-1.0) }() + url := urls.Value.(string) + + d.sendToEndpoint(url, msg) +} + +func (d *StreamDispatcher) sendToEndpoint(url string, msg *wrp.Message) { // Send it d.obs.logger.Debug("attempting to send event", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination)) msgs := []*wrp.Message{msg} - failedRecordCount, err := d.sender.OnEvent(msgs) + failedRecordCount, err := d.sender.OnEvent(msgs, url) eventType := strings.ToValidUTF8(msg.FindEventStringSubMatch(), "") var deliveryCounterLabels prometheus.Labels @@ -67,20 +73,20 @@ func (d *StreamDispatcher) Send(urls *ring.Ring, secret, acceptType string, msg l := d.obs.logger if err != nil { reason = "send_error" - l = d.obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) - deliveryCounterLabels = prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} - d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason}).Add(1) + d.obs.logger.Error("error writing to stream", zap.String(reasonLabel, reason), zap.Error(err)) + deliveryCounterLabels = prometheus.Labels{urlLabel: url, reasonLabel: reason, codeLabel: code, eventLabel: eventType} + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: url, reasonLabel: reason}).Add(1) } else if failedRecordCount > 0 { reason = "some_records_failed" - l = d.obs.logger.With(zap.String(reasonLabel, reason), zap.Error(err)) - deliveryCounterLabels = prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason, codeLabel: code, eventLabel: eventType} - d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason}).Add(1) + d.obs.logger.Error("some records failed to write to stream", zap.String(reasonLabel, reason), zap.Int("failedRecordCount", failedRecordCount)) + deliveryCounterLabels = prometheus.Labels{urlLabel: url, reasonLabel: reason, codeLabel: code, eventLabel: eventType} + d.obs.metrics.droppedMessage.With(prometheus.Labels{urlLabel: url, reasonLabel: reason}).Add(1) } else { - deliveryCounterLabels = prometheus.Labels{urlLabel: d.sender.GetUrl(), reasonLabel: reason, codeLabel: "success", eventLabel: eventType} + deliveryCounterLabels = prometheus.Labels{urlLabel: url, reasonLabel: reason, codeLabel: "200", eventLabel: eventType} } d.obs.metrics.deliveryCounter.With(deliveryCounterLabels).Add(1.0) - l.Debug("event sent-ish", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination), zap.String("code", code), zap.String("url", d.sender.GetUrl())) + l.Debug("event sent-ish", zap.String("event.source", msg.Source), zap.String("event.destination", msg.Destination), zap.String("code", code), zap.String("url", url)) } // queueOverflow handles the logic of what to do when a queue overflows: diff --git a/stream_dispatcher_test.go b/stream_dispatcher_test.go new file mode 100644 index 00000000..fa53b628 --- /dev/null +++ b/stream_dispatcher_test.go @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +const testEvent = "event:test" + +type StreamDispatcherSuite struct { + suite.Suite + obs *streamOutboundSender + caduceusOutboundSender *CaduceusOutboundSender + dispatcher *StreamDispatcher + mockStreamSender *mockStreamSender + mockMetrics *OutboundSenderMetrics +} + +func TestStreamDispatcherSuite(t *testing.T) { + suite.Run(t, new(StreamDispatcherSuite)) +} + +func (suite *StreamDispatcherSuite) SetupTest() { + mockStreamSender := new(mockStreamSender) + trans := &transport{} + obs, err := simpleFactorySetup(trans, time.Second, nil, true).New() + suite.NoError(err) + dispatcher, err := NewStreamDispatcher(obs.(*streamOutboundSender).obs) // TODO - this is an awkward pattern + suite.NoError(err) + suite.mockStreamSender = mockStreamSender + suite.dispatcher = dispatcher.(*StreamDispatcher) + suite.dispatcher.sender = mockStreamSender + suite.obs = obs.(*streamOutboundSender) + suite.caduceusOutboundSender = suite.obs.obs + mockMetrics := getMockMetrics() + suite.caduceusOutboundSender.metrics = mockMetrics + suite.mockMetrics = &mockMetrics +} + +func getMockMetrics() OutboundSenderMetrics { + fakeDC := new(mockCounter) + fakeDC.On("With", mock.Anything).Return(fakeDC) + fakeDC.On("Add", mock.Anything).Return() + + // test slow metric + fakeSlow := new(mockCounter) + fakeSlow.On("Add", mock.Anything).Return() + fakeSlow.On("With", mock.Anything).Return(fakeSlow) + + // test dropped metric + fakeDroppedSlow := new(mockCounter) + fakeDroppedSlow.On("Add", mock.Anything).Return() + fakeDroppedSlow.On("With", mock.Anything).Return(fakeDroppedSlow) + + // IncomingContentType cases + fakeContentType := new(mockCounter) + fakeContentType.On("Add", mock.Anything).Return() + fakeContentType.On("With", mock.Anything).Return(fakeContentType) + + // QueueDepth case + fakeQdepth := new(mockGauge) + fakeQdepth.On("Add", mock.Anything).Return() + fakeQdepth.On("Set", mock.Anything).Return() + fakeQdepth.On("With", mock.Anything).Return(fakeQdepth) + + // Fake Latency + fakeLatency := new(mockHistogram) + fakeLatency.On("Observe", mock.Anything).Return() + + return OutboundSenderMetrics{ + queryLatency: fakeLatency, + deliveryCounter: fakeDC, + deliveryRetryCounter: fakeDC, + droppedMessage: fakeDroppedSlow, + cutOffCounter: fakeSlow, + queueDepthGauge: fakeQdepth, + renewalTimeGauge: fakeQdepth, + deliverUntilGauge: fakeQdepth, + dropUntilGauge: fakeQdepth, + maxWorkersGauge: fakeQdepth, + currentWorkersGauge: fakeQdepth, + deliveryRetryMaxGauge: fakeQdepth, + } +} + +func (suite *StreamDispatcherSuite) TestSend() { + args := suite.mockStreamSender.On("OnEvent", mock.Anything, mock.Anything).Return(0, nil) + msg := simpleRequestWithPartnerIDs() + msg.Destination = "testEvent" + + suite.obs.obs.workers.Acquire() + suite.dispatcher.Send(suite.obs.obs.urls, "", "", msg) + + args.Parent.AssertCalled(suite.T(), "OnEvent", mock.Anything, mock.Anything) +} + +func (suite *StreamDispatcherSuite) TestSendError() { + fakeDC := suite.mockMetrics.deliveryCounter.(*mockCounter) + metricArgs := fakeDC.On("With", mock.Anything).Return(fakeDC) + + args := suite.mockStreamSender.On("OnEvent", mock.Anything, mock.Anything).Return(0, errors.New("network_err")) + msg := simpleRequestWithPartnerIDs() + msg.Destination = testEvent + + suite.obs.obs.workers.Acquire() + suite.dispatcher.Send(suite.obs.obs.urls, "", "", msg) + + args.Parent.AssertCalled(suite.T(), "OnEvent", mock.Anything, mock.Anything) + labels := metricArgs.Parent.Calls[0].Arguments[0].(prometheus.Labels) + suite.Equal("message_dropped", labels[codeLabel]) + suite.Equal("send_error", labels[reasonLabel]) +} + +func (suite *StreamDispatcherSuite) TestSomeFailedRecordsError() { + fakeDC := suite.mockMetrics.deliveryCounter.(*mockCounter) + metricArgs := fakeDC.On("With", mock.Anything).Return(fakeDC) + + args := suite.mockStreamSender.On("OnEvent", mock.Anything, mock.Anything).Return(1, nil) + msg := simpleRequestWithPartnerIDs() + msg.Destination = "testEvent" + + suite.obs.obs.workers.Acquire() + suite.dispatcher.Send(suite.obs.obs.urls, "", "", msg) + + args.Parent.AssertCalled(suite.T(), "OnEvent", mock.Anything, mock.Anything) + labels := metricArgs.Parent.Calls[0].Arguments[0].(prometheus.Labels) + suite.Equal("message_dropped", labels[codeLabel]) + suite.Equal("some_records_failed", labels[reasonLabel]) +} + +func (suite *StreamDispatcherSuite) TestQueueOverflow() { + fakeSlow := suite.mockMetrics.cutOffCounter.(*mockCounter) + metricArgs := fakeSlow.On("With", mock.Anything).Return(fakeSlow) + + suite.dispatcher.QueueOverflow() + metricArgs.Parent.AssertCalled(suite.T(), "Add", 1.0) +} diff --git a/stream_outbound_sender_test.go b/stream_outbound_sender_test.go index 775b7e2b..bdd3074e 100644 --- a/stream_outbound_sender_test.go +++ b/stream_outbound_sender_test.go @@ -13,6 +13,8 @@ import ( "github.com/xmidt-org/wrp-go/v3" ) +const iotEvent = "event:iot" + type StreamOutboundSenderSuite struct { suite.Suite streamOutboundSender *streamOutboundSender @@ -51,7 +53,7 @@ func (suite *StreamOutboundSenderSuite) TestShutdown() { func (suite *StreamOutboundSenderSuite) TestQueue() { suite.mockDispatcher.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything) req := simpleRequestWithPartnerIDs() - req.Destination = "event:iot" + req.Destination = iotEvent suite.streamOutboundSender.Queue(req) time.Sleep(100 * time.Millisecond) suite.mockDispatcher.AssertCalled(suite.T(), "Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything)