state, rev: %d, compactRev: %d
", state.Revision, state.CompactRevision)) + + keys := []string{} + for i, v := range state.KeyValues { + if v == nil { + continue + } + keys = append(keys, state.Keys[i]) + } + + if len(keys) > 0 { + descHTML = append(descHTML, "keys:" + html.EscapeString(string(data)) + "" + }, + } } type EtcdState struct { - Revision int64 - CompactRevision int64 - KeyValues map[string]ValueRevision - KeyLeases map[string]int64 - Leases map[int64]EtcdLease + Revision int64 `json:",omitempty"` + CompactRevision int64 `json:",omitempty"` + // Slices below are positionally aligned. If KeyValue is nil on index i, + // it means the key `Keys[i]` doesn't exist. + Keys []string `json:",omitempty"` + KeyValues []*ValueRevision `json:",omitempty"` + KeyLeases []*int64 `json:",omitempty"` + // All leases sorted by LeaseID. + Leases []int64 `json:",omitempty"` +} + +func (s EtcdState) Equal(other EtcdState) bool { + if s.Revision != other.Revision { + return false + } + if s.CompactRevision != other.CompactRevision { + return false + } + if unsafe.SliceData(s.Keys) != unsafe.SliceData(other.Keys) { + panic("Can only compare states created from the same key slice") + } + return slices.EqualFunc(s.KeyValues, other.KeyValues, equalPtr) && + slices.EqualFunc(s.KeyLeases, other.KeyLeases, equalPtr) && + slices.Equal(s.Leases, other.Leases) +} + +func equalPtr[T comparable](a, b *T) bool { + if a == b { + return true + } + if a == nil || b == nil { + return false + } + return *a == *b } func (s EtcdState) apply(request EtcdRequest, response EtcdResponse) (bool, EtcdState) { @@ -85,144 +117,171 @@ func (s EtcdState) DeepCopy() EtcdState { CompactRevision: s.CompactRevision, } - newState.KeyValues = maps.Clone(s.KeyValues) - newState.KeyLeases = maps.Clone(s.KeyLeases) - - newLeases := map[int64]EtcdLease{} - for key, val := range s.Leases { - newLeases[key] = val.DeepCopy() - } - newState.Leases = newLeases + newState.Keys = s.Keys + newState.KeyValues = slices.Clone(s.KeyValues) + newState.KeyLeases = slices.Clone(s.KeyLeases) + newState.Leases = slices.Clone(s.Leases) return newState } -func freshEtcdState() EtcdState { +func freshEtcdState(keys []string) EtcdState { return EtcdState{ Revision: 1, // Start from CompactRevision equal -1 as etcd allows client to compact revision 0 for some reason. CompactRevision: -1, - KeyValues: map[string]ValueRevision{}, - KeyLeases: map[string]int64{}, - Leases: map[int64]EtcdLease{}, + Keys: keys, + KeyValues: make([]*ValueRevision, len(keys)), + KeyLeases: make([]*int64, len(keys)), + Leases: make([]int64, 0), } } // Step handles a successful request, returning updated state and response it would generate. func (s EtcdState) Step(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { - newState := s.DeepCopy() - switch request.Type { case Range: - if request.Range.Revision == 0 || request.Range.Revision == newState.Revision { - resp := newState.getRange(request.Range.RangeOptions) - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &resp, Revision: newState.Revision}} - } - if request.Range.Revision > newState.Revision { - return newState, MaybeEtcdResponse{Error: ErrEtcdFutureRev.Error()} - } - if request.Range.Revision < newState.CompactRevision { - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrCompacted.Error()}} - } - return newState, MaybeEtcdResponse{Persisted: true, PersistedRevision: newState.Revision} + return s.stepRange(request) case Txn: - failure := false - for _, cond := range request.Txn.Conditions { - val := newState.KeyValues[cond.Key] - if cond.ExpectedVersion > 0 { - if val.Version != cond.ExpectedVersion { - failure = true - break - } - } else if val.ModRevision != cond.ExpectedRevision { + return s.stepTxn(request) + case LeaseGrant: + return s.stepLeaseGrant(request) + case LeaseRevoke: + return s.stepLeaseRevoke(request) + case Defragment: + return s.stepDefragment() + case Compact: + return s.stepCompact(request) + default: + panic(fmt.Sprintf("Unknown request type: %v", request.Type)) + } +} + +func (s EtcdState) stepRange(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { + if request.Range.Revision == 0 || request.Range.Revision == s.Revision { + resp := s.getRange(request.Range.RangeOptions) + return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &resp, Revision: s.Revision}} + } + if request.Range.Revision > s.Revision { + return s, MaybeEtcdResponse{Error: ErrEtcdFutureRev.Error()} + } + if request.Range.Revision < s.CompactRevision { + return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrCompacted.Error()}} + } + return s, MaybeEtcdResponse{Persisted: true, PersistedRevision: s.Revision} +} + +func (s EtcdState) stepTxn(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { + // TODO: Avoid copying when TXN only has read operations + newState := s.DeepCopy() + failure := false + for _, cond := range request.Txn.Conditions { + val, ok := newState.GetValue(cond.Key) + if !ok { + val = &ValueRevision{} + } + if cond.ExpectedVersion > 0 { + if val.Version != cond.ExpectedVersion { failure = true break } + } else if val.ModRevision != cond.ExpectedRevision { + failure = true + break } - operations := request.Txn.OperationsOnSuccess - if failure { - operations = request.Txn.OperationsOnFailure - } - opResp := make([]EtcdOperationResult, len(operations)) - increaseRevision := false - for i, op := range operations { - switch op.Type { - case RangeOperation: - opResp[i] = EtcdOperationResult{ - RangeResponse: newState.getRange(op.Range), - } - case PutOperation: - _, leaseExists := newState.Leases[op.Put.LeaseID] - if op.Put.LeaseID != 0 && !leaseExists { + } + operations := request.Txn.OperationsOnSuccess + if failure { + operations = request.Txn.OperationsOnFailure + } + opResp := make([]EtcdOperationResult, len(operations)) + increaseRevision := false + for i, op := range operations { + switch op.Type { + case RangeOperation: + opResp[i] = EtcdOperationResult{ + RangeResponse: newState.getRange(op.Range), + } + case PutOperation: + var leaseID *int64 + if op.Put.LeaseID != 0 { + if !newState.leaseExists(op.Put.LeaseID) { break } - ver := int64(1) - if val, exists := newState.KeyValues[op.Put.Key]; exists && val.Version > 0 { - ver = val.Version + 1 - } - newState.KeyValues[op.Put.Key] = ValueRevision{ - Value: op.Put.Value, - ModRevision: newState.Revision + 1, - Version: ver, - } - increaseRevision = true - newState = detachFromOldLease(newState, op.Put.Key) - if leaseExists { - newState = attachToNewLease(newState, op.Put.LeaseID, op.Put.Key) - } - case DeleteOperation: - if _, ok := newState.KeyValues[op.Delete.Key]; ok { - delete(newState.KeyValues, op.Delete.Key) - increaseRevision = true - newState = detachFromOldLease(newState, op.Delete.Key) - opResp[i].Deleted = 1 - } - default: - panic("unsupported operation") + leaseID = &op.Put.LeaseID } - } - if increaseRevision { - newState.Revision++ - } - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Failure: failure, Results: opResp}, Revision: newState.Revision}} - case LeaseGrant: - lease := EtcdLease{ - LeaseID: request.LeaseGrant.LeaseID, - Keys: map[string]struct{}{}, - } - newState.Leases[request.LeaseGrant.LeaseID] = lease - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseGrant: &LeaseGrantReponse{}}} - case LeaseRevoke: - // Delete the keys attached to the lease - keyDeleted := false - for key := range newState.Leases[request.LeaseRevoke.LeaseID].Keys { - // same as delete. - if _, ok := newState.KeyValues[key]; ok { - if !keyDeleted { - keyDeleted = true - } - delete(newState.KeyValues, key) - delete(newState.KeyLeases, key) + ver := int64(1) + if val, exists := newState.GetValue(op.Put.Key); exists && val.Version > 0 { + ver = val.Version + 1 } + newState.setValueLease(op.Put.Key, ValueRevision{ + Value: op.Put.Value, + ModRevision: newState.Revision + 1, + Version: ver, + }, leaseID) + increaseRevision = true + case DeleteOperation: + if _, ok := newState.GetValue(op.Delete.Key); ok { + newState.deleteKey(op.Delete.Key) + increaseRevision = true + opResp[i].Deleted = 1 + } + default: + panic("unsupported operation") } - // delete the lease - delete(newState.Leases, request.LeaseRevoke.LeaseID) - if keyDeleted { - newState.Revision++ + } + if increaseRevision { + newState.Revision++ + } + return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Failure: failure, Results: opResp}, Revision: newState.Revision}} +} + +func (s EtcdState) stepLeaseGrant(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { + newState := s.DeepCopy() + // Empty LeaseID means the request failed and client didn't get response. Ignore it as client cannot use lease without knowing its id. + if request.LeaseGrant.LeaseID == 0 { + return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseGrant: &LeaseGrantResponse{}}} + } + newState.Leases = append(newState.Leases, request.LeaseGrant.LeaseID) + sort.Slice(newState.Leases, func(i, j int) bool { return newState.Leases[i] < newState.Leases[j] }) + return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseGrant: &LeaseGrantResponse{}}} +} + +func (s EtcdState) stepLeaseRevoke(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { + newState := s.DeepCopy() + keyDeleted := false + for i, l := range newState.KeyLeases { + if l != nil && *l == request.LeaseRevoke.LeaseID { + keyDeleted = true + newState.KeyValues[i] = nil + newState.KeyLeases[i] = nil } - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseRevoke: &LeaseRevokeResponse{}}} - case Defragment: - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: newState.Revision}} - case Compact: - if request.Compact.Revision <= newState.CompactRevision { - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrCompacted.Error()}} + } + for i, l := range newState.Leases { + if l == request.LeaseRevoke.LeaseID { + newState.Leases = append(newState.Leases[:i], newState.Leases[i+1:]...) + break } - newState.CompactRevision = request.Compact.Revision - // Set fake revision as compaction returns non-linearizable revision. - // TODO: Model non-linearizable response revision in model. - return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Compact: &CompactResponse{}, Revision: -1}} - default: - panic(fmt.Sprintf("Unknown request type: %v", request.Type)) } + if keyDeleted { + newState.Revision++ + } + return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseRevoke: &LeaseRevokeResponse{}}} +} + +func (s EtcdState) stepDefragment() (EtcdState, MaybeEtcdResponse) { + return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: RevisionForNonLinearizableResponse}} +} + +func (s EtcdState) stepCompact(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { + newState := s.DeepCopy() + if request.Compact.Revision <= newState.CompactRevision { + return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrCompacted.Error()}} + } + if request.Compact.Revision > newState.Revision { + return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrFutureRev.Error()}} + } + newState.CompactRevision = request.Compact.Revision + return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Compact: &CompactResponse{}, Revision: RevisionForNonLinearizableResponse}} } func (s EtcdState) getRange(options RangeOptions) RangeResponse { @@ -231,9 +290,17 @@ func (s EtcdState) getRange(options RangeOptions) RangeResponse { } if options.End != "" { var count int64 - for k, v := range s.KeyValues { + for i, v := range s.KeyValues { + if v == nil { + continue + } + k := s.Keys[i] if k >= options.Start && k < options.End { - response.KVs = append(response.KVs, KeyValue{Key: k, ValueRevision: v}) + kv := KeyValue{Key: k, ValueRevision: *v} + if options.KeysOnly { + kv.ValueRevision.Value = ValueOrHash{} + } + response.KVs = append(response.KVs, kv) count++ } } @@ -245,11 +312,11 @@ func (s EtcdState) getRange(options RangeOptions) RangeResponse { } response.Count = count } else { - value, ok := s.KeyValues[options.Start] + value, ok := s.GetValue(options.Start) if ok { response.KVs = append(response.KVs, KeyValue{ Key: options.Start, - ValueRevision: value, + ValueRevision: *value, }) response.Count = 1 } @@ -257,217 +324,76 @@ func (s EtcdState) getRange(options RangeOptions) RangeResponse { return response } -func detachFromOldLease(s EtcdState, key string) EtcdState { - if oldLeaseID, ok := s.KeyLeases[key]; ok { - delete(s.Leases[oldLeaseID].Keys, key) - delete(s.KeyLeases, key) - } - return s -} - -func attachToNewLease(s EtcdState, leaseID int64, key string) EtcdState { - s.KeyLeases[key] = leaseID - s.Leases[leaseID].Keys[key] = leased - return s -} - -type RequestType string - -const ( - Range RequestType = "range" - Txn RequestType = "txn" - LeaseGrant RequestType = "leaseGrant" - LeaseRevoke RequestType = "leaseRevoke" - Defragment RequestType = "defragment" - Compact RequestType = "compact" -) - -type EtcdRequest struct { - Type RequestType - LeaseGrant *LeaseGrantRequest - LeaseRevoke *LeaseRevokeRequest - Range *RangeRequest - Txn *TxnRequest - Defragment *DefragmentRequest - Compact *CompactRequest -} +func (s EtcdState) KeysValueLeases() (keys []string, values []ValueRevision, leases []int64) { + keys = make([]string, 0, len(s.KeyValues)) + values = make([]ValueRevision, 0, len(s.KeyValues)) + leases = make([]int64, 0, len(s.KeyLeases)) -func (r *EtcdRequest) IsRead() bool { - if r.Type == Range { - return true - } - if r.Type != Txn { - return false - } - for _, op := range append(r.Txn.OperationsOnSuccess, r.Txn.OperationsOnFailure...) { - if op.Type != RangeOperation { - return false + for i, v := range s.KeyValues { + if v == nil { + continue + } + keys = append(keys, s.Keys[i]) + values = append(values, *v) + lease := int64(0) + if s.KeyLeases[i] != nil { + lease = *s.KeyLeases[i] } + leases = append(leases, lease) } - return true -} - -type RangeRequest struct { - RangeOptions - Revision int64 -} - -type RangeOptions struct { - Start string - End string - Limit int64 + return keys, values, leases } -type PutOptions struct { - Key string - Value ValueOrHash - LeaseID int64 +func (s EtcdState) leases() []int64 { + return slices.Clone(s.Leases) } -type DeleteOptions struct { - Key string -} - -type TxnRequest struct { - Conditions []EtcdCondition - OperationsOnSuccess []EtcdOperation - OperationsOnFailure []EtcdOperation -} - -type EtcdCondition struct { - Key string - ExpectedRevision int64 - ExpectedVersion int64 -} - -type EtcdOperation struct { - Type OperationType - Range RangeOptions - Put PutOptions - Delete DeleteOptions -} - -type OperationType string - -const ( - RangeOperation OperationType = "range-operation" - PutOperation OperationType = "put-operation" - DeleteOperation OperationType = "delete-operation" -) - -type LeaseGrantRequest struct { - LeaseID int64 -} -type LeaseRevokeRequest struct { - LeaseID int64 -} -type DefragmentRequest struct{} - -// MaybeEtcdResponse extends EtcdResponse to include partial information about responses to a request. -// Possible response state information: -// * Normal response. Client observed response. Only EtcdResponse is set. -// * Persisted. Client didn't observe response, but we know it was persisted by etcd. Only Persisted is set -// * Persisted with Revision. Client didn't observe response, but we know that it was persisted, and it's revision. Both Persisted and PersistedRevision is set. -// * Error response. Client observed error, but we don't know if it was persisted. Only Error is set. -type MaybeEtcdResponse struct { - EtcdResponse - Persisted bool - PersistedRevision int64 - Error string -} - -var ErrEtcdFutureRev = errors.New("future rev") - -type EtcdResponse struct { - Txn *TxnResponse - Range *RangeResponse - LeaseGrant *LeaseGrantReponse - LeaseRevoke *LeaseRevokeResponse - Defragment *DefragmentResponse - Compact *CompactResponse - ClientError string - Revision int64 -} - -func Match(r1, r2 MaybeEtcdResponse) bool { - r1Revision := r1.Revision - if r1.Persisted { - r1Revision = r1.PersistedRevision - } - r2Revision := r2.Revision - if r2.Persisted { - r2Revision = r2.PersistedRevision +func (s EtcdState) GetValue(key string) (*ValueRevision, bool) { + for i, k := range s.Keys { + if k == key { + return s.KeyValues[i], s.KeyValues[i] != nil + } } - return (r1.Persisted && r1.PersistedRevision == 0) || (r2.Persisted && r2.PersistedRevision == 0) || ((r1.Persisted || r2.Persisted) && (r1.Error != "" || r2.Error != "" || r1Revision == r2Revision)) || reflect.DeepEqual(r1, r2) -} - -type TxnResponse struct { - Failure bool - Results []EtcdOperationResult -} - -type RangeResponse struct { - KVs []KeyValue - Count int64 -} - -type LeaseGrantReponse struct { - LeaseID int64 -} -type ( - LeaseRevokeResponse struct{} - DefragmentResponse struct{} -) - -type EtcdOperationResult struct { - RangeResponse - Deleted int64 -} - -type KeyValue struct { - Key string - ValueRevision + return nil, false } -var leased = struct{}{} - -type EtcdLease struct { - LeaseID int64 - Keys map[string]struct{} -} - -func (el EtcdLease) DeepCopy() EtcdLease { - return EtcdLease{ - LeaseID: el.LeaseID, - Keys: maps.Clone(el.Keys), +func (s EtcdState) leaseExists(lease int64) bool { + for _, l := range s.Leases { + if l == lease { + return true + } } + return false } -type ValueRevision struct { - Value ValueOrHash - ModRevision int64 - Version int64 -} - -type ValueOrHash struct { - Value string - Hash uint32 +func (s EtcdState) setValueLease(key string, val ValueRevision, lease *int64) { + for i, k := range s.Keys { + if k == key { + s.KeyValues[i] = &val + s.KeyLeases[i] = lease + return + } + } + panic(fmt.Sprintf("couldn't find key %s in EtcdState (%v) when calling setValue", key, s.Keys)) } -func ToValueOrHash(value string) ValueOrHash { - v := ValueOrHash{} - if len(value) < 20 { - v.Value = value - } else { - h := fnv.New32a() - h.Write([]byte(value)) - v.Hash = h.Sum32() +func (s EtcdState) deleteKey(key string) { + for i, k := range s.Keys { + if k == key { + s.KeyValues[i] = nil + s.KeyLeases[i] = nil + return + } } - return v + panic(fmt.Sprintf("couldn't find key %s in EtcdState (%v) when calling setValue", key, s.Keys)) } -type CompactResponse struct{} - -type CompactRequest struct { - Revision int64 +func (s EtcdState) leaseKeys(leaseID int64) []string { + keys := []string{} + for i, l := range s.KeyLeases { + if l != nil && *l == leaseID { + keys = append(keys, s.Keys[i]) + } + } + return keys } diff --git a/tests/robustness/model/deterministic_test.go b/tests/robustness/model/deterministic_test.go index 6e75126239db..f5737e8859d0 100644 --- a/tests/robustness/model/deterministic_test.go +++ b/tests/robustness/model/deterministic_test.go @@ -15,11 +15,11 @@ package model import ( - "encoding/json" + "math/rand" + "slices" "testing" "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" ) @@ -28,15 +28,15 @@ func TestModelDeterministic(t *testing.T) { for _, tc := range commonTestScenarios { tc := tc t.Run(tc.name, func(t *testing.T) { - state := DeterministicModel.Init() + keys := keysFromTestOperations(tc.operations) + model := DeterministicModel(keys) + state := model.Init() for _, op := range tc.operations { - ok, newState := DeterministicModel.Step(state, op.req, op.resp.EtcdResponse) + ok, newState := model.Step(state, op.req, op.resp.EtcdResponse) if op.expectFailure == ok { t.Logf("state: %v", state) - t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, DeterministicModel.DescribeOperation(op.req, op.resp.EtcdResponse)) - var loadedState EtcdState - err := json.Unmarshal([]byte(state.(string)), &loadedState) - require.NoErrorf(t, err, "Failed to load state") + t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, model.DescribeOperation(op.req, op.resp.EtcdResponse)) + loadedState := state.(EtcdState) _, resp := loadedState.Step(op.req) t.Errorf("Response diff: %s", cmp.Diff(op.resp, resp)) break @@ -50,6 +50,170 @@ func TestModelDeterministic(t *testing.T) { } } +func TestEtcdStateEqual(t *testing.T) { + keys := []string{"key"} + testCases := []struct { + name string + s1 EtcdState + s2 EtcdState + equal bool + }{ + { + name: "Fresh states should be equal", + s1: freshEtcdState(keys), + s2: freshEtcdState(keys), + equal: true, + }, + { + name: "States from identical history should be equal", + s1: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "1")) + s, _ = s.Step(putRequest("key", "2")) + return s + }(), + s2: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "1")) + s, _ = s.Step(putRequest("key", "2")) + return s + }(), + equal: true, + }, + { + name: "States from different history should not be equal", + s1: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "1")) + s, _ = s.Step(putRequest("key", "2")) + return s + }(), + s2: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "2")) + s, _ = s.Step(putRequest("key", "1")) + return s + }(), + equal: false, + }, + { + name: "Empty states with higher revision should be equal", + s1: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "1")) + s, _ = s.Step(putRequest("key", "2")) + s, _ = s.Step(deleteRequest("key")) + return s + }(), + s2: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "2")) + s, _ = s.Step(putRequest("key", "1")) + s, _ = s.Step(deleteRequest("key")) + return s + }(), + equal: true, + }, + { + name: "Grant and Revoke empty lease should be equal to fresh state", + s1: freshEtcdState(keys), + s2: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(leaseGrantRequest(1)) + s, _ = s.Step(leaseRevokeRequest(1)) + return s + }(), + equal: true, + }, + { + name: "Delete via Revoke vs Delete directly should be equal", + s1: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(leaseGrantRequest(1)) + s, _ = s.Step(putWithLeaseRequest("key", "val", 1)) + s, _ = s.Step(leaseRevokeRequest(1)) + return s + }(), + s2: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "val")) + s, _ = s.Step(deleteRequest("key")) + return s + }(), + equal: true, + }, + { + name: "Put via Txn vs Put directly should be equal", + s1: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(compareRevisionAndPutRequest("key", 0, "val")) + return s + }(), + s2: func() EtcdState { + s := freshEtcdState(keys) + s, _ = s.Step(putRequest("key", "val")) + return s + }(), + equal: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.s1.Equal(tc.s2) != tc.equal { + t.Errorf("Expected equal=%v, got %v", tc.equal, !tc.equal) + t.Errorf("Diff:\n%v", cmp.Diff(tc.s1, tc.s2)) + } + }) + } +} + +func TestEtcdStateEqualCommutativeRequests(t *testing.T) { + commutativeRequests := []EtcdRequest{ + leaseGrantRequest(1), + leaseGrantRequest(2), + leaseRevokeRequest(3), + leaseRevokeRequest(4), + getRequest("key1"), + getRequest("key2"), + defragmentRequest(), + defragmentRequest(), + compactRequest(1), + compactRequest(2), + } + keys := []string{"key1", "key2"} + + baseState := applyRequests(keys, commutativeRequests) + + for i := 0; i < 10_000; i++ { + perm := slices.Clone(commutativeRequests) + rand.Shuffle(len(perm), func(i, j int) { + perm[i], perm[j] = perm[j], perm[i] + }) + s2 := applyRequests(keys, perm) + + if !baseState.Equal(s2) { + t.Errorf("Expected states to be equal after random reordering, but they are not") + } + } +} + +func applyRequests(keys []string, reqs []EtcdRequest) EtcdState { + state := freshEtcdState(keys) + for _, req := range reqs { + state, _ = state.Step(req) + } + return state +} + +func keysFromTestOperations(ops []testOperation) []string { + requests := make([]EtcdRequest, 0, len(ops)) + for _, op := range ops { + requests = append(requests, op.req) + } + return keysFromRequests(requests) +} + type modelTestCase struct { name string operations []testOperation @@ -264,6 +428,13 @@ var commonTestScenarios = []modelTestCase{ {req: getRequest("key"), resp: getResponse("key", "2", 2, 2)}, }, }, + { + name: "Lease grant returns current revision", + operations: []testOperation{ + {req: putRequest("key", "1"), resp: putResponse(2)}, + {req: leaseGrantRequest(1), resp: leaseGrantResponse(2)}, + }, + }, { name: "Put with valid lease id should succeed. Put with expired lease id should fail", operations: []testOperation{ @@ -381,27 +552,27 @@ var commonTestScenarios = []modelTestCase{ {req: getRequest("key"), resp: getResponse("key", "4", 4, 4)}, {req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)}, {req: deleteRequest("key"), resp: deleteResponse(1, 6)}, - {req: defragmentRequest(), resp: defragmentResponse(6)}, + {req: defragmentRequest(), resp: defragmentResponse()}, }, }, { name: "Defragment success between all other request types", operations: []testOperation{ - {req: defragmentRequest(), resp: defragmentResponse(1)}, + {req: defragmentRequest(), resp: defragmentResponse()}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, - {req: defragmentRequest(), resp: defragmentResponse(1)}, + {req: defragmentRequest(), resp: defragmentResponse()}, {req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)}, - {req: defragmentRequest(), resp: defragmentResponse(2)}, + {req: defragmentRequest(), resp: defragmentResponse()}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, - {req: defragmentRequest(), resp: defragmentResponse(3)}, + {req: defragmentRequest(), resp: defragmentResponse()}, {req: putRequest("key", "4"), resp: putResponse(4)}, - {req: defragmentRequest(), resp: defragmentResponse(4)}, + {req: defragmentRequest(), resp: defragmentResponse()}, {req: getRequest("key"), resp: getResponse("key", "4", 4, 4)}, - {req: defragmentRequest(), resp: defragmentResponse(4)}, + {req: defragmentRequest(), resp: defragmentResponse()}, {req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)}, - {req: defragmentRequest(), resp: defragmentResponse(5)}, + {req: defragmentRequest(), resp: defragmentResponse()}, {req: deleteRequest("key"), resp: deleteResponse(1, 6)}, - {req: defragmentRequest(), resp: defragmentResponse(6)}, + {req: defragmentRequest(), resp: defragmentResponse()}, }, }, } diff --git a/tests/robustness/model/history.go b/tests/robustness/model/history.go index fc224a8afffe..a41ffeac82fb 100644 --- a/tests/robustness/model/history.go +++ b/tests/robustness/model/history.go @@ -28,44 +28,49 @@ import ( "go.etcd.io/etcd/tests/v3/robustness/identity" ) -// AppendableHistory allows to collect history of sequential operations. +// AppendableHistory allows collecting the history of sequential operations. // -// Ensures that operation history is compatible with porcupine library, by preventing concurrent requests sharing the -// same stream id. For failed requests, we don't know their return time, so generate new stream id. +// Ensures that the operation history is compatible with the porcupine library by preventing concurrent requests from sharing the +// same stream id. For failed requests, we don't know their return time, so we generate a new stream id. // // Appending needs to be done in order of operation execution time (start, end time). -// Operations time should be calculated as time.Since common base time to ensure that Go monotonic time is used. +// Operation time should be calculated as time.Since a common base time to ensure that Go monotonic time is used. // More in https://github.com/golang/go/blob/96add980ad27faed627f26ef1ab09e8fe45d6bd1/src/time/time.go#L10. type AppendableHistory struct { // streamID for the next operation. Used for porcupine.Operation.ClientId as porcupine assumes no concurrent requests. streamID int // If needed a new streamId is requested from idProvider. idProvider identity.Provider + // lastOperation holds the last operation of each stream. + lastOperation map[int]*porcupine.Operation History } func NewAppendableHistory(ids identity.Provider) *AppendableHistory { return &AppendableHistory{ - streamID: ids.NewStreamID(), - idProvider: ids, + streamID: ids.NewStreamID(), + idProvider: ids, + lastOperation: make(map[int]*porcupine.Operation), History: History{ operations: []porcupine.Operation{}, }, } } -func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit int64, start, end time.Duration, resp *clientv3.GetResponse, err error) { - request := staleRangeRequest(startKey, endKey, limit, revision) +func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit int64, keysOnly bool, start, end time.Duration, resp *clientv3.GetResponse, err error) { + request := staleRangeRequest(startKey, endKey, limit, revision, keysOnly) if err != nil { h.appendFailed(request, start, end, err) return } var respRevision int64 + var respMemberID uint64 if resp != nil && resp.Header != nil { respRevision = resp.Header.Revision + respMemberID = resp.Header.MemberId } - h.appendSuccessful(request, start, end, rangeResponse(resp.Kvs, resp.Count, respRevision)) + h.appendSuccessful(request, start, end, rangeResponseWithMemberID(resp.Kvs, resp.Count, respRevision, MemberID(respMemberID))) } func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duration, resp *clientv3.PutResponse, err error) { @@ -75,10 +80,12 @@ func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duratio return } var revision int64 + var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision + memberID = MemberID(resp.Header.MemberId) } - h.appendSuccessful(request, start, end, putResponse(revision)) + h.appendSuccessful(request, start, end, putResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendPutWithLease(key, value string, leaseID int64, start, end time.Duration, resp *clientv3.PutResponse, err error) { @@ -88,10 +95,12 @@ func (h *AppendableHistory) AppendPutWithLease(key, value string, leaseID int64, return } var revision int64 + var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision + memberID = MemberID(resp.Header.MemberId) } - h.appendSuccessful(request, start, end, putResponse(revision)) + h.appendSuccessful(request, start, end, putResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *clientv3.LeaseGrantResponse, err error) { @@ -105,10 +114,12 @@ func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *cli return } var revision int64 + var memberID MemberID if resp != nil && resp.ResponseHeader != nil { revision = resp.ResponseHeader.Revision + memberID = MemberID(resp.ResponseHeader.MemberId) } - h.appendSuccessful(request, start, end, leaseGrantResponse(revision)) + h.appendSuccessful(request, start, end, leaseGrantResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration, resp *clientv3.LeaseRevokeResponse, err error) { @@ -118,10 +129,12 @@ func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration return } var revision int64 + var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision + memberID = MemberID(resp.Header.MemberId) } - h.appendSuccessful(request, start, end, leaseRevokeResponse(revision)) + h.appendSuccessful(request, start, end, leaseRevokeResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, resp *clientv3.DeleteResponse, err error) { @@ -131,12 +144,14 @@ func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, r return } var revision int64 + var memberID MemberID var deleted int64 if resp != nil && resp.Header != nil { revision = resp.Header.Revision + memberID = MemberID(resp.Header.MemberId) deleted = resp.Deleted } - h.appendSuccessful(request, start, end, deleteResponse(deleted, revision)) + h.appendSuccessful(request, start, end, deleteResponseWithMemberID(deleted, revision, memberID)) } func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, clientOnFailure []clientv3.Op, start, end time.Duration, resp *clientv3.TxnResponse, err error) { @@ -158,14 +173,22 @@ func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, cl return } var revision int64 + var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision + memberID = MemberID(resp.Header.MemberId) } results := []EtcdOperationResult{} for _, resp := range resp.Responses { results = append(results, toEtcdOperationResult(resp)) } - h.appendSuccessful(request, start, end, txnResponse(results, resp.Succeeded, revision)) + h.appendSuccessful(request, start, end, txnResponseWithMemberID(results, resp.Succeeded, revision, memberID)) +} + +func (h *AppendableHistory) appendClientError(request EtcdRequest, start, end time.Duration, err error) { + h.appendSuccessful(request, start, end, MaybeEtcdResponse{ + EtcdResponse: EtcdResponse{ClientError: err.Error()}, + }) } func (h *AppendableHistory) appendSuccessful(request EtcdRequest, start, end time.Duration, response MaybeEtcdResponse) { @@ -175,20 +198,22 @@ func (h *AppendableHistory) appendSuccessful(request EtcdRequest, start, end tim Call: start.Nanoseconds(), Output: response, Return: end.Nanoseconds(), + Metadata: response, } h.append(op) } func toEtcdCondition(cmp clientv3.Cmp) (cond EtcdCondition) { + compare := cmp.GetCompare() switch { - case cmp.Result == etcdserverpb.Compare_EQUAL && cmp.Target == etcdserverpb.Compare_MOD: + case compare.GetResult() == etcdserverpb.Compare_EQUAL && compare.GetTarget() == etcdserverpb.Compare_MOD: cond.Key = string(cmp.KeyBytes()) - cond.ExpectedRevision = cmp.TargetUnion.(*etcdserverpb.Compare_ModRevision).ModRevision - case cmp.Result == etcdserverpb.Compare_EQUAL && cmp.Target == etcdserverpb.Compare_VERSION: - cond.ExpectedVersion = cmp.TargetUnion.(*etcdserverpb.Compare_Version).Version + cond.ExpectedRevision = compare.GetTargetUnion().(*etcdserverpb.Compare_ModRevision).ModRevision + case compare.GetResult() == etcdserverpb.Compare_EQUAL && compare.GetTarget() == etcdserverpb.Compare_VERSION: + cond.ExpectedVersion = compare.GetTargetUnion().(*etcdserverpb.Compare_Version).Version cond.Key = string(cmp.KeyBytes()) default: - panic(fmt.Sprintf("Compare not supported, target: %q, result: %q", cmp.Target, cmp.Result)) + panic(fmt.Sprintf("Compare not supported, target: %q, result: %q", compare.GetTarget(), compare.GetResult())) } return cond } @@ -256,42 +281,38 @@ func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *cli h.appendFailed(request, start, end, err) return } - var revision int64 - if resp != nil && resp.Header != nil { - revision = resp.Header.Revision - } - h.appendSuccessful(request, start, end, defragmentResponse(revision)) + h.appendSuccessful(request, start, end, defragmentResponse()) } func (h *AppendableHistory) AppendCompact(rev int64, start, end time.Duration, resp *clientv3.CompactResponse, err error) { request := compactRequest(rev) if err != nil { if strings.Contains(err.Error(), mvcc.ErrCompacted.Error()) { - h.appendSuccessful(request, start, end, MaybeEtcdResponse{ - EtcdResponse: EtcdResponse{ClientError: mvcc.ErrCompacted.Error()}, - }) + h.appendClientError(request, start, end, mvcc.ErrCompacted) + return + } + if strings.Contains(err.Error(), mvcc.ErrFutureRev.Error()) { + h.appendClientError(request, start, end, mvcc.ErrFutureRev) return } h.appendFailed(request, start, end, err) return } - // Set fake revision as compaction returns non-linearizable revision. - // TODO: Model non-linearizable response revision in model. - h.appendSuccessful(request, start, end, compactResponse(-1)) + h.appendSuccessful(request, start, end, compactResponse()) } func (h *AppendableHistory) appendFailed(request EtcdRequest, start, end time.Duration, err error) { + resp := failedResponse(err) op := porcupine.Operation{ ClientId: h.streamID, Input: request, Call: start.Nanoseconds(), - Output: failedResponse(err), + Output: resp, Return: end.Nanoseconds(), + Metadata: resp, } isRead := request.IsRead() if !isRead { - // Failed writes can still be persisted, setting -1 for now as don't know when request has took effect. - op.Return = -1 // Operations of single client needs to be sequential. // As we don't know return time of failed operations, all new writes need to be done with new stream id. h.streamID = h.idProvider.NewStreamID() @@ -300,11 +321,11 @@ func (h *AppendableHistory) appendFailed(request EtcdRequest, start, end time.Du } func (h *AppendableHistory) append(op porcupine.Operation) { - if op.Return != -1 && op.Call >= op.Return { + if op.Call >= op.Return { panic(fmt.Sprintf("Invalid operation, call(%d) >= return(%d)", op.Call, op.Return)) } - if len(h.operations) > 0 { - prev := h.operations[len(h.operations)-1] + + if prev, ok := h.lastOperation[op.ClientId]; ok { if op.Call <= prev.Call { panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prev.Call)) } @@ -312,6 +333,8 @@ func (h *AppendableHistory) append(op porcupine.Operation) { panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prev.Return)) } } + h.lastOperation[op.ClientId] = &op + h.operations = append(h.operations, op) } @@ -320,11 +343,11 @@ func getRequest(key string) EtcdRequest { } func staleGetRequest(key string, revision int64) EtcdRequest { - return staleRangeRequest(key, "", 0, revision) + return staleRangeRequest(key, "", 0, revision, false) } func rangeRequest(start, end string, limit int64) EtcdRequest { - return staleRangeRequest(start, end, limit, 0) + return staleRangeRequest(start, end, limit, 0, false) } func listRequest(key string, limit int64) EtcdRequest { @@ -332,11 +355,11 @@ func listRequest(key string, limit int64) EtcdRequest { } func staleListRequest(key string, limit, revision int64) EtcdRequest { - return staleRangeRequest(key, clientv3.GetPrefixRangeEnd(key), limit, revision) + return staleRangeRequest(key, clientv3.GetPrefixRangeEnd(key), limit, revision, false) } -func staleRangeRequest(start, end string, limit, revision int64) EtcdRequest { - return EtcdRequest{Type: Range, Range: &RangeRequest{RangeOptions: RangeOptions{Start: start, End: end, Limit: limit}, Revision: revision}} +func staleRangeRequest(start, end string, limit, revision int64, keysOnly bool) EtcdRequest { + return EtcdRequest{Type: Range, Range: &RangeRequest{RangeOptions: RangeOptions{Start: start, End: end, Limit: limit, KeysOnly: keysOnly}, Revision: revision}} } func emptyGetResponse(revision int64) MaybeEtcdResponse { @@ -352,6 +375,10 @@ func getResponseWithVer(key, value string, modRevision, ver, revision int64) May } func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) MaybeEtcdResponse { + return rangeResponseWithMemberID(kvs, count, revision, 0) +} + +func rangeResponseWithMemberID(kvs []*mvccpb.KeyValue, count int64, revision int64, memberID MemberID) MaybeEtcdResponse { result := RangeResponse{KVs: make([]KeyValue, len(kvs)), Count: count} for i, kv := range kvs { @@ -364,7 +391,7 @@ func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) MaybeEtc }, } } - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &result, Revision: revision}} + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &result, Revision: revision, MemberID: memberID}} } func failedResponse(err error) MaybeEtcdResponse { @@ -380,7 +407,11 @@ func putRequest(key, value string) EtcdRequest { } func putResponse(revision int64) MaybeEtcdResponse { - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{}}}, Revision: revision}} + return putResponseWithMemberID(revision, 0) +} + +func putResponseWithMemberID(revision int64, memberID MemberID) MaybeEtcdResponse { + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{}}}, Revision: revision, MemberID: memberID}} } func deleteRequest(key string) EtcdRequest { @@ -388,7 +419,11 @@ func deleteRequest(key string) EtcdRequest { } func deleteResponse(deleted int64, revision int64) MaybeEtcdResponse { - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}} + return deleteResponseWithMemberID(deleted, revision, 0) +} + +func deleteResponseWithMemberID(deleted int64, revision int64, memberID MemberID) MaybeEtcdResponse { + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision, MemberID: memberID}} } func compareRevisionAndPutRequest(key string, expectedRevision int64, value string) EtcdRequest { @@ -439,7 +474,11 @@ func txnEmptyResponse(succeeded bool, revision int64) MaybeEtcdResponse { } func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) MaybeEtcdResponse { - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: result, Failure: !succeeded}, Revision: revision}} + return txnResponseWithMemberID(result, succeeded, revision, 0) +} + +func txnResponseWithMemberID(result []EtcdOperationResult, succeeded bool, revision int64, memberID MemberID) MaybeEtcdResponse { + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: result, Failure: !succeeded}, Revision: revision, MemberID: memberID}} } func putWithLeaseRequest(key, value string, leaseID int64) EtcdRequest { @@ -451,7 +490,11 @@ func leaseGrantRequest(leaseID int64) EtcdRequest { } func leaseGrantResponse(revision int64) MaybeEtcdResponse { - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantReponse{}, Revision: revision}} + return leaseGrantResponseWithMemberID(revision, 0) +} + +func leaseGrantResponseWithMemberID(revision int64, memberID MemberID) MaybeEtcdResponse { + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantResponse{}, Revision: revision, MemberID: memberID}} } func leaseRevokeRequest(leaseID int64) EtcdRequest { @@ -459,23 +502,27 @@ func leaseRevokeRequest(leaseID int64) EtcdRequest { } func leaseRevokeResponse(revision int64) MaybeEtcdResponse { - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision}} + return leaseRevokeResponseWithMemberID(revision, 0) +} + +func leaseRevokeResponseWithMemberID(revision int64, memberID MemberID) MaybeEtcdResponse { + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision, MemberID: memberID}} } func defragmentRequest() EtcdRequest { return EtcdRequest{Type: Defragment, Defragment: &DefragmentRequest{}} } -func defragmentResponse(revision int64) MaybeEtcdResponse { - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: revision}} +func defragmentResponse() MaybeEtcdResponse { + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: RevisionForNonLinearizableResponse}} } func compactRequest(rev int64) EtcdRequest { return EtcdRequest{Type: Compact, Compact: &CompactRequest{Revision: rev}} } -func compactResponse(revision int64) MaybeEtcdResponse { - return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Compact: &CompactResponse{}, Revision: revision}} +func compactResponse() MaybeEtcdResponse { + return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Compact: &CompactResponse{}, Revision: RevisionForNonLinearizableResponse}} } type History struct { @@ -488,34 +535,8 @@ func (h History) Len() int { func (h History) Operations() []porcupine.Operation { operations := make([]porcupine.Operation, 0, len(h.operations)) - maxTime := h.lastObservedTime() - for _, op := range h.operations { - // Failed requests don't have a known return time. - if op.Return == -1 { - // Simulate Infinity by using last observed time. - op.Return = maxTime + time.Second.Nanoseconds() - } - operations = append(operations, op) - } - return operations -} -func (h History) lastObservedTime() int64 { - var maxTime int64 - for _, op := range h.operations { - if op.Return == -1 { - // Collect call time from failed operations - if op.Call > maxTime { - maxTime = op.Call - } - } else { - // Collect return time from successful operations - if op.Return > maxTime { - maxTime = op.Return - } - } - } - return maxTime + return append(operations, h.operations...) } func (h History) MaxRevision() int64 { diff --git a/tests/robustness/model/history_test.go b/tests/robustness/model/history_test.go new file mode 100644 index 000000000000..263441dca1f1 --- /dev/null +++ b/tests/robustness/model/history_test.go @@ -0,0 +1,245 @@ +// Copyright 2025 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "github.com/anishathalye/porcupine" + "github.com/stretchr/testify/assert" + + "go.etcd.io/etcd/tests/v3/robustness/identity" +) + +func TestHistoryAppendSuccess(t *testing.T) { + tcs := []struct { + name string + operations []porcupine.Operation + }{ + { + name: "append non-overlapping operations for the same clientId", + operations: []porcupine.Operation{ + { + ClientId: 1, + Input: nil, + Call: 100, + Output: nil, + Return: 200, + }, + { + ClientId: 2, + Input: nil, + Call: 1, + Output: nil, + Return: 2, + }, + }, + }, + { + name: "append overlapping operations in different ClientId", + operations: []porcupine.Operation{ + { + ClientId: 1, + Input: nil, + Call: 1, + Output: nil, + Return: 100, + }, + { + ClientId: 2, + Input: nil, + Call: 1, + Output: nil, + Return: 2, + }, + { + ClientId: 3, + Input: nil, + Call: 10, + Output: nil, + Return: 20, + }, + { + ClientId: 1, + Input: nil, + Call: 101, + Output: nil, + Return: 200, + }, + { + ClientId: 2, + Input: nil, + Call: 3, + Output: nil, + Return: 4, + }, + { + ClientId: 3, + Input: nil, + Call: 30, + Output: nil, + Return: 40, + }, + }, + }, + } + + for _, tc := range tcs { + h := NewAppendableHistory(identity.NewIDProvider()) + + for _, operation := range tc.operations[:len(tc.operations)-1] { + h.append(operation) + } + } +} + +func TestHistoryAppendFailure(t *testing.T) { + tcs := []struct { + name string + operations []porcupine.Operation + expectError string + }{ + { + name: "append operation with call time > return time", + operations: []porcupine.Operation{ + { + ClientId: 1, + Input: nil, + Call: 2, + Output: nil, + Return: 1, + }, + }, + expectError: "Invalid operation, call(2) >= return(1)", + }, + { + name: "out of order append in the same ClientId", + operations: []porcupine.Operation{ + { + ClientId: 1, + Input: nil, + Call: 10, + Output: nil, + Return: 100, + }, + { + ClientId: 1, + Input: nil, + Call: 1, + Output: nil, + Return: 10, + }, + }, + expectError: "Out of order append, new.call(1) <= prev.call(10)", + }, + { + name: "out of order append in one of the ClientIds", + operations: []porcupine.Operation{ + { + ClientId: 1, + Input: nil, + Call: 10, + Output: nil, + Return: 100, + }, + { + ClientId: 1, + Input: nil, + Call: 101, + Output: nil, + Return: 200, + }, + { + ClientId: 2, + Input: nil, + Call: 10, + Output: nil, + Return: 100, + }, + { + ClientId: 2, + Input: nil, + Call: 1, + Output: nil, + Return: 10, + }, + }, + expectError: "Out of order append, new.call(1) <= prev.call(10)", + }, + { + name: "append overlapping operations in the same ClientId", + operations: []porcupine.Operation{ + { + ClientId: 1, + Input: nil, + Call: 1, + Output: nil, + Return: 100, + }, + { + ClientId: 1, + Input: nil, + Call: 10, + Output: nil, + Return: 20, + }, + }, + expectError: "Overlapping operations, new.call(10) <= prev.return(100)", + }, + { + name: "append overlapping operations in one of the ClientIds", + operations: []porcupine.Operation{ + { + ClientId: 1, + Input: nil, + Call: 1, + Output: nil, + Return: 100, + }, + { + ClientId: 1, + Input: nil, + Call: 101, + Output: nil, + Return: 200, + }, + { + ClientId: 2, + Input: nil, + Call: 1, + Output: nil, + Return: 100, + }, + { + ClientId: 2, + Input: nil, + Call: 10, + Output: nil, + Return: 20, + }, + }, + expectError: "Overlapping operations, new.call(10) <= prev.return(100)", + }, + } + + for _, tc := range tcs { + h := NewAppendableHistory(identity.NewIDProvider()) + + for _, operation := range tc.operations[:len(tc.operations)-1] { + h.append(operation) + } + assert.PanicsWithValue(t, tc.expectError, func() { h.append(tc.operations[len(tc.operations)-1]) }) + } +} diff --git a/tests/robustness/model/non_deterministic.go b/tests/robustness/model/non_deterministic.go index 3167e340fd16..b9a28abe00b1 100644 --- a/tests/robustness/model/non_deterministic.go +++ b/tests/robustness/model/non_deterministic.go @@ -15,46 +15,153 @@ package model import ( - "encoding/json" + "cmp" "fmt" - "reflect" + "slices" + "strings" + "sync/atomic" "github.com/anishathalye/porcupine" ) -// NonDeterministicModel extends DeterministicModel to allow for clients with imperfect knowledge of request destiny. -// Unknown/error response doesn't inform whether request was persisted or not, so model -// considers both cases. This is represented as multiple equally possible deterministic states. +// LinearizationDeadlineTripped is set when linearization validation exceeds +// its timeout. +// +// Porcupine currently does not propagate its timeout to the model Step +// callback. A single non-deterministic Step can take seconds or minutes, so +// CheckOperationsVerbose may not return promptly after its timeout expires. +// Until etcd depends on a Porcupine version with timeout propagation, validation +// uses this atomic flag to let Step and its helpers return early. +// +// See https://github.com/anishathalye/porcupine/pull/45 and https://github.com/etcd-io/etcd/pull/21715. +var LinearizationDeadlineTripped atomic.Int32 + +// NonDeterministicModel extends DeterministicModel to allow for clients with imperfect knowledge of the request's destiny. +// An unknown/error response doesn't inform whether the request was persisted or not, so the model +// considers both cases. This is represented as multiple, equally possible deterministic states. // Failed requests fork the possible states, while successful requests merge and filter them. -var NonDeterministicModel = porcupine.Model{ - Init: func() any { - data, err := json.Marshal(nonDeterministicState{freshEtcdState()}) - if err != nil { - panic(err) - } - return string(data) - }, - Step: func(st any, in any, out any) (bool, any) { - var states nonDeterministicState - err := json.Unmarshal([]byte(st.(string)), &states) - if err != nil { - panic(err) - } - ok, states := states.apply(in.(EtcdRequest), out.(MaybeEtcdResponse)) - data, err := json.Marshal(states) - if err != nil { - panic(err) - } - return ok, string(data) - }, - DescribeOperation: func(in, out any) string { - return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse))) - }, +var NonDeterministicModel = func(keys []string) porcupine.Model { + return porcupine.Model{ + Init: func() any { + return nonDeterministicState{freshEtcdState(keys)} + }, + Step: func(st any, in any, out any) (bool, any) { + return st.(nonDeterministicState).apply(in.(EtcdRequest), out.(MaybeEtcdResponse)) + }, + Equal: func(st1, st2 any) bool { + return st1.(nonDeterministicState).Equal(st2.(nonDeterministicState)) + }, + DescribeOperation: func(in, out any) string { + return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse))) + }, + DescribeOperationMetadata: func(info any) string { + if info == nil { + return "" + } + return DescribeOperationMetadata(info.(MaybeEtcdResponse)) + }, + DescribeState: func(st any) string { + etcdStates := st.(nonDeterministicState) + desc := make([]string, 0, len(etcdStates)) + + slices.SortFunc(etcdStates, compareStates) + + for i, s := range etcdStates { + // Describe just 3 first states before truncating + if i >= 3 { + desc = append(desc, "...truncated...") + break + } + desc = append(desc, describeEtcdState(s)) + } + + return strings.Join(desc, "\n") + }, + } } type nonDeterministicState []EtcdState +func (states nonDeterministicState) Equal(other nonDeterministicState) bool { + if len(states) != len(other) { + return false + } + slices.SortFunc(states, compareStates) + slices.SortFunc(other, compareStates) + + for i := range states { + if !states[i].Equal(other[i]) { + return false + } + } + return true +} + +func compareStates(first, second EtcdState) int { + if c := cmp.Compare(first.Revision, second.Revision); c != 0 { + return c + } + if c := cmp.Compare(first.CompactRevision, second.CompactRevision); c != 0 { + return c + } + if c := cmp.Compare(len(first.KeyValues), len(second.KeyValues)); c != 0 { + return c + } + for i := range first.KeyValues { + if (first.KeyValues[i] == nil) != (second.KeyValues[i] == nil) { + if first.KeyValues[i] == nil { + return -1 + } + return 1 + } + if first.KeyValues[i] == nil { + continue + } + if c := cmp.Compare(first.KeyValues[i].ModRevision, second.KeyValues[i].ModRevision); c != 0 { + return c + } + if c := cmp.Compare(first.KeyValues[i].Version, second.KeyValues[i].Version); c != 0 { + return c + } + if c := cmp.Compare(first.KeyValues[i].Value.Value, second.KeyValues[i].Value.Value); c != 0 { + return c + } + if c := cmp.Compare(first.KeyValues[i].Value.Hash, second.KeyValues[i].Value.Hash); c != 0 { + return c + } + } + if c := cmp.Compare(len(first.KeyLeases), len(second.KeyLeases)); c != 0 { + return c + } + for i := range first.KeyLeases { + if (first.KeyLeases[i] == nil) != (second.KeyLeases[i] == nil) { + if first.KeyLeases[i] == nil { + return -1 + } + return 1 + } + if first.KeyLeases[i] == nil { + continue + } + if c := cmp.Compare(*first.KeyLeases[i], *second.KeyLeases[i]); c != 0 { + return c + } + } + if c := cmp.Compare(len(first.Leases), len(second.Leases)); c != 0 { + return c + } + for i := range first.Leases { + if c := cmp.Compare(first.Leases[i], second.Leases[i]); c != 0 { + return c + } + } + return 0 +} + func (states nonDeterministicState) apply(request EtcdRequest, response MaybeEtcdResponse) (bool, nonDeterministicState) { + if LinearizationDeadlineTripped.Load() != 0 { + return false, nil + } var newStates nonDeterministicState switch { case response.Error != "": @@ -73,9 +180,12 @@ func (states nonDeterministicState) apply(request EtcdRequest, response MaybeEtc func (states nonDeterministicState) applyFailedRequest(request EtcdRequest) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)*2) for _, s := range states { + if LinearizationDeadlineTripped.Load() != 0 { + return nil + } newStates = append(newStates, s) newState, _ := s.Step(request) - if !reflect.DeepEqual(newState, s) { + if !newState.Equal(s) { newStates = append(newStates, newState) } } @@ -86,6 +196,9 @@ func (states nonDeterministicState) applyFailedRequest(request EtcdRequest) nonD func (states nonDeterministicState) applyPersistedRequest(request EtcdRequest) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)) for _, s := range states { + if LinearizationDeadlineTripped.Load() != 0 { + return nil + } newState, _ := s.Step(request) newStates = append(newStates, newState) } @@ -96,6 +209,9 @@ func (states nonDeterministicState) applyPersistedRequest(request EtcdRequest) n func (states nonDeterministicState) applyPersistedRequestWithRevision(request EtcdRequest, responseRevision int64) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)) for _, s := range states { + if LinearizationDeadlineTripped.Load() != 0 { + return nil + } newState, modelResponse := s.Step(request) if modelResponse.Revision == responseRevision { newStates = append(newStates, newState) @@ -108,6 +224,9 @@ func (states nonDeterministicState) applyPersistedRequestWithRevision(request Et func (states nonDeterministicState) applyRequestWithResponse(request EtcdRequest, response EtcdResponse) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)) for _, s := range states { + if LinearizationDeadlineTripped.Load() != 0 { + return nil + } newState, modelResponse := s.Step(request) if Match(modelResponse, MaybeEtcdResponse{EtcdResponse: response}) { newStates = append(newStates, newState) diff --git a/tests/robustness/model/non_deterministic_test.go b/tests/robustness/model/non_deterministic_test.go index b6622e6fa812..1960f75e9c7c 100644 --- a/tests/robustness/model/non_deterministic_test.go +++ b/tests/robustness/model/non_deterministic_test.go @@ -15,13 +15,11 @@ package model import ( - "encoding/json" "errors" "testing" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" ) @@ -328,15 +326,15 @@ func TestModelNonDeterministic(t *testing.T) { for _, tc := range nonDeterministicTestScenarios { tc := tc t.Run(tc.name, func(t *testing.T) { - state := NonDeterministicModel.Init() + keys := keysFromTestOperations(tc.operations) + model := NonDeterministicModel(keys) + state := model.Init() for _, op := range tc.operations { - ok, newState := NonDeterministicModel.Step(state, op.req, op.resp) + ok, newState := model.Step(state, op.req, op.resp) if ok != !op.expectFailure { t.Logf("state: %v", state) - t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, NonDeterministicModel.DescribeOperation(op.req, op.resp)) - var loadedState nonDeterministicState - err := json.Unmarshal([]byte(state.(string)), &loadedState) - require.NoErrorf(t, err, "Failed to load state") + t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, model.DescribeOperation(op.req, op.resp)) + loadedState := state.(nonDeterministicState) for i, s := range loadedState { _, resp := s.Step(op.req) t.Errorf("For state %d, response diff: %s", i, cmp.Diff(op.resp, resp)) @@ -543,6 +541,11 @@ func TestModelResponseMatch(t *testing.T) { resp2: MaybeEtcdResponse{Persisted: true, PersistedRevision: 3}, expectMatch: false, }, + { + resp1: putResponseWithMemberID(2, 1), + resp2: putResponseWithMemberID(2, 2), + expectMatch: true, + }, { resp1: failedResponse(errors.New("failed request")), resp2: MaybeEtcdResponse{Persisted: true}, diff --git a/tests/robustness/model/replay.go b/tests/robustness/model/replay.go index 7f2be26e8ed2..a9f3653b1db5 100644 --- a/tests/robustness/model/replay.go +++ b/tests/robustness/model/replay.go @@ -16,13 +16,25 @@ package model import ( "fmt" - "sort" + "maps" + "slices" "strings" + + "github.com/anishathalye/porcupine" ) +func NewReplayFromOperations(ops []porcupine.Operation) *EtcdReplay { + requests := []EtcdRequest{} + for _, op := range ops { + requests = append(requests, op.Input.(EtcdRequest)) + } + return NewReplay(requests) +} + func NewReplay(persistedRequests []EtcdRequest) *EtcdReplay { - state := freshEtcdState() - // Padding for index 0 and 1, so index matches revision.. + keys := keysFromRequests(persistedRequests) + state := freshEtcdState(keys) + // Padding for index 0 and 1, so the index matches the revision. revisionToEtcdState := []EtcdState{state, state} var events []PersistedEvent for _, request := range persistedRequests { @@ -74,7 +86,7 @@ func toWatchEvents(prevState *EtcdState, request EtcdRequest, response MaybeEtcd } else { ops = request.Txn.OperationsOnSuccess } - for _, op := range ops { + for i, op := range ops { switch op.Type { case RangeOperation: case DeleteOperation: @@ -85,11 +97,11 @@ func toWatchEvents(prevState *EtcdState, request EtcdRequest, response MaybeEtcd }, Revision: response.Revision, } - if _, ok := prevState.KeyValues[op.Delete.Key]; ok { + if response.Txn.Results[i].Deleted != 0 { events = append(events, e) } case PutOperation: - _, leaseExists := prevState.Leases[op.Put.LeaseID] + leaseExists := prevState.leaseExists(op.Put.LeaseID) if op.Put.LeaseID != 0 && !leaseExists { break } @@ -102,7 +114,7 @@ func toWatchEvents(prevState *EtcdState, request EtcdRequest, response MaybeEtcd }, Revision: response.Revision, } - if _, ok := prevState.KeyValues[op.Put.Key]; !ok { + if _, ok := prevState.GetValue(op.Put.Key); !ok { e.IsCreate = true } events = append(events, e) @@ -111,15 +123,7 @@ func toWatchEvents(prevState *EtcdState, request EtcdRequest, response MaybeEtcd } } case LeaseRevoke: - deletedKeys := []string{} - for key := range prevState.Leases[request.LeaseRevoke.LeaseID].Keys { - if _, ok := prevState.KeyValues[key]; ok { - deletedKeys = append(deletedKeys, key) - } - } - - sort.Strings(deletedKeys) - for _, key := range deletedKeys { + for _, key := range prevState.leaseKeys(request.LeaseRevoke.LeaseID) { e := PersistedEvent{ Event: Event{ Type: DeleteOperation, @@ -134,20 +138,20 @@ func toWatchEvents(prevState *EtcdState, request EtcdRequest, response MaybeEtcd } type WatchEvent struct { - PersistedEvent - PrevValue *ValueRevision + PersistedEvent `json:",omitempty"` + PrevValue *ValueRevision `json:",omitempty"` } type PersistedEvent struct { - Event - Revision int64 - IsCreate bool + Event `json:",omitempty"` + Revision int64 `json:",omitempty"` + IsCreate bool `json:",omitempty"` } type Event struct { - Type OperationType - Key string - Value ValueOrHash + Type OperationType `json:",omitempty"` + Key string `json:",omitempty"` + Value ValueOrHash `json:",omitempty"` } func (e Event) Match(request WatchRequest) bool { @@ -164,3 +168,41 @@ type WatchRequest struct { WithProgressNotify bool WithPrevKV bool } + +func ModelKeys(operations []porcupine.Operation) []string { + requests := []EtcdRequest{} + for _, op := range operations { + requests = append(requests, op.Input.(EtcdRequest)) + } + return keysFromRequests(requests) +} + +func keysFromRequests(requests []EtcdRequest) []string { + keysMap := map[string]bool{} + for _, request := range requests { + switch request.Type { + case Range: + keysMap[request.Range.Start] = true + if request.Range.End != "" { + keysMap[request.Range.End] = true + } + case Txn: + for _, op := range slices.Concat(request.Txn.OperationsOnSuccess, request.Txn.OperationsOnFailure) { + switch op.Type { + case RangeOperation: + keysMap[op.Range.Start] = true + if op.Range.End != "" { + keysMap[op.Range.End] = true + } + case PutOperation: + keysMap[op.Put.Key] = true + case DeleteOperation: + keysMap[op.Delete.Key] = true + } + } + } + } + keys := slices.Collect(maps.Keys(keysMap)) + slices.Sort(keys) + return keys +} diff --git a/tests/robustness/model/types.go b/tests/robustness/model/types.go new file mode 100644 index 000000000000..280a90464744 --- /dev/null +++ b/tests/robustness/model/types.go @@ -0,0 +1,251 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "errors" + "hash/fnv" + "reflect" + "slices" + + "go.etcd.io/etcd/client/pkg/v3/types" +) + +// RevisionForNonLinearizableResponse is a fake revision value used to +// separate responses for requests that are not linearizable, +// thus we need to ignore it in model when checking linearization. +const RevisionForNonLinearizableResponse = -1 + +type RequestType string + +const ( + Range RequestType = "range" + Txn RequestType = "txn" + LeaseGrant RequestType = "leaseGrant" + LeaseRevoke RequestType = "leaseRevoke" + Defragment RequestType = "defragment" + Compact RequestType = "compact" +) + +type EtcdRequest struct { + Type RequestType + LeaseGrant *LeaseGrantRequest + LeaseRevoke *LeaseRevokeRequest + Range *RangeRequest + Txn *TxnRequest + Defragment *DefragmentRequest + Compact *CompactRequest +} + +func (r *EtcdRequest) IsRead() bool { + if r.Type == Range { + return true + } + if r.Type != Txn { + return false + } + for _, op := range append(r.Txn.OperationsOnSuccess, r.Txn.OperationsOnFailure...) { + if op.Type != RangeOperation { + return false + } + } + return true +} + +type RangeRequest struct { + RangeOptions + Revision int64 +} + +type RangeOptions struct { + Start string + End string + Limit int64 + KeysOnly bool +} + +type PutOptions struct { + Key string + Value ValueOrHash + LeaseID int64 +} + +type DeleteOptions struct { + Key string +} + +type TxnRequest struct { + Conditions []EtcdCondition + OperationsOnSuccess []EtcdOperation + OperationsOnFailure []EtcdOperation +} + +func (txn *TxnRequest) AllOperations() []EtcdOperation { + return slices.Concat(txn.OperationsOnSuccess, txn.OperationsOnFailure) +} + +type EtcdCondition struct { + Key string + ExpectedRevision int64 + ExpectedVersion int64 +} + +type EtcdOperation struct { + Type OperationType + Range RangeOptions + Put PutOptions + Delete DeleteOptions +} + +type OperationType string + +const ( + RangeOperation OperationType = "range-operation" + PutOperation OperationType = "put-operation" + DeleteOperation OperationType = "delete-operation" +) + +type LeaseGrantRequest struct { + LeaseID int64 +} +type LeaseRevokeRequest struct { + LeaseID int64 +} +type DefragmentRequest struct{} + +// MaybeEtcdResponse extends EtcdResponse to include partial information about responses to a request. +// Possible response state information: +// * Normal response. Client observed response. Only EtcdResponse is set. +// * Persisted. Client didn't observe response, but we know it was persisted by etcd. Only Persisted is set +// * Persisted with Revision. Client didn't observe a response, but we know that it was persisted, and its revision. Both Persisted and PersistedRevision is set. +// * Error response. Client observed error, but we don't know if it was persisted. Only Error is set. +type MaybeEtcdResponse struct { + EtcdResponse + Persisted bool + PersistedRevision int64 + Error string +} + +var ErrEtcdFutureRev = errors.New("future rev") + +type MemberID uint64 + +// MarshalJSON implements the json.Marshaler interface for MemberID. +// It marshals the MemberID as a hexadecimal string. +func (m MemberID) MarshalJSON() ([]byte, error) { + return json.Marshal(types.ID(m).String()) +} + +// UnmarshalJSON implements the json.Unmarshaler interface for MemberID. +// It unmarshals the MemberID from a hexadecimal string. +func (m *MemberID) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + id, err := types.IDFromString(s) + if err != nil { + return err + } + *m = MemberID(id) + return nil +} + +type EtcdResponse struct { + Txn *TxnResponse + Range *RangeResponse + LeaseGrant *LeaseGrantResponse + LeaseRevoke *LeaseRevokeResponse + Defragment *DefragmentResponse + Compact *CompactResponse + ClientError string + Revision int64 + MemberID MemberID `json:"member-id,omitempty"` +} + +func Match(r1, r2 MaybeEtcdResponse) bool { + r1Revision := r1.Revision + if r1.Persisted { + r1Revision = r1.PersistedRevision + } + r2Revision := r2.Revision + if r2.Persisted { + r2Revision = r2.PersistedRevision + } + + r1.EtcdResponse.MemberID = 0 + r2.EtcdResponse.MemberID = 0 + + return (r1.Persisted && r1.PersistedRevision == 0) || (r2.Persisted && r2.PersistedRevision == 0) || ((r1.Persisted || r2.Persisted) && (r1.Error != "" || r2.Error != "" || r1Revision == r2Revision)) || reflect.DeepEqual(r1, r2) +} + +type TxnResponse struct { + Failure bool + Results []EtcdOperationResult +} + +type RangeResponse struct { + KVs []KeyValue + Count int64 +} + +type LeaseGrantResponse struct { + LeaseID int64 +} +type ( + LeaseRevokeResponse struct{} + DefragmentResponse struct{} +) + +type EtcdOperationResult struct { + RangeResponse + Deleted int64 +} + +type KeyValue struct { + Key string + ValueRevision +} + +type ValueRevision struct { + Value ValueOrHash `json:",omitempty"` + ModRevision int64 `json:",omitempty"` + Version int64 `json:",omitempty"` +} + +type ValueOrHash struct { + Value string `json:",omitempty"` + Hash uint32 `json:",omitempty"` +} + +func ToValueOrHash(value string) ValueOrHash { + v := ValueOrHash{} + if len(value) < 20 { + v.Value = value + } else { + h := fnv.New32a() + h.Write([]byte(value)) + v.Hash = h.Sum32() + } + return v +} + +type CompactResponse struct{} + +type CompactRequest struct { + Revision int64 +} diff --git a/tests/robustness/model/types_test.go b/tests/robustness/model/types_test.go new file mode 100644 index 000000000000..c0cd72d2781d --- /dev/null +++ b/tests/robustness/model/types_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd/client/pkg/v3/types" +) + +func TestMemberIDMatchesTypesID(t *testing.T) { + raw := uint64(13770331908272521436) + + mID := MemberID(raw) + jsonBytes, err := json.Marshal(mID) + require.NoError(t, err) + jsonStr := string(jsonBytes) + + tID := types.ID(raw) + expectedHex := tID.String() + require.JSONEq(t, "\""+expectedHex+"\"", jsonStr) +} + +func TestMemberIDMatchesTypesIDUnmarshal(t *testing.T) { + raw := uint64(13770331908272521999) + mID := MemberID(raw) + + jsonBytes, err := json.Marshal(mID) + require.NoError(t, err) + + tID := types.ID(raw) + expectedHex := "\"" + tID.String() + "\"" + require.JSONEq(t, expectedHex, string(jsonBytes)) + + var parsedID MemberID + err = json.Unmarshal(jsonBytes, &parsedID) + require.NoError(t, err) + require.Equal(t, mID, parsedID) +} diff --git a/tests/robustness/model/watch.go b/tests/robustness/model/watch.go index fc880e30ede6..de2c54d1cc44 100644 --- a/tests/robustness/model/watch.go +++ b/tests/robustness/model/watch.go @@ -17,14 +17,14 @@ package model import "time" type WatchOperation struct { - Request WatchRequest - Responses []WatchResponse + Request WatchRequest `json:",omitempty"` + Responses []WatchResponse `json:",omitempty"` } type WatchResponse struct { - Events []WatchEvent - IsProgressNotify bool - Revision int64 - Time time.Duration - Error string + Events []WatchEvent `json:",omitempty"` + IsProgressNotify bool `json:",omitempty"` + Revision int64 `json:",omitempty"` + Time time.Duration `json:",omitempty"` + Error string `json:",omitempty"` } diff --git a/tests/robustness/options/cluster_options.go b/tests/robustness/options/cluster_options.go index 01031e218469..455d78322416 100644 --- a/tests/robustness/options/cluster_options.go +++ b/tests/robustness/options/cluster_options.go @@ -25,8 +25,8 @@ var internalRand = rand.New(rand.NewSource(time.Now().UnixNano())) type ClusterOptions []e2e.EPClusterOption -// WithClusterOptionGroups takes an array of EPClusterOption arrays, and randomly picks one EPClusterOption array when constructing the config. -// This function is mainly used to group strongly coupled config options together, so that we can dynamically test different groups of options. +// WithClusterOptionGroups takes an array of EPClusterOption arrays and randomly picks one EPClusterOption array when constructing the config. +// This function is mainly used to group strongly coupled config options together so that we can dynamically test different groups of options. func WithClusterOptionGroups(input ...ClusterOptions) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { optsPicked := input[internalRand.Intn(len(input))] @@ -36,10 +36,10 @@ func WithClusterOptionGroups(input ...ClusterOptions) e2e.EPClusterOption { } } -// WithSubsetOptions randomly select a subset of input options, and apply the subset to the cluster config. +// WithSubsetOptions randomly selects a subset of input options and applies the subset to the cluster config. func WithSubsetOptions(input ...e2e.EPClusterOption) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { - // selects random subsetLen (0 to len(input)) elements from the input array. + // selects a random subsetLen (0 to len(input)) elements from the input array. subsetLen := internalRand.Intn(len(input) + 1) perm := internalRand.Perm(len(input)) for i := 0; i < subsetLen; i++ { diff --git a/tests/robustness/options/server_config_options.go b/tests/robustness/options/server_config_options.go index ade51592cd1c..a0a6b5d5a875 100644 --- a/tests/robustness/options/server_config_options.go +++ b/tests/robustness/options/server_config_options.go @@ -26,9 +26,9 @@ func WithSnapshotCount(input ...uint64) e2e.EPClusterOption { } } -func WithExperimentalCompactionBatchLimit(input ...int) e2e.EPClusterOption { +func WithCompactionBatchLimit(input ...int) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { - c.ServerConfig.ExperimentalCompactionBatchLimit = input[internalRand.Intn(len(input))] + c.ServerConfig.CompactionBatchLimit = input[internalRand.Intn(len(input))] } } @@ -50,9 +50,9 @@ func WithElectionMs(input ...uint) e2e.EPClusterOption { } } -func WithExperimentalWatchProgressNotifyInterval(input ...time.Duration) e2e.EPClusterOption { +func WithWatchProgressNotifyInterval(input ...time.Duration) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { - c.ServerConfig.ExperimentalWatchProgressNotifyInterval = input[internalRand.Intn(len(input))] + c.ServerConfig.WatchProgressNotifyInterval = input[internalRand.Intn(len(input))] } } diff --git a/tests/robustness/report/client.go b/tests/robustness/report/client.go index 48d29b8ae818..4585f91ec013 100644 --- a/tests/robustness/report/client.go +++ b/tests/robustness/report/client.go @@ -22,10 +22,8 @@ import ( "sort" "strconv" "strings" - "testing" "github.com/anishathalye/porcupine" - "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" @@ -47,25 +45,32 @@ func (r ClientReport) WatchEventCount() int { return count } -func persistClientReports(t *testing.T, lg *zap.Logger, path string, reports []ClientReport) { +func persistClientReports(lg *zap.Logger, path string, reports []ClientReport) error { sort.Slice(reports, func(i, j int) bool { return reports[i].ClientID < reports[j].ClientID }) for _, r := range reports { clientDir := filepath.Join(path, fmt.Sprintf("client-%d", r.ClientID)) - err := os.MkdirAll(clientDir, 0o700) - require.NoError(t, err) + if err := os.MkdirAll(clientDir, 0o700); err != nil { + return err + } + if len(r.Watch) != 0 { - persistWatchOperations(t, lg, filepath.Join(clientDir, "watch.json"), r.Watch) + if err := persistWatchOperations(lg, filepath.Join(clientDir, "watch.json"), r.Watch); err != nil { + return err + } } else { lg.Info("no watch operations for client, skip persisting", zap.Int("client-id", r.ClientID)) } if len(r.KeyValue) != 0 { - persistKeyValueOperations(t, lg, filepath.Join(clientDir, "operations.json"), r.KeyValue) + if err := persistKeyValueOperations(lg, filepath.Join(clientDir, "operations.json"), r.KeyValue); err != nil { + return err + } } else { lg.Info("no KV operations for client, skip persisting", zap.Int("client-id", r.ClientID)) } } + return nil } func LoadClientReports(path string) ([]ClientReport, error) { @@ -138,11 +143,11 @@ func loadKeyValueOperations(path string) (operations []porcupine.Operation, err if os.IsNotExist(err) { return nil, nil } - return nil, fmt.Errorf("failed to open watch operation file: %q, err: %w", path, err) + return nil, fmt.Errorf("failed to open KV operation file: %q, err: %w", path, err) } file, err := os.OpenFile(path, os.O_RDONLY, 0o755) if err != nil { - return nil, fmt.Errorf("failed to open watch operation file: %q, err: %w", path, err) + return nil, fmt.Errorf("failed to open KV operation file: %q, err: %w", path, err) } defer file.Close() decoder := json.NewDecoder(file) @@ -156,7 +161,7 @@ func loadKeyValueOperations(path string) (operations []porcupine.Operation, err } err = decoder.Decode(&operation) if err != nil { - return nil, fmt.Errorf("failed to decode watch operation, err: %w", err) + return nil, fmt.Errorf("failed to decode KV operation, err: %w", err) } operations = append(operations, porcupine.Operation{ ClientId: operation.ClientID, @@ -164,41 +169,57 @@ func loadKeyValueOperations(path string) (operations []porcupine.Operation, err Call: operation.Call, Output: operation.Output, Return: operation.Return, + Metadata: operation.Output, }) } return operations, nil } -func persistWatchOperations(t *testing.T, lg *zap.Logger, path string, responses []model.WatchOperation) { +func persistWatchOperations(lg *zap.Logger, path string, responses []model.WatchOperation) error { lg.Info("Saving watch operations", zap.String("path", path)) file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) if err != nil { - t.Errorf("Failed to save watch operations: %v", err) - return + return fmt.Errorf("failed to save watch operations: %w", err) } defer file.Close() - encoder := json.NewEncoder(file) for _, resp := range responses { - err := encoder.Encode(resp) + data, err := json.MarshalIndent(resp, "", " ") if err != nil { - t.Errorf("Failed to encode operation: %v", err) + return fmt.Errorf("failed to encode operation: %w", err) } + file.Write(data) + file.WriteString("\n") } + return nil } -func persistKeyValueOperations(t *testing.T, lg *zap.Logger, path string, operations []porcupine.Operation) { +func persistKeyValueOperations(lg *zap.Logger, path string, operations []porcupine.Operation) error { lg.Info("Saving operation history", zap.String("path", path)) file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) if err != nil { - t.Errorf("Failed to save operation history: %v", err) - return + return fmt.Errorf("Failed to save KV operation history: %w", err) } defer file.Close() - encoder := json.NewEncoder(file) for _, op := range operations { - err := encoder.Encode(op) + data, err := json.MarshalIndent(op, "", " ") if err != nil { - t.Errorf("Failed to encode operation: %v", err) + return fmt.Errorf("Failed to encode KV operation: %w", err) + } + file.Write(data) + file.WriteString("\n") + } + return nil +} + +func OperationsMaxRevision(reports []ClientReport) int64 { + var maxRevision int64 + for _, r := range reports { + for _, op := range r.KeyValue { + resp := op.Output.(model.MaybeEtcdResponse) + if resp.Revision > maxRevision { + maxRevision = resp.Revision + } } } + return maxRevision } diff --git a/tests/robustness/report/client_test.go b/tests/robustness/report/client_test.go index 07da049c74ea..3e00dc990436 100644 --- a/tests/robustness/report/client_test.go +++ b/tests/robustness/report/client_test.go @@ -38,7 +38,7 @@ func TestPersistLoadClientReports(t *testing.T) { start := time.Since(baseTime) time.Sleep(time.Nanosecond) stop := time.Since(baseTime) - h.AppendRange("key", "", 0, 0, start, stop, &clientv3.GetResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}, Count: 2, Kvs: []*mvccpb.KeyValue{{ + h.AppendRange("key", "", 0, 0, false, start, stop, &clientv3.GetResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}, Count: 2, Kvs: []*mvccpb.KeyValue{{ Key: []byte("key"), ModRevision: 2, Value: []byte("value"), @@ -133,7 +133,8 @@ func TestPersistLoadClientReports(t *testing.T) { }, } path := t.TempDir() - persistClientReports(t, zaptest.NewLogger(t), path, reports) + err := persistClientReports(zaptest.NewLogger(t), path, reports) + require.NoError(t, err) got, err := LoadClientReports(path) require.NoError(t, err) if diff := cmp.Diff(reports, got, cmpopts.EquateEmpty()); diff != "" { diff --git a/tests/robustness/report/report.go b/tests/robustness/report/report.go index b3d3d5a5e32c..d1bfbc87b914 100644 --- a/tests/robustness/report/report.go +++ b/tests/robustness/report/report.go @@ -15,71 +15,107 @@ package report import ( + "encoding/json" "fmt" "os" + "path" "path/filepath" - "strings" - "testing" - "time" - "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/framework/e2e" ) type TestReport struct { - Logger *zap.Logger - Cluster *e2e.EtcdProcessCluster - Client []ClientReport - Visualize func(path string) error + logger *zap.Logger + reportPath string + serverDataPaths map[string]string + clientReports []ClientReport + visualize func(lg *zap.Logger, path string) error + traffic *TrafficDetail + dataSaved bool } -func testResultsDirectory(t *testing.T) string { - resultsDirectory, ok := os.LookupEnv("RESULTS_DIR") - if !ok { - resultsDirectory = "/tmp/" +func NewTestReport(lg *zap.Logger, reportPath string, serverDataPaths map[string]string, traffic *TrafficDetail) (*TestReport, error) { + if reportPath == "" { + return nil, fmt.Errorf("reportPath is not set") } - resultsDirectory, err := filepath.Abs(resultsDirectory) - if err != nil { - panic(err) - } - path, err := filepath.Abs(filepath.Join( - resultsDirectory, strings.ReplaceAll(t.Name(), "/", "_"), fmt.Sprintf("%v", time.Now().UnixNano()))) - require.NoError(t, err) - err = os.RemoveAll(path) - require.NoError(t, err) - err = os.MkdirAll(path, 0o700) - require.NoError(t, err) - return path + return &TestReport{ + logger: lg, + reportPath: reportPath, + serverDataPaths: serverDataPaths, + traffic: traffic, + }, nil +} + +func (r *TestReport) SetClientReports(reports []ClientReport) { + r.clientReports = reports } -func (r *TestReport) Report(t *testing.T, force bool) { - _, persistResultsEnvSet := os.LookupEnv("PERSIST_RESULTS") - if !t.Failed() && !force && !persistResultsEnvSet { - return +func (r *TestReport) SetVisualizer(visualize func(lg *zap.Logger, path string) error) { + r.visualize = visualize +} + +func (r *TestReport) SaveEtcdData() error { + if r.dataSaved { + return nil + } + r.logger.Info("Saving etcd data", zap.String("path", r.reportPath)) + err := os.RemoveAll(r.reportPath) + if err != nil { + r.logger.Error("Failed to remove report dir", zap.Error(err)) } - path := testResultsDirectory(t) - r.Logger.Info("Saving robustness test report", zap.String("path", path)) - for _, member := range r.Cluster.Procs { - memberDataDir := filepath.Join(path, fmt.Sprintf("server-%s", member.Config().Name)) - persistMemberDataDir(t, r.Logger, member, memberDataDir) + for server, dataPath := range r.serverDataPaths { + serverReportPath := filepath.Join(r.reportPath, fmt.Sprintf("server-%s", server)) + r.logger.Info("Saving member data dir", zap.String("member", server), zap.String("data-dir", dataPath), zap.String("path", serverReportPath)) + if err := os.CopyFS(serverReportPath, os.DirFS(dataPath)); err != nil { + return err + } } - if r.Client != nil { - persistClientReports(t, r.Logger, path, r.Client) + if r.clientReports != nil { + if err := persistClientReports(r.logger, r.reportPath, r.clientReports); err != nil { + return err + } } - if r.Visualize != nil { - err := r.Visualize(filepath.Join(path, "history.html")) - if err != nil { - t.Error(err) + if r.traffic != nil { + if err := persistTrafficDetail(r.logger, r.reportPath, *r.traffic); err != nil { + return err } } + r.dataSaved = true + return nil } -func persistMemberDataDir(t *testing.T, lg *zap.Logger, member e2e.EtcdProcess, path string) { - lg.Info("Saving member data dir", zap.String("member", member.Config().Name), zap.String("path", path)) - err := os.Rename(memberDataDir(member), path) - require.NoError(t, err) +func (r *TestReport) Finalize(tFailed bool, panicked bool) error { + _, persistResults := os.LookupEnv("PERSIST_RESULTS") + keep := tFailed || panicked || persistResults + if !keep { + if !r.dataSaved { + return nil + } + r.logger.Info("Removing robustness test report", zap.String("path", r.reportPath)) + return os.RemoveAll(r.reportPath) + } + + if err := r.SaveEtcdData(); err != nil { + return fmt.Errorf("failed to save etcd data: %w", err) + } + + if r.visualize == nil { + r.logger.Info("No visualization available to be saved", zap.String("path", r.reportPath)) + return nil + } + r.logger.Info("Adding visualization to test report", zap.String("path", r.reportPath)) + return r.visualize(r.logger, filepath.Join(r.reportPath, "history.html")) +} + +func ServerDataPaths(c *e2e.EtcdProcessCluster) map[string]string { + dataPaths := make(map[string]string) + for _, member := range c.Procs { + dataPaths[member.Config().Name] = memberDataDir(member) + } + + return dataPaths } func memberDataDir(member e2e.EtcdProcess) string { @@ -89,3 +125,28 @@ func memberDataDir(member e2e.EtcdProcess) string { } return member.Config().DataDirPath } + +type TrafficDetail struct { + ExpectUniqueRevision bool `json:"expectuniquerevision,omitempty"` +} + +const trafficDetailFileName = "traffic.json" + +func persistTrafficDetail(lg *zap.Logger, p string, td TrafficDetail) error { + lg.Info("Saving traffic configuration details", zap.String("path", path.Join(p, trafficDetailFileName))) + b, err := json.Marshal(td) + if err != nil { + return nil + } + return os.WriteFile(filepath.Join(p, trafficDetailFileName), b, 0o644) +} + +func LoadTrafficDetail(p string) (TrafficDetail, error) { + var detail TrafficDetail + b, err := os.ReadFile(filepath.Join(p, trafficDetailFileName)) + if err != nil { + return TrafficDetail{}, err + } + err = json.Unmarshal(b, &detail) + return detail, err +} diff --git a/tests/robustness/report/wal.go b/tests/robustness/report/wal.go index 077ddc8896aa..6abde5d44ede 100644 --- a/tests/robustness/report/wal.go +++ b/tests/robustness/report/wal.go @@ -15,18 +15,22 @@ package report import ( + "bytes" "errors" "fmt" "io" + "math" "os" "path/filepath" "strings" "github.com/google/go-cmp/cmp" "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" pb "go.etcd.io/etcd/api/v3/etcdserverpb" - "go.etcd.io/etcd/pkg/v3/pbutil" + "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" @@ -46,7 +50,7 @@ func LoadClusterPersistedRequests(lg *zap.Logger, path string) ([]model.EtcdRequ dataDirs = append(dataDirs, filepath.Join(path, file.Name())) } } - return PersistedRequestsDirs(lg, dataDirs) + return PersistedRequests(lg, dataDirs) } func PersistedRequestsCluster(lg *zap.Logger, cluster *e2e.EtcdProcessCluster) ([]model.EtcdRequest, error) { @@ -54,70 +58,157 @@ func PersistedRequestsCluster(lg *zap.Logger, cluster *e2e.EtcdProcessCluster) ( for _, proc := range cluster.Procs { dataDirs = append(dataDirs, memberDataDir(proc)) } - return PersistedRequestsDirs(lg, dataDirs) + return PersistedRequests(lg, dataDirs) } -func PersistedRequestsDirs(lg *zap.Logger, dataDirs []string) ([]model.EtcdRequest, error) { - persistedRequests := []model.EtcdRequest{} - // Allow failure in minority of etcd cluster. - // 0 failures in 1 node cluster, 1 failure in 3 node cluster - allowedFailures := len(dataDirs) / 2 - for _, dir := range dataDirs { - memberRequests, err := requestsPersistedInWAL(lg, dir) +func PersistedRequests(lg *zap.Logger, dataDirs []string) ([]model.EtcdRequest, error) { + if len(dataDirs) == 0 { + return nil, errors.New("no data dirs") + } + entriesPersistedInWAL := make([][]*raftpb.Entry, len(dataDirs)) + var minCommitIndex uint64 = math.MaxUint64 + for i, dir := range dataDirs { + state, entries, err := ReadWAL(lg, dir) if err != nil { - if allowedFailures < 1 { - return nil, err - } - allowedFailures-- + lg.Error("Failed to read WAL", zap.Error(err), zap.String("data-dir", dir)) continue } - minLength := min(len(persistedRequests), len(memberRequests)) - if diff := cmp.Diff(memberRequests[:minLength], persistedRequests[:minLength]); diff != "" { - return nil, fmt.Errorf("unexpected differences between wal entries, diff:\n%s", diff) - } - if len(memberRequests) > len(persistedRequests) { - persistedRequests = memberRequests - } + minCommitIndex = min(minCommitIndex, state.GetCommit()) + entriesPersistedInWAL[i] = entries } - return persistedRequests, nil -} - -func requestsPersistedInWAL(lg *zap.Logger, dataDir string) ([]model.EtcdRequest, error) { - _, ents, err := ReadWAL(lg, dataDir) + entries, err := mergeMembersEntries(minCommitIndex, entriesPersistedInWAL) if err != nil { return nil, err } - requests := make([]model.EtcdRequest, 0, len(ents)) - for _, ent := range ents { - if ent.Type != raftpb.EntryNormal || len(ent.Data) == 0 { + persistedRequests := make([]model.EtcdRequest, 0, len(entries)) + for _, e := range entries { + if e.GetType() != raftpb.EntryNormal { continue } - request, err := parseEntryNormal(ent) + request, err := parseEntryNormal(e) if err != nil { return nil, err } if request != nil { - requests = append(requests, *request) + persistedRequests = append(persistedRequests, *request) } } - return requests, nil + return persistedRequests, nil } -func ReadWAL(lg *zap.Logger, dataDir string) (state raftpb.HardState, ents []raftpb.Entry, err error) { +func mergeMembersEntries(minCommitIndex uint64, memberEntries [][]*raftpb.Entry) ([]*raftpb.Entry, error) { + for _, entries := range memberEntries { + var lastIndex uint64 + for _, e := range entries { + if e.GetIndex() <= lastIndex { + return nil, fmt.Errorf("raft index should increase, got: %d, previous: %d", e.GetIndex(), lastIndex) + } + lastIndex = e.GetIndex() + } + } + memberIndices := make([]int, len(memberEntries)) + mergedHistory := []*raftpb.Entry{} + var raftIndex uint64 + for { + // Find entry with raftIndex. + raftIndex++ + entriesLeft := false + for i, entries := range memberEntries { + memberIndex := memberIndices[i] + for memberIndex < len(entries) && entries[memberIndex].GetIndex() < raftIndex { + memberIndex++ + } + if memberIndex < len(entries) { + entriesLeft = true + } + memberIndices[i] = memberIndex + } + if !entriesLeft { + break + } + // Entries collects votes from matching entries. + votes := make([]int, len(memberEntries)) + for i := 0; i < len(memberEntries); i++ { + if len(memberEntries[i]) <= memberIndices[i] { + continue + } + entry1 := memberEntries[i][memberIndices[i]] + if entry1.GetIndex() != raftIndex { + continue + } + for j := i; j < len(memberEntries); j++ { + if i == j { + votes[i]++ + continue + } + if len(memberEntries[j]) <= memberIndices[j] { + continue + } + entry2 := memberEntries[j][memberIndices[j]] + if entry2.GetIndex() != raftIndex { + continue + } + if areEntriesEqual(entry1, entry2) { + votes[i]++ + votes[j]++ + } + } + } + // Select entry with most votes + topVotes := 0 + for _, vote := range votes { + if vote > topVotes { + topVotes = vote + } + } + if topVotes == 0 { + return nil, fmt.Errorf("no entry for raft index %d", raftIndex) + } + var entryWithMostVotes *raftpb.Entry + for i, vote := range votes { + if vote != topVotes { + continue + } + entry := memberEntries[i][memberIndices[i]] + if entryWithMostVotes == nil { + entryWithMostVotes = entry + continue + } + if entryWithMostVotes.GetTerm() != entry.GetTerm() && entry.GetIndex() > minCommitIndex { + if entryWithMostVotes.GetTerm() < entry.GetTerm() { + entryWithMostVotes = entry + } + continue + } + if !areEntriesEqual(entryWithMostVotes, entry) { + diff := cmp.Diff(entryWithMostVotes, entry, protocmp.Transform(), protocmp.IgnoreDefaultScalars(), cmp.Comparer(bytes.Equal)) + return nil, fmt.Errorf("mismatching entries on raft index %d, diff: %s", raftIndex, diff) + } + } + mergedHistory = append(mergedHistory, proto.Clone(entryWithMostVotes).(*raftpb.Entry)) + } + if len(mergedHistory) == 0 { + return nil, errors.New("no WAL entries matched") + } + return mergedHistory, nil +} + +func areEntriesEqual(e1, e2 *raftpb.Entry) bool { + if e1 == nil || e2 == nil { + return e1 == e2 + } + return e1.GetIndex() == e2.GetIndex() && + e1.GetTerm() == e2.GetTerm() && + e1.GetType() == e2.GetType() && + bytes.Equal(e1.GetData(), e2.GetData()) +} + +func ReadWAL(lg *zap.Logger, dataDir string) (state *raftpb.HardState, ents []*raftpb.Entry, err error) { walDir := datadir.ToWALDir(dataDir) repaired := false for { - w, err := wal.OpenForRead(lg, walDir, walpb.Snapshot{Index: 0}) - if err != nil { - return state, nil, fmt.Errorf("failed to open WAL, err: %w", err) - } - _, state, ents, err = w.ReadAll() - w.Close() + state, ents, err = ReadAllWALEntries(lg, walDir) if err != nil { - if errors.Is(err, wal.ErrSnapshotNotFound) || errors.Is(err, wal.ErrSliceOutOfRange) { - lg.Info("Error occurred when reading WAL entries", zap.Error(err)) - return state, ents, nil - } // we can only repair ErrUnexpectedEOF and we never repair twice. if repaired || !errors.Is(err, io.ErrUnexpectedEOF) { return state, nil, fmt.Errorf("failed to read WAL, cannot be repaired, err: %w", err) @@ -133,16 +224,24 @@ func ReadWAL(lg *zap.Logger, dataDir string) (state raftpb.HardState, ents []raf } } -func parseEntryNormal(ent raftpb.Entry) (*model.EtcdRequest, error) { +func parseEntryNormal(ent *raftpb.Entry) (*model.EtcdRequest, error) { var raftReq pb.InternalRaftRequest - if err := raftReq.Unmarshal(ent.Data); err != nil { - var r pb.Request - isV2Entry := pbutil.MaybeUnmarshal(&r, ent.Data) - if !isV2Entry { - return nil, err - } + if len(ent.Data) == 0 { return nil, nil } + if err := proto.Unmarshal(ent.Data, &raftReq); err != nil { + // PR https://github.com/etcd-io/etcd/pull/21263 removed v2 requests + // in etcd v3.7. However, robustness always uses the latest protobuf + // definitions to parse WAL entries generated by all previous versions. + // etcd v3.4 and v3.5 generate v2 requests during bootstrap, which the + // v3.7 protobuf can no longer parse. As a result, robustness fails when + // parsing those WAL entries. We intentionally ignore this error here. + // See https://github.com/etcd-io/etcd/pull/21263#discussion_r2776042340 + if strings.Contains(err.Error(), "proto: wrong wireType") { + return nil, nil + } + return nil, err + } switch { case raftReq.Put != nil: op := model.PutOptions{ @@ -226,7 +325,7 @@ func parseEntryNormal(ent raftpb.Entry) (*model.EtcdRequest, error) { } return &request, nil default: - panic(fmt.Sprintf("Unhandled raft request: %+v", raftReq)) + panic(fmt.Sprintf("Unhandled raft request: %s", raftReq.String())) } } @@ -237,9 +336,10 @@ func toEtcdOperation(op *pb.RequestOp) (operation model.EtcdOperation) { operation = model.EtcdOperation{ Type: model.RangeOperation, Range: model.RangeOptions{ - Start: string(rangeOp.Key), - End: string(rangeOp.RangeEnd), - Limit: rangeOp.Limit, + Start: string(rangeOp.Key), + End: string(rangeOp.RangeEnd), + Limit: rangeOp.Limit, + KeysOnly: rangeOp.GetKeysOnly(), }, } case op.GetRequestPut() != nil: @@ -264,3 +364,56 @@ func toEtcdOperation(op *pb.RequestOp) (operation model.EtcdOperation) { } return operation } + +func ReadAllWALEntries(lg *zap.Logger, dirpath string) (state *raftpb.HardState, ents []*raftpb.Entry, err error) { + names, err := fileutil.ReadDir(dirpath) + if err != nil { + return state, nil, err + } + files := make([]fileutil.FileReader, 0, len(names)) + for _, name := range names { + if !strings.HasSuffix(name, ".wal") { + continue + } + p := filepath.Join(dirpath, name) + var f *os.File + f, err = os.OpenFile(p, os.O_RDONLY, fileutil.PrivateFileMode) + if err != nil { + return state, nil, fmt.Errorf("os.OpenFile failed (%q): %w", p, err) + } + defer f.Close() + files = append(files, fileutil.NewFileReader(f)) + } + rec := &walpb.Record{} + decoder := wal.NewDecoder(files...) + for err = decoder.Decode(rec); err == nil; err = decoder.Decode(rec) { + switch rec.GetType() { + case wal.EntryType: + e := wal.MustUnmarshalEntry(rec.Data) + i := len(ents) + for ; i > 0 && ents[i-1].GetIndex() >= e.GetIndex(); i-- { + } + // The line below is potentially overriding some 'uncommitted' entries. + ents = append(ents[:i], e) + case wal.StateType: + state = wal.MustUnmarshalState(rec.Data) + case wal.MetadataType: + case wal.CrcType: + crc := decoder.LastCRC() + // current crc of decoder must match the crc of the record. + // do no need to match 0 crc, since the decoder is a new one at this case. + if crc != 0 && rec.Validate(crc) != nil { + state.Reset() + return state, nil, wal.ErrCRCMismatch + } + decoder.UpdateCRC(rec.GetCrc()) + case wal.SnapshotType: + default: + return state, nil, fmt.Errorf("unexpected block type %d", rec.Type) + } + } + if err != nil && !errors.Is(err, io.EOF) { + return state, nil, err + } + return state, ents, nil +} diff --git a/tests/robustness/report/wal_test.go b/tests/robustness/report/wal_test.go new file mode 100644 index 000000000000..9e27644e1ed0 --- /dev/null +++ b/tests/robustness/report/wal_test.go @@ -0,0 +1,748 @@ +// Copyright 2025 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:govet +package report + +import ( + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + "google.golang.org/protobuf/proto" + + "go.etcd.io/etcd/server/v3/storage/wal" + "go.etcd.io/etcd/server/v3/storage/wal/walpb" + "go.etcd.io/raft/v3/raftpb" +) + +func TestMergeMemberEntries(t *testing.T) { + tcs := []struct { + name string + minCommitIndex uint64 + memberEntries [][]*raftpb.Entry + expectErr string + expectEntries []*raftpb.Entry + }{ + { + name: "Error when empty data dir", + memberEntries: [][]*raftpb.Entry{}, + expectErr: "no WAL entries matched", + }, + { + name: "Success when no entries", + memberEntries: [][]*raftpb.Entry{ + {}, + }, + expectErr: "no WAL entries matched", + }, + { + name: "Error when one member cluster didn't observe the index", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectErr: "no entry for raft index 2", + }, + { + name: "Error when entries index unordered", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + }, + }, + expectErr: "raft index should increase, got: 1, previous: 3", + }, + { + name: "Error when entries index duplicated", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + }, + }, + expectErr: "raft index should increase, got: 1, previous: 1", + }, + { + name: "Success when one member cluster", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Success when three members agree on entries", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Success when three members have no entries", + memberEntries: [][]*raftpb.Entry{ + {}, {}, {}, + }, + expectErr: "no WAL entries matched", + }, + { + name: "Success when one member has no entries in three node cluster", + memberEntries: [][]*raftpb.Entry{ + {}, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Success if two members have no entries in three node cluster", + memberEntries: [][]*raftpb.Entry{ + {}, + {}, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Success if members didn't observe the whole history", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + }, + { + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Success if members observed only one part of history", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + }, + { + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + }, + { + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Error when in three member cluster if no members observed index", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectErr: "no entry for raft index 2", + }, + { + name: "Success if only one member observed history", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + {}, + {}, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Success when one member observed different last entry", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("x")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Data: []byte("b")}, + {Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + { + name: "Error when one member didn't observe whole history and others observed different last entry", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("x")}, + }, + }, + expectErr: "mismatching entries on raft index 3", + }, + { + name: "Error when three members observed different last entry", + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("x")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("y")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("z")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectErr: "mismatching entries on raft index 2", + }, + { + name: "Error when one member observed empty history and others differ on last entry", + memberEntries: [][]*raftpb.Entry{ + {}, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("x")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Data: []byte("y")}, + &raftpb.Entry{Index: new(uint64(2)), Data: []byte("b")}, + &raftpb.Entry{Index: new(uint64(3)), Data: []byte("c")}, + }, + }, + expectErr: "mismatching entries on raft index 1", + }, + { + name: "Error if entries mismatch on index before minCommitIndex", + minCommitIndex: 2, + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Term: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Term: new(uint64(1)), Data: []byte("b")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Term: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Term: new(uint64(2)), Data: []byte("c")}, + }, + }, + expectErr: "mismatching entries on raft index 2", + }, + { + name: "Select entry with higher term if they conflict on uncommitted index", + minCommitIndex: 1, + memberEntries: [][]*raftpb.Entry{ + { + &raftpb.Entry{Index: new(uint64(1)), Term: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Term: new(uint64(1)), Data: []byte("b")}, + }, + { + &raftpb.Entry{Index: new(uint64(1)), Term: new(uint64(1)), Data: []byte("a")}, + &raftpb.Entry{Index: new(uint64(2)), Term: new(uint64(2)), Data: []byte("x")}, + }, + }, + expectEntries: []*raftpb.Entry{ + {Index: new(uint64(1)), Term: new(uint64(1)), Data: []byte("a")}, + {Index: new(uint64(2)), Term: new(uint64(2)), Data: []byte("x")}, + }, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + entries, err := mergeMembersEntries(tc.minCommitIndex, tc.memberEntries) + if tc.expectErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.expectErr) + } + if diff := cmp.Diff(tc.expectEntries, entries, cmpopts.IgnoreUnexported(raftpb.Entry{}, raftpb.HardState{})); diff != "" { + t.Errorf("expectEntries mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestWriteReadWAL(t *testing.T) { + type batch struct { + state *raftpb.HardState + entries []*raftpb.Entry + snapshot *walpb.Snapshot + } + type want struct { + wantState raftpb.HardState + wantEntries []*raftpb.Entry + wantError string + } + + tcs := []struct { + name string + operations []batch + readAt *walpb.Snapshot + walReadAll want + readAllEntries want + }{ + { + name: "single batch", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(5))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + walReadAll: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + { + name: "multiple committed batches", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(2))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(4))}, + entries: []*raftpb.Entry{{Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(5))}, + entries: []*raftpb.Entry{{Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + walReadAll: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + { + name: "uncommitted ovewritten entries", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(1))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("a")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(3))}, + entries: []*raftpb.Entry{{Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("b")}, {Index: new(uint64(4)), Data: []byte("b")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(4))}, + entries: []*raftpb.Entry{{Index: new(uint64(4)), Data: []byte("c")}, {Index: new(uint64(5)), Data: []byte("c")}}, + }, + }, + walReadAll: want{ + wantState: raftpb.HardState{Commit: new(uint64(4))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("b")}, {Index: new(uint64(4)), Data: []byte("c")}, {Index: new(uint64(5)), Data: []byte("c")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(4))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("b")}, {Index: new(uint64(4)), Data: []byte("c")}, {Index: new(uint64(5)), Data: []byte("c")}}, + }, + }, + { + name: "entries in bad order", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(2))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(6))}, + entries: []*raftpb.Entry{{Index: new(uint64(5)), Data: []byte("e")}, {Index: new(uint64(6)), Data: []byte("f")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(4))}, + entries: []*raftpb.Entry{{Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}}, + }, + }, + walReadAll: want{ + wantError: "slice bounds out of range", + wantState: raftpb.HardState{Commit: new(uint64(2))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(4))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}}, + }, + }, + { + name: "read before snapshot", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(1))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + { + snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(5))}, + entries: []*raftpb.Entry{{Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + walReadAll: want{ + wantError: "slice bounds out of range", + wantState: raftpb.HardState{Commit: new(uint64(1))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + { + name: "read at snapshot", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(1))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + { + snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(5))}, + entries: []*raftpb.Entry{{Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + readAt: &walpb.Snapshot{Index: new(uint64(3))}, + walReadAll: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + { + name: "uncommitted entries before snapshot", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(1))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(3))}, + entries: []*raftpb.Entry{{Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}}, + }, + { + snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(4))}, + entries: []*raftpb.Entry{{Index: new(uint64(4)), Data: []byte("e")}, {Index: new(uint64(5)), Data: []byte("f")}}, + }, + }, + walReadAll: want{ + wantState: raftpb.HardState{Commit: new(uint64(4))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("e")}, {Index: new(uint64(5)), Data: []byte("f")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(4))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("e")}, {Index: new(uint64(5)), Data: []byte("f")}}, + }, + }, + { + name: "entries preceding snapshot", + operations: []batch{ + { + snapshot: &walpb.Snapshot{Index: new(uint64(4)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(2))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(4))}, + entries: []*raftpb.Entry{{Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(6))}, + entries: []*raftpb.Entry{{Index: new(uint64(5)), Data: []byte("e")}, {Index: new(uint64(6)), Data: []byte("f")}}, + }, + }, + readAt: &walpb.Snapshot{Index: new(uint64(4))}, + walReadAll: want{ + wantState: raftpb.HardState{Commit: new(uint64(6))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(5)), Data: []byte("e")}, {Index: new(uint64(6)), Data: []byte("f")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(6))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(3)), Data: []byte("c")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}, {Index: new(uint64(6)), Data: []byte("f")}}, + }, + }, + { + name: "read after snapshot", + operations: []batch{ + { + state: &raftpb.HardState{Commit: new(uint64(1))}, + entries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}}, + }, + { + snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, + }, + { + state: &raftpb.HardState{Commit: new(uint64(5))}, + entries: []*raftpb.Entry{{Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + readAt: &walpb.Snapshot{Index: new(uint64(4))}, + walReadAll: want{ + wantError: "snapshot not found", + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(5)), Data: []byte("e")}}, + }, + readAllEntries: want{ + wantState: raftpb.HardState{Commit: new(uint64(5))}, + wantEntries: []*raftpb.Entry{{Index: new(uint64(1)), Data: []byte("a")}, {Index: new(uint64(2)), Data: []byte("b")}, {Index: new(uint64(4)), Data: []byte("d")}, {Index: new(uint64(5)), Data: []byte("e")}}, + }, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + lg := zaptest.NewLogger(t) + w, err := wal.Create(lg, dir, nil) + require.NoError(t, err) + for _, op := range tc.operations { + if op.state != nil { + err = w.Save(op.state, op.entries) + require.NoError(t, err) + } + if op.snapshot != nil { + err = w.SaveSnapshot(op.snapshot) + require.NoError(t, err) + } + } + w.Close() + + w2, err := wal.OpenForRead(lg, dir, tc.readAt) + require.NoError(t, err) + defer w2.Close() + t.Run("wal.ReadAll", func(t *testing.T) { + _, state, entries, err := w2.ReadAll() + if tc.walReadAll.wantError != "" { + require.ErrorContains(t, err, tc.walReadAll.wantError) + } else { + require.NoError(t, err) + } + if !proto.Equal(&tc.walReadAll.wantState, state) { + t.Errorf("wantState mismatch\n got %+v\n want %+v", state, tc.walReadAll.wantState) + } + if diff := cmp.Diff(tc.walReadAll.wantEntries, entries, cmpopts.IgnoreUnexported(raftpb.Entry{}, raftpb.HardState{})); diff != "" { + t.Errorf("wantEntries mismatch (-want +got):\n%s", diff) + } + }) + t.Run("ReadAllEntries", func(t *testing.T) { + state, entries, err := ReadAllWALEntries(lg, dir) + if tc.readAllEntries.wantError != "" { + require.ErrorContains(t, err, tc.walReadAll.wantError) + } else { + require.NoError(t, err) + } + if !proto.Equal(&tc.readAllEntries.wantState, state) { + t.Errorf("wantState mismatch\n got %+v\n want %+v", state, tc.readAllEntries.wantState) + } + if diff := cmp.Diff(tc.readAllEntries.wantEntries, entries, cmpopts.IgnoreUnexported(raftpb.Entry{}, raftpb.HardState{})); diff != "" { + t.Errorf("wantEntries mismatch (-want +got):\n%s", diff) + } + }) + }) + } +} + +func TestAreEntriesEqualVerifyAllFields(t *testing.T) { + base := &raftpb.Entry{ + Index: new(uint64(1)), + Term: new(uint64(1)), + Type: raftpb.EntryNormal.Enum(), + Data: []byte("data"), + } + + val := reflect.ValueOf(base).Elem() + typ := val.Type() + + for i := 0; i < val.NumField(); i++ { + field := typ.Field(i) + if !field.IsExported() { + continue + } + + t.Run(field.Name, func(t *testing.T) { + modified := proto.Clone(base).(*raftpb.Entry) + modVal := reflect.ValueOf(modified).Elem() + + f := modVal.Field(i) + switch f.Kind() { + case reflect.Pointer: + if f.Type().Elem().Kind() == reflect.Uint64 { + v := uint64(999) + f.Set(reflect.ValueOf(&v)) + } else if f.Type().Elem().Kind() == reflect.Int32 { + v := int32(999) + f.Set(reflect.ValueOf(&v).Convert(f.Type())) + } + case reflect.Slice: + if f.Type().Elem().Kind() == reflect.Uint8 { + f.SetBytes([]byte("different-data-payload")) + } + default: + t.Fatalf("Unhandled field type for reflection: %s", f.Type()) + } + + if areEntriesEqual(base, modified) { + t.Errorf("areEntriesEqual returned true after mutating field %q! This means changes to %q are not being compared.", field.Name, field.Name) + } + }) + } +} + +func BenchmarkMergeMemberEntries(b *testing.B) { + const numEntries = 1000 + memberEntries := make([][]*raftpb.Entry, 3) + for i := 0; i < 3; i++ { + memberEntries[i] = make([]*raftpb.Entry, numEntries) + for j := 0; j < numEntries; j++ { + var entryType *raftpb.EntryType + if i == 0 { + entryType = nil + } else { + entryType = raftpb.EntryNormal.Enum() + } + memberEntries[i][j] = &raftpb.Entry{ + Index: new(uint64(j + 1)), + Term: new(uint64(1)), + Type: entryType, + Data: []byte("some realistic data payload for entry"), + } + } + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := mergeMembersEntries(0, memberEntries) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/tests/robustness/scenarios/scenarios.go b/tests/robustness/scenarios/scenarios.go index ba33add20c2e..64f4ab3e2497 100644 --- a/tests/robustness/scenarios/scenarios.go +++ b/tests/robustness/scenarios/scenarios.go @@ -15,6 +15,7 @@ package scenarios import ( + "os" "path/filepath" "testing" "time" @@ -25,7 +26,6 @@ import ( "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/e2e" - "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/failpoint" "go.etcd.io/etcd/tests/v3/robustness/options" "go.etcd.io/etcd/tests/v3/robustness/random" @@ -42,22 +42,38 @@ var trafficProfiles = []TrafficProfile{ { Name: "EtcdHighTraffic", Traffic: traffic.EtcdPut, - Profile: traffic.HighTrafficProfile, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueHigh, + Watch: &traffic.WatchDefault, + Compaction: &traffic.CompactionDefault, + }, }, { Name: "EtcdTrafficDeleteLeases", Traffic: traffic.EtcdPutDeleteLease, - Profile: traffic.LowTraffic, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueMedium, + Watch: &traffic.WatchDefault, + Compaction: &traffic.CompactionDefault, + }, }, { Name: "KubernetesHighTraffic", Traffic: traffic.Kubernetes, - Profile: traffic.HighTrafficProfile, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueHigh, + Watch: &traffic.WatchDefault, + Compaction: &traffic.CompactionDefault, + }, }, { Name: "KubernetesLowTraffic", Traffic: traffic.Kubernetes, - Profile: traffic.LowTraffic, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueMedium, + Watch: &traffic.WatchDefault, + Compaction: &traffic.CompactionDefault, + }, }, } @@ -67,7 +83,6 @@ type TestScenario struct { Cluster e2e.EtcdProcessClusterConfig Traffic traffic.Traffic Profile traffic.Profile - Watch client.WatchConfig } func Exploratory(_ *testing.T) []TestScenario { @@ -96,9 +111,13 @@ func Exploratory(_ *testing.T) []TestScenario { options.WithSnapshotCount(50, 100, 1000), options.WithSubsetOptions(randomizableOptions...), e2e.WithGoFailEnabled(true), - // Set low minimal compaction batch limit to allow for triggering multi batch compaction failpoints. - options.WithExperimentalCompactionBatchLimit(10, 100, 1000), - e2e.WithExperimentalWatchProcessNotifyInterval(100 * time.Millisecond), + // Set a low minimal compaction batch limit to allow for triggering multi batch compaction failpoints. + options.WithCompactionBatchLimit(10, 100, 1000), + e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), + } + + if addr := os.Getenv("TRACING_SERVER_ADDR"); addr != "" { + baseOptions = append(baseOptions, e2e.WithEnableDistributedTracing(addr)) } if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { @@ -135,17 +154,18 @@ func Exploratory(_ *testing.T) []TestScenario { if e2e.BinPath.LazyFSAvailable() { newScenarios := scenarios for _, s := range scenarios { - // LazyFS increases the load on CPU, so we run it with more lightweight case. - if s.Profile.MinimalQPS <= 100 && s.Cluster.ClusterSize == 1 { + // LazyFS increases the load on the CPU, so we run it with a more lightweight case. + if s.Profile.KeyValue.MinimalQPS <= 100 && s.Cluster.ClusterSize == 1 { lazyfsCluster := s.Cluster lazyfsCluster.LazyFSEnabled = true + profileWithoutCompaction := s.Profile + profileWithoutCompaction.Compaction = nil newScenarios = append(newScenarios, TestScenario{ Name: filepath.Join(s.Name, "LazyFS"), Failpoint: s.Failpoint, Cluster: lazyfsCluster, Traffic: s.Traffic, - Profile: s.Profile.WithoutCompaction(), - Watch: s.Watch, + Profile: profileWithoutCompaction, }) } } @@ -162,8 +182,11 @@ func Regression(t *testing.T) []TestScenario { scenarios = append(scenarios, TestScenario{ Name: "Issue14370", Failpoint: failpoint.RaftBeforeSavePanic, - Profile: traffic.LowTraffic, - Traffic: traffic.EtcdPutDeleteLease, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueMedium, + Compaction: &traffic.CompactionDefault, + }, + Traffic: traffic.EtcdPutDeleteLease, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), @@ -172,8 +195,11 @@ func Regression(t *testing.T) []TestScenario { scenarios = append(scenarios, TestScenario{ Name: "Issue14685", Failpoint: failpoint.DefragBeforeCopyPanic, - Profile: traffic.LowTraffic, - Traffic: traffic.EtcdPutDeleteLease, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueMedium, + Compaction: &traffic.CompactionDefault, + }, + Traffic: traffic.EtcdPutDeleteLease, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), @@ -182,18 +208,22 @@ func Regression(t *testing.T) []TestScenario { scenarios = append(scenarios, TestScenario{ Name: "Issue13766", Failpoint: failpoint.KillFailpoint, - Profile: traffic.HighTrafficProfile, - Traffic: traffic.EtcdPut, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueHigh, + Compaction: &traffic.CompactionDefault, + }, + Traffic: traffic.EtcdPut, Cluster: *e2e.NewConfig( e2e.WithSnapshotCount(100), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue15220", - Watch: client.WatchConfig{ - RequestProgress: true, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueMedium, + Watch: &traffic.WatchDefault, + Compaction: &traffic.CompactionDefault, }, - Profile: traffic.LowTraffic, Traffic: traffic.EtcdPutDeleteLease, Failpoint: failpoint.KillFailpoint, Cluster: *e2e.NewConfig( @@ -201,8 +231,12 @@ func Regression(t *testing.T) []TestScenario { ), }) scenarios = append(scenarios, TestScenario{ - Name: "Issue17529", - Profile: traffic.HighTrafficProfile, + Name: "Issue17529", + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueHigh, + Watch: &traffic.WatchDefault, + Compaction: &traffic.CompactionDefault, + }, Traffic: traffic.Kubernetes, Failpoint: failpoint.SleepBeforeSendWatchResponse, Cluster: *e2e.NewConfig( @@ -213,13 +247,15 @@ func Regression(t *testing.T) []TestScenario { }) scenarios = append(scenarios, TestScenario{ - Name: "Issue17780", - Profile: traffic.LowTraffic.WithoutCompaction(), + Name: "Issue17780", + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueMedium, + }, Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic, Traffic: traffic.Kubernetes, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), - e2e.WithExperimentalCompactionBatchLimit(300), + e2e.WithCompactionBatchLimit(300), e2e.WithSnapshotCount(1000), e2e.WithGoFailEnabled(true), ), @@ -240,24 +276,25 @@ func Regression(t *testing.T) []TestScenario { scenarios = append(scenarios, TestScenario{ Name: "Issue19179", Profile: traffic.Profile{ - MinimalQPS: 50, - MaximalQPS: 100, - BurstableQPS: 100, - ClientCount: 8, - MaxNonUniqueRequestConcurrency: 3, - }.WithoutCompaction(), + KeyValue: &traffic.KeyValueVeryLow, + Watch: &traffic.WatchDefault, + }, Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic, Traffic: traffic.KubernetesCreateDelete, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), - e2e.WithExperimentalCompactionBatchLimit(50), + e2e.WithCompactionBatchLimit(50), e2e.WithSnapshotCount(1000), e2e.WithGoFailEnabled(true), ), }) scenarios = append(scenarios, TestScenario{ - Name: "Issue18089", - Profile: traffic.LowTraffic.WithCompactionPeriod(100 * time.Millisecond), // Use frequent compaction for high reproduce rate + Name: "Issue18089", + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueMedium, + Compaction: &traffic.CompactionFrequent, // Use frequent compaction for high reproduce rate + Watch: &traffic.WatchDefault, + }, Failpoint: failpoint.SleepBeforeSendWatchResponse, Traffic: traffic.EtcdDelete, Cluster: *e2e.NewConfig( @@ -265,6 +302,26 @@ func Regression(t *testing.T) []TestScenario { e2e.WithGoFailEnabled(true), ), }) + scenarios = append(scenarios, TestScenario{ + Name: "Issue20221", + Failpoint: failpoint.BlackholeUntilSnapshot, + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueHigh, + Watch: &traffic.Watch{ + MemberClientCount: 6, + ClusterClientCount: 2, + RevisionOffsetRange: traffic.Range{Min: 50, Max: 100}, + }, + }, + Traffic: traffic.EtcdPut, + Cluster: *e2e.NewConfig( + e2e.WithSnapshotCount(10), + e2e.WithPeerProxy(true), + e2e.WithIsPeerTLS(true), + e2e.WithWatchProcessNotifyInterval(10*time.Millisecond), + e2e.WithSnapshotCatchUpEntries(10), + ), + }) if v.Compare(version.V3_5) >= 0 { opts := []e2e.EPClusterOption{ e2e.WithSnapshotCount(100), @@ -277,9 +334,13 @@ func Regression(t *testing.T) []TestScenario { scenarios = append(scenarios, TestScenario{ Name: "Issue15271", Failpoint: failpoint.BlackholeUntilSnapshot, - Profile: traffic.HighTrafficProfile, - Traffic: traffic.EtcdPut, - Cluster: *e2e.NewConfig(opts...), + Profile: traffic.Profile{ + KeyValue: &traffic.KeyValueHigh, + Compaction: &traffic.CompactionDefault, + Watch: &traffic.WatchDefault, + }, + Traffic: traffic.EtcdPut, + Cluster: *e2e.NewConfig(opts...), }) } return scenarios diff --git a/tests/robustness/traffic/etcd.go b/tests/robustness/traffic/etcd.go index 906f3d2cb465..a34c072f2ac8 100644 --- a/tests/robustness/traffic/etcd.go +++ b/tests/robustness/traffic/etcd.go @@ -32,43 +32,46 @@ import ( var ( EtcdPutDeleteLease Traffic = etcdTraffic{ - keyCount: 10, - leaseTTL: DefaultLeaseTTL, - largePutSize: 32769, + keyCount: 10, + leaseTTL: DefaultLeaseTTL, // Please keep the sum of weights equal 100. requests: []random.ChoiceWeight[etcdRequestType]{ {Choice: Get, Weight: 15}, - {Choice: List, Weight: 15}, + {Choice: List, Weight: 4}, + {Choice: ListKeyOnly, Weight: 4}, + {Choice: ListStream, Weight: 4}, + {Choice: ListStreamKeyOnly, Weight: 3}, {Choice: StaleGet, Weight: 10}, - {Choice: StaleList, Weight: 10}, + {Choice: StaleList, Weight: 5}, + {Choice: StaleListStream, Weight: 5}, {Choice: Delete, Weight: 5}, - {Choice: MultiOpTxn, Weight: 5}, + {Choice: MultiOpTxn, Weight: 10}, {Choice: PutWithLease, Weight: 5}, {Choice: LeaseRevoke, Weight: 5}, {Choice: CompareAndSet, Weight: 5}, {Choice: Put, Weight: 20}, - {Choice: LargePut, Weight: 5}, }, } EtcdPut Traffic = etcdTraffic{ - keyCount: 10, - largePutSize: 32769, - leaseTTL: DefaultLeaseTTL, + keyCount: 10, + leaseTTL: DefaultLeaseTTL, // Please keep the sum of weights equal 100. requests: []random.ChoiceWeight[etcdRequestType]{ {Choice: Get, Weight: 15}, - {Choice: List, Weight: 15}, + {Choice: List, Weight: 4}, + {Choice: ListKeyOnly, Weight: 4}, + {Choice: ListStream, Weight: 4}, + {Choice: ListStreamKeyOnly, Weight: 3}, {Choice: StaleGet, Weight: 10}, - {Choice: StaleList, Weight: 10}, - {Choice: MultiOpTxn, Weight: 5}, - {Choice: LargePut, Weight: 5}, + {Choice: StaleList, Weight: 5}, + {Choice: StaleListStream, Weight: 5}, + {Choice: MultiOpTxn, Weight: 10}, {Choice: Put, Weight: 40}, }, } EtcdDelete Traffic = etcdTraffic{ - keyCount: 10, - largePutSize: 32769, - leaseTTL: DefaultLeaseTTL, + keyCount: 10, + leaseTTL: DefaultLeaseTTL, // Please keep the sum of weights equal 100. requests: []random.ChoiceWeight[etcdRequestType]{ {Choice: Put, Weight: 50}, @@ -78,10 +81,9 @@ var ( ) type etcdTraffic struct { - keyCount int - requests []random.ChoiceWeight[etcdRequestType] - leaseTTL int64 - largePutSize int + keyCount int + requests []random.ChoiceWeight[etcdRequestType] + leaseTTL int64 } func (t etcdTraffic) ExpectUniqueRevision() bool { @@ -91,41 +93,44 @@ func (t etcdTraffic) ExpectUniqueRevision() bool { type etcdRequestType string const ( - Get etcdRequestType = "get" - StaleGet etcdRequestType = "staleGet" - List etcdRequestType = "list" - StaleList etcdRequestType = "staleList" - Put etcdRequestType = "put" - LargePut etcdRequestType = "largePut" - Delete etcdRequestType = "delete" - MultiOpTxn etcdRequestType = "multiOpTxn" - PutWithLease etcdRequestType = "putWithLease" - LeaseRevoke etcdRequestType = "leaseRevoke" - CompareAndSet etcdRequestType = "compareAndSet" - Defragment etcdRequestType = "defragment" + Get etcdRequestType = "get" + StaleGet etcdRequestType = "staleGet" + List etcdRequestType = "list" + ListKeyOnly etcdRequestType = "listKeyOnly" + StaleList etcdRequestType = "staleList" + ListStream etcdRequestType = "listStream" + ListStreamKeyOnly etcdRequestType = "listStreamKeyOnly" + StaleListStream etcdRequestType = "staleListStream" + Put etcdRequestType = "put" + Delete etcdRequestType = "delete" + MultiOpTxn etcdRequestType = "multiOpTxn" + PutWithLease etcdRequestType = "putWithLease" + LeaseRevoke etcdRequestType = "leaseRevoke" + CompareAndSet etcdRequestType = "compareAndSet" + Defragment etcdRequestType = "defragment" ) func (t etcdTraffic) Name() string { return "Etcd" } -func (t etcdTraffic) RunTrafficLoop(ctx context.Context, c *client.RecordingClient, limiter *rate.Limiter, ids identity.Provider, lm identity.LeaseIDStorage, nonUniqueWriteLimiter ConcurrencyLimiter, finish <-chan struct{}) { +func (t etcdTraffic) RunKeyValueLoop(ctx context.Context, c *client.RecordingClient, p RunTrafficLoopParam) { lastOperationSucceeded := true var lastRev int64 var requestType etcdRequestType client := etcdTrafficClient{ etcdTraffic: t, - keyPrefix: "key", + keyStore: p.KeyStore, client: c, - limiter: limiter, - idProvider: ids, - leaseStorage: lm, + limiter: p.QPSLimiter, + idProvider: p.IDs, + leaseStorage: p.LeaseIDStorage, } for { select { case <-ctx.Done(): return - case <-finish: + case <-p.Finish: return default: } @@ -134,16 +139,16 @@ func (t etcdTraffic) RunTrafficLoop(ctx context.Context, c *client.RecordingClie // Avoid multiple failed writes in a row if lastOperationSucceeded { choices := t.requests - if shouldReturn = nonUniqueWriteLimiter.Take(); !shouldReturn { + if shouldReturn = p.NonUniqueRequestConcurrencyLimiter.Take(); !shouldReturn { choices = filterOutNonUniqueEtcdWrites(choices) } requestType = random.PickRandom(choices) } else { - requestType = Get + requestType = List } rev, err := client.Request(ctx, requestType, lastRev) if shouldReturn { - nonUniqueWriteLimiter.Return() + p.NonUniqueRequestConcurrencyLimiter.Return() } lastOperationSucceeded = err == nil if err != nil { @@ -152,19 +157,25 @@ func (t etcdTraffic) RunTrafficLoop(ctx context.Context, c *client.RecordingClie if rev != 0 { lastRev = rev } - limiter.Wait(ctx) + p.QPSLimiter.Wait(ctx) } } -func (t etcdTraffic) RunCompactLoop(ctx context.Context, c *client.RecordingClient, period time.Duration, finish <-chan struct{}) { +func (t etcdTraffic) RunWatchLoop(ctx context.Context, c *client.RecordingClient, p RunWatchLoopParam) { + runWatchLoop(ctx, c, p, watchLoopConfig{ + key: p.KeyStore.GetPrefix(), + }) +} + +func (t etcdTraffic) RunCompactLoop(ctx context.Context, c *client.RecordingClient, param RunCompactLoopParam) { var lastRev int64 = 2 - ticker := time.NewTicker(period) + ticker := time.NewTicker(param.Period) defer ticker.Stop() for { select { case <-ctx.Done(): return - case <-finish: + case <-param.Finish: return case <-ticker.C: } @@ -196,7 +207,7 @@ func filterOutNonUniqueEtcdWrites(choices []random.ChoiceWeight[etcdRequestType] type etcdTrafficClient struct { etcdTraffic - keyPrefix string + keyStore *keyStore client *client.RecordingClient limiter *rate.Limiter idProvider identity.Provider @@ -211,57 +222,79 @@ func (c etcdTrafficClient) Request(ctx context.Context, request etcdRequestType, switch request { case StaleGet: var resp *clientv3.GetResponse - resp, err = c.client.Get(opCtx, c.randomKey(), clientv3.WithRev(lastRev)) + resp, err = c.client.Get(opCtx, c.keyStore.GetKey(), clientv3.WithRev(lastRev)) if err == nil { rev = resp.Header.Revision } case Get: var resp *clientv3.GetResponse - resp, err = c.client.Get(opCtx, c.randomKey(), clientv3.WithRev(0)) + resp, err = c.client.Get(opCtx, c.keyStore.GetKey(), clientv3.WithRev(0)) if err == nil { rev = resp.Header.Revision } case List: var resp *clientv3.GetResponse - resp, err = c.client.Range(ctx, c.keyPrefix, clientv3.GetPrefixRangeEnd(c.keyPrefix), 0, limit) + resp, err = c.client.Range(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), 0, limit, false) + if resp != nil { + c.keyStore.SyncKeys(resp) + rev = resp.Header.Revision + } + case ListKeyOnly: + var resp *clientv3.GetResponse + resp, err = c.client.Range(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), 0, limit, true) if resp != nil { + c.keyStore.SyncKeys(resp) rev = resp.Header.Revision } case StaleList: var resp *clientv3.GetResponse - resp, err = c.client.Range(ctx, c.keyPrefix, clientv3.GetPrefixRangeEnd(c.keyPrefix), lastRev, limit) + resp, err = c.client.Range(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), lastRev, limit, false) if resp != nil { rev = resp.Header.Revision } - case Put: - var resp *clientv3.PutResponse - resp, err = c.client.Put(opCtx, c.randomKey(), fmt.Sprintf("%d", c.idProvider.NewRequestID())) + case ListStream: + var resp *clientv3.GetResponse + resp, err = c.client.RangeStream(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), 0, limit, false) if resp != nil { + c.keyStore.SyncKeys(resp) rev = resp.Header.Revision } - case LargePut: + case ListStreamKeyOnly: + var resp *clientv3.GetResponse + resp, err = c.client.RangeStream(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), 0, limit, true) + if resp != nil { + c.keyStore.SyncKeys(resp) + rev = resp.Header.Revision + } + case StaleListStream: + var resp *clientv3.GetResponse + resp, err = c.client.RangeStream(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), lastRev, limit, false) + if resp != nil { + rev = resp.Header.Revision + } + case Put: var resp *clientv3.PutResponse - resp, err = c.client.Put(opCtx, c.randomKey(), random.RandString(c.largePutSize)) + resp, err = c.client.Put(opCtx, c.keyStore.GetKey(), fmt.Sprintf("%d", c.idProvider.NewRequestID())) if resp != nil { rev = resp.Header.Revision } case Delete: var resp *clientv3.DeleteResponse - resp, err = c.client.Delete(opCtx, c.randomKey()) + resp, err = c.client.Delete(opCtx, c.keyStore.GetKeyForDelete()) if resp != nil { rev = resp.Header.Revision } case MultiOpTxn: var resp *clientv3.TxnResponse resp, err = c.client.Txn(opCtx).Then( - c.pickMultiTxnOps()..., + c.pickMultiTxnOps(c.keyStore)..., ).Commit() if resp != nil { rev = resp.Header.Revision } case CompareAndSet: var kv *mvccpb.KeyValue - key := c.randomKey() + key := c.keyStore.GetKey() var resp *clientv3.GetResponse resp, err = c.client.Get(opCtx, key, clientv3.WithRev(0)) if err == nil { @@ -303,7 +336,7 @@ func (c etcdTrafficClient) Request(ctx context.Context, request etcdRequestType, if leaseID != 0 { putCtx, putCancel := context.WithTimeout(ctx, RequestTimeout) var resp *clientv3.PutResponse - resp, err = c.client.PutWithLease(putCtx, c.randomKey(), fmt.Sprintf("%d", c.idProvider.NewRequestID()), leaseID) + resp, err = c.client.PutWithLease(putCtx, c.keyStore.GetKey(), fmt.Sprintf("%d", c.idProvider.NewRequestID()), leaseID) putCancel() if resp != nil { rev = resp.Header.Revision @@ -334,8 +367,7 @@ func (c etcdTrafficClient) Request(ctx context.Context, request etcdRequestType, return rev, err } -func (c etcdTrafficClient) pickMultiTxnOps() (ops []clientv3.Op) { - keys := rand.Perm(c.keyCount) +func (c etcdTrafficClient) pickMultiTxnOps(keyStore *keyStore) (ops []clientv3.Op) { opTypes := make([]model.OperationType, 4) atLeastOnePut := false @@ -350,8 +382,10 @@ func (c etcdTrafficClient) pickMultiTxnOps() (ops []clientv3.Op) { opTypes[0] = model.PutOperation } + keys := keyStore.GetKeysForMultiTxnOps(opTypes) + for i, opType := range opTypes { - key := c.key(keys[i]) + key := keys[i] switch opType { case model.RangeOperation: ops = append(ops, clientv3.OpGet(key)) @@ -361,20 +395,12 @@ func (c etcdTrafficClient) pickMultiTxnOps() (ops []clientv3.Op) { case model.DeleteOperation: ops = append(ops, clientv3.OpDelete(key)) default: - panic("unsuported choice type") + panic("unsupported choice type") } } return ops } -func (c etcdTrafficClient) randomKey() string { - return c.key(rand.Int()) -} - -func (c etcdTrafficClient) key(i int) string { - return fmt.Sprintf("%s%d", c.keyPrefix, i%c.keyCount) -} - func (t etcdTraffic) pickOperationType() model.OperationType { roll := rand.Int() % 100 if roll < 10 { diff --git a/tests/robustness/traffic/key_store.go b/tests/robustness/traffic/key_store.go new file mode 100644 index 000000000000..a95bc194be8e --- /dev/null +++ b/tests/robustness/traffic/key_store.go @@ -0,0 +1,164 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package traffic + +import ( + "fmt" + "math/rand" + "slices" + "sort" + "strconv" + "sync" + + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/robustness/model" +) + +// Stores the key pool to use for operations. This allows keys used in Put and Delete operations to be more unique by probabilistically swapping out used keys. +type keyStore struct { + mu sync.Mutex + + counter int + keys []string + keyPrefix string + latestRevision int64 +} + +func NewKeyStore(size int, keyPrefix string) *keyStore { + k := &keyStore{ + keys: make([]string, size), + counter: 0, + keyPrefix: keyPrefix, + } + + // Fill with default values i.e. key0-key9 + for ; k.counter < len(k.keys); k.counter++ { + k.keys[k.counter] = fmt.Sprintf("%s%d", k.keyPrefix, k.counter) + } + + return k +} + +func (k *keyStore) GetKey() string { + k.mu.Lock() + defer k.mu.Unlock() + + useKey := k.keys[rand.Intn(len(k.keys))] + + return useKey +} + +func (k *keyStore) GetKeyForDelete() string { + k.mu.Lock() + defer k.mu.Unlock() + + useKeyIndex := rand.Intn(len(k.keys)) + useKey := k.keys[useKeyIndex] + + k.replaceKey(useKeyIndex) + + return useKey +} + +func (k *keyStore) GetKeysForMultiTxnOps(ops []model.OperationType) []string { + k.mu.Lock() + defer k.mu.Unlock() + + numOps := len(ops) + + if numOps > len(k.keys) { + panic("GetKeysForMultiTxnOps: number of operations is more than the key pool size") + } + + keys := make([]string, numOps) + + permutedKeyIndexes := rand.Perm(len(k.keys)) + for i, op := range ops { + keys[i] = k.keys[permutedKeyIndexes[i]] + + if op == model.DeleteOperation { + k.replaceKey(permutedKeyIndexes[i]) + } + } + + return keys +} + +func (k *keyStore) GetPrefix() string { + k.mu.Lock() + defer k.mu.Unlock() + + return k.keyPrefix +} + +// SyncKeys reconciles our local key store with the keys currently in etcd. +// +// SyncKeys resolves this by: +// 1. Getting the list request result of all the keys. +// 2. Adding any keys that exist in etcd but are missing in the key store. +// 3. Maintaining the key store size by removing the higher numbered keys. +// +// Notice that higher numbered keys will eventually be added +// again into the keystore, so it is safe to temporarily remove them from the +// key store. +func (k *keyStore) SyncKeys(resp *clientv3.GetResponse) { + k.mu.Lock() + defer k.mu.Unlock() + + if resp.Header.GetRevision() < k.latestRevision { + return + } + + k.latestRevision = resp.Header.GetRevision() + + listKeys := make([]string, len(resp.Kvs)) + for i, kv := range resp.Kvs { + listKeys[i] = string(kv.Key) + } + + for _, key := range k.keys { + if !slices.Contains(listKeys, key) { + listKeys = append(listKeys, key) + } + } + + sort.Slice(listKeys, func(i, j int) bool { + keyNumI := k.getKeyNum(listKeys[i]) + keyNumJ := k.getKeyNum(listKeys[j]) + return keyNumI < keyNumJ + }) + + k.keys = listKeys[:len(k.keys)] + + lastKeyNum := k.getKeyNum(k.keys[len(k.keys)-1]) + k.counter = lastKeyNum + 1 +} + +func (k *keyStore) getKeyNum(key string) int { + if len(key) < len(k.keyPrefix) { + return 0 + } + + numStr := key[len(k.keyPrefix):] + num, _ := strconv.Atoi(numStr) + return num +} + +func (k *keyStore) replaceKey(index int) { + if rand.Intn(100) < 90 { + k.keys[index] = fmt.Sprintf("%s%d", k.keyPrefix, k.counter) + k.counter++ + } +} diff --git a/tests/robustness/traffic/kubernetes.go b/tests/robustness/traffic/kubernetes.go index fe1386fa243c..e45171c03fa1 100644 --- a/tests/robustness/traffic/kubernetes.go +++ b/tests/robustness/traffic/kubernetes.go @@ -41,6 +41,14 @@ var ( resource: "pods", namespace: "default", // Please keep the sum of weights equal 100. + readChoices: []random.ChoiceWeight[KubernetesRequestType]{ + {Choice: KubernetesGet, Weight: 5}, + {Choice: KubernetesGetStale, Weight: 2}, + {Choice: KubernetesGetRev, Weight: 8}, + {Choice: KubernetesListStale, Weight: 5}, + {Choice: KubernetesListAndWatch, Weight: 80}, + }, + // Please keep the sum of weights equal 100. writeChoices: []random.ChoiceWeight[KubernetesRequestType]{ {Choice: KubernetesUpdate, Weight: 90}, {Choice: KubernetesDelete, Weight: 5}, @@ -53,6 +61,14 @@ var ( resource: "pods", namespace: "default", // Please keep the sum of weights equal 100. + readChoices: []random.ChoiceWeight[KubernetesRequestType]{ + {Choice: KubernetesGet, Weight: 5}, + {Choice: KubernetesGetStale, Weight: 2}, + {Choice: KubernetesGetRev, Weight: 8}, + {Choice: KubernetesListStale, Weight: 5}, + {Choice: KubernetesListAndWatch, Weight: 80}, + }, + // Please keep the sum of weights equal 100. writeChoices: []random.ChoiceWeight[KubernetesRequestType]{ {Choice: KubernetesDelete, Weight: 40}, {Choice: KubernetesCreate, Weight: 60}, @@ -64,6 +80,7 @@ type kubernetesTraffic struct { averageKeyCount int resource string namespace string + readChoices []random.ChoiceWeight[KubernetesRequestType] writeChoices []random.ChoiceWeight[KubernetesRequestType] } @@ -71,27 +88,25 @@ func (t kubernetesTraffic) ExpectUniqueRevision() bool { return true } -func (t kubernetesTraffic) RunTrafficLoop(ctx context.Context, c *client.RecordingClient, limiter *rate.Limiter, ids identity.Provider, lm identity.LeaseIDStorage, nonUniqueWriteLimiter ConcurrencyLimiter, finish <-chan struct{}) { +func (t kubernetesTraffic) RunKeyValueLoop(ctx context.Context, c *client.RecordingClient, p RunTrafficLoopParam) { kc := kubernetes.Client{Client: &clientv3.Client{KV: c}} - s := newStorage() + s := p.Storage keyPrefix := "/registry/" + t.resource + "/" g := errgroup.Group{} - readLimit := t.averageKeyCount g.Go(func() error { for { select { case <-ctx.Done(): return ctx.Err() - case <-finish: + case <-p.Finish: return nil default: } - rev, err := t.Read(ctx, kc, s, limiter, keyPrefix, readLimit) + err := t.Read(ctx, c, s, p.QPSLimiter, keyPrefix) if err != nil { continue } - t.Watch(ctx, c, s, limiter, keyPrefix, rev+1) } }) g.Go(func() error { @@ -100,18 +115,18 @@ func (t kubernetesTraffic) RunTrafficLoop(ctx context.Context, c *client.Recordi select { case <-ctx.Done(): return ctx.Err() - case <-finish: + case <-p.Finish: return nil default: } // Avoid multiple failed writes in a row if lastWriteFailed { - _, err := t.Read(ctx, kc, s, limiter, keyPrefix, 0) + _, err := t.List(ctx, kc, s, p.QPSLimiter, keyPrefix, t.averageKeyCount, 0) if err != nil { continue } } - err := t.Write(ctx, kc, ids, s, limiter, nonUniqueWriteLimiter) + err := t.Write(ctx, kc, p.IDs, s, p.QPSLimiter, p.NonUniqueRequestConcurrencyLimiter) lastWriteFailed = err != nil if err != nil { continue @@ -121,10 +136,53 @@ func (t kubernetesTraffic) RunTrafficLoop(ctx context.Context, c *client.Recordi g.Wait() } -func (t kubernetesTraffic) Read(ctx context.Context, kc kubernetes.Interface, s *storage, limiter *rate.Limiter, keyPrefix string, limit int) (rev int64, err error) { +func (t kubernetesTraffic) RunWatchLoop(ctx context.Context, c *client.RecordingClient, p RunWatchLoopParam) { + runWatchLoop(ctx, c, p, watchLoopConfig{ + key: "/registry/" + t.resource + "/", + requireLeader: true, + }) +} + +func (t kubernetesTraffic) Read(ctx context.Context, c *client.RecordingClient, s *storage, limiter *rate.Limiter, keyPrefix string) error { + kc := kubernetes.Client{Client: &clientv3.Client{KV: c}} + op := random.PickRandom(t.readChoices) + switch op { + case KubernetesGet: + key, unusedRev := s.PickRandom() + if unusedRev == 0 { + return errors.New("storage empty") + } + return t.Get(ctx, kc, s, limiter, key, 0) + case KubernetesGetStale: + key, rev := s.KeyWithUnrelatedRev() + return t.Get(ctx, kc, s, limiter, key, rev) + case KubernetesGetRev: + return t.Get(ctx, kc, s, limiter, "/registry/"+t.resource, 0) + case KubernetesListStale: + _, rev := s.PickRandom() + _, err := t.List(ctx, kc, s, limiter, keyPrefix, t.averageKeyCount, rev) + return err + case KubernetesListAndWatch: + rev, err := t.List(ctx, kc, s, limiter, keyPrefix, t.averageKeyCount, 0) + if err != nil { + return err + } + t.Watch(ctx, c, s, limiter, keyPrefix, rev+1) + return nil + default: + panic(fmt.Sprintf("invalid choice: %q", op)) + } +} + +func (t kubernetesTraffic) Get(ctx context.Context, kc kubernetes.Interface, s *storage, limiter *rate.Limiter, key string, rev int64) error { + _, err := kc.Get(ctx, key, kubernetes.GetOptions{Revision: rev}) + limiter.Wait(ctx) + return err +} + +func (t kubernetesTraffic) List(ctx context.Context, kc kubernetes.Interface, s *storage, limiter *rate.Limiter, keyPrefix string, limit int, revision int64) (rev int64, err error) { hasMore := true var kvs []*mvccpb.KeyValue - var revision int64 var cont string for hasMore { @@ -220,17 +278,17 @@ func (t kubernetesTraffic) generateKey() string { return fmt.Sprintf("/registry/%s/%s/%s", t.resource, t.namespace, stringutil.RandString(5)) } -func (t kubernetesTraffic) RunCompactLoop(ctx context.Context, c *client.RecordingClient, interval time.Duration, finish <-chan struct{}) { +func (t kubernetesTraffic) RunCompactLoop(ctx context.Context, c *client.RecordingClient, param RunCompactLoopParam) { // Based on https://github.com/kubernetes/apiserver/blob/7dd4904f1896e11244ba3c5a59797697709de6b6/pkg/storage/etcd3/compact.go#L112-L127 var compactTime int64 var rev int64 var err error for { select { - case <-time.After(interval): + case <-time.After(param.Period): case <-ctx.Done(): return - case <-finish: + case <-param.Finish: return } @@ -277,9 +335,14 @@ func compact(ctx context.Context, client *client.RecordingClient, t, rev int64) type KubernetesRequestType string const ( - KubernetesDelete KubernetesRequestType = "delete" - KubernetesUpdate KubernetesRequestType = "update" - KubernetesCreate KubernetesRequestType = "create" + KubernetesDelete KubernetesRequestType = "delete" + KubernetesUpdate KubernetesRequestType = "update" + KubernetesCreate KubernetesRequestType = "create" + KubernetesGet KubernetesRequestType = "get" + KubernetesGetStale KubernetesRequestType = "get_stale" + KubernetesGetRev KubernetesRequestType = "get_rev" + KubernetesListStale KubernetesRequestType = "list_stale" + KubernetesListAndWatch KubernetesRequestType = "list_watch" ) type storage struct { @@ -288,7 +351,9 @@ type storage struct { revision int64 } -func newStorage() *storage { +// NewKubernetesStorage creates a new storage instance for tracking Kubernetes resource revisions. +// This storage is used by Kubernetes traffic to maintain consistency in read and write operations. +func NewKubernetesStorage() *storage { return &storage{ keyRevision: map[string]int64{}, } @@ -303,9 +368,9 @@ func (s *storage) Update(resp clientv3.WatchResponse) { } s.revision = e.Kv.ModRevision switch e.Type { - case mvccpb.PUT: + case mvccpb.Event_PUT: s.keyRevision[string(e.Kv.Key)] = e.Kv.ModRevision - case mvccpb.DELETE: + case mvccpb.Event_DELETE: delete(s.keyRevision, string(e.Kv.Key)) } } @@ -333,7 +398,15 @@ func (s *storage) Count() int { func (s *storage) PickRandom() (key string, rev int64) { s.mux.RLock() defer s.mux.RUnlock() - n := rand.Intn(len(s.keyRevision)) + return s.pickRandomLocked() +} + +func (s *storage) pickRandomLocked() (key string, rev int64) { + l := len(s.keyRevision) + if l == 0 { + return "", 0 + } + n := rand.Intn(l) i := 0 for k, v := range s.keyRevision { if i == n { @@ -343,3 +416,15 @@ func (s *storage) PickRandom() (key string, rev int64) { } return "", 0 } + +func (s *storage) KeyWithUnrelatedRev() (key string, rev int64) { + s.mux.RLock() + defer s.mux.RUnlock() + l := len(s.keyRevision) + if l == 0 { + return "", 0 + } + key, _ = s.pickRandomLocked() + _, rev = s.pickRandomLocked() + return key, rev +} diff --git a/tests/robustness/traffic/traffic.go b/tests/robustness/traffic/traffic.go index c66aa4543620..04159c6ecc99 100644 --- a/tests/robustness/traffic/traffic.go +++ b/tests/robustness/traffic/traffic.go @@ -24,90 +24,121 @@ import ( "go.uber.org/zap" "golang.org/x/time/rate" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/model" + "go.etcd.io/etcd/tests/v3/robustness/random" "go.etcd.io/etcd/tests/v3/robustness/report" + "go.etcd.io/etcd/tests/v3/robustness/validate" ) +type Range struct { + Min int64 + Max int64 +} + +func (r Range) Rand() int64 { + if r.Min == r.Max { + return r.Min + } + return random.RandRange(r.Min, r.Max+1) +} + var ( DefaultLeaseTTL int64 = 7200 RequestTimeout = 200 * time.Millisecond - WatchTimeout = time.Second + WatchTimeout = 500 * time.Millisecond MultiOpTxnOpCount = 4 - DefaultCompactionPeriod = 200 * time.Millisecond + MinimalCompactionPeriod = 100 * time.Millisecond - LowTraffic = Profile{ + KeyValueVeryLow = KeyValue{ + MinimalQPS: 50, + MaximalQPS: 100, + BurstableQPS: 100, + MemberClientCount: 6, + ClusterClientCount: 2, + MaxNonUniqueRequestConcurrency: 3, + } + KeyValueMedium = KeyValue{ MinimalQPS: 100, MaximalQPS: 200, BurstableQPS: 1000, - ClientCount: 8, + MemberClientCount: 6, + ClusterClientCount: 2, MaxNonUniqueRequestConcurrency: 3, } - HighTrafficProfile = Profile{ + KeyValueHigh = KeyValue{ MinimalQPS: 100, MaximalQPS: 1000, BurstableQPS: 1000, - ClientCount: 8, + MemberClientCount: 6, + ClusterClientCount: 2, MaxNonUniqueRequestConcurrency: 3, } + WatchDefault = Watch{ + MemberClientCount: 6, + ClusterClientCount: 2, + RevisionOffsetRange: Range{Min: -100, Max: 100}, + } + CompactionDefault = Compaction{ + Period: 200 * time.Millisecond, + } + CompactionFrequent = Compaction{ + Period: 100 * time.Millisecond, + } ) -func SimulateTraffic(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, profile Profile, traffic Traffic, failpointInjected <-chan report.FailpointInjection, baseTime time.Time, ids identity.Provider) []report.ClientReport { - mux := sync.Mutex{} +func SimulateTraffic(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, profile Profile, traffic Traffic, failpointInjected <-chan report.FailpointInjection, clientSet *client.ClientSet) []report.ClientReport { endpoints := clus.EndpointsGRPC() lm := identity.NewLeaseIDStorage() - reports := []report.ClientReport{} // Use the highest MaximalQPS of all traffic profiles as burst otherwise actual traffic may be accidentally limited - limiter := rate.NewLimiter(rate.Limit(profile.MaximalQPS), profile.BurstableQPS) + limiter := rate.NewLimiter(rate.Limit(profile.KeyValue.MaximalQPS), profile.KeyValue.BurstableQPS) - cc, err := client.NewRecordingClient(endpoints, ids, baseTime) + err := CheckEmptyDatabaseAtStart(ctx, lg, endpoints, clientSet) require.NoError(t, err) - defer cc.Close() - // Ensure that first operation succeeds - _, err = cc.Put(ctx, "start", "true") - require.NoErrorf(t, err, "First operation failed, validation requires first operation to succeed") + wg := sync.WaitGroup{} - nonUniqueWriteLimiter := NewConcurrencyLimiter(profile.MaxNonUniqueRequestConcurrency) + nonUniqueWriteLimiter := NewConcurrencyLimiter(profile.KeyValue.MaxNonUniqueRequestConcurrency) finish := make(chan struct{}) - lg.Info("Start traffic") - startTime := time.Since(baseTime) - for i := 0; i < profile.ClientCount; i++ { - wg.Add(1) - c, nerr := client.NewRecordingClient([]string{endpoints[i%len(endpoints)]}, ids, baseTime) - require.NoError(t, nerr) - go func(c *client.RecordingClient) { - defer wg.Done() - defer c.Close() - traffic.RunTrafficLoop(ctx, c, limiter, ids, lm, nonUniqueWriteLimiter, finish) - mux.Lock() - reports = append(reports, c.Report()) - mux.Unlock() - }(c) + keyStore := NewKeyStore(10, "key") + kubernetesStorage := NewKubernetesStorage() + + lg.Info("Start traffic") + startTime := time.Since(clientSet.BaseTime()) + err = SimulateKeyValueTraffic(ctx, &wg, profile.KeyValue, endpoints, clientSet, traffic, RunTrafficLoopParam{ + QPSLimiter: limiter, + IDs: clientSet.IdentityProvider(), + LeaseIDStorage: lm, + NonUniqueRequestConcurrencyLimiter: nonUniqueWriteLimiter, + KeyStore: keyStore, + Storage: kubernetesStorage, + Finish: finish, + }) + require.NoError(t, err) + if profile.Watch != nil { + err = SimulateWatchTraffic(ctx, &wg, profile.Watch, endpoints, clientSet, traffic, RunWatchLoopParam{ + Config: *profile.Watch, + QPSLimiter: limiter, + KeyStore: keyStore, + Storage: kubernetesStorage, + Finish: finish, + Logger: lg, + }) + require.NoError(t, err) } - if !profile.ForbidCompaction { - wg.Add(1) - c, nerr := client.NewRecordingClient(endpoints, ids, baseTime) - if nerr != nil { - t.Fatal(nerr) + if profile.Compaction != nil { + if profile.Compaction.Period < MinimalCompactionPeriod { + t.Fatalf("Compaction period %v below minimal %v", profile.Compaction.Period, MinimalCompactionPeriod) } - go func(c *client.RecordingClient) { - defer wg.Done() - defer c.Close() - - compactionPeriod := DefaultCompactionPeriod - if profile.CompactPeriod != time.Duration(0) { - compactionPeriod = profile.CompactPeriod - } - - traffic.RunCompactLoop(ctx, c, compactionPeriod, finish) - mux.Lock() - reports = append(reports, c.Report()) - mux.Unlock() - }(c) + err = SimulateCompactionTraffic(ctx, &wg, profile.Compaction, endpoints, clientSet, traffic, RunCompactLoopParam{ + Period: profile.Compaction.Period, + Finish: finish, + }) + require.NoError(t, err) } var fr *report.FailpointInjection select { @@ -120,33 +151,186 @@ func SimulateTraffic(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2 close(finish) wg.Wait() lg.Info("Finished traffic") - endTime := time.Since(baseTime) + endTime := time.Since(clientSet.BaseTime()) time.Sleep(time.Second) // Ensure that last operation succeeds + cc, err := clientSet.NewClient(endpoints) + require.NoError(t, err) + defer cc.Close() _, err = cc.Put(ctx, "tombstone", "true") require.NoErrorf(t, err, "Last operation failed, validation requires last operation to succeed") - reports = append(reports, cc.Report()) + reports := clientSet.Reports() + + totalStats := CalculateStats(reports, startTime, endTime) + beforeFailpointStats := CalculateStats(reports, startTime, fr.Start) + duringFailpointStats := CalculateStats(reports, fr.Start, fr.End) + afterFailpointStats := CalculateStats(reports, fr.End, endTime) - totalStats := calculateStats(reports, startTime, endTime) - beforeFailpointStats := calculateStats(reports, startTime, fr.Start) - duringFailpointStats := calculateStats(reports, fr.Start, fr.End) - afterFailpointStats := calculateStats(reports, fr.End, endTime) + lg.Info("Reporting complete traffic", zap.Int("successes", totalStats.Successes), zap.Int("failures", totalStats.Failures), zap.Float64("successRate", totalStats.SuccessRate()), zap.Duration("period", totalStats.Period), zap.Float64("qps", totalStats.QPS())) + lg.Info("Reporting traffic before failure injection", zap.Int("successes", beforeFailpointStats.Successes), zap.Int("failures", beforeFailpointStats.Failures), zap.Float64("successRate", beforeFailpointStats.SuccessRate()), zap.Duration("period", beforeFailpointStats.Period), zap.Float64("qps", beforeFailpointStats.QPS())) + lg.Info("Reporting traffic during failure injection", zap.Int("successes", duringFailpointStats.Successes), zap.Int("failures", duringFailpointStats.Failures), zap.Float64("successRate", duringFailpointStats.SuccessRate()), zap.Duration("period", duringFailpointStats.Period), zap.Float64("qps", duringFailpointStats.QPS())) + lg.Info("Reporting traffic after failure injection", zap.Int("successes", afterFailpointStats.Successes), zap.Int("failures", afterFailpointStats.Failures), zap.Float64("successRate", afterFailpointStats.SuccessRate()), zap.Duration("period", afterFailpointStats.Period), zap.Float64("qps", afterFailpointStats.QPS())) - lg.Info("Reporting complete traffic", zap.Int("successes", totalStats.successes), zap.Int("failures", totalStats.failures), zap.Float64("successRate", totalStats.successRate()), zap.Duration("period", totalStats.period), zap.Float64("qps", totalStats.QPS())) - lg.Info("Reporting traffic before failure injection", zap.Int("successes", beforeFailpointStats.successes), zap.Int("failures", beforeFailpointStats.failures), zap.Float64("successRate", beforeFailpointStats.successRate()), zap.Duration("period", beforeFailpointStats.period), zap.Float64("qps", beforeFailpointStats.QPS())) - lg.Info("Reporting traffic during failure injection", zap.Int("successes", duringFailpointStats.successes), zap.Int("failures", duringFailpointStats.failures), zap.Float64("successRate", duringFailpointStats.successRate()), zap.Duration("period", duringFailpointStats.period), zap.Float64("qps", duringFailpointStats.QPS())) - lg.Info("Reporting traffic after failure injection", zap.Int("successes", afterFailpointStats.successes), zap.Int("failures", afterFailpointStats.failures), zap.Float64("successRate", afterFailpointStats.successRate()), zap.Duration("period", afterFailpointStats.period), zap.Float64("qps", afterFailpointStats.QPS())) + watchTotal := CalculateWatchStats(reports, startTime, endTime) + lg.Info("Reporting complete watch", zap.Int("requests", watchTotal.Requests), zap.Int("events", watchTotal.Events), zap.Float64("eventsQPS", watchTotal.EventsQPS()), zap.Int("progressNotifies", watchTotal.ProgressNotifies), zap.Int("immediateClosures", watchTotal.ImmediateClosures), zap.Duration("period", watchTotal.Period), zap.Duration("avgDuration", watchTotal.AvgDuration())) - if beforeFailpointStats.QPS() < profile.MinimalQPS { - t.Errorf("Requiring minimal %f qps before failpoint injection for test results to be reliable, got %f qps", profile.MinimalQPS, beforeFailpointStats.QPS()) + if beforeFailpointStats.QPS() < profile.KeyValue.MinimalQPS { + t.Errorf("Requiring minimal %f qps before failpoint injection for test results to be reliable, got %f qps", profile.KeyValue.MinimalQPS, beforeFailpointStats.QPS()) } - // TODO: Validate QPS post failpoint injection to ensure the that we sufficiently cover period when cluster recovers. + // TODO: Validate QPS post failpoint injection to ensure that we sufficiently cover the period when the cluster recovers. return reports } -func calculateStats(reports []report.ClientReport, start, end time.Duration) (ts trafficStats) { - ts.period = end - start +func SimulateKeyValueTraffic(ctx context.Context, wg *sync.WaitGroup, profile *KeyValue, endpoints []string, clientSet *client.ClientSet, tf Traffic, baseParam RunTrafficLoopParam) error { + for i := range profile.MemberClientCount { + c, err := clientSet.NewClient([]string{endpoints[i%len(endpoints)]}) + if err != nil { + return err + } + wg.Add(1) + go func(c *client.RecordingClient) { + defer wg.Done() + defer c.Close() + + tf.RunKeyValueLoop(ctx, c, baseParam) + }(c) + } + for range profile.ClusterClientCount { + c, err := clientSet.NewClient(endpoints) + if err != nil { + return err + } + wg.Add(1) + go func(c *client.RecordingClient) { + defer wg.Done() + defer c.Close() + + tf.RunKeyValueLoop(ctx, c, baseParam) + }(c) + } + return nil +} + +func SimulateWatchTraffic(ctx context.Context, wg *sync.WaitGroup, profile *Watch, endpoints []string, clientSet *client.ClientSet, tf Traffic, baseParam RunWatchLoopParam) error { + for i := range profile.MemberClientCount { + c, err := clientSet.NewClient([]string{endpoints[i%len(endpoints)]}) + if err != nil { + return err + } + wg.Add(1) + go func(c *client.RecordingClient) { + defer wg.Done() + defer c.Close() + tf.RunWatchLoop(ctx, c, baseParam) + }(c) + } + for range profile.ClusterClientCount { + c, err := clientSet.NewClient(endpoints) + if err != nil { + return err + } + wg.Add(1) + go func(c *client.RecordingClient) { + defer wg.Done() + defer c.Close() + tf.RunWatchLoop(ctx, c, baseParam) + }(c) + } + return nil +} + +func SimulateCompactionTraffic(ctx context.Context, wg *sync.WaitGroup, profile *Compaction, endpoints []string, clientSet *client.ClientSet, tf Traffic, baseParam RunCompactLoopParam) error { + c, err := clientSet.NewClient(endpoints) + if err != nil { + return err + } + wg.Add(1) + go func(c *client.RecordingClient) { + defer wg.Done() + defer c.Close() + tf.RunCompactLoop(ctx, c, baseParam) + }(c) + return nil +} + +func CalculateWatchStats(reports []report.ClientReport, start, end time.Duration) (ws watchStats) { + ws.Period = end - start + if ws.Period <= 0 { + return ws + } + + for _, r := range reports { + for _, w := range r.Watch { + var ( + firstInWindow time.Duration + lastInWindow time.Duration + haveInWindow bool + noEventsYetInWindow = true + closedCounted = false + ) + + for _, resp := range w.Responses { + if resp.Time < start || resp.Time > end { + continue + } + if !haveInWindow { + firstInWindow = resp.Time + haveInWindow = true + } + lastInWindow = resp.Time + if resp.IsProgressNotify { + ws.ProgressNotifies++ + } + if len(resp.Events) > 0 { + ws.Events += len(resp.Events) + noEventsYetInWindow = false + } + if resp.Error != "" && noEventsYetInWindow && !closedCounted { + ws.ImmediateClosures++ + closedCounted = true + } + } + + if haveInWindow { + ws.Requests++ + if lastInWindow > firstInWindow { + ws.SumDuration += lastInWindow - firstInWindow + ws.DurationsCount++ + } + } + } + } + + return ws +} + +type watchStats struct { + Period time.Duration + Requests int + Events int + ProgressNotifies int + ImmediateClosures int + SumDuration time.Duration + DurationsCount int +} + +func (ws *watchStats) AvgDuration() time.Duration { + if ws.DurationsCount == 0 { + return 0 + } + return ws.SumDuration / time.Duration(ws.DurationsCount) +} + +func (ws *watchStats) EventsQPS() float64 { + if ws.Period <= 0 { + return 0 + } + return float64(ws.Events) / ws.Period.Seconds() +} + +func CalculateStats(reports []report.ClientReport, start, end time.Duration) (ts trafficStats) { + ts.Period = end - start for _, r := range reports { for _, op := range r.KeyValue { @@ -155,9 +339,9 @@ func calculateStats(reports []report.ClientReport, start, end time.Duration) (ts } resp := op.Output.(model.MaybeEtcdResponse) if resp.Error == "" { - ts.successes++ + ts.Successes++ } else { - ts.failures++ + ts.Failures++ } } } @@ -165,40 +349,148 @@ func calculateStats(reports []report.ClientReport, start, end time.Duration) (ts } type trafficStats struct { - successes, failures int - period time.Duration + Successes, Failures int + Period time.Duration } -func (ts *trafficStats) successRate() float64 { - return float64(ts.successes) / float64(ts.successes+ts.failures) +func (ts *trafficStats) SuccessRate() float64 { + return float64(ts.Successes) / float64(ts.Successes+ts.Failures) } func (ts *trafficStats) QPS() float64 { - return float64(ts.successes) / ts.period.Seconds() + return float64(ts.Successes) / ts.Period.Seconds() } type Profile struct { + KeyValue *KeyValue + Watch *Watch + Compaction *Compaction +} + +type KeyValue struct { MinimalQPS float64 MaximalQPS float64 BurstableQPS int MaxNonUniqueRequestConcurrency int - ClientCount int - ForbidCompaction bool - CompactPeriod time.Duration + MemberClientCount int + ClusterClientCount int +} + +type Watch struct { + MemberClientCount int + ClusterClientCount int + RevisionOffsetRange Range +} + +type Compaction struct { + Period time.Duration +} + +type RunTrafficLoopParam struct { + QPSLimiter *rate.Limiter + IDs identity.Provider + LeaseIDStorage identity.LeaseIDStorage + NonUniqueRequestConcurrencyLimiter ConcurrencyLimiter + KeyStore *keyStore + Storage *storage + Finish <-chan struct{} } -func (p Profile) WithoutCompaction() Profile { - p.ForbidCompaction = true - return p +type RunCompactLoopParam struct { + Period time.Duration + Finish <-chan struct{} } -func (p Profile) WithCompactionPeriod(cp time.Duration) Profile { - p.CompactPeriod = cp - return p +type RunWatchLoopParam struct { + Config Watch + QPSLimiter *rate.Limiter + // TODO: merge 2 key stores into 1 + KeyStore *keyStore + Storage *storage + Finish <-chan struct{} + Logger *zap.Logger } type Traffic interface { - RunTrafficLoop(ctx context.Context, c *client.RecordingClient, qpsLimiter *rate.Limiter, ids identity.Provider, lm identity.LeaseIDStorage, nonUniqueWriteLimiter ConcurrencyLimiter, finish <-chan struct{}) - RunCompactLoop(ctx context.Context, c *client.RecordingClient, period time.Duration, finish <-chan struct{}) + RunKeyValueLoop(ctx context.Context, c *client.RecordingClient, param RunTrafficLoopParam) + RunWatchLoop(ctx context.Context, c *client.RecordingClient, param RunWatchLoopParam) + RunCompactLoop(ctx context.Context, c *client.RecordingClient, param RunCompactLoopParam) ExpectUniqueRevision() bool } + +func runWatchLoop(ctx context.Context, c *client.RecordingClient, p RunWatchLoopParam, cfg watchLoopConfig) { + for { + select { + case <-ctx.Done(): + return + case <-p.Finish: + return + default: + } + err := p.QPSLimiter.Wait(ctx) + if err != nil { + return + } + // Client.get may fail when the blackhole is injected. + // We suppress logging for these expected failures to avoid polluting the logs. + _ = runWatch(ctx, c, p, cfg) + } +} + +func runWatch(ctx context.Context, c *client.RecordingClient, p RunWatchLoopParam, cfg watchLoopConfig) error { + getCtx, getCancel := context.WithTimeout(ctx, RequestTimeout) + defer getCancel() + + resp, err := c.Get(getCtx, cfg.key) + if err != nil { + return err + } + rev := resp.Header.Revision + p.Config.RevisionOffsetRange.Rand() + + watchCtx, watchCancel := context.WithTimeout(ctx, WatchTimeout) + defer watchCancel() + + if cfg.requireLeader { + watchCtx = clientv3.WithRequireLeader(watchCtx) + } + w := c.Watch(watchCtx, cfg.key, rev, true, true, true) + for { + select { + case <-ctx.Done(): + return nil + case <-p.Finish: + return nil + case _, ok := <-w: + if !ok { + return nil + } + } + } +} + +type watchLoopConfig struct { + key string + requireLeader bool +} + +func CheckEmptyDatabaseAtStart(ctx context.Context, lg *zap.Logger, endpoints []string, cs *client.ClientSet) error { + c, err := cs.NewClient(endpoints) + if err != nil { + return err + } + defer c.Close() + for { + rCtx, cancel := context.WithTimeout(ctx, RequestTimeout) + resp, err := c.Get(rCtx, "key") + cancel() + if err != nil { + lg.Warn("Failed to check if database empty at start, retrying", zap.Error(err)) + continue + } + if resp.Header.Revision != 1 { + return validate.ErrNotEmptyDatabase + } + break + } + return nil +} diff --git a/tests/robustness/validate/operations.go b/tests/robustness/validate/operations.go index 9f39407ad818..d86f60fff26b 100644 --- a/tests/robustness/validate/operations.go +++ b/tests/robustness/validate/operations.go @@ -24,7 +24,6 @@ import ( "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" - "go.etcd.io/etcd/tests/v3/robustness/report" ) var ( @@ -32,32 +31,71 @@ var ( errFutureRevRespRequested = errors.New("request about a future rev with response") ) -func validateLinearizableOperationsAndVisualize(lg *zap.Logger, operations []porcupine.Operation, timeout time.Duration) (result porcupine.CheckResult, visualize func(basepath string) error) { +func validateLinearizableOperationsAndVisualize(lg *zap.Logger, keys []string, operations []porcupine.Operation, timeout time.Duration) LinearizationResult { lg.Info("Validating linearizable operations", zap.Duration("timeout", timeout)) start := time.Now() - result, info := porcupine.CheckOperationsVerbose(model.NonDeterministicModel, operations, timeout) - switch result { - case porcupine.Illegal: - lg.Error("Linearization failed", zap.Duration("duration", time.Since(start))) - case porcupine.Unknown: - lg.Error("Linearization has timed out", zap.Duration("duration", time.Since(start))) + + model.LinearizationDeadlineTripped.Store(0) + + var timer *time.Timer + if timeout > 0 { + // Porcupine timeout is not always enforced (see https://github.com/anishathalye/porcupine/issues/44) + // Give it a small grace period before forcing the deadline from inside model execution. + timer = time.AfterFunc(timeout*11/10, func() { + model.LinearizationDeadlineTripped.Store(1) + }) + } + + m := model.NonDeterministicModel(keys) + check, info := porcupine.CheckOperationsVerbose(m, operations, timeout) + if timer != nil { + timer.Stop() + } + duration := time.Since(start) + + result := LinearizationResult{ + Info: info, + Model: m, + } + + if model.LinearizationDeadlineTripped.Load() != 0 { + result.Status = DeadlineExceeded + result.Message = "deadline exceeded" + lg.Error("Linearization deadline exceeded", zap.Duration("duration", duration)) + return result + } + + switch check { case porcupine.Ok: - lg.Info("Linearization success", zap.Duration("duration", time.Since(start))) + result.Status = Success + lg.Info("Linearization success", zap.Duration("duration", duration)) + case porcupine.Unknown: + result.Status = Timeout + lg.Error("Linearization timed out", zap.Duration("duration", duration)) + case porcupine.Illegal: + result.Status = Failure + result.Message = "illegal" + lg.Error("Linearization illegal", zap.Duration("duration", duration)) default: - panic(fmt.Sprintf("Unknown Linearization result %s", result)) - } - return result, func(path string) error { - lg.Info("Saving visualization", zap.String("path", path)) - err := porcupine.VisualizePath(model.NonDeterministicModel, info, path) - if err != nil { - return fmt.Errorf("failed to visualize, err: %w", err) - } - return nil + result.Status = Failure + result.Message = fmt.Sprintf("unknown results from porcupine: %s", check) } + return result } -func validateSerializableOperations(lg *zap.Logger, operations []porcupine.Operation, replay *model.EtcdReplay) (lastErr error) { +func validateSerializableOperations(lg *zap.Logger, operations []porcupine.Operation, replay *model.EtcdReplay) Result { lg.Info("Validating serializable operations") + start := time.Now() + err := validateSerializableOperationsError(lg, operations, replay) + if err != nil { + lg.Error("Serializable validation failed", zap.Duration("duration", time.Since(start)), zap.Error(err)) + } else { + lg.Info("Serializable validation success", zap.Duration("duration", time.Since(start))) + } + return ResultFromError(err) +} + +func validateSerializableOperationsError(lg *zap.Logger, operations []porcupine.Operation, replay *model.EtcdReplay) (lastErr error) { for _, read := range operations { request := read.Input.(model.EtcdRequest) response := read.Output.(model.MaybeEtcdResponse) @@ -69,19 +107,6 @@ func validateSerializableOperations(lg *zap.Logger, operations []porcupine.Opera return lastErr } -func filterSerializableOperations(clients []report.ClientReport) []porcupine.Operation { - resp := []porcupine.Operation{} - for _, client := range clients { - for _, op := range client.KeyValue { - request := op.Input.(model.EtcdRequest) - if request.Type == model.Range && request.Range.Revision != 0 { - resp = append(resp, op) - } - } - } - return resp -} - func validateSerializableRead(lg *zap.Logger, replay *model.EtcdReplay, request model.EtcdRequest, response model.MaybeEtcdResponse) error { if response.Persisted || response.Error != "" { return nil diff --git a/tests/robustness/validate/operations_test.go b/tests/robustness/validate/operations_test.go index a244d9f75339..21f366769871 100644 --- a/tests/robustness/validate/operations_test.go +++ b/tests/robustness/validate/operations_test.go @@ -17,9 +17,12 @@ package validate import ( "fmt" + "math/rand/v2" "testing" + "time" "github.com/anishathalye/porcupine" + "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/tests/v3/robustness/model" @@ -238,13 +241,9 @@ func TestValidateSerializableOperations(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { replay := model.NewReplay(tc.persistedRequests) - err := validateSerializableOperations(zaptest.NewLogger(t), tc.operations, replay) - var errStr string - if err != nil { - errStr = err.Error() - } - if errStr != tc.expectError { - t.Errorf("validateSerializableOperations(...), got: %q, want: %q", err, tc.expectError) + result := validateSerializableOperations(zaptest.NewLogger(t), tc.operations, replay) + if result.Message != tc.expectError { + t.Errorf("validateSerializableOperations(...), got: %q, want: %q", result.Message, tc.expectError) } }) } @@ -294,3 +293,235 @@ func keyValueRevision(key, value string, rev int64) model.KeyValue { }, } } + +func TestValidateLinearizableOperationsTimeoutIsRespected(t *testing.T) { + timeout := time.Second + const failedPutCount = 17 + // Repeat the range read to make the final applyRequestWithResponse step slow. + const rangeReadCount = 3072 + + history := []porcupine.Operation{} + for i := 0; i < failedPutCount; i++ { + history = append(history, porcupine.Operation{ + ClientId: i, + Input: putRequest(fmt.Sprintf("failed%03d", i), "value"), + Output: errorResponse(fmt.Errorf("timeout")), + Call: int64(i), + Return: int64(failedPutCount + i), + }) + } + + rangeOperations := make([]model.EtcdOperation, 0, rangeReadCount) + for i := 0; i < rangeReadCount; i++ { + rangeOperations = append(rangeOperations, model.EtcdOperation{ + Type: model.RangeOperation, + Range: model.RangeOptions{ + Start: "failed", + End: "faile~", + }, + }) + } + readRequest := model.EtcdRequest{ + Type: model.Txn, + Txn: &model.TxnRequest{ + OperationsOnSuccess: rangeOperations, + }, + } + + // Each failed put can be lost or persisted, so the large read-only + // transaction reaches applyRequestWithResponse with many possible states in + // a single Step call. + replay := model.NewReplayFromOperations(history) + state, err := replay.StateForRevision(failedPutCount + 1) + if err != nil { + t.Fatal(err) + } + _, readResponse := state.Step(readRequest) + history = append(history, porcupine.Operation{ + ClientId: failedPutCount, + Input: readRequest, + Output: readResponse, + Call: int64(2 * failedPutCount), + Return: int64(2*failedPutCount + 1), + }) + keys := model.ModelKeys(history) + + start := time.Now() + result := validateLinearizableOperationsAndVisualize(zap.NewNop(), keys, history, timeout) + elapsed := time.Since(start) + + if result.Status != DeadlineExceeded { + t.Fatalf("validateLinearizableOperationsAndVisualize(...) status = %q, want %q", result.Status, DeadlineExceeded) + } + if result.Message != "deadline exceeded" { + t.Fatalf("validateLinearizableOperationsAndVisualize(...) message = %q, want %q", result.Message, "deadline exceeded") + } + if elapsed > timeout+250*time.Millisecond { + t.Fatalf("validateLinearizableOperationsAndVisualize(...) does not respect timeout: %v, timeout was %v", elapsed, timeout) + } +} + +func BenchmarkValidateLinearizableOperations(b *testing.B) { + lg := zap.NewNop() + b.Run("SequentialSuccessPuts", func(b *testing.B) { + history := sequentialSuccessPuts(7000, 2) + shuffles := shuffleHistory(history, b.N) + b.ResetTimer() + validateShuffles(b, lg, shuffles, time.Second) + }) + b.Run("SequentialFailedPuts", func(b *testing.B) { + history := sequentialFailedPuts(14, 1) + shuffles := shuffleHistory(history, b.N) + b.ResetTimer() + validateShuffles(b, lg, shuffles, time.Second) + }) + b.Run("ConcurrentFailedPutsWithRead", func(b *testing.B) { + history := concurrentFailedPutsWithRead(b, 13) + shuffles := shuffleHistory(history, b.N) + b.ResetTimer() + validateShuffles(b, lg, shuffles, time.Second) + }) + b.Run("BacktrackingHeavy", func(b *testing.B) { + history := backtrackingHeavy(b) + shuffles := shuffleHistory(history, b.N) + keys := model.ModelKeys(history) + b.ResetTimer() + for i := 0; i < len(shuffles); i++ { + validateLinearizableOperationsAndVisualize(lg, keys, shuffles[i], time.Second) + } + }) +} + +func sequentialSuccessPuts(count int, startRevision int64) []porcupine.Operation { + ops := []porcupine.Operation{} + for i := 0; i < count; i++ { + ops = append(ops, porcupine.Operation{ + ClientId: i, + Input: putRequest("key", "value"), + Output: txnResponse(startRevision+int64(i), model.EtcdOperationResult{}), + Call: int64(i * 2), + Return: int64(i*2 + 1), + }) + } + return ops +} + +func concurrentFailedPutsWithRead(b *testing.B, concurrencyCount int) []porcupine.Operation { + ops := []porcupine.Operation{} + for i := 0; i < concurrencyCount; i++ { + ops = append(ops, porcupine.Operation{ + ClientId: i, + Input: putRequest("key", "value"), + Output: errorResponse(fmt.Errorf("timeout")), + Call: int64(i), + Return: int64(i) + int64(concurrencyCount), + }) + } + replay := model.NewReplayFromOperations(ops) + state, err := replay.StateForRevision(int64(concurrencyCount) + 1) + if err != nil { + b.Fatal(err) + } + request := rangeRequest("key", "kez", 0, 0) + _, resp := state.Step(request) + ops = append(ops, porcupine.Operation{ + ClientId: 0, + Input: request, + Output: resp, + Call: int64(concurrencyCount) + 1, + Return: int64(concurrencyCount) + 2, + }) + return ops +} + +func sequentialFailedPuts(count int, keyCount int) []porcupine.Operation { + ops := []porcupine.Operation{} + for i := 0; i < count; i++ { + key := "key0" + if keyCount > 1 { + key = fmt.Sprintf("key%d", i%keyCount) + } + ops = append(ops, porcupine.Operation{ + ClientId: i, + Input: putRequest(key, "value"), + Output: errorResponse(fmt.Errorf("timeout")), + Call: int64(i * 2), + Return: int64(i*2 + 1), + }) + } + return ops +} + +func backtrackingHeavy(b *testing.B) (ops []porcupine.Operation) { + for i := 0; i < 30; i++ { + ops = append(ops, porcupine.Operation{ + ClientId: -1, + Input: putRequest(fmt.Sprintf("key%d", i+1000), "value"), + Output: txnResponse(int64(i+2), model.EtcdOperationResult{}), + Call: int64(i), + Return: int64(i) + 1, + }) + } + startTime := int64(1000) + + failedPuts := 4 + for i := 0; i < failedPuts; i++ { + ops = append(ops, porcupine.Operation{ + ClientId: i, + Input: putRequest(fmt.Sprintf("key%d", i), "value"), + Output: errorResponse(fmt.Errorf("timeout")), + Call: startTime + int64(i), + Return: startTime + 1000 + int64(i), + }) + } + replay := model.NewReplayFromOperations(ops) + state, err := replay.StateForRevision(int64(30 + 1)) + if err != nil { + b.Fatal(err) + } + + concurrentReads := 3 + for i := 0; i < concurrentReads; i++ { + request := rangeRequest(fmt.Sprintf("key%d", i), "", 0, 0) + _, resp := state.Step(request) + ops = append(ops, porcupine.Operation{ + ClientId: failedPuts + i, + Input: request, + Output: resp, + Call: startTime + 1100, + Return: startTime + 2100, + }) + } + + ops = append(ops, porcupine.Operation{ + ClientId: 99, + Input: rangeRequest("key0", "", 0, 0), + Output: rangeResponse(0, keyValueRevision("key0", "wrong", 9999)), + Call: startTime + 3000, + Return: startTime + 4000, + }) + return ops +} + +func shuffleHistory(history []porcupine.Operation, shuffleCount int) [][]porcupine.Operation { + shuffles := make([][]porcupine.Operation, shuffleCount) + for i := 0; i < shuffleCount; i++ { + historyCopy := make([]porcupine.Operation, len(history)) + copy(historyCopy, history) + rand.Shuffle(len(historyCopy), func(i, j int) { + historyCopy[i], historyCopy[j] = historyCopy[j], historyCopy[i] + }) + shuffles[i] = historyCopy + } + return shuffles +} + +func validateShuffles(b *testing.B, lg *zap.Logger, shuffles [][]porcupine.Operation, duration time.Duration) { + keys := model.ModelKeys(shuffles[0]) + for i := 0; i < len(shuffles); i++ { + result := validateLinearizableOperationsAndVisualize(lg, keys, shuffles[i], duration) + if err := result.Error(); err != nil { + b.Fatalf("Not linearizable: %v", err) + } + } +} diff --git a/tests/robustness/validate/patch_history.go b/tests/robustness/validate/patch_history.go index 5ec3bd2ae2ed..98c8711d5af9 100644 --- a/tests/robustness/validate/patch_history.go +++ b/tests/robustness/validate/patch_history.go @@ -16,6 +16,7 @@ package validate import ( "fmt" + "math" "github.com/anishathalye/porcupine" @@ -23,32 +24,60 @@ import ( "go.etcd.io/etcd/tests/v3/robustness/report" ) -func patchLinearizableOperations(reports []report.ClientReport, persistedRequests []model.EtcdRequest) []porcupine.Operation { - allOperations := relevantOperations(reports) - putRevision := putRevision(reports) - putReturnTime := putReturnTime(allOperations, reports, persistedRequests) - clientPutCount := countClientPuts(reports) - persistedPutCount := countPersistedPuts(persistedRequests) - return patchOperations(allOperations, putRevision, putReturnTime, clientPutCount, persistedPutCount) +type patchArgs struct { + returnTime int64 + clientCount int64 + persistedCount int64 + revision int64 } -func relevantOperations(reports []report.ClientReport) []porcupine.Operation { - var ops []porcupine.Operation - for _, r := range reports { - for _, op := range r.KeyValue { - request := op.Input.(model.EtcdRequest) - resp := op.Output.(model.MaybeEtcdResponse) - // Remove failed read requests as they are not relevant for linearization. - if resp.Error == "" || !request.IsRead() { - ops = append(ops, op) - } +func patchLinearizableOperations(operations []porcupine.Operation, reports []report.ClientReport, persistedRequests []model.EtcdRequest) []porcupine.Operation { + putRevision := watchRevisions(reports) + persistedPutCount := countPersistedPuts(persistedRequests) + clientPutCount := countClientPuts(reports) + + persistedDeleteCount := countPersistedDeletes(persistedRequests) + clientDeleteCount := countClientDeletes(reports) + + persistedCompactCount := countPersistedCompacts(persistedRequests) + clientCompactCount := countClientCompacts(reports) + + putReturnTime, delReturnTime, compactReturnTime := uniqueOperationReturnTime(operations, persistedRequests, clientPutCount, clientDeleteCount, clientCompactCount) + + putArgs := make(map[model.PutOptions]patchArgs) + for opts, c := range clientPutCount { + putArgs[opts] = patchArgs{ + clientCount: c, + persistedCount: persistedPutCount[opts], + returnTime: putReturnTime[opts], + revision: putRevision[opts], + } + } + delArgs := make(map[model.DeleteOptions]patchArgs) + for opts, c := range clientDeleteCount { + delArgs[opts] = patchArgs{ + clientCount: c, + persistedCount: persistedDeleteCount[opts], + returnTime: delReturnTime[opts], + } + } + compactArgs := make(map[model.CompactRequest]patchArgs) + for opts, c := range clientCompactCount { + compactArgs[opts] = patchArgs{ + clientCount: c, + persistedCount: persistedCompactCount[opts], + returnTime: compactReturnTime[opts], } } - return ops + + return patchOperations( + operations, putArgs, delArgs, compactArgs, + ) } -func putRevision(reports []report.ClientReport) map[keyValue]int64 { - requestRevision := map[keyValue]int64{} +func watchRevisions(reports []report.ClientReport) map[model.PutOptions]int64 { + putRevisions := map[model.PutOptions]int64{} + for _, client := range reports { for _, watch := range client.Watch { for _, resp := range watch.Responses { @@ -56,9 +85,10 @@ func putRevision(reports []report.ClientReport) map[keyValue]int64 { switch event.Type { case model.RangeOperation: case model.PutOperation: - kv := keyValue{Key: event.Key, Value: event.Value} - requestRevision[kv] = event.Revision + kv := model.PutOptions{Key: event.Key, Value: event.Value} + putRevisions[kv] = event.Revision case model.DeleteOperation: + // Delete events are also created by leaseRevoke request. default: panic(fmt.Sprintf("unknown event type %q", event.Type)) } @@ -66,15 +96,36 @@ func putRevision(reports []report.ClientReport) map[keyValue]int64 { } } } - return requestRevision + return putRevisions } -func patchOperations(operations []porcupine.Operation, watchRevision, putReturnTime, clientPutCount, persistedPutCount map[keyValue]int64) []porcupine.Operation { +func patchOperations( + operations []porcupine.Operation, + putArgs map[model.PutOptions]patchArgs, + delArgs map[model.DeleteOptions]patchArgs, + compactArgs map[model.CompactRequest]patchArgs, +) []porcupine.Operation { newOperations := make([]porcupine.Operation, 0, len(operations)) for _, op := range operations { request := op.Input.(model.EtcdRequest) resp := op.Output.(model.MaybeEtcdResponse) + + if request.Type == model.Compact { + kv := model.CompactRequest{Revision: request.Compact.Revision} + if arg, ok := compactArgs[kv]; ok && arg.clientCount == 1 { + if arg.persistedCount == 0 && resp.Error != "" { + // the failed compact should be dropped + continue + } + if arg.returnTime > 0 { + op.Return = min(op.Return, arg.returnTime) + } + } + newOperations = append(newOperations, op) + continue + } + if resp.Error == "" || request.Type != model.Txn { // Cannot patch those requests. newOperations = append(newOperations, op) @@ -82,29 +133,50 @@ func patchOperations(operations []porcupine.Operation, watchRevision, putReturnT } var txnRevision int64 var persisted bool - for _, etcdOp := range append(request.Txn.OperationsOnSuccess, request.Txn.OperationsOnFailure...) { + for _, etcdOp := range request.Txn.AllOperations() { switch etcdOp.Type { case model.PutOperation: - kv := keyValue{Key: etcdOp.Put.Key, Value: etcdOp.Put.Value} - if _, ok := persistedPutCount[kv]; ok { + kv := model.PutOptions{Key: etcdOp.Put.Key, Value: etcdOp.Put.Value} + arg, ok := putArgs[kv] + if !ok { + continue + } + if arg.persistedCount > 0 { persisted = true } - if count := clientPutCount[kv]; count != 1 { + if arg.clientCount != 1 { continue } - if revision, ok := watchRevision[kv]; ok { - txnRevision = revision + if arg.revision > 0 { + txnRevision = arg.revision } - if returnTime, ok := putReturnTime[kv]; ok { - op.Return = min(op.Return, returnTime) + if arg.returnTime > 0 { + op.Return = min(op.Return, arg.returnTime) } case model.DeleteOperation: + kv := model.DeleteOptions{Key: etcdOp.Delete.Key} + arg, ok := delArgs[kv] + if !ok { + continue + } + if arg.persistedCount > 0 { + persisted = true + } + if arg.clientCount != 1 { + continue + } + if arg.revision > 0 { + txnRevision = arg.revision + } + if arg.returnTime > 0 { + op.Return = min(op.Return, arg.returnTime) + } case model.RangeOperation: default: panic(fmt.Sprintf("unknown operation type %q", etcdOp.Type)) } } - if isUniqueTxn(request.Txn, clientPutCount) { + if isUniqueTxn(request.Txn, putArgs, delArgs) { if !persisted { // Remove non persisted operations continue @@ -121,12 +193,12 @@ func patchOperations(operations []porcupine.Operation, watchRevision, putReturnT return newOperations } -func isUniqueTxn(request *model.TxnRequest, clientRequestCount map[keyValue]int64) bool { - return isUniqueOps(request.OperationsOnSuccess, clientRequestCount) && isUniqueOps(request.OperationsOnFailure, clientRequestCount) +func isUniqueTxn(request *model.TxnRequest, putArgs map[model.PutOptions]patchArgs, delArgs map[model.DeleteOptions]patchArgs) bool { + return isUniqueOps(request.OperationsOnSuccess, putArgs, delArgs) && isUniqueOps(request.OperationsOnFailure, putArgs, delArgs) } -func isUniqueOps(ops []model.EtcdOperation, clientRequestCount map[keyValue]int64) bool { - return hasUniqueWriteOperation(ops, clientRequestCount) || !hasWriteOperation(ops) +func isUniqueOps(ops []model.EtcdOperation, putArgs map[model.PutOptions]patchArgs, delArgs map[model.DeleteOptions]patchArgs) bool { + return hasUniqueWriteOperation(ops, putArgs, delArgs) || !hasWriteOperation(ops) } func hasWriteOperation(ops []model.EtcdOperation) bool { @@ -138,15 +210,19 @@ func hasWriteOperation(ops []model.EtcdOperation) bool { return false } -func hasUniqueWriteOperation(ops []model.EtcdOperation, clientRequestCount map[keyValue]int64) bool { +func hasUniqueWriteOperation(ops []model.EtcdOperation, putArgs map[model.PutOptions]patchArgs, delArgs map[model.DeleteOptions]patchArgs) bool { for _, operation := range ops { switch operation.Type { case model.PutOperation: - kv := keyValue{Key: operation.Put.Key, Value: operation.Put.Value} - if count := clientRequestCount[kv]; count == 1 { + kv := model.PutOptions{Key: operation.Put.Key, Value: operation.Put.Value} + if arg, ok := putArgs[kv]; ok && arg.clientCount == 1 { return true } case model.DeleteOperation: + kv := model.DeleteOptions{Key: operation.Delete.Key} + if arg, ok := delArgs[kv]; ok && arg.clientCount == 1 { + return true + } case model.RangeOperation: default: panic(fmt.Sprintf("unknown operation type %q", operation.Type)) @@ -155,27 +231,56 @@ func hasUniqueWriteOperation(ops []model.EtcdOperation, clientRequestCount map[k return false } -func putReturnTime(allOperations []porcupine.Operation, reports []report.ClientReport, persistedRequests []model.EtcdRequest) map[keyValue]int64 { - earliestReturnTime := map[keyValue]int64{} +func uniqueOperationReturnTime( + allOperations []porcupine.Operation, + persistedRequests []model.EtcdRequest, + clientPutCount map[model.PutOptions]int64, + clientDeleteCount map[model.DeleteOptions]int64, + clientCompactCount map[model.CompactRequest]int64, +) ( + map[model.PutOptions]int64, + map[model.DeleteOptions]int64, + map[model.CompactRequest]int64, +) { + putTimes := map[model.PutOptions]int64{} + delTimes := map[model.DeleteOptions]int64{} + compactTimes := map[model.CompactRequest]int64{} var lastReturnTime int64 for _, op := range allOperations { request := op.Input.(model.EtcdRequest) switch request.Type { case model.Txn: - for _, etcdOp := range append(request.Txn.OperationsOnSuccess, request.Txn.OperationsOnFailure...) { - if etcdOp.Type != model.PutOperation { - continue - } - kv := keyValue{Key: etcdOp.Put.Key, Value: etcdOp.Put.Value} - if returnTime, ok := earliestReturnTime[kv]; !ok || returnTime > op.Return { - earliestReturnTime[kv] = op.Return + for _, etcdOp := range request.Txn.AllOperations() { + switch etcdOp.Type { + case model.PutOperation: + kv := model.PutOptions{Key: etcdOp.Put.Key, Value: etcdOp.Put.Value} + if clientPutCount[kv] > 1 { + continue + } + if returnTime, ok := putTimes[kv]; !ok || returnTime > op.Return { + putTimes[kv] = op.Return + } + case model.DeleteOperation: + kv := model.DeleteOptions{Key: etcdOp.Delete.Key} + if clientDeleteCount[kv] > 1 { + continue + } + if returnTime, ok := delTimes[kv]; !ok || returnTime > op.Return { + delTimes[kv] = op.Return + } } - earliestReturnTime[kv] = op.Return } case model.Range: case model.LeaseGrant: case model.LeaseRevoke: case model.Compact: + if clientCompactCount[*request.Compact] > 1 { + continue + } + if returnTime, ok := compactTimes[*request.Compact]; !ok || returnTime > op.Return { + compactTimes[*request.Compact] = op.Return + } + case model.Defragment: default: panic(fmt.Sprintf("Unknown request type: %q", request.Type)) } @@ -184,54 +289,57 @@ func putReturnTime(allOperations []porcupine.Operation, reports []report.ClientR } } - for _, client := range reports { - for _, watch := range client.Watch { - for _, resp := range watch.Responses { - for _, event := range resp.Events { - switch event.Type { - case model.RangeOperation: - case model.PutOperation: - kv := keyValue{Key: event.Key, Value: event.Value} - if t, ok := earliestReturnTime[kv]; !ok || t > resp.Time.Nanoseconds() { - earliestReturnTime[kv] = resp.Time.Nanoseconds() - } - case model.DeleteOperation: - default: - panic(fmt.Sprintf("unknown event type %q", event.Type)) - } - } - } - } - } - for i := len(persistedRequests) - 1; i >= 0; i-- { request := persistedRequests[i] switch request.Type { case model.Txn: - lastReturnTime-- - for _, op := range request.Txn.OperationsOnSuccess { - if op.Type != model.PutOperation { - continue - } - kv := keyValue{Key: op.Put.Key, Value: op.Put.Value} - returnTime, ok := earliestReturnTime[kv] - if ok { - lastReturnTime = min(returnTime, lastReturnTime) - earliestReturnTime[kv] = lastReturnTime + if lastReturnTime != math.MaxInt64 { + lastReturnTime-- + } + for _, op := range request.Txn.AllOperations() { + switch op.Type { + case model.PutOperation: + kv := model.PutOptions{Key: op.Put.Key, Value: op.Put.Value} + if clientPutCount[kv] > 1 { + continue + } + if returnTime, ok := putTimes[kv]; ok { + lastReturnTime = min(returnTime, lastReturnTime) + putTimes[kv] = lastReturnTime + } + case model.DeleteOperation: + kv := model.DeleteOptions{Key: op.Delete.Key} + if clientDeleteCount[kv] > 1 { + continue + } + if returnTime, ok := delTimes[kv]; ok { + lastReturnTime = min(returnTime, lastReturnTime) + delTimes[kv] = lastReturnTime + } } } case model.LeaseGrant: case model.LeaseRevoke: case model.Compact: + if lastReturnTime != math.MaxInt64 { + lastReturnTime-- + } + if clientCompactCount[*request.Compact] > 1 { + continue + } + if returnTime, ok := compactTimes[*request.Compact]; ok { + lastReturnTime = min(returnTime, lastReturnTime) + compactTimes[*request.Compact] = lastReturnTime + } default: panic(fmt.Sprintf("Unknown request type: %q", request.Type)) } } - return earliestReturnTime + return putTimes, delTimes, compactTimes } -func countClientPuts(reports []report.ClientReport) map[keyValue]int64 { - counter := map[keyValue]int64{} +func countClientPuts(reports []report.ClientReport) map[model.PutOptions]int64 { + counter := map[model.PutOptions]int64{} for _, client := range reports { for _, op := range client.KeyValue { request := op.Input.(model.EtcdRequest) @@ -241,21 +349,21 @@ func countClientPuts(reports []report.ClientReport) map[keyValue]int64 { return counter } -func countPersistedPuts(requests []model.EtcdRequest) map[keyValue]int64 { - counter := map[keyValue]int64{} +func countPersistedPuts(requests []model.EtcdRequest) map[model.PutOptions]int64 { + counter := map[model.PutOptions]int64{} for _, request := range requests { countPuts(counter, request) } return counter } -func countPuts(counter map[keyValue]int64, request model.EtcdRequest) { +func countPuts(counter map[model.PutOptions]int64, request model.EtcdRequest) { switch request.Type { case model.Txn: - for _, operation := range append(request.Txn.OperationsOnSuccess, request.Txn.OperationsOnFailure...) { + for _, operation := range request.Txn.AllOperations() { switch operation.Type { case model.PutOperation: - kv := keyValue{Key: operation.Put.Key, Value: operation.Put.Value} + kv := model.PutOptions{Key: operation.Put.Key, Value: operation.Put.Value} counter[kv]++ case model.DeleteOperation: case model.RangeOperation: @@ -273,7 +381,57 @@ func countPuts(counter map[keyValue]int64, request model.EtcdRequest) { } } -type keyValue struct { - Key string - Value model.ValueOrHash +func countClientDeletes(reports []report.ClientReport) map[model.DeleteOptions]int64 { + counter := map[model.DeleteOptions]int64{} + for _, client := range reports { + for _, op := range client.KeyValue { + request := op.Input.(model.EtcdRequest) + countDeletes(counter, request) + } + } + return counter +} + +func countPersistedDeletes(requests []model.EtcdRequest) map[model.DeleteOptions]int64 { + counter := map[model.DeleteOptions]int64{} + for _, req := range requests { + countDeletes(counter, req) + } + return counter +} + +func countDeletes(counter map[model.DeleteOptions]int64, request model.EtcdRequest) { + if request.Type != model.Txn { + return + } + for _, operation := range request.Txn.AllOperations() { + if operation.Type == model.DeleteOperation { + counter[operation.Delete]++ + } + } +} + +func countClientCompacts(reports []report.ClientReport) map[model.CompactRequest]int64 { + counter := map[model.CompactRequest]int64{} + for _, client := range reports { + for _, op := range client.KeyValue { + request := op.Input.(model.EtcdRequest) + countCompacts(counter, request) + } + } + return counter +} + +func countPersistedCompacts(requests []model.EtcdRequest) map[model.CompactRequest]int64 { + counter := map[model.CompactRequest]int64{} + for _, req := range requests { + countCompacts(counter, req) + } + return counter +} + +func countCompacts(counter map[model.CompactRequest]int64, request model.EtcdRequest) { + if request.Type == model.Compact { + counter[*request.Compact]++ + } } diff --git a/tests/robustness/validate/patch_history_test.go b/tests/robustness/validate/patch_history_test.go index bb104b0125fe..5e9b0dc84f1a 100644 --- a/tests/robustness/validate/patch_history_test.go +++ b/tests/robustness/validate/patch_history_test.go @@ -17,6 +17,7 @@ package validate import ( "errors" + "math" "testing" "time" @@ -30,7 +31,7 @@ import ( "go.etcd.io/etcd/tests/v3/robustness/report" ) -const infinite = 1000000000 +const infinite = math.MaxInt64 func TestPatchHistory(t *testing.T) { for _, tc := range []struct { @@ -43,7 +44,7 @@ func TestPatchHistory(t *testing.T) { { name: "successful range remains", historyFunc: func(h *model.AppendableHistory) { - h.AppendRange("key", "", 0, 0, 100, 200, &clientv3.GetResponse{}, nil) + h.AppendRange("key", "", 0, 0, false, 100, 200, &clientv3.GetResponse{}, nil) }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: rangeResponse(0)}, @@ -58,25 +59,25 @@ func TestPatchHistory(t *testing.T) { putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: 200, Output: putResponse(model.EtcdOperationResult{})}, + {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put remains if there is a matching event, return time untouched", historyFunc: func(h *model.AppendableHistory) { - h.AppendPut("key", "value", 100, infinite, nil, errors.New("failed")) + h.AppendPut("key", "value", 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 99, Output: model.MaybeEtcdResponse{Persisted: true}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed put remains if there is a matching event, uniqueness allows for return time to be based on next persisted request", historyFunc: func(h *model.AppendableHistory) { - h.AppendPut("key1", "value", 100, infinite, nil, errors.New("failed")) + h.AppendPut("key1", "value", 100, 200, nil, errors.New("failed")) h.AppendPut("key2", "value", 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ @@ -85,20 +86,20 @@ func TestPatchHistory(t *testing.T) { }, expectedRemainingOperations: []porcupine.Operation{ {Return: 399, Output: model.MaybeEtcdResponse{Persisted: true}}, - {Return: 400, Output: putResponse(model.EtcdOperationResult{})}, + {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { - name: "failed put remains if there is a matching persisted request, uniqueness allows for revision and return time to be based on watch", + name: "failed put remains if there is a matching persisted request, uniqueness allows for revision to be based on watch", historyFunc: func(h *model.AppendableHistory) { - h.AppendPut("key", "value", 100, infinite, nil, errors.New("failed")) + h.AppendPut("key", "value", 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, watchOperations: watchResponse(300, putEvent("key", "value", 2)), expectedRemainingOperations: []porcupine.Operation{ - {Return: 300, Output: model.MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}}, }, }, { @@ -113,52 +114,52 @@ func TestPatchHistory(t *testing.T) { }, watchOperations: watchResponse(3, putEvent("key", "value", 2), putEvent("key", "value", 3)), expectedRemainingOperations: []porcupine.Operation{ - {Return: 1000000004, Output: model.MaybeEtcdResponse{Error: "failed"}}, - {Return: 4, Output: putResponse(model.EtcdOperationResult{})}, + {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: 4, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put is dropped if event has different key", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key2", "value", 100, 200, &clientv3.PutResponse{}, nil) - h.AppendPut("key1", "value", 300, infinite, nil, errors.New("failed")) + h.AppendPut("key1", "value", 300, 400, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key2", "value"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: 200, Output: putResponse(model.EtcdOperationResult{})}, + {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put is dropped if event has different value", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key", "value2", 100, 200, &clientv3.PutResponse{}, nil) - h.AppendPut("key", "value1", 300, infinite, nil, errors.New("failed")) + h.AppendPut("key", "value1", 300, 400, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value2"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: 200, Output: putResponse(model.EtcdOperationResult{})}, + {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put with lease remains if there is a matching event, return time untouched", historyFunc: func(h *model.AppendableHistory) { - h.AppendPutWithLease("key", "value", 123, 100, infinite, nil, errors.New("failed")) + h.AppendPutWithLease("key", "value", 123, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequestWithLease("key", "value", 123), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 99, Output: model.MaybeEtcdResponse{Persisted: true}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed put with lease remains if there is a matching event, uniqueness allows return time to be based on next persisted request", historyFunc: func(h *model.AppendableHistory) { - h.AppendPutWithLease("key1", "value", 123, 100, infinite, nil, errors.New("failed")) + h.AppendPutWithLease("key1", "value", 123, 100, 200, nil, errors.New("failed")) h.AppendPutWithLease("key2", "value", 234, 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ @@ -167,11 +168,11 @@ func TestPatchHistory(t *testing.T) { }, expectedRemainingOperations: []porcupine.Operation{ {Return: 399, Output: model.MaybeEtcdResponse{Persisted: true}}, - {Return: 400, Output: putResponse(model.EtcdOperationResult{})}, + {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { - name: "failed put with lease remains if there is a matching event, uniqueness allows for revision and return time to be based on watch", + name: "failed put with lease remains if there is a matching event, uniqueness allows for revision to be based on watch", historyFunc: func(h *model.AppendableHistory) { h.AppendPutWithLease("key", "value", 123, 1, 2, nil, errors.New("failed")) }, @@ -180,7 +181,7 @@ func TestPatchHistory(t *testing.T) { }, watchOperations: watchResponse(3, putEvent("key", "value", 2)), expectedRemainingOperations: []porcupine.Operation{ - {Return: 3, Output: model.MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}}, }, }, { @@ -195,21 +196,21 @@ func TestPatchHistory(t *testing.T) { }, watchOperations: watchResponse(3, putEvent("key", "value", 2), putEvent("key", "value", 3)), expectedRemainingOperations: []porcupine.Operation{ - {Return: 1000000004, Output: model.MaybeEtcdResponse{Error: "failed"}}, - {Return: 4, Output: putResponse(model.EtcdOperationResult{})}, + {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: 4, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendPut("key", "value", 100, infinite, nil, errors.New("failed")) + h.AppendPut("key", "value", 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed put with lease is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendPutWithLease("key", "value", 123, 100, infinite, nil, errors.New("failed")) + h.AppendPutWithLease("key", "value", 123, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, @@ -219,57 +220,110 @@ func TestPatchHistory(t *testing.T) { h.AppendDelete("key", 100, 200, &clientv3.DeleteResponse{}, nil) }, expectedRemainingOperations: []porcupine.Operation{ - {Return: 200, Output: putResponse(model.EtcdOperationResult{})}, + {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { - name: "failed delete remains, time untouched regardless of persisted event and watch", + name: "failed delete with lease is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendDelete("key", 100, infinite, nil, errors.New("failed")) + h.AppendDelete("key", 100, 200, nil, errors.New("failed")) + }, + expectedRemainingOperations: []porcupine.Operation{}, + }, + { + name: "failed delete remains, if there is a persisted request", + historyFunc: func(h *model.AppendableHistory) { + h.AppendDelete("key", 100, 200, nil, errors.New("failed")) + }, + persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, + }, + }, + { + name: "failed delete remains, if there is a persisted request, revision is not patched based on watch due to leaseRevoke also triggering delete watch events", + historyFunc: func(h *model.AppendableHistory) { + h.AppendDelete("key", 100, 200, nil, errors.New("failed")) + }, + persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), + }, + watchOperations: watchResponse(3, deleteEvent("key", 2)), + expectedRemainingOperations: []porcupine.Operation{ + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, + }, + }, + { + name: "failed delete remains, if there is a matching persisted request, uniqueness of this operation and following operation allows patching based on following operation", + historyFunc: func(h *model.AppendableHistory) { + h.AppendDelete("key", 100, 200, nil, errors.New("failed")) h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), putRequest("key", "value"), }, - watchOperations: watchResponse(3, deleteEvent("key", 2)), expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 400, Output: model.MaybeEtcdResponse{Error: "failed"}}, - {Return: 400, Output: putResponse(model.EtcdOperationResult{})}, + {Return: 399, Output: model.MaybeEtcdResponse{Persisted: true}}, + {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, + }, + }, + { + name: "failed delete remains if there is a matching persisted request, lack of uniqueness of this operation prevents patching based on following operation", + historyFunc: func(h *model.AppendableHistory) { + h.AppendDelete("key", 100, 200, nil, errors.New("failed")) + h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) + h.AppendDelete("key", 500, 600, &clientv3.DeleteResponse{}, nil) + }, + persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), + putRequest("key", "value"), + deleteRequest("key"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, + {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed empty txn is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn put is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn put remains if there is a matching event", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 99, Output: model.MaybeEtcdResponse{Persisted: true}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed txn delete remains", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) + }, + persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 100, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { @@ -277,91 +331,235 @@ func TestPatchHistory(t *testing.T) { historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{clientv3.OpDelete("key")}, 100, 200, &clientv3.TxnResponse{Succeeded: true}, nil) }, + persistedRequest: []model.EtcdRequest{ + putRequest("key", "value"), + }, expectedRemainingOperations: []porcupine.Operation{ - {Return: 200, Output: putResponse()}, + // It's successful, so it remains as-is with its original response + {Return: 200, Output: txnResponse(0)}, }, }, { - name: "failed txn put/delete remains", + name: "failed txn empty/delete is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{clientv3.OpDelete("key")}, 100, infinite, nil, errors.New("failed")) - }, - expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 100, Output: model.MaybeEtcdResponse{Error: "failed"}}, + h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) }, + expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn delete/put remains", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{clientv3.OpPut("key", "value")}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{clientv3.OpPut("key", "value")}, 100, 200, nil, errors.New("failed")) + }, + persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 100, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed txn empty/put is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpPut("key", "value")}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpPut("key", "value")}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn empty/put remains if there is a matching event", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 99, Output: model.MaybeEtcdResponse{Persisted: true}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, + }, + }, + { + name: "failed put remains if there is a matching persisted request, uniqueness of this operation and following operation allows patching based on following operation", + historyFunc: func(h *model.AppendableHistory) { + h.AppendPut("key1", "value1", 300, 400, nil, errors.New("failed")) + h.AppendPut("key2", "value2", 500, 600, &clientv3.PutResponse{}, nil) + }, + persistedRequest: []model.EtcdRequest{ + putRequest("key1", "value1"), + putRequest("key2", "value2"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: 599, Output: model.MaybeEtcdResponse{Persisted: true}}, + {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, + }, + }, + { + name: "failed put remains if there is a matching persisted request, lack of uniqueness of this operation prevents patching based on following operation", + historyFunc: func(h *model.AppendableHistory) { + h.AppendPut("key1", "value1", 100, 200, &clientv3.PutResponse{}, nil) + h.AppendPut("key1", "value1", 300, 400, nil, errors.New("failed")) + h.AppendPut("key2", "value2", 500, 600, &clientv3.PutResponse{}, nil) + }, + persistedRequest: []model.EtcdRequest{ + putRequest("key1", "value1"), + putRequest("key1", "value1"), + putRequest("key2", "value2"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, + {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, + }, + }, + { + name: "failed put remains if there is a matching persisted request, lack of uniqueness of following operation prevents patching based on following operation", + historyFunc: func(h *model.AppendableHistory) { + h.AppendPut("key2", "value2", 100, 200, &clientv3.PutResponse{}, nil) + h.AppendPut("key1", "value1", 300, 400, nil, errors.New("failed")) + h.AppendPut("key2", "value2", 500, 600, &clientv3.PutResponse{}, nil) + }, + persistedRequest: []model.EtcdRequest{ + putRequest("key2", "value2"), + putRequest("key1", "value1"), + putRequest("key2", "value2"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, + // TODO: We can infer that failed operation finished before last operation matching following. + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, + {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed txn empty/delete remains", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpDelete("key")}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) + }, + persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ - {Return: infinite + 100, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed txn put&delete is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, []clientv3.Op{}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn empty/put&delete is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn put&delete/put&delete is dropped", historyFunc: func(h *model.AppendableHistory) { - h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, []clientv3.Op{clientv3.OpPut("key", "value2"), clientv3.OpDelete("key")}, 100, infinite, nil, errors.New("failed")) + h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, []clientv3.Op{clientv3.OpPut("key", "value2"), clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) + }, + expectedRemainingOperations: []porcupine.Operation{}, + }, + { + name: "failed delete remains, time untouched due to non-uniqueness", + historyFunc: func(h *model.AppendableHistory) { + h.AppendDelete("key", 100, 200, nil, errors.New("failed")) + h.AppendDelete("key", 300, 400, &clientv3.DeleteResponse{}, nil) + h.AppendPut("key", "value", 500, 600, &clientv3.PutResponse{}, nil) + }, + persistedRequest: []model.EtcdRequest{ + deleteRequest("key"), + deleteRequest("key"), + putRequest("key", "value"), + }, + watchOperations: watchResponse(250, deleteEvent("key", 2)), + expectedRemainingOperations: []porcupine.Operation{ + {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, + {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, + }, + }, + { + name: "failed txn delete is dropped when not persisted", + historyFunc: func(h *model.AppendableHistory) { + h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) + }, + expectedRemainingOperations: []porcupine.Operation{}, + }, + { + name: "successful compact with unique revision keeps original return time", + historyFunc: func(h *model.AppendableHistory) { + h.AppendCompact(5, 100, 200, &clientv3.CompactResponse{}, nil) + h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) + }, + persistedRequest: []model.EtcdRequest{ + compactRequest(5), + putRequest("key", "value"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: 200, Output: model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: -1, Compact: &model.CompactResponse{}}}}, + {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, + }, + }, + { + name: "failed compact with unique revision is patched with return time", + historyFunc: func(h *model.AppendableHistory) { + h.AppendCompact(5, 100, 200, nil, errors.New("failed")) + h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) + }, + persistedRequest: []model.EtcdRequest{ + compactRequest(5), + putRequest("key", "value"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: 399, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, + }, + }, + { + name: "failed compact with non-unique revision remains unchanged", + historyFunc: func(h *model.AppendableHistory) { + h.AppendCompact(5, 100, 200, nil, errors.New("failed")) + h.AppendCompact(5, 300, 400, &clientv3.CompactResponse{}, nil) + h.AppendPut("key", "value", 500, 600, &clientv3.PutResponse{}, nil) }, + persistedRequest: []model.EtcdRequest{ + compactRequest(5), + compactRequest(5), + putRequest("key", "value"), + }, + expectedRemainingOperations: []porcupine.Operation{ + {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, + {Return: 400, Output: model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: -1, Compact: &model.CompactResponse{}}}}, + {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, + }, + }, + { + name: "failed compact with unique revision is dropped when not persisted", + historyFunc: func(h *model.AppendableHistory) { + h.AppendCompact(5, 100, 200, nil, errors.New("failed")) + }, + persistedRequest: []model.EtcdRequest{}, expectedRemainingOperations: []porcupine.Operation{}, }, } { t.Run(tc.name, func(t *testing.T) { history := model.NewAppendableHistory(identity.NewIDProvider()) tc.historyFunc(history) - operations := patchLinearizableOperations([]report.ClientReport{ + reports := []report.ClientReport{ { ClientID: 0, KeyValue: history.History.Operations(), Watch: tc.watchOperations, }, - }, tc.persistedRequest) - if diff := cmp.Diff(tc.expectedRemainingOperations, operations, + } + operations, _, _ := prepareAndCategorizeOperations(reports) + patched := patchLinearizableOperations(operations, reports, tc.persistedRequest) + if diff := cmp.Diff(tc.expectedRemainingOperations, patched, cmpopts.EquateEmpty(), - cmpopts.IgnoreFields(porcupine.Operation{}, "Input", "Call", "ClientId"), + cmpopts.IgnoreFields(porcupine.Operation{}, "Input", "Call", "ClientId", "Metadata"), ); diff != "" { t.Errorf("Response didn't match expected, diff:\n%s", diff) } @@ -369,8 +567,8 @@ func TestPatchHistory(t *testing.T) { } } -func putResponse(result ...model.EtcdOperationResult) model.MaybeEtcdResponse { - return model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Txn: &model.TxnResponse{Results: result}}} +func txnResponse(rev int64, result ...model.EtcdOperationResult) model.MaybeEtcdResponse { + return model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: rev, Txn: &model.TxnResponse{Results: result}}} } func watchResponse(responseTime int64, events ...model.WatchEvent) []model.WatchOperation { @@ -410,3 +608,12 @@ func deleteEvent(key string, revision int64) model.WatchEvent { }, } } + +func compactRequest(revision int64) model.EtcdRequest { + return model.EtcdRequest{ + Type: model.Compact, + Compact: &model.CompactRequest{ + Revision: revision, + }, + } +} diff --git a/tests/robustness/validate/result.go b/tests/robustness/validate/result.go new file mode 100644 index 000000000000..aac8d36a9bfe --- /dev/null +++ b/tests/robustness/validate/result.go @@ -0,0 +1,125 @@ +// Copyright 2025 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "errors" + "fmt" + + "github.com/anishathalye/porcupine" + "go.uber.org/zap" +) + +type RobustnessResult struct { + Assumptions Result + Linearization LinearizationResult + Watch Result + Serializable Result +} + +type Result struct { + Status ResultStatus + Message string +} + +type ResultStatus string + +var ( + Unknown ResultStatus + Success ResultStatus = "Success" + Failure ResultStatus = "Failure" + Timeout ResultStatus = "Timeout" + // DeadlineExceeded is a workaround for Porcupine not always enforcing its timeout. It does not support visualization. + DeadlineExceeded ResultStatus = "DeadlineExceeded" +) + +func (r RobustnessResult) Error() error { + if err := r.Assumptions.Error(); err != nil { + return fmt.Errorf("assumptions: %w", err) + } + if err := r.Linearization.Error(); err != nil { + return fmt.Errorf("linearization: %w", err) + } + if err := r.Watch.Error(); err != nil { + return fmt.Errorf("watch: %w", err) + } + if err := r.Serializable.Error(); err != nil { + return fmt.Errorf("serializable: %w", err) + } + return nil +} + +func ResultFromError(err error) Result { + if err != nil { + return Result{ + Status: Failure, + Message: err.Error(), + } + } + return Result{ + Status: Success, + } +} + +func (r Result) Error() error { + switch r.Status { + case Success, Unknown: + return nil + default: + return errors.New(r.String()) + } +} + +func (r Result) String() string { + if r.Message != "" { + return fmt.Sprintf("%s: %s", r.Status, r.Message) + } + return string(r.Status) +} + +type LinearizationResult struct { + Info porcupine.LinearizationInfo + Model porcupine.Model + Result +} + +func (r *LinearizationResult) Visualize(lg *zap.Logger, path string) error { + err := r.Error() + if err != nil { + lg.Info("Skipping linearization visualization", zap.Error(err)) + return nil + } + + lg.Info("Saving visualization", zap.String("path", path)) + err = porcupine.VisualizePath(r.Model, r.Info, path) + if err != nil { + return fmt.Errorf("failed to visualize, err: %w", err) + } + return nil +} + +func (r *LinearizationResult) AddToVisualization(serializable []porcupine.Operation) { + annotations := []porcupine.Annotation{} + for _, op := range serializable { + annotations = append(annotations, porcupine.Annotation{ + ClientId: op.ClientId, + Start: op.Call, + End: op.Return, + Description: r.Model.DescribeOperation(op.Input, op.Output), + Details: r.Model.DescribeOperationMetadata(op.Metadata), + }) + } + r.Info.AddAnnotations(annotations) +} diff --git a/tests/robustness/validate/validate.go b/tests/robustness/validate/validate.go index 5918ec0df83c..0a8b370c2abb 100644 --- a/tests/robustness/validate/validate.go +++ b/tests/robustness/validate/validate.go @@ -15,59 +15,108 @@ package validate import ( - "encoding/json" + "errors" "fmt" - "testing" + "math" "time" "github.com/anishathalye/porcupine" - "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/report" ) -// ValidateAndReturnVisualize returns visualize as porcupine.linearizationInfo used to generate visualization is private. -func ValidateAndReturnVisualize(t *testing.T, lg *zap.Logger, cfg Config, reports []report.ClientReport, persistedRequests []model.EtcdRequest, timeout time.Duration) (visualize func(basepath string) error) { - err := checkValidationAssumptions(reports, persistedRequests) - require.NoErrorf(t, err, "Broken validation assumptions") - linearizableOperations := patchLinearizableOperations(reports, persistedRequests) - serializableOperations := filterSerializableOperations(reports) +var ErrNotEmptyDatabase = errors.New("non empty database at start, required by model used for linearizability validation") - linearizable, visualize := validateLinearizableOperationsAndVisualize(lg, linearizableOperations, timeout) - if linearizable != porcupine.Ok { - t.Error("Failed linearization, skipping further validation") - return visualize +func ValidateAndReturnVisualize(lg *zap.Logger, cfg Config, reports []report.ClientReport, persistedRequests []model.EtcdRequest, timeout time.Duration) (result RobustnessResult) { + result.Assumptions = ResultFromError(checkValidationAssumptions(reports)) + if result.Assumptions.Error() != nil { + return result } - // TODO: Use requests from linearization for replay. - replay := model.NewReplay(persistedRequests) - - err = validateWatch(lg, cfg, reports, replay) - if err != nil { - t.Errorf("Failed validating watch history, err: %s", err) + linearizableOperations, serializableOperations, operationsForVisualization := prepareAndCategorizeOperations(reports) + // We are passing in the original reports and linearizableOperations with modified return time. + // The reason is that linearizableOperations are those dedicated for linearization, which requires them to have returnTime set to infinity as required by pourcupine. + // As for the report, the original report is used so the consumer doesn't need to track what patching was done or not. + if len(persistedRequests) != 0 { + linearizableOperations = patchLinearizableOperations(linearizableOperations, reports, persistedRequests) } - err = validateSerializableOperations(lg, serializableOperations, replay) - if err != nil { - t.Errorf("Failed validating serializable operations, err: %s", err) + keys := model.ModelKeys(linearizableOperations) + result.Linearization = validateLinearizableOperationsAndVisualize(lg, keys, linearizableOperations, timeout) + result.Linearization.AddToVisualization(operationsForVisualization) + // Skip other validations if model is not linearizable, as they are expected to fail too and obfuscate the logs. + if result.Linearization.Error() != nil { + lg.Info("Skipping other validations as linearization failed") + return result } - return visualize + if len(persistedRequests) == 0 { + lg.Info("Skipping other validations as persisted requests were empty") + return result + } + replay := model.NewReplay(persistedRequests) + result.Watch = validateWatch(lg, cfg, reports, replay) + result.Serializable = validateSerializableOperations(lg, serializableOperations, replay) + return result } type Config struct { ExpectRevisionUnique bool } -func checkValidationAssumptions(reports []report.ClientReport, persistedRequests []model.EtcdRequest) error { - err := validateEmptyDatabaseAtStart(reports) - if err != nil { - return err +func prepareAndCategorizeOperations(reports []report.ClientReport) (linearizable, serializable, forVisualization []porcupine.Operation) { + for _, report := range reports { + for _, op := range report.KeyValue { + request := op.Input.(model.EtcdRequest) + response := op.Output.(model.MaybeEtcdResponse) + if isSerializable(request, response) { + serializable = append(serializable, op) + } + // Operations that will not be linearized need to be added separately to the visualization. + if !isLinearizable(request, response) { + forVisualization = append(forVisualization, op) + continue + } + // For linearization, we set the return time of failed requests to MaxInt64. + // Failed requests can still be persisted, however we don't know when the request has taken effect. + if response.Error != "" { + op.Return = math.MaxInt64 + } + linearizable = append(linearizable, op) + } } + return linearizable, serializable, forVisualization +} + +func isLinearizable(request model.EtcdRequest, response model.MaybeEtcdResponse) bool { + // Cannot test response for request without side effect. + if request.IsRead() && response.Error != "" { + return false + } + // Defragment is not linearizable + if request.Type == model.Defragment { + return false + } + return true +} + +func isSerializable(request model.EtcdRequest, response model.MaybeEtcdResponse) bool { + // Cannot test response for request without side effect. + if request.IsRead() && response.Error != "" { + return false + } + // Test range requests about stale revision + if request.Type == model.Range && request.Range.Revision != 0 { + return true + } + return false +} - err = validatePersistedRequestMatchClientRequests(reports, persistedRequests) +func checkValidationAssumptions(reports []report.ClientReport) error { + err := validateEmptyDatabaseAtStart(reports) if err != nil { return err } + err = validateNonConcurrentClientRequests(reports) if err != nil { return err @@ -76,84 +125,19 @@ func checkValidationAssumptions(reports []report.ClientReport, persistedRequests } func validateEmptyDatabaseAtStart(reports []report.ClientReport) error { - for _, r := range reports { - for _, op := range r.KeyValue { - request := op.Input.(model.EtcdRequest) - response := op.Output.(model.MaybeEtcdResponse) - if response.Revision == 2 && !request.IsRead() { - return nil - } - } - } - return fmt.Errorf("non empty database at start or first write didn't succeed, required by model implementation") -} - -func validatePersistedRequestMatchClientRequests(reports []report.ClientReport, persistedRequests []model.EtcdRequest) error { - persistedRequestSet := map[string]model.EtcdRequest{} - for _, request := range persistedRequests { - data, err := json.Marshal(request) - if err != nil { - return err - } - persistedRequestSet[string(data)] = request + if len(reports) == 0 { + return nil } - clientRequests := map[string]porcupine.Operation{} - for _, r := range reports { - for _, op := range r.KeyValue { - request := op.Input.(model.EtcdRequest) - data, err := json.Marshal(request) - if err != nil { - return err - } - clientRequests[string(data)] = op - } - } - - for requestDump, request := range persistedRequestSet { - _, found := clientRequests[requestDump] - // We cannot validate if persisted leaseGrant was sent by client as failed leaseGrant will not return LeaseID to clients. - if request.Type == model.LeaseGrant { - continue - } - - if !found { - return fmt.Errorf("request %+v was not sent by client, required to validate", requestDump) - } - } - - var firstOp, lastOp porcupine.Operation for _, r := range reports { for _, op := range r.KeyValue { request := op.Input.(model.EtcdRequest) response := op.Output.(model.MaybeEtcdResponse) - if response.Error != "" || request.IsRead() { - continue - } - if firstOp.Call == 0 || op.Call < firstOp.Call { - firstOp = op - } - if lastOp.Call == 0 || op.Call > lastOp.Call { - lastOp = op + if response.Revision == 1 && request.IsRead() { + return nil } } } - firstOpData, err := json.Marshal(firstOp.Input.(model.EtcdRequest)) - if err != nil { - return err - } - _, found := persistedRequestSet[string(firstOpData)] - if !found { - return fmt.Errorf("first succesful client write %s was not persisted, required to validate", firstOpData) - } - lastOpData, err := json.Marshal(lastOp.Input.(model.EtcdRequest)) - if err != nil { - return err - } - _, found = persistedRequestSet[string(lastOpData)] - if !found { - return fmt.Errorf("last succesful client write %s was not persisted, required to validate", lastOpData) - } - return nil + return ErrNotEmptyDatabase } func validateNonConcurrentClientRequests(reports []report.ClientReport) error { diff --git a/tests/robustness/validate/validate_test.go b/tests/robustness/validate/validate_test.go index e847af672a90..2842eb997363 100644 --- a/tests/robustness/validate/validate_test.go +++ b/tests/robustness/validate/validate_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/anishathalye/porcupine" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" @@ -34,7 +35,7 @@ func TestDataReports(t *testing.T) { files, err := os.ReadDir(testdataPath) require.NoError(t, err) for _, file := range files { - if file.Name() == ".gitignore" { + if !file.IsDir() { continue } t.Run(file.Name(), func(t *testing.T) { @@ -44,15 +45,168 @@ func TestDataReports(t *testing.T) { require.NoError(t, err) persistedRequests, err := report.LoadClusterPersistedRequests(lg, path) - require.NoError(t, err) - visualize := ValidateAndReturnVisualize(t, zaptest.NewLogger(t), Config{}, reports, persistedRequests, 5*time.Minute) + if err != nil { + t.Error(err) + } + result := ValidateAndReturnVisualize(zaptest.NewLogger(t), Config{}, reports, persistedRequests, 5*time.Minute) + err = result.Error() + if err != nil { + t.Error(err) + } + + err = result.Linearization.Visualize(lg, filepath.Join(path, "history.html")) + if err != nil { + t.Error(err) + } + }) + } +} - err = visualize(filepath.Join(path, "history.html")) +func TestValidateAndReturnVisualize(t *testing.T) { + tcs := []struct { + name string + reports []report.ClientReport + persistedRequests []model.EtcdRequest + config Config + expectError string + }{ + { + name: "Success with no persisted requests", + reports: []report.ClientReport{ + { + KeyValue: []porcupine.Operation{ + { + ClientId: 0, + Input: getRequest("key"), + Call: 100, + Output: getResponse(1), + Return: 200, + }, + }, + }, + }, + expectError: "", + }, + { + name: "Failure with not empty database", + reports: []report.ClientReport{ + { + KeyValue: []porcupine.Operation{ + { + ClientId: 0, + Input: getRequest("key"), + Call: 100, + // Empty database should have revision 1 + Output: getResponse(2), + Return: 200, + }, + }, + }, + }, + expectError: "non empty database at start, required by model used for linearizability validation", + }, + { + name: "Success", + reports: []report.ClientReport{ + { + KeyValue: []porcupine.Operation{ + { + ClientId: 0, + Input: getRequest("key"), + Call: 100, + Output: getResponse(1), + Return: 200, + }, + { + ClientId: 0, + Input: putRequest("key", "value"), + Call: 300, + Output: txnResponse(2, model.EtcdOperationResult{}), + Return: 400, + }, + }, + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{Key: "key"}, + Responses: []model.WatchResponse{ + {Events: []model.WatchEvent{watchEvent(2, true, model.PutOperation, "key", "value")}}, + }, + }, + }, + }, + }, + persistedRequests: []model.EtcdRequest{putRequest("key", "value")}, + expectError: "", + }, + { + name: "Failure of watch", + reports: []report.ClientReport{ + { + KeyValue: []porcupine.Operation{ + { + ClientId: 0, + Input: getRequest("key"), + Call: 100, + Output: getResponse(1), + Return: 200, + }, + { + ClientId: 0, + Input: putRequest("key", "value"), + Call: 300, + Output: txnResponse(2, model.EtcdOperationResult{}), + Return: 400, + }, + }, + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{Key: "key"}, + Responses: []model.WatchResponse{ + {Events: []model.WatchEvent{watchEvent(2, true, model.PutOperation, "key", "value2")}}, + }, + }, + }, + }, + }, + persistedRequests: []model.EtcdRequest{putRequest("key", "value")}, + expectError: "watch: Failure: broke Reliable", + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + lg := zaptest.NewLogger(t) + result := ValidateAndReturnVisualize(lg, Config{}, tc.reports, tc.persistedRequests, 5*time.Second) + + if tc.expectError != "" { + require.ErrorContains(t, result.Error(), tc.expectError) + } else { + require.NoError(t, result.Error()) + } + err := result.Linearization.Visualize(lg, filepath.Join(t.TempDir(), "history.html")) require.NoError(t, err) }) } } +func TestLinearizationVisualizeSkipsDeadlineExceeded(t *testing.T) { + lg := zaptest.NewLogger(t) + path := filepath.Join(t.TempDir(), "history.html") + result := LinearizationResult{ + Result: Result{ + Status: DeadlineExceeded, + Message: "deadline exceeded", + }, + } + + require.NoError(t, result.Visualize(lg, path)) + _, err := os.Stat(path) + require.Truef(t, os.IsNotExist(err), "deadline exceeded should not produce visualization") +} + +func watchEvent(rev int64, isCreate bool, eventType model.OperationType, key, value string) model.WatchEvent { + return model.WatchEvent{PersistedEvent: model.PersistedEvent{Revision: rev, IsCreate: isCreate, Event: model.Event{Type: eventType, Key: key, Value: model.ToValueOrHash(value)}}} +} + func TestValidateWatch(t *testing.T) { tcs := []struct { name string @@ -87,6 +241,39 @@ func TestValidateWatch(t *testing.T) { putRequest("b", "2"), }, }, + { + name: "Reliable - Put with non-existent lease doesn't generate watch event - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "a", + }, + Responses: []model.WatchResponse{}, + }, + }, + }, + }, + persistedRequests: []model.EtcdRequest{ + { + Type: model.Txn, + Txn: &model.TxnRequest{ + OperationsOnSuccess: []model.EtcdOperation{ + { + Type: model.PutOperation, + Put: model.PutOptions{ + Key: "a", + Value: model.ToValueOrHash("1"), + LeaseID: 1, + }, + }, + }, + }, + }, + }, + expectError: "", + }, { name: "Ordered, Unique - unique ordered events in separate response - pass", reports: []report.ClientReport{ @@ -1412,7 +1599,7 @@ func TestValidateWatch(t *testing.T) { }, }, { - name: "Resumable - missing first matching event - pass", + name: "Resumable - missing first matching event - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ @@ -1440,7 +1627,7 @@ func TestValidateWatch(t *testing.T) { expectError: errBrokeResumable.Error(), }, { - name: "Resumable - missing first matching event with prefix - pass", + name: "Resumable - missing first matching event with prefix - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ @@ -1468,7 +1655,7 @@ func TestValidateWatch(t *testing.T) { expectError: errBrokeResumable.Error(), }, { - name: "Resumable - missing first matching event with prefix - pass", + name: "Resumable - missing first matching event with prefix - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ @@ -1558,7 +1745,7 @@ func TestValidateWatch(t *testing.T) { expectError: errBrokeIsCreate.Error(), }, { - name: "IsCreate - put after delete marked as not created - pass", + name: "IsCreate - put after delete marked as not created - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ @@ -1839,13 +2026,9 @@ func TestValidateWatch(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { replay := model.NewReplay(tc.persistedRequests) - err := validateWatch(zaptest.NewLogger(t), tc.config, tc.reports, replay) - var errStr string - if err != nil { - errStr = err.Error() - } - if errStr != tc.expectError { - t.Errorf("validateWatch(...), got: %q, want: %q", err, tc.expectError) + result := validateWatch(zaptest.NewLogger(t), tc.config, tc.reports, replay) + if result.Message != tc.expectError { + t.Errorf("validateWatch(...), got: %q, want: %q", result.Message, tc.expectError) } }) } @@ -1915,6 +2098,21 @@ func deletePersistedEvent(key string, rev int64) model.PersistedEvent { } } +func getRequest(key string) model.EtcdRequest { + return model.EtcdRequest{ + Type: model.Range, + Range: &model.RangeRequest{ + RangeOptions: model.RangeOptions{ + Start: key, + }, + }, + } +} + +func getResponse(rev int64) model.MaybeEtcdResponse { + return model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: rev, Range: &model.RangeResponse{KVs: []model.KeyValue{}}}} +} + func putRequest(key, value string) model.EtcdRequest { return model.EtcdRequest{ Type: model.Txn, diff --git a/tests/robustness/validate/watch.go b/tests/robustness/validate/watch.go index 506cbeca431f..59d7a8f51f2d 100644 --- a/tests/robustness/validate/watch.go +++ b/tests/robustness/validate/watch.go @@ -16,9 +16,11 @@ package validate import ( "errors" + "fmt" + "reflect" + "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" @@ -26,7 +28,7 @@ import ( ) var ( - errBrokeBookmarkable = errors.New("broke Bookmarkable - Progress notification events guarantee that all events up to a revision have been already delivered") + errBrokeBookmarkable = errors.New("broke Bookmarkable - Progress notification events guarantee that all events up to a revision have already been delivered") errBrokeOrdered = errors.New("broke Ordered - events are ordered by revision; an event will never appear on a watch if it precedes an event in time that has already been posted") errBrokeUnique = errors.New("broke Unique - an event will never appear on a watch twice") errBrokeAtomic = errors.New("broke Atomic - a list of events is guaranteed to encompass complete revisions; updates in the same revision over multiple keys will not be split over several lists of events") @@ -37,8 +39,19 @@ var ( errBrokeFilter = errors.New("event not matching watch filter") ) -func validateWatch(lg *zap.Logger, cfg Config, reports []report.ClientReport, replay *model.EtcdReplay) error { +func validateWatch(lg *zap.Logger, cfg Config, reports []report.ClientReport, replay *model.EtcdReplay) Result { lg.Info("Validating watch") + start := time.Now() + err := validateWatchError(lg, cfg, reports, replay) + if err != nil { + lg.Error("Watch validation failed", zap.Duration("duration", time.Since(start)), zap.Error(err)) + } else { + lg.Info("Watch validation success", zap.Duration("duration", time.Since(start))) + } + return ResultFromError(err) +} + +func validateWatchError(lg *zap.Logger, cfg Config, reports []report.ClientReport, replay *model.EtcdReplay) error { // Validate etcd watch properties defined in https://etcd.io/docs/v3.6/learning/api_guarantees/#watch-apis for _, r := range reports { err := validateFilter(lg, r) @@ -65,15 +78,15 @@ func validateWatch(lg *zap.Logger, cfg Config, reports []report.ClientReport, re if err != nil { return err } - err = validateReliable(lg, replay, r) + err = validateIsCreate(lg, replay, r) if err != nil { return err } - err = validatePrevKV(lg, replay, r) + err = validateReliable(lg, replay, r) if err != nil { return err } - err = validateIsCreate(lg, replay, r) + err = validatePrevKV(lg, replay, r) if err != nil { return err } @@ -205,8 +218,10 @@ func validateReliable(lg *zap.Logger, replay *model.EtcdReplay, report report.Cl gotEvents = append(gotEvents, event.PersistedEvent) } } - if diff := cmp.Diff(wantEvents, gotEvents, cmpopts.IgnoreFields(model.PersistedEvent{}, "IsCreate")); diff != "" { - lg.Error("Broke watch guarantee", zap.String("guarantee", "reliable"), zap.Int("client", report.ClientID), zap.String("diff", diff)) + if !reflect.DeepEqual(wantEvents, gotEvents) { + lg.Error("Broke watch guarantee", zap.String("guarantee", "reliable"), zap.Int("client", report.ClientID)) + // Directly print to console to avoid escaping newline. + fmt.Print(cmp.Diff(wantEvents, gotEvents)) err = errBrokeReliable } } @@ -245,7 +260,7 @@ func validatePrevKV(lg *zap.Logger, replay *model.EtcdReplay, report report.Clie } for _, resp := range op.Responses { for _, event := range resp.Events { - // Get state state just before the current event. + // Get state just before the current event. state, err2 := replay.StateForRevision(event.Revision - 1) if err2 != nil { panic(err2) @@ -263,9 +278,15 @@ func validatePrevKV(lg *zap.Logger, replay *model.EtcdReplay, report report.Clie // We allow PrevValue to be nil since in the face of compaction, etcd does not // guarantee its presence. - if event.PrevValue != nil && *event.PrevValue != state.KeyValues[event.Key] { - lg.Error("Incorrect event prevValue field", zap.Int("client", report.ClientID), zap.Any("event", event), zap.Any("previousValue", state.KeyValues[event.Key])) - err = errBrokePrevKV + if event.PrevValue != nil { + val, ok := state.GetValue(event.Key) + if !ok { + lg.Error("Incorrect event prevValue field", zap.Int("client", report.ClientID), zap.Any("event", event), zap.Any("previousValue", nil)) + err = errBrokePrevKV + } else if *event.PrevValue != *val { + lg.Error("Incorrect event prevValue field", zap.Int("client", report.ClientID), zap.Any("event", event), zap.Any("previousValue", *val)) + err = errBrokePrevKV + } } } } @@ -277,14 +298,14 @@ func validateIsCreate(lg *zap.Logger, replay *model.EtcdReplay, report report.Cl for _, op := range report.Watch { for _, resp := range op.Responses { for _, event := range resp.Events { - // Get state state just before the current event. + // Get state just before the current event. state, err2 := replay.StateForRevision(event.Revision - 1) if err2 != nil { panic(err2) } // A create event will not have an entry in our history and a non-create // event *should* have an entry in our history. - if _, prevKeyExists := state.KeyValues[event.Key]; event.IsCreate == prevKeyExists { + if _, prevKeyExists := state.GetValue(event.Key); event.IsCreate == prevKeyExists { lg.Error("Incorrect event IsCreate field", zap.Int("client", report.ClientID), zap.Any("event", event)) err = errBrokeIsCreate } diff --git a/tools/.golangci.yaml b/tools/.golangci.yaml index b4251f1556b3..02090fdd447a 100644 --- a/tools/.golangci.yaml +++ b/tools/.golangci.yaml @@ -3,19 +3,44 @@ version: "2" linters: default: none enable: # please keep this alphabetized - # Disabled after Go 1.24 / golangci-lint v2 update: - # - errorlint # Disabled as we want to preserve errors from the stable release branch. - # - testifylint # Disabled as it needs changes in test files to properly + - errorlint + - goheader + - govet - ineffassign + - misspell - nakedret - - revive # Disabled as revive from golangci-lint v2 is more strict, it would need more changes in the stable release branch. + - revive - staticcheck + - testifylint + - thelper - unconvert # Remove unnecessary type conversions - unparam - unused - usestdlibvars + - usetesting - whitespace settings: + goheader: + values: + regexp: + ORIGINAL_YEAR: 20[1-2][0-9] + template: |- + Copyright {{ORIGINAL_YEAR}} The etcd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + govet: + enable: + - shadow nakedret: # Align with https://github.com/alexkohler/nakedret/blob/v1.0.2/cmd/nakedret/main.go#L10 max-func-lines: 5 @@ -43,17 +68,19 @@ linters: arguments: - preserveScope - name: time-naming + - name: unnecessary-stmt - name: use-any - name: var-declaration - # Disabled after Go 1.24 / golangci-lint v2 update - name: var-naming - disabled: true arguments: # The following is the configuration for var-naming rule, the first element is the allow list and the second element is the deny list. - [] # AllowList: leave it empty to use the default (empty, too). This means that we're not relaxing the rule in any way, i.e. elementId will raise a violation, it should be elementID, refer to the next line to see the list of denied initialisms. # DenyList: Add GRPC and WAL to strict the rule not allowing instances like Wal or Grpc. The default values are located at commonInitialisms, refer to: https://github.com/mgechev/revive/blob/v1.3.7/lint/utils.go#L93-L133. - - GRPC - WAL + # Disable checks on package names that collide with Go standard library packages. See example error: + # var-naming: avoid package names that conflict with Go standard library package names (revive) + - - skip-package-name-collision-with-go-std: true - name: exported disabled: true - name: unexported-return @@ -61,7 +88,6 @@ linters: staticcheck: checks: - all - - -SA1019 # TODO(fix) Using a deprecated function, variable, constant or field - -SA2002 # TODO(fix) Called testing.T.FailNow or SkipNow in a goroutine, which isn’t allowed - -QF1001 # TODO(fix) Apply De Morgan’s law - -QF1002 # TODO(fix) Convert untagged switch to tagged switch @@ -87,6 +113,21 @@ linters: # to always require f-functions for stretchr/testify, but not for golang standard lib. # Also refer to https://github.com/etcd-io/etcd/pull/18741#issuecomment-2422395914 require-f-funcs: true + thelper: + test: + first: false + begin: false + fuzz: + first: false + begin: false + benchmark: + first: false + begin: false + tb: + first: false + begin: false + usetesting: + os-mkdir-temp: false exclusions: generated: lax presets: @@ -113,9 +154,15 @@ issues: max-same-issues: 0 formatters: enable: + - gci - gofmt - goimports settings: # please keep this alphabetized + gci: + sections: + - standard + - default + - prefix(go.etcd.io) goimports: local-prefixes: - go.etcd.io # Put imports beginning with prefix after 3rd-party packages. diff --git a/tools/.markdownlint.jsonc b/tools/.markdownlint.jsonc new file mode 100644 index 000000000000..d6b36639cfc1 --- /dev/null +++ b/tools/.markdownlint.jsonc @@ -0,0 +1,282 @@ +// Example markdownlint configuration with all properties set to their default value +{ + + // Default state for all rules + "default": true, + + // Path to configuration file to extend + "extends": null, + + // MD001/heading-increment : Heading levels should only increment by one level at a time : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md001.md + "MD001": true, + + // MD003/heading-style : Heading style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md003.md + "MD003": { + // Heading style + "style": "consistent" + }, + + // MD004/ul-style : Unordered list style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md004.md + "MD004": { + // List style + "style": "consistent" + }, + + // MD005/list-indent : Inconsistent indentation for list items at the same level : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md005.md + "MD005": true, + + // MD007/ul-indent : Unordered list indentation : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md007.md + "MD007": { + // Spaces for indent + "indent": 2, + // Whether to indent the first level of the list + "start_indented": false, + // Spaces for first level indent (when start_indented is set) + "start_indent": 2 + }, + + // MD009/no-trailing-spaces : Trailing spaces : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md009.md + "MD009": { + // Spaces for line break + "br_spaces": 2, + // Allow spaces for empty lines in list items + "list_item_empty_lines": false, + // Include unnecessary breaks + "strict": false + }, + + // MD010/no-hard-tabs : Hard tabs : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md010.md + "MD010": { + // Include code blocks + "code_blocks": true, + // Fenced code languages to ignore + "ignore_code_languages": [], + // Number of spaces for each hard tab + "spaces_per_tab": 1 + }, + + // MD011/no-reversed-links : Reversed link syntax : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md011.md + "MD011": true, + + // MD012/no-multiple-blanks : Multiple consecutive blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md012.md + "MD012": { + // Consecutive blank lines + "maximum": 1 + }, + + "MD013": false, + + // MD014/commands-show-output : Dollar signs used before commands without showing output : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md014.md + "MD014": true, + + // MD018/no-missing-space-atx : No space after hash on atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md018.md + "MD018": true, + + // MD019/no-multiple-space-atx : Multiple spaces after hash on atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md019.md + "MD019": true, + + // MD020/no-missing-space-closed-atx : No space inside hashes on closed atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md020.md + "MD020": true, + + // MD021/no-multiple-space-closed-atx : Multiple spaces inside hashes on closed atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md021.md + "MD021": true, + + // MD022/blanks-around-headings : Headings should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md022.md + "MD022": { + // Blank lines above heading + "lines_above": 1, + // Blank lines below heading + "lines_below": 1 + }, + + // MD023/heading-start-left : Headings must start at the beginning of the line : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md023.md + "MD023": true, + + // MD024/no-duplicate-heading : Multiple headings with the same content : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md024.md + "MD024": { + // Only check sibling headings + "siblings_only": false + }, + + // MD025/single-title/single-h1 : Multiple top-level headings in the same document : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md025.md + "MD025": { + // Heading level + "level": 1, + // RegExp for matching title in front matter + "front_matter_title": "^\\s*title\\s*[:=]" + }, + + // MD026/no-trailing-punctuation : Trailing punctuation in heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md026.md + "MD026": { + // Punctuation characters + "punctuation": ".,;:!。，；：！" + }, + + // MD027/no-multiple-space-blockquote : Multiple spaces after blockquote symbol : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md027.md + "MD027": true, + + // MD028/no-blanks-blockquote : Blank line inside blockquote : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md028.md + "MD028": true, + + // MD029/ol-prefix : Ordered list item prefix : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md029.md + "MD029": { + // List style + "style": "one_or_ordered" + }, + + // MD030/list-marker-space : Spaces after list markers : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md030.md + "MD030": { + // Spaces for single-line unordered list items + "ul_single": 1, + // Spaces for single-line ordered list items + "ol_single": 1, + // Spaces for multi-line unordered list items + "ul_multi": 1, + // Spaces for multi-line ordered list items + "ol_multi": 1 + }, + + // MD031/blanks-around-fences : Fenced code blocks should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md031.md + "MD031": { + // Include list items + "list_items": true + }, + + // MD032/blanks-around-lists : Lists should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md032.md + "MD032": true, + + // MD033/no-inline-html : Inline HTML : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md033.md + "MD033": { + // Allowed elements + "allowed_elements": [] + }, + + // MD034/no-bare-urls : Bare URL used : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md034.md + "MD034": true, + + // MD035/hr-style : Horizontal rule style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md035.md + "MD035": { + // Horizontal rule style + "style": "consistent" + }, + + // MD036/no-emphasis-as-heading : Emphasis used instead of a heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md036.md + "MD036": { + // Punctuation characters + "punctuation": ".,;:!?。，；：！？" + }, + + // MD037/no-space-in-emphasis : Spaces inside emphasis markers : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md037.md + "MD037": true, + + // MD038/no-space-in-code : Spaces inside code span elements : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md038.md + "MD038": true, + + // MD039/no-space-in-links : Spaces inside link text : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md039.md + "MD039": true, + + // MD040/fenced-code-language : Fenced code blocks should have a language specified : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md040.md + "MD040": { + // List of languages + "allowed_languages": [], + // Require language only + "language_only": false + }, + + // MD041/first-line-heading/first-line-h1 : First line in a file should be a top-level heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md041.md + "MD041": false, + + // MD042/no-empty-links : No empty links : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md042.md + "MD042": true, + + // MD043/required-headings : Required heading structure : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md043.md + "MD043": false, + + // MD044/proper-names : Proper names should have the correct capitalization : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md044.md + "MD044": { + // List of proper names + "names": [], + // Include code blocks + "code_blocks": true, + // Include HTML elements + "html_elements": true + }, + + // MD045/no-alt-text : Images should have alternate text (alt text) : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md045.md + "MD045": true, + + // MD046/code-block-style : Code block style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md046.md + "MD046": { + // Block style + "style": "consistent" + }, + + // MD047/single-trailing-newline : Files should end with a single newline character : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md047.md + "MD047": true, + + // MD048/code-fence-style : Code fence style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md048.md + "MD048": { + // Code fence style + "style": "consistent" + }, + + // MD049/emphasis-style : Emphasis style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md049.md + "MD049": { + // Emphasis style + "style": "consistent" + }, + + // MD050/strong-style : Strong style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md050.md + "MD050": { + // Strong style + "style": "consistent" + }, + + // MD051/link-fragments : Link fragments should be valid : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md051.md + "MD051": { + // Ignore case of fragments + "ignore_case": false + }, + + // MD052/reference-links-images : Reference links and images should use a label that is defined : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md052.md + "MD052": { + // Include shortcut syntax + "shortcut_syntax": false + }, + + // MD053/link-image-reference-definitions : Link and image reference definitions should be needed : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md053.md + "MD053": { + // Ignored definitions + "ignored_definitions": [ + "//" + ] + }, + + // MD054/link-image-style : Link and image style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md054.md + "MD054": { + // Allow autolinks + "autolink": true, + // Allow inline links and images + "inline": true, + // Allow full reference links and images + "full": true, + // Allow collapsed reference links and images + "collapsed": true, + // Allow shortcut reference links and images + "shortcut": true, + // Allow URLs as inline links + "url_inline": true + }, + + // MD055/table-pipe-style : Table pipe style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md055.md + "MD055": { + // Table pipe style + "style": "consistent" + }, + + // MD056/table-column-count : Table column count : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md056.md + "MD056": true, + + // MD058/blanks-around-tables : Tables should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md058.md + "MD058": true + } diff --git a/tools/benchmark/cmd/lease.go b/tools/benchmark/cmd/lease.go index 5ed3dbe4b3e0..6eb26bff1e74 100644 --- a/tools/benchmark/cmd/lease.go +++ b/tools/benchmark/cmd/lease.go @@ -40,14 +40,14 @@ func init() { leaseKeepaliveCmd.Flags().IntVar(&leaseKeepaliveTotal, "total", 10000, "Total number of lease keepalive requests") } -func leaseKeepaliveFunc(_ *cobra.Command, _ []string) { +func leaseKeepaliveFunc(cmd *cobra.Command, _ []string) { requests := make(chan struct{}) clients := mustCreateClients(totalClients, totalConns) bar = pb.New(leaseKeepaliveTotal) bar.Start() - r := newReport() + r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c v3.Lease) { @@ -65,14 +65,12 @@ func leaseKeepaliveFunc(_ *cobra.Command, _ []string) { }(clients[i]) } - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for i := 0; i < leaseKeepaliveTotal; i++ { requests <- struct{}{} } close(requests) - }() + }) rc := r.Run() wg.Wait() diff --git a/tools/benchmark/cmd/mvcc-put.go b/tools/benchmark/cmd/mvcc-put.go index 25cdb61d5417..b6a6379c23c1 100644 --- a/tools/benchmark/cmd/mvcc-put.go +++ b/tools/benchmark/cmd/mvcc-put.go @@ -21,11 +21,11 @@ import ( "runtime/pprof" "time" + "github.com/spf13/cobra" + "go.etcd.io/etcd/pkg/v3/report" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" - - "github.com/spf13/cobra" ) // mvccPutCmd represents a storage put performance benchmarking tool @@ -69,7 +69,7 @@ func createBytesSlice(bytesN, sliceN int) [][]byte { return rs } -func mvccPutFunc(_ *cobra.Command, _ []string) { +func mvccPutFunc(cmd *cobra.Command, _ []string) { if cpuProfPath != "" { f, err := os.Create(cpuProfPath) if err != nil { @@ -105,7 +105,7 @@ func mvccPutFunc(_ *cobra.Command, _ []string) { vals := createBytesSlice(valueSize, mvccTotalRequests*nrTxnOps) weight := float64(nrTxnOps) - r := newWeightedReport() + r := newWeightedReport(cmd.Name()) rrc := r.Results() rc := r.Run() diff --git a/tools/benchmark/cmd/mvcc.go b/tools/benchmark/cmd/mvcc.go index baf2bf1a1bb6..67e3f7400705 100644 --- a/tools/benchmark/cmd/mvcc.go +++ b/tools/benchmark/cmd/mvcc.go @@ -18,13 +18,12 @@ import ( "os" "time" + "github.com/spf13/cobra" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" - - "github.com/spf13/cobra" ) var ( diff --git a/tools/benchmark/cmd/put.go b/tools/benchmark/cmd/put.go index 62b9a3dc83cc..7f66046f2109 100644 --- a/tools/benchmark/cmd/put.go +++ b/tools/benchmark/cmd/put.go @@ -88,7 +88,7 @@ func putFunc(cmd *cobra.Command, _ []string) { bar = pb.New(putTotal) bar.Start() - r := newReport() + r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c *v3.Client) { diff --git a/tools/benchmark/cmd/range.go b/tools/benchmark/cmd/range.go index 111d69bd0558..4d5b520ca703 100644 --- a/tools/benchmark/cmd/range.go +++ b/tools/benchmark/cmd/range.go @@ -22,13 +22,19 @@ import ( "time" "github.com/cheggaaa/pb/v3" + "github.com/golang/protobuf/proto" //nolint:staticcheck "github.com/spf13/cobra" "golang.org/x/time/rate" + etcdserverpb "go.etcd.io/etcd/api/v3/etcdserverpb" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) +// k8sWatchCachePageSize matches the default page size used by the Kubernetes +// watch cache when paginating List requests against etcd. +const k8sWatchCachePageSize = 10000 + // rangeCmd represents the range command var rangeCmd = &cobra.Command{ Use: "range key [end-range]", @@ -43,6 +49,10 @@ var ( rangeConsistency string rangeLimit int64 rangeCountOnly bool + rangeKeysOnly bool + rangeStream bool + rangePaginate bool + rangePrefix bool ) func init() { @@ -52,20 +62,37 @@ func init() { rangeCmd.Flags().StringVar(&rangeConsistency, "consistency", "l", "Linearizable(l) or Serializable(s)") rangeCmd.Flags().Int64Var(&rangeLimit, "limit", 0, "Maximum number of results to return from range request (0 is no limit)") rangeCmd.Flags().BoolVar(&rangeCountOnly, "count-only", false, "Only returns the count of keys") + rangeCmd.Flags().BoolVar(&rangeKeysOnly, "keys-only", false, "Only returns the keys") + rangeCmd.Flags().BoolVar(&rangeStream, "stream", false, "Use RangeStream instead of unary Range") + rangeCmd.Flags().BoolVar(&rangePaginate, "paginate", false, "Use paginated unary range with 10k-key pages") + rangeCmd.Flags().BoolVar(&rangePrefix, "prefix", false, "Range over all keys with the given key as prefix") } func rangeFunc(cmd *cobra.Command, args []string) { - if len(args) == 0 || len(args) > 2 { + if len(args) > 2 || (len(args) == 0 && !rangePrefix) { fmt.Fprintln(os.Stderr, cmd.Usage()) os.Exit(1) } - k := args[0] + key := "" end := "" + if len(args) >= 1 { + key = args[0] + } if len(args) == 2 { end = args[1] } + if rangePaginate && rangeLimit > 0 { + fmt.Fprintln(os.Stderr, "--paginate and --limit are mutually exclusive") + os.Exit(1) + } + + if rangeCountOnly && rangeKeysOnly { + fmt.Fprintln(os.Stderr, "`--keys-only` and `--count-only` cannot be set at the same time") + os.Exit(1) + } + if rangeConsistency == "l" { fmt.Println("bench with linearizable range") } else if rangeConsistency == "s" { @@ -80,22 +107,63 @@ func rangeFunc(cmd *cobra.Command, args []string) { } limit := rate.NewLimiter(rate.Limit(rangeRate), 1) - requests := make(chan v3.Op, totalClients) + requests := make(chan struct{}, totalClients) clients := mustCreateClients(totalClients, totalConns) bar = pb.New(rangeTotal) bar.Start() - r := newReport() + var baseOpts []v3.OpOption + if rangeLimit > 0 { + baseOpts = append(baseOpts, v3.WithLimit(rangeLimit)) + } + + switch { + case rangeCountOnly: + baseOpts = append(baseOpts, v3.WithCountOnly()) + case rangeKeysOnly: + baseOpts = append(baseOpts, v3.WithKeysOnly()) + } + + if rangeConsistency == "s" { + baseOpts = append(baseOpts, v3.WithSerializable()) + } + if rangePrefix { + if rangePaginate { + // Pin the prefix's range end once. Otherwise WithPrefix would + // re-derive it from the advancing start key on every page. + baseOpts = append(baseOpts, v3.WithRange(v3.GetPrefixRangeEnd(key))) + if key == "" { + key = "\x00" + } + } else { + baseOpts = append(baseOpts, v3.WithPrefix()) + } + } else if end != "" { + baseOpts = append(baseOpts, v3.WithRange(end)) + } + + r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c *v3.Client) { defer wg.Done() - for op := range requests { + for range requests { limit.Wait(context.Background()) - st := time.Now() - _, err := c.Do(context.Background(), op) + var err error + switch { + case rangeStream: + var stream v3.GetStreamChan + stream, err = c.GetStream(context.Background(), key, baseOpts...) + if err == nil { + _, err = v3.GetStreamToGetResponse(stream) + } + case rangePaginate: + err = paginatedRange(c, key, k8sWatchCachePageSize, baseOpts) + default: + _, err = c.Get(context.Background(), key, baseOpts...) + } r.Results() <- report.Result{Err: err, Start: st, End: time.Now()} bar.Increment() } @@ -104,15 +172,7 @@ func rangeFunc(cmd *cobra.Command, args []string) { go func() { for i := 0; i < rangeTotal; i++ { - opts := []v3.OpOption{v3.WithRange(end), v3.WithLimit(rangeLimit)} - if rangeCountOnly { - opts = append(opts, v3.WithCountOnly()) - } - if rangeConsistency == "s" { - opts = append(opts, v3.WithSerializable()) - } - op := v3.OpGet(k, opts...) - requests <- op + requests <- struct{}{} } close(requests) }() @@ -123,3 +183,27 @@ func rangeFunc(cmd *cobra.Command, args []string) { bar.Finish() fmt.Printf("%s", <-rc) } + +func paginatedRange(c *v3.Client, key string, pageSize int64, baseOpts []v3.OpOption) error { + merged := &etcdserverpb.RangeResponse{} + var rev int64 + for { + opts := append([]v3.OpOption{v3.WithLimit(pageSize)}, baseOpts...) + if rev != 0 { + opts = append(opts, v3.WithRev(rev)) + } + resp, err := c.Get(context.Background(), key, opts...) + if err != nil { + return err + } + if rev == 0 { + rev = resp.Header.Revision + } + proto.Merge(merged, (*etcdserverpb.RangeResponse)(resp)) + if !resp.More || len(resp.Kvs) == 0 { + return nil + } + last := resp.Kvs[len(resp.Kvs)-1].Key + key = string(append(last, '\x00')) + } +} diff --git a/tools/benchmark/cmd/root.go b/tools/benchmark/cmd/root.go index c85beb9dbc65..f3768db1006c 100644 --- a/tools/benchmark/cmd/root.go +++ b/tools/benchmark/cmd/root.go @@ -53,8 +53,9 @@ var ( dialTimeout time.Duration - targetLeader bool autoSyncInterval time.Duration + + generatePerfReport bool ) func init() { @@ -72,6 +73,7 @@ func init() { RootCmd.PersistentFlags().StringVar(&user, "user", "", "provide username[:password] and prompt if password is not supplied.") RootCmd.PersistentFlags().DurationVar(&dialTimeout, "dial-timeout", 0, "dial timeout for client connections") - RootCmd.PersistentFlags().BoolVar(&targetLeader, "target-leader", false, "connect only to the leader node") RootCmd.PersistentFlags().DurationVar(&autoSyncInterval, "auto-sync-interval", time.Duration(0), "AutoSyncInterval is the interval to update endpoints with its latest members") + + RootCmd.PersistentFlags().BoolVar(&generatePerfReport, "report-perfdash", false, "Generate benchmark report in perfdash format") } diff --git a/tools/benchmark/cmd/stm.go b/tools/benchmark/cmd/stm.go index d6dfba0d397e..22dc88657142 100644 --- a/tools/benchmark/cmd/stm.go +++ b/tools/benchmark/cmd/stm.go @@ -110,7 +110,7 @@ func stmFunc(cmd *cobra.Command, _ []string) { bar = pb.New(stmTotal) bar.Start() - r := newReport() + r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go doSTM(clients[i], requests, r.Results()) diff --git a/tools/benchmark/cmd/txn_mixed.go b/tools/benchmark/cmd/txn_mixed.go index ffc004ecfe20..bbd88d599c09 100644 --- a/tools/benchmark/cmd/txn_mixed.go +++ b/tools/benchmark/cmd/txn_mixed.go @@ -95,8 +95,8 @@ func mixedTxnFunc(cmd *cobra.Command, _ []string) { bar = pb.New(mixedTxnTotal) bar.Start() - reportRead := newReport() - reportWrite := newReport() + reportRead := newReport(cmd.Name() + "-read") + reportWrite := newReport(cmd.Name() + "-write") for i := range clients { wg.Add(1) go func(c *v3.Client) { diff --git a/tools/benchmark/cmd/txn_put.go b/tools/benchmark/cmd/txn_put.go index d69293b838a8..5e2d9d7a4073 100644 --- a/tools/benchmark/cmd/txn_put.go +++ b/tools/benchmark/cmd/txn_put.go @@ -55,7 +55,7 @@ func init() { txnPutCmd.Flags().IntVar(&keySpaceSize, "key-space-size", 1, "Maximum possible keys") } -func txnPutFunc(_ *cobra.Command, _ []string) { +func txnPutFunc(cmd *cobra.Command, _ []string) { if keySpaceSize <= 0 { fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize) os.Exit(1) @@ -78,7 +78,7 @@ func txnPutFunc(_ *cobra.Command, _ []string) { bar = pb.New(txnPutTotal) bar.Start() - r := newReport() + r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c *v3.Client) { diff --git a/tools/benchmark/cmd/util.go b/tools/benchmark/cmd/util.go index a6b999390671..63fa9802da25 100644 --- a/tools/benchmark/cmd/util.go +++ b/tools/benchmark/cmd/util.go @@ -15,7 +15,6 @@ package cmd import ( - "context" "crypto/rand" "fmt" "os" @@ -29,50 +28,18 @@ import ( ) var ( - // dialTotal counts the number of mustCreateConn calls so that endpoint - // connections can be handed out in round-robin order - dialTotal int - - // leaderEps is a cache for holding endpoints of a leader node - leaderEps []string - // cache the username and password for multiple connections globalUserName string globalPassword string ) -func mustFindLeaderEndpoints(c *clientv3.Client) { - resp, lerr := c.MemberList(context.TODO()) - if lerr != nil { - fmt.Fprintf(os.Stderr, "failed to get a member list: %s\n", lerr) - os.Exit(1) - } - - leaderID := uint64(0) - for _, ep := range c.Endpoints() { - if sresp, serr := c.Status(context.TODO(), ep); serr == nil { - leaderID = sresp.Leader - break - } - } - - for _, m := range resp.Members { - if m.ID == leaderID { - leaderEps = m.ClientURLs - return - } - } - - fmt.Fprint(os.Stderr, "failed to find a leader endpoint\n") - os.Exit(1) -} - func getUsernamePassword(usernameFlag string) (string, string, error) { if globalUserName != "" && globalPassword != "" { return globalUserName, globalPassword, nil } - colon := strings.Index(usernameFlag, ":") - if colon == -1 { + var ok bool + globalUserName, globalPassword, ok = strings.Cut(usernameFlag, ":") + if !ok { // Prompt for the password. password, err := speakeasy.Ask("Password: ") if err != nil { @@ -80,22 +47,14 @@ func getUsernamePassword(usernameFlag string) (string, string, error) { } globalUserName = usernameFlag globalPassword = password - } else { - globalUserName = usernameFlag[:colon] - globalPassword = usernameFlag[colon+1:] } return globalUserName, globalPassword, nil } func mustCreateConn() *clientv3.Client { - connEndpoints := leaderEps - if len(connEndpoints) == 0 { - connEndpoints = []string{endpoints[dialTotal%len(endpoints)]} - dialTotal++ - } cfg := clientv3.Config{ AutoSyncInterval: autoSyncInterval, - Endpoints: connEndpoints, + Endpoints: endpoints, DialTimeout: dialTimeout, } if !tls.Empty() || tls.TrustedCAFile != "" { @@ -118,12 +77,6 @@ func mustCreateConn() *clientv3.Client { } client, err := clientv3.New(cfg) - if targetLeader && len(leaderEps) == 0 { - mustFindLeaderEndpoints(client) - client.Close() - return mustCreateConn() - } - grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr)) if err != nil { @@ -157,24 +110,24 @@ func mustRandBytes(n int) []byte { return rb } -func newReport() report.Report { +func newReport(benchmarkOp string) report.Report { p := "%4.4f" if precise { p = "%g" } if sample { - return report.NewReportSample(p) + return report.NewReportSample(p, benchmarkOp, generatePerfReport) } - return report.NewReport(p) + return report.NewReport(p, benchmarkOp, generatePerfReport) } -func newWeightedReport() report.Report { +func newWeightedReport(benchmarkOp string) report.Report { p := "%4.4f" if precise { p = "%g" } if sample { - return report.NewReportSample(p) + return report.NewReportSample(p, benchmarkOp, generatePerfReport) } - return report.NewWeightedReport(report.NewReport(p), p) + return report.NewWeightedReport(report.NewReport(p, benchmarkOp, generatePerfReport), p, benchmarkOp, generatePerfReport) } diff --git a/tools/benchmark/cmd/watch.go b/tools/benchmark/cmd/watch.go index a52a34360f2b..7b707bf96aae 100644 --- a/tools/benchmark/cmd/watch.go +++ b/tools/benchmark/cmd/watch.go @@ -117,7 +117,7 @@ func benchMakeWatches(clients []*clientv3.Client, wk *watchedKeys) { bar = pb.New(watchStreams * watchWatchesPerStream) bar.Start() - r := newReport() + r := newReport("watch-make") rch := r.Results() wg.Add(len(streams) + 1) @@ -189,7 +189,7 @@ func benchPutWatches(clients []*clientv3.Client, wk *watchedKeys) { bar = pb.New(eventsTotal) bar.Start() - r := newReport() + r := newReport("watch-put") wg.Add(len(wk.watches)) nrRxed := int32(eventsTotal) diff --git a/tools/benchmark/cmd/watch_get.go b/tools/benchmark/cmd/watch_get.go index b9a8bc3363d5..96abad665c75 100644 --- a/tools/benchmark/cmd/watch_get.go +++ b/tools/benchmark/cmd/watch_get.go @@ -49,7 +49,7 @@ func init() { watchGetCmd.Flags().IntVar(&watchEvents, "events", 8, "Number of events per watcher") } -func watchGetFunc(_ *cobra.Command, _ []string) { +func watchGetFunc(cmd *cobra.Command, _ []string) { clients := mustCreateClients(totalClients, totalConns) getClient := mustCreateClients(1, 1) @@ -75,7 +75,7 @@ func watchGetFunc(_ *cobra.Command, _ []string) { bar.Start() // report from trying to do serialized gets with concurrent watchers - r := newReport() + r := newReport(cmd.Name()) ctx, cancel := context.WithCancel(context.TODO()) f := func() { defer close(r.Results()) diff --git a/tools/benchmark/cmd/watch_latency.go b/tools/benchmark/cmd/watch_latency.go index 2e60860b460d..59fa689b4148 100644 --- a/tools/benchmark/cmd/watch_latency.go +++ b/tools/benchmark/cmd/watch_latency.go @@ -60,7 +60,7 @@ func init() { watchLatencyCmd.Flags().IntVar(&watchLValueSize, "val-size", 32, "Value size of watch response") } -func watchLatencyFunc(_ *cobra.Command, _ []string) { +func watchLatencyFunc(cmd *cobra.Command, _ []string) { key := string(mustRandBytes(watchLKeySize)) value := string(mustRandBytes(watchLValueSize)) wchs := setupWatchChannels(key) @@ -75,12 +75,8 @@ func watchLatencyFunc(_ *cobra.Command, _ []string) { eventTimes := make([][]time.Time, len(wchs)) for i, wch := range wchs { - wch := wch - i := i eventTimes[i] = make([]time.Time, watchLPutTotal) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { eventCount := 0 for eventCount < watchLPutTotal { resp := <-wch @@ -90,12 +86,12 @@ func watchLatencyFunc(_ *cobra.Command, _ []string) { bar.Increment() } } - }() + }) } - putReport := newReport() + putReport := newReport(cmd.Name() + "-put") putReportResults := putReport.Run() - watchReport := newReport() + watchReport := newReport(cmd.Name() + "-watch") watchReportResults := watchReport.Run() for i := 0; i < watchLPutTotal; i++ { // limit key put as per reqRate diff --git a/tools/check-grpc-experimental/allowlist.txt b/tools/check-grpc-experimental/allowlist.txt new file mode 100644 index 000000000000..6c4b395db0d2 --- /dev/null +++ b/tools/check-grpc-experimental/allowlist.txt @@ -0,0 +1,21 @@ +# Allowlist for experimental gRPC APIs +# Format: PackageName.Symbol +# Remove items from this list as they are migrated or stabilized. + +grpc.NewContextWithServerTransportStream +grpc.ServeHTTP +grpc.WithResolvers +resolver.Address +resolver.BuildOptions +resolver.Builder +resolver.ClientConn +resolver.Endpoint +resolver.ParseServiceConfig +resolver.ResolveNowOptions +resolver.Resolver +resolver.State +resolver.Target +resolver.URL +resolver.UpdateState +serviceconfig.Err +serviceconfig.ParseResult diff --git a/tools/check-grpc-experimental/doc.go b/tools/check-grpc-experimental/doc.go new file mode 100644 index 000000000000..b1f4273b393e --- /dev/null +++ b/tools/check-grpc-experimental/doc.go @@ -0,0 +1,16 @@ +// Copyright 2026 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// check-grpc-experimental checks for experimental gRPC APIs in etcd. +package main diff --git a/tools/check-grpc-experimental/main.go b/tools/check-grpc-experimental/main.go new file mode 100644 index 000000000000..a41c79e6f2f8 --- /dev/null +++ b/tools/check-grpc-experimental/main.go @@ -0,0 +1,325 @@ +// Copyright 2026 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + + "golang.org/x/tools/go/packages" +) + +var ( + debugMode = flag.Bool("debug", false, "enable verbose debug logging") + allowListFile = flag.String("allow-list", "", "path to a file containing allowed APIs (one per line)") +) + +// Map to store allowed signatures (e.g., "grpc.WithResolvers" -> true) +var allowList = make(map[string]bool) + +func main() { + flag.Parse() + patterns := flag.Args() + if len(patterns) == 0 { + patterns = []string{"./..."} + } + + if *allowListFile != "" { + if err := loadAllowList(*allowListFile); err != nil { + log.Fatalf("Failed to load allow list: %v", err) + } + } + + // Load source with type info. + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports | packages.NeedDeps, + Tests: true, + } + + if *debugMode { + log.Println("Loading packages...") + } + + pkgs, err := packages.Load(cfg, patterns...) + if err != nil { + log.Fatalf("failed to load packages: %v", err) + } + if n := packages.PrintErrors(pkgs); n > 0 { + os.Exit(1) + } + + if *debugMode { + log.Printf("Loaded %d packages. Scanning for gRPC usage...", len(pkgs)) + } + + foundExperimental := false + + for _, pkg := range pkgs { + for _, file := range pkg.Syntax { + ast.Inspect(file, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + + obj := pkg.TypesInfo.Uses[sel.Sel] + if obj == nil || obj.Pkg() == nil { + return true + } + + // Strict filter for gRPC + if !strings.Contains(obj.Pkg().Path(), "google.golang.org/grpc") { + return true + } + + // Check Allowlist + // Construct the signature: PackageName.Symbol (e.g. "grpc.WithResolvers", "resolver.Address") + signature := obj.Pkg().Name() + "." + obj.Name() + if allowList[signature] { + if *debugMode { + log.Printf("Ignoring allowed usage: %s", signature) + } + return true + } + + if *debugMode { + log.Printf("Checking reference: %s", signature) + } + + if isExperimental(pkg.Fset, obj) { + pos := pkg.Fset.Position(sel.Pos()) + fmt.Printf("%s:%d:%d: usage of experimental gRPC API: %s\n", + pos.Filename, pos.Line, pos.Column, signature) + foundExperimental = true + } + + return true + }) + } + } + + if foundExperimental { + os.Exit(1) + } +} + +var ( + experimentalRegex = regexp.MustCompile(`(?i)(#\s*Experimental|All APIs in this package are experimental|This API is EXPERIMENTAL|is currently experimental)`) +) + +func loadAllowList(fpath string) error { + f, err := os.Open(fpath) + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + allowList[line] = true + } + return scanner.Err() +} + +func isExperimental(mainFset *token.FileSet, obj types.Object) bool { + pos := obj.Pos() + if !pos.IsValid() { + return false + } + + // Get the absolute path to the dependency file + position := mainFset.Position(pos) + filename := position.Filename + + // Skip if it's not a Go file (e.g. built-in types might have no file) + if !strings.HasSuffix(filename, ".go") { + return false + } + + if *debugMode { + // Log checking definition + log.Printf(" -> Definition found at: %s:%d", filename, position.Line) + } + + return checkFileForExperimental(filename, position.Line) +} + +// Cached file structure +type cachedFile struct { + f *ast.File + fset *token.FileSet + src []byte + hasDocs bool +} + +var ( + cacheMu sync.RWMutex + advancedCache = make(map[string]*cachedFile) +) + +func getParsedFile(filename string) (*cachedFile, error) { + cacheMu.RLock() + if v, ok := advancedCache[filename]; ok { + cacheMu.RUnlock() + return v, nil + } + cacheMu.RUnlock() + + // Parse the file + fset := token.NewFileSet() + // We read the file content manually to help with debugging if needed + src, err := os.ReadFile(filename) + if err != nil { + if *debugMode { + log.Printf("ERROR reading file %s: %v", filename, err) + } + return nil, err + } + + f, err := parser.ParseFile(fset, filename, src, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + return nil, err + } + + res := &cachedFile{f: f, fset: fset, src: src, hasDocs: f.Doc != nil} + + cacheMu.Lock() + advancedCache[filename] = res + cacheMu.Unlock() + + return res, nil +} + +func checkFileForExperimental(filename string, targetLine int) bool { + cf, err := getParsedFile(filename) + if err != nil { + return false + } + + // check package-level comments + if cf.f.Doc != nil { + if experimentalRegex.MatchString(cf.f.Doc.Text()) { + if *debugMode { + log.Printf(" -> [MATCH] Package experimental (doc in file): %s", filename) + } + return true + } + } + + // check package-level comment in doc.go + // If the current file didn't have the experimental tag, check if a doc.go exists in the same folder + dir := filepath.Dir(filename) + docPath := filepath.Join(dir, "doc.go") + // Only check doc.go if we aren't already looking at it + if docPath != filename { + if docContent, err := os.ReadFile(docPath); err == nil { + if experimentalRegex.Match(docContent) { + if *debugMode { + log.Printf(" -> [MATCH] Package experimental (found in doc.go): %s", docPath) + } + return true + } + } + } + + // check specific object comments + found := false + + ast.Inspect(cf.f, func(n ast.Node) bool { + if found { + return false + } + if n == nil { + return true + } + + // Helper to check a comment group + checkDoc := func(doc *ast.CommentGroup, name string) { + if doc != nil && experimentalRegex.MatchString(doc.Text()) { + found = true + if *debugMode { + log.Printf(" -> [MATCH] Object experimental: %s", name) + } + } + } + + switch decl := n.(type) { + case *ast.FuncDecl: + // Match if the target line is within the function declaration lines + // Actually, we want the definition line exactly, or close to it. + start := cf.fset.Position(decl.Pos()).Line + // The object.Pos() points to the name, not the 'func' keyword, usually. + namePos := cf.fset.Position(decl.Name.Pos()).Line + + if namePos == targetLine || start == targetLine { + checkDoc(decl.Doc, decl.Name.Name) + } + + case *ast.GenDecl: + // GenDecl covers `type X struct`, `var X`, `const X` + // The GenDecl doc applies to all specs inside it usually. + + // If the GenDecl itself starts on the line (e.g. `type ( ...`) + // or if it contains our line. + start := cf.fset.Position(decl.Pos()).Line + end := cf.fset.Position(decl.End()).Line + + if targetLine >= start && targetLine <= end { + // Check the top-level GenDecl doc (e.g. "// Experimental\n var ( ... )") + if decl.Doc != nil && experimentalRegex.MatchString(decl.Doc.Text()) { + found = true + if *debugMode { + log.Printf(" -> [MATCH] GenDecl experimental block around line %d", targetLine) + } + return false + } + + // Check individual specs + for _, spec := range decl.Specs { + switch s := spec.(type) { + case *ast.TypeSpec: + if cf.fset.Position(s.Name.Pos()).Line == targetLine { + checkDoc(s.Doc, s.Name.Name) + } + case *ast.ValueSpec: // var/const + for _, name := range s.Names { + if cf.fset.Position(name.Pos()).Line == targetLine { + checkDoc(s.Doc, name.Name) + } + } + } + } + } + } + return true + }) + + return found +} diff --git a/tools/container-images/README.md b/tools/container-images/README.md new file mode 100644 index 000000000000..dfbc748ffe6b --- /dev/null +++ b/tools/container-images/README.md @@ -0,0 +1,10 @@ +# Container Images + +The container images defined in this directory are used to maintain +depenendencies up to date. **These images are not used to build the project.** + +These images cause [Dependabot] to do a version bump. After that, a [GitHub +action] will update other artifacts in the repository. + +[Dependabot]: ../../.github/dependabot.yml +[GitHub action]: ../../.github/workflows/bump-devcontainer-version.yml diff --git a/tools/container-images/devcontainer/Dockerfile b/tools/container-images/devcontainer/Dockerfile new file mode 100644 index 000000000000..3496e71c69c3 --- /dev/null +++ b/tools/container-images/devcontainer/Dockerfile @@ -0,0 +1 @@ +FROM mcr.microsoft.com/devcontainers/go:dev-1.25-bookworm@sha256:c9a8c52ad9c962655ef43c9e333954b2a4d22c18b184479cefec2833ae02b92e diff --git a/tools/etcd-dump-db/backend.go b/tools/etcd-dump-db/backend.go index 875997ad2476..ef88fb0ba33a 100644 --- a/tools/etcd-dump-db/backend.go +++ b/tools/etcd-dump-db/backend.go @@ -20,6 +20,7 @@ import ( "path/filepath" "go.uber.org/zap" + "google.golang.org/protobuf/proto" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/authpb" @@ -71,7 +72,7 @@ func defaultDecoder(k, v []byte) { func keyDecoder(k, v []byte) { rev := mvcc.BytesToBucketKey(k) var kv mvccpb.KeyValue - if err := kv.Unmarshal(v); err != nil { + if err := proto.Unmarshal(v, &kv); err != nil { panic(err) } fmt.Printf("rev=%+v, value=[key %q | val %q | created %d | mod %d | ver %d]\n", rev, string(kv.Key), string(kv.Value), kv.CreateRevision, kv.ModRevision, kv.Version) @@ -87,7 +88,7 @@ func bytesToLeaseID(bytes []byte) int64 { func leaseDecoder(k, v []byte) { leaseID := bytesToLeaseID(k) var lpb leasepb.Lease - if err := lpb.Unmarshal(v); err != nil { + if err := proto.Unmarshal(v, &lpb); err != nil { panic(err) } fmt.Printf("lease ID=%016x, TTL=%ds, remaining TTL=%ds\n", leaseID, lpb.TTL, lpb.RemainingTTL) @@ -104,7 +105,7 @@ func authDecoder(k, v []byte) { func authRolesDecoder(_, v []byte) { role := &authpb.Role{} - err := role.Unmarshal(v) + err := proto.Unmarshal(v, role) if err != nil { panic(err) } @@ -113,7 +114,7 @@ func authRolesDecoder(_, v []byte) { func authUsersDecoder(_, v []byte) { user := &authpb.User{} - err := user.Unmarshal(v) + err := proto.Unmarshal(v, user) if err != nil { panic(err) } diff --git a/tools/etcd-dump-db/main.go b/tools/etcd-dump-db/main.go index 6be6d83ac739..530e64092732 100644 --- a/tools/etcd-dump-db/main.go +++ b/tools/etcd-dump-db/main.go @@ -24,6 +24,8 @@ import ( "time" "github.com/spf13/cobra" + + "go.etcd.io/etcd/client/pkg/v3/fileutil" ) var ( @@ -85,7 +87,7 @@ func listBucketCommandFunc(_ *cobra.Command, args []string) { if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } - if !existFileOrDir(dp) { + if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } @@ -106,7 +108,7 @@ func iterateBucketCommandFunc(_ *cobra.Command, args []string) { if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } - if !existFileOrDir(dp) { + if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } bucket := args[1] @@ -124,7 +126,7 @@ func scanKeysCommandFunc(_ *cobra.Command, args []string) { if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } - if !existFileOrDir(dp) { + if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } startRev, err := strconv.ParseInt(args[1], 10, 64) @@ -145,7 +147,7 @@ func getHashCommandFunc(_ *cobra.Command, args []string) { if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } - if !existFileOrDir(dp) { + if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } diff --git a/tools/etcd-dump-db/scan.go b/tools/etcd-dump-db/scan.go index 3898459d2608..838ccab11ac1 100644 --- a/tools/etcd-dump-db/scan.go +++ b/tools/etcd-dump-db/scan.go @@ -31,7 +31,7 @@ func scanKeys(dbPath string, startRev int64) error { for pageID := uint64(2); pageID < hwm; { p, _, err := readPage(dbPath, pgSize, pageID) if err != nil { - fmt.Fprintf(os.Stderr, "Reading page %d failed: %v. Continuting...\n", pageID, err) + fmt.Fprintf(os.Stderr, "Reading page %d failed: %v. Continuing...\n", pageID, err) pageID++ continue } diff --git a/tools/etcd-dump-db/utils.go b/tools/etcd-dump-db/utils.go index 184cb5181c47..35ba3d5c10cf 100644 --- a/tools/etcd-dump-db/utils.go +++ b/tools/etcd-dump-db/utils.go @@ -15,15 +15,9 @@ package main import ( - "os" "unsafe" ) -func existFileOrDir(name string) bool { - _, err := os.Stat(name) - return err == nil -} - func unsafeAdd(base unsafe.Pointer, offset uintptr) unsafe.Pointer { return unsafe.Pointer(uintptr(base) + offset) } diff --git a/tools/etcd-dump-logs/README.md b/tools/etcd-dump-logs/README.md index 0922f2878fb1..58ed80df6d6c 100644 --- a/tools/etcd-dump-logs/README.md +++ b/tools/etcd-dump-logs/README.md @@ -52,7 +52,10 @@ Flags: IRRRange, IRRPut, IRRDeleteRange, IRRTxn, IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke -start-index uint - The index to start dumping + The index to start dumping (inclusive) + If unspecified, dumps from the index of the last snapshot. + -end-index uint + The index to stop dumping (exclusive) -start-snap string The base name of snapshot file to start dumping -stream-decoder string @@ -146,11 +149,11 @@ Entry types () count is : 8 ``` #### etcd-dump-logs -start-index