diff --git a/api/v1/api.go b/api/v1/api.go index 7b2981d4b0..004b58232f 100644 --- a/api/v1/api.go +++ b/api/v1/api.go @@ -15,11 +15,12 @@ package v1 import ( "bytes" + "context" "encoding/json" + stdErrors "errors" "io" "net/http" "strconv" - "sync/atomic" "github.com/gin-gonic/gin" "github.com/pingcap/log" @@ -192,25 +193,41 @@ func (o *OpenAPIV1) rebalanceTables(c *gin.Context) { // drainCapture drains all tables from a capture. // Usage: // curl -X PUT http://127.0.0.1:8300/api/v1/captures/drain -// TODO: Implement this API in the future, currently it is a no-op. +// +// It is kept for API v1 compatibility. In the new architecture, `capture_id` is +// treated as a node ID, and the returned `current_table_count` represents the +// remaining drain work rather than a literal table count. func (o *OpenAPIV1) drainCapture(c *gin.Context) { var req drainCaptureRequest if err := c.ShouldBindJSON(&req); err != nil { _ = c.Error(errors.ErrAPIInvalidParam.Wrap(err)) return } - drainCaptureCounter.Add(1) - if drainCaptureCounter.Load()%10 == 0 { - log.Info("api v1 drainCapture", zap.Any("captureID", req.CaptureID), zap.Int64("currentTableCount", drainCaptureCounter.Load())) - c.JSON(http.StatusAccepted, &drainCaptureResp{ - CurrentTableCount: 10, - }) - } else { - log.Info("api v1 drainCapture done", zap.Any("captureID", req.CaptureID), zap.Int64("currentTableCount", drainCaptureCounter.Load())) - c.JSON(http.StatusAccepted, &drainCaptureResp{ - CurrentTableCount: 0, - }) + + coordinator, err := o.server.GetCoordinator() + if err != nil { + _ = c.Error(err) + return + } + drainable, ok := coordinator.(interface { + DrainNode(ctx context.Context, nodeID string) (int, error) + }) + if !ok { + _ = c.Error(stdErrors.New("coordinator does not support node drain")) + return } + + remaining, err := drainable.DrainNode(c.Request.Context(), req.CaptureID) + if err != nil { + _ = c.Error(err) + return + } + log.Info("api v1 drainCapture", + zap.String("captureID", req.CaptureID), + zap.Int("remaining", remaining)) + c.JSON(http.StatusAccepted, &drainCaptureResp{ + CurrentTableCount: remaining, + }) } func getV2ChangefeedConfig(changefeedConfig changefeedConfig) *v2.ChangefeedConfig { @@ -258,8 +275,6 @@ type drainCaptureRequest struct { CaptureID string `json:"capture_id"` } -var drainCaptureCounter atomic.Int64 - // drainCaptureResp is response for manual `DrainCapture` type drainCaptureResp struct { CurrentTableCount int `json:"current_table_count"` diff --git a/api/v2/health.go b/api/v2/health.go index 873fc50e57..b8c87a9829 100644 --- a/api/v2/health.go +++ b/api/v2/health.go @@ -30,7 +30,8 @@ import ( // @Router /api/v2/health [get] func (h *OpenAPIV2) ServerHealth(c *gin.Context) { liveness := h.server.Liveness() - if liveness != api.LivenessCaptureAlive { + // Draining is a pre-offline state and should not be treated as unhealthy. + if liveness != api.LivenessCaptureAlive && liveness != api.LivenessCaptureDraining { err := errors.ErrClusterIsUnhealthy.FastGenByArgs() _ = c.Error(err) return diff --git a/coordinator/controller.go b/coordinator/controller.go index 4285e3624d..e708e4d886 100644 --- a/coordinator/controller.go +++ b/coordinator/controller.go @@ -20,6 +20,8 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/drain" + "github.com/pingcap/ticdc/coordinator/nodeliveness" "github.com/pingcap/ticdc/coordinator/operator" coscheduler "github.com/pingcap/ticdc/coordinator/scheduler" "github.com/pingcap/ticdc/heartbeatpb" @@ -86,6 +88,9 @@ type Controller struct { changefeedChangeCh chan []*changefeedChange apiLock sync.RWMutex + + livenessView *nodeliveness.View + drainController *drain.Controller } type changefeedChange struct { @@ -116,7 +121,9 @@ func NewController( balanceInterval time.Duration, pdClient pd.Client, ) *Controller { + mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) changefeedDB := changefeed.NewChangefeedDB(version) + livenessView := nodeliveness.NewView(30 * time.Second) oc := operator.NewOperatorController(selfNode, changefeedDB, backend, batchSize) c := &Controller{ @@ -129,6 +136,7 @@ func NewController( batchSize, oc, changefeedDB, + livenessView, ), scheduler.BalanceScheduler: coscheduler.NewBalanceScheduler( selfNode.ID.String(), @@ -136,11 +144,19 @@ func NewController( oc, changefeedDB, balanceInterval, + livenessView, + ), + scheduler.DrainScheduler: coscheduler.NewDrainScheduler( + selfNode.ID.String(), + batchSize, + oc, + changefeedDB, + livenessView, ), }), eventCh: eventCh, operatorController: oc, - messageCenter: appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter), + messageCenter: mc, changefeedDB: changefeedDB, nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), taskScheduler: threadpool.NewThreadPoolDefault(), @@ -148,8 +164,10 @@ func NewController( changefeedChangeCh: changefeedChangeCh, pdClient: pdClient, pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), + livenessView: livenessView, } c.nodeChanged.changed = false + c.drainController = drain.NewController(mc, livenessView, changefeedDB, oc) c.bootstrapper = bootstrap.NewBootstrapper[heartbeatpb.CoordinatorBootstrapResponse]( bootstrapperID, @@ -273,6 +291,16 @@ func (c *Controller) onMessage(ctx context.Context, msg *messaging.TargetMessage req := msg.Message[0].(*heartbeatpb.MaintainerHeartbeat) c.handleMaintainerStatus(msg.From, req.Statuses) } + case messaging.TypeNodeHeartbeatRequest: + if c.livenessView != nil { + req := msg.Message[0].(*heartbeatpb.NodeHeartbeat) + c.livenessView.HandleNodeHeartbeat(msg.From, req, time.Now()) + } + case messaging.TypeSetNodeLivenessResponse: + if c.livenessView != nil { + resp := msg.Message[0].(*heartbeatpb.SetNodeLivenessResponse) + c.livenessView.HandleSetNodeLivenessResponse(msg.From, resp, time.Now()) + } case messaging.TypeLogCoordinatorResolvedTsResponse: c.onLogCoordinatorReportResolvedTs(msg) default: @@ -327,6 +355,17 @@ func (c *Controller) RequestResolvedTsFromLogCoordinator(ctx context.Context, ch } } +// DrainNode requests draining a node and returns the remaining drain work. +// +// It is exposed for API v1 compatibility and should be safe to call repeatedly. +func (c *Controller) DrainNode(nodeID node.ID) int { + if c.drainController == nil { + return 1 + } + c.drainController.RequestDrain(nodeID) + return c.drainController.Remaining(nodeID) +} + func (c *Controller) onNodeChanged(ctx context.Context) { addedNodes, removedNodes, requests, responses := c.bootstrapper.HandleNodesChange(c.nodeManager.GetAliveNodes()) log.Info("controller detects node changed", @@ -584,7 +623,8 @@ func (c *Controller) finishBootstrap(ctx context.Context, runningChangefeeds map defer c.taskHandlerMutex.Unlock() c.taskHandlers = append(c.taskHandlers, c.scheduler.Start(c.taskScheduler)...) operatorControllerHandle := c.taskScheduler.Submit(c.operatorController, time.Now()) - c.taskHandlers = append(c.taskHandlers, operatorControllerHandle) + drainControllerHandle := c.taskScheduler.Submit(c.drainController, time.Now()) + c.taskHandlers = append(c.taskHandlers, operatorControllerHandle, drainControllerHandle) c.initialized.Store(true) log.Info("coordinator bootstrapped", zap.Any("nodeID", c.selfNode.ID)) } diff --git a/coordinator/coordinator.go b/coordinator/coordinator.go index b3b7751255..79effe7e34 100644 --- a/coordinator/coordinator.go +++ b/coordinator/coordinator.go @@ -416,6 +416,14 @@ func (c *coordinator) RequestResolvedTsFromLogCoordinator(ctx context.Context, c c.controller.RequestResolvedTsFromLogCoordinator(ctx, changefeedDisplayName) } +// DrainNode requests draining a node and returns the remaining drain work. +// +// This is used by API v1 compatibility handler. It is intentionally not part of +// pkg/server.Coordinator interface to avoid broad interface changes. +func (c *coordinator) DrainNode(_ context.Context, nodeID string) (int, error) { + return c.controller.DrainNode(node.ID(nodeID)), nil +} + func (c *coordinator) sendMessages(msgs []*messaging.TargetMessage) { for _, msg := range msgs { err := c.mc.SendCommand(msg) diff --git a/coordinator/coordinator_test.go b/coordinator/coordinator_test.go index 797cefbf30..598cb83ce9 100644 --- a/coordinator/coordinator_test.go +++ b/coordinator/coordinator_test.go @@ -20,6 +20,7 @@ import ( "fmt" "net" "net/http" + "net/http/httptest" "net/http/pprof" "strconv" "sync" @@ -290,6 +291,19 @@ func (m *mockMaintainerManager) sendHeartbeat() { } } +func newTestNodeWithListener(t *testing.T) (*node.Info, net.Listener) { + t.Helper() + + // Use a random loopback port to avoid collisions when running tests from + // different packages in parallel. + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = lis.Close() }) + + n := node.NewInfo(lis.Addr().String(), "") + return n, lis +} + func TestCoordinatorScheduling(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/debug/pprof/", pprof.Index) @@ -297,9 +311,8 @@ func TestCoordinatorScheduling(t *testing.T) { mux.HandleFunc("/debug/pprof/profile", pprof.Profile) mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - go func() { - t.Fatal(http.ListenAndServe(":38300", mux)) - }() + pprofServer := httptest.NewServer(mux) + defer pprofServer.Close() ctx := context.Background() info := node.NewInfo("127.0.0.1:8700", "") @@ -373,8 +386,10 @@ func TestCoordinatorScheduling(t *testing.T) { } func TestScaleNode(t *testing.T) { - ctx := context.Background() - info := node.NewInfo("127.0.0.1:28300", "") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + info, lis1 := newTestNodeWithListener(t) etcdClient := newMockEtcdClient(string(info.ID)) nodeManager := watcher.NewNodeManager(nil, etcdClient) appcontext.SetService(watcher.NodeManagerName, nodeManager) @@ -382,13 +397,10 @@ func TestScaleNode(t *testing.T) { cfg := config.NewDefaultMessageCenterConfig(info.AdvertiseAddr) mc1 := messaging.NewMessageCenter(ctx, info.ID, cfg, nil) mc1.Run(ctx) - defer func() { - mc1.Close() - log.Info("close message center 1") - }() appcontext.SetService(appcontext.MessageCenter, mc1) - startMaintainerNode(ctx, info, mc1, nodeManager) + n1 := startMaintainerNode(ctx, info, lis1, mc1, nodeManager) + defer n1.stop() serviceID := "default" @@ -426,23 +438,17 @@ func TestScaleNode(t *testing.T) { }, waitTime, time.Millisecond*5) // add two nodes - info2 := node.NewInfo("127.0.0.1:28400", "") + info2, lis2 := newTestNodeWithListener(t) mc2 := messaging.NewMessageCenter(ctx, info2.ID, config.NewDefaultMessageCenterConfig(info2.AdvertiseAddr), nil) mc2.Run(ctx) - defer func() { - mc2.Close() - log.Info("close message center 2") - }() - startMaintainerNode(ctx, info2, mc2, nodeManager) - info3 := node.NewInfo("127.0.0.1:28500", "") + n2 := startMaintainerNode(ctx, info2, lis2, mc2, nodeManager) + defer n2.stop() + + info3, lis3 := newTestNodeWithListener(t) mc3 := messaging.NewMessageCenter(ctx, info3.ID, config.NewDefaultMessageCenterConfig(info3.AdvertiseAddr), nil) mc3.Run(ctx) - defer func() { - mc3.Close() - log.Info("close message center 3") - }() - - startMaintainerNode(ctx, info3, mc3, nodeManager) + n3 := startMaintainerNode(ctx, info3, lis3, mc3, nodeManager) + defer n3.stop() log.Info("Start maintainer node", zap.Stringer("id", info3.ID), @@ -488,7 +494,8 @@ func TestScaleNode(t *testing.T) { func TestBootstrapWithUnStoppedChangefeed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - info := node.NewInfo("127.0.0.1:28301", "") + + info, lis := newTestNodeWithListener(t) etcdClient := newMockEtcdClient(string(info.ID)) nodeManager := watcher.NewNodeManager(nil, etcdClient) appcontext.SetService(watcher.NodeManagerName, nodeManager) @@ -498,10 +505,10 @@ func TestBootstrapWithUnStoppedChangefeed(t *testing.T) { mc1 := messaging.NewMessageCenter(ctx, info.ID, config.NewDefaultMessageCenterConfig(info.AdvertiseAddr), nil) mc1.Run(ctx) - defer mc1.Close() appcontext.SetService(appcontext.MessageCenter, mc1) - mNode := startMaintainerNode(ctx, info, mc1, nodeManager) + mNode := startMaintainerNode(ctx, info, lis, mc1, nodeManager) + defer mNode.stop() removingCf1 := &changefeed.ChangefeedMetaWrapper{ Info: &config.ChangeFeedInfo{ @@ -718,7 +725,9 @@ func (d *maintainNode) stop() { } func startMaintainerNode(ctx context.Context, - node *node.Info, mc messaging.MessageCenter, + node *node.Info, + lis net.Listener, + mc messaging.MessageCenter, nodeManager *watcher.NodeManager, ) *maintainNode { nodeManager.RegisterNodeChangeHandler(node.ID, mc.OnNodeChanges) @@ -729,15 +738,12 @@ func startMaintainerNode(ctx context.Context, grpcServer := grpc.NewServer(opts...) mcs := messaging.NewMessageCenterServer(mc) proto.RegisterMessageServiceServer(grpcServer, mcs) - lis, err := net.Listen("tcp", node.AdvertiseAddr) - if err != nil { - panic(err) - } go func() { _ = grpcServer.Serve(lis) }() _ = maintainerM.Run(ctx) grpcServer.Stop() + _ = lis.Close() }() return &maintainNode{ cancel: cancel, diff --git a/coordinator/drain/controller.go b/coordinator/drain/controller.go new file mode 100644 index 0000000000..5a6276f068 --- /dev/null +++ b/coordinator/drain/controller.go @@ -0,0 +1,245 @@ +package drain + +import ( + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/nodeliveness" + "github.com/pingcap/ticdc/coordinator/operator" + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/messaging" + "github.com/pingcap/ticdc/pkg/node" + "go.uber.org/zap" +) + +const ( + defaultResendInterval = time.Second +) + +type nodeState struct { + drainRequested bool + + lastSendDrain time.Time + lastSendStopping time.Time +} + +// Controller drives node drain by sending SetNodeLiveness requests and computing remaining work. +// +// It is purely in-memory and relies on node-reported liveness to survive coordinator failover. +type Controller struct { + mc messaging.MessageCenter + + livenessView *nodeliveness.View + changefeedDB *changefeed.ChangefeedDB + oc *operator.Controller + + resendInterval time.Duration + + mu sync.Mutex + nodes map[node.ID]*nodeState +} + +func NewController( + mc messaging.MessageCenter, + livenessView *nodeliveness.View, + changefeedDB *changefeed.ChangefeedDB, + oc *operator.Controller, +) *Controller { + return &Controller{ + mc: mc, + livenessView: livenessView, + changefeedDB: changefeedDB, + oc: oc, + resendInterval: defaultResendInterval, + nodes: make(map[node.ID]*nodeState), + } +} + +// RequestDrain marks nodeID as requested to drain and sends SetNodeLiveness(DRAINING) eagerly. +func (c *Controller) RequestDrain(nodeID node.ID) { + now := time.Now() + + c.mu.Lock() + st := c.mustGetStateLocked(nodeID) + st.drainRequested = true + c.mu.Unlock() + + c.sendSetNodeLiveness(nodeID, heartbeatpb.NodeLiveness_DRAINING, now) + c.mu.Lock() + st.lastSendDrain = now + c.mu.Unlock() +} + +// Remaining returns the current drain remaining for nodeID. +func (c *Controller) Remaining(nodeID node.ID) int { + return c.remaining(nodeID, time.Now()) +} + +func (c *Controller) remaining(nodeID node.ID, now time.Time) int { + state := nodeliveness.StateAlive + if c.livenessView != nil { + state = c.livenessView.GetState(nodeID, now) + } + + maintainersOnTarget := 0 + if c.changefeedDB != nil { + maintainersOnTarget = len(c.changefeedDB.GetByNodeID(nodeID)) + } + inflightOps := 0 + if c.oc != nil { + inflightOps = c.oc.CountOperatorsInvolvingNode(nodeID) + } + remaining := maxInt(maintainersOnTarget, inflightOps) + + drainingObserved := state == nodeliveness.StateDraining || state == nodeliveness.StateStopping + stoppingObserved := state == nodeliveness.StateStopping + + // Safety rules: never return 0 before drain is truly proven complete. + if state == nodeliveness.StateUnknown { + remaining = maxInt(remaining, 1) + } + if !drainingObserved { + remaining = maxInt(remaining, 1) + } + + if stoppingObserved && maintainersOnTarget == 0 && inflightOps == 0 { + return 0 + } + if remaining == 0 { + return 1 + } + return remaining +} + +// Execute implements threadpool.Task. +func (c *Controller) Execute() time.Time { + now := time.Now() + c.tick(now) + return now.Add(c.resendInterval) +} + +func (c *Controller) tick(now time.Time) { + if c.livenessView == nil { + return + } + + // Ensure draining/stopping nodes are tracked so drain can continue after coordinator failover. + for _, id := range c.livenessView.GetNodesByState(nodeliveness.StateDraining, now) { + c.mu.Lock() + c.mustGetStateLocked(id) + c.mu.Unlock() + } + for _, id := range c.livenessView.GetNodesByState(nodeliveness.StateStopping, now) { + c.mu.Lock() + c.mustGetStateLocked(id) + c.mu.Unlock() + } + + c.mu.Lock() + ids := make([]node.ID, 0, len(c.nodes)) + for id := range c.nodes { + ids = append(ids, id) + } + c.mu.Unlock() + + for _, id := range ids { + c.tickNode(id, now) + } +} + +func (c *Controller) tickNode(nodeID node.ID, now time.Time) { + state := c.livenessView.GetState(nodeID, now) + + c.mu.Lock() + st := c.nodes[nodeID] + requested := st != nil && st.drainRequested + lastSendDrain := time.Time{} + lastSendStopping := time.Time{} + if st != nil { + lastSendDrain = st.lastSendDrain + lastSendStopping = st.lastSendStopping + } + c.mu.Unlock() + + drainingObserved := state == nodeliveness.StateDraining || state == nodeliveness.StateStopping + stoppingObserved := state == nodeliveness.StateStopping + + if requested && !drainingObserved && now.Sub(lastSendDrain) >= c.resendInterval { + c.sendSetNodeLiveness(nodeID, heartbeatpb.NodeLiveness_DRAINING, now) + c.mu.Lock() + if st := c.nodes[nodeID]; st != nil { + st.lastSendDrain = now + } + c.mu.Unlock() + } + + if stoppingObserved || !drainingObserved { + return + } + + maintainersOnTarget := 0 + if c.changefeedDB != nil { + maintainersOnTarget = len(c.changefeedDB.GetByNodeID(nodeID)) + } + inflightOps := 0 + if c.oc != nil { + inflightOps = c.oc.CountOperatorsInvolvingNode(nodeID) + } + if maintainersOnTarget != 0 || inflightOps != 0 { + return + } + if now.Sub(lastSendStopping) < c.resendInterval { + return + } + + c.sendSetNodeLiveness(nodeID, heartbeatpb.NodeLiveness_STOPPING, now) + c.mu.Lock() + if st := c.nodes[nodeID]; st != nil { + st.lastSendStopping = now + } + c.mu.Unlock() +} + +func (c *Controller) mustGetStateLocked(id node.ID) *nodeState { + st, ok := c.nodes[id] + if !ok { + st = &nodeState{} + c.nodes[id] = st + } + return st +} + +func (c *Controller) sendSetNodeLiveness(nodeID node.ID, target heartbeatpb.NodeLiveness, now time.Time) { + epoch := uint64(0) + if c.livenessView != nil { + if e, ok := c.livenessView.GetNodeEpoch(nodeID); ok { + epoch = e + } + } + + req := &heartbeatpb.SetNodeLivenessRequest{ + Target: target, + NodeEpoch: epoch, + } + msg := messaging.NewSingleTargetMessage(nodeID, messaging.MaintainerManagerTopic, req) + if err := c.mc.SendCommand(msg); err != nil { + log.Warn("send set node liveness request failed", + zap.Stringer("target", nodeID), + zap.String("liveness", target.String()), + zap.Error(err)) + return + } + log.Info("send set node liveness request", + zap.Stringer("target", nodeID), + zap.String("liveness", target.String()), + zap.Uint64("nodeEpoch", epoch)) +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/coordinator/drain/controller_test.go b/coordinator/drain/controller_test.go new file mode 100644 index 0000000000..c7b0c85ddb --- /dev/null +++ b/coordinator/drain/controller_test.go @@ -0,0 +1,105 @@ +package drain + +import ( + "testing" + "time" + + "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/nodeliveness" + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/messaging" + "github.com/pingcap/ticdc/pkg/node" + "github.com/stretchr/testify/require" +) + +func TestRemainingSafetyRules(t *testing.T) { + mc := messaging.NewMockMessageCenter() + view := nodeliveness.NewView(30 * time.Second) + db := changefeed.NewChangefeedDB(1) + + c := NewController(mc, view, db, nil) + target := node.ID("n1") + + now := time.Unix(0, 0) + view.HandleNodeHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_ALIVE, + NodeEpoch: 101, + }, now) + + c.RequestDrain(target) + require.GreaterOrEqual(t, c.Remaining(target), 1) + + msg := recvMessage(t, mc) + require.Equal(t, messaging.TypeSetNodeLivenessRequest, msg.Type) + req := msg.Message[0].(*heartbeatpb.SetNodeLivenessRequest) + require.Equal(t, heartbeatpb.NodeLiveness_DRAINING, req.Target) + require.Equal(t, uint64(101), req.NodeEpoch) + + // Draining observed, but STOPPING not observed => remaining must stay non-zero. + view.HandleSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ + Applied: heartbeatpb.NodeLiveness_DRAINING, + NodeEpoch: 101, + }, now) + require.Equal(t, 1, c.remaining(target, now)) + + // STOPPING observed and no work => remaining can be 0. + view.HandleSetNodeLivenessResponse(target, &heartbeatpb.SetNodeLivenessResponse{ + Applied: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: 101, + }, now) + require.Equal(t, 0, c.remaining(target, now)) +} + +func TestTickPromoteToStoppingWhenQuiescent(t *testing.T) { + mc := messaging.NewMockMessageCenter() + view := nodeliveness.NewView(30 * time.Second) + db := changefeed.NewChangefeedDB(1) + + c := NewController(mc, view, db, nil) + target := node.ID("n1") + + now := time.Unix(0, 0) + view.HandleNodeHeartbeat(target, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_DRAINING, + NodeEpoch: 101, + }, now) + + cfID := common.NewChangeFeedIDWithName("cf1", common.DefaultKeyspaceName) + cf := changefeed.NewChangefeed(cfID, &config.ChangeFeedInfo{ + ChangefeedID: cfID, + Config: config.GetDefaultReplicaConfig(), + SinkURI: "mysql://127.0.0.1:3306", + }, 1, true) + db.AddReplicatingMaintainer(cf, target) + + // Not quiescent => no STOPPING request. + c.tick(now) + select { + case msg := <-mc.GetMessageChannel(): + t.Fatalf("unexpected message: %v", msg) + default: + } + + // Quiescent => should promote to STOPPING. + db.StopByChangefeedID(cfID, true) + c.tick(now.Add(2 * time.Second)) + msg := recvMessage(t, mc) + req := msg.Message[0].(*heartbeatpb.SetNodeLivenessRequest) + require.Equal(t, heartbeatpb.NodeLiveness_STOPPING, req.Target) +} + +func recvMessage(t *testing.T, mc interface { + GetMessageChannel() chan *messaging.TargetMessage +}, +) *messaging.TargetMessage { + t.Helper() + select { + case msg := <-mc.GetMessageChannel(): + return msg + case <-time.After(5 * time.Second): + t.Fatal("timeout waiting message") + return nil + } +} diff --git a/coordinator/nodeliveness/view.go b/coordinator/nodeliveness/view.go new file mode 100644 index 0000000000..70023ccd12 --- /dev/null +++ b/coordinator/nodeliveness/view.go @@ -0,0 +1,190 @@ +package nodeliveness + +import ( + "sync" + "time" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/node" +) + +// State is the coordinator-derived view of a node's liveness. +// +// UNKNOWN is derived when a node's heartbeat exceeds the configured TTL after it has +// ever been observed via node liveness messages. +type State int + +const ( + StateAlive State = iota + StateDraining + StateStopping + StateUnknown +) + +type record struct { + lastSeen time.Time + nodeEpoch uint64 + liveness heartbeatpb.NodeLiveness + everSeenHeartbeat bool +} + +// View maintains an in-memory view of node-reported liveness. +// +// It is used by schedulers to filter destination nodes and by the drain controller +// to drive drain state transitions. +type View struct { + mu sync.RWMutex + ttl time.Duration + data map[node.ID]*record +} + +func NewView(ttl time.Duration) *View { + return &View{ + ttl: ttl, + data: make(map[node.ID]*record), + } +} + +func (v *View) HandleNodeHeartbeat(id node.ID, hb *heartbeatpb.NodeHeartbeat, now time.Time) { + if hb == nil { + return + } + v.mu.Lock() + defer v.mu.Unlock() + + r, ok := v.data[id] + if !ok { + r = &record{} + v.data[id] = r + } + r.lastSeen = now + r.nodeEpoch = hb.NodeEpoch + r.liveness = hb.Liveness + r.everSeenHeartbeat = true +} + +func (v *View) HandleSetNodeLivenessResponse(id node.ID, resp *heartbeatpb.SetNodeLivenessResponse, now time.Time) { + if resp == nil { + return + } + v.mu.Lock() + defer v.mu.Unlock() + + r, ok := v.data[id] + if !ok { + r = &record{} + v.data[id] = r + } + r.lastSeen = now + r.nodeEpoch = resp.NodeEpoch + r.liveness = resp.Applied + r.everSeenHeartbeat = true +} + +// GetState returns a derived liveness state for node id. +// +// Note: Nodes that have never sent node liveness messages are treated as ALIVE for +// compatibility during rollout. +func (v *View) GetState(id node.ID, now time.Time) State { + var ( + lastSeen time.Time + liveness heartbeatpb.NodeLiveness + everSeen bool + ) + v.mu.RLock() + r := v.data[id] + if r != nil { + lastSeen = r.lastSeen + liveness = r.liveness + everSeen = r.everSeenHeartbeat + } + v.mu.RUnlock() + + if r == nil || !everSeen { + return StateAlive + } + if v.ttl > 0 && now.Sub(lastSeen) > v.ttl { + return StateUnknown + } + switch liveness { + case heartbeatpb.NodeLiveness_DRAINING: + return StateDraining + case heartbeatpb.NodeLiveness_STOPPING: + return StateStopping + default: + return StateAlive + } +} + +// GetNodeEpoch returns the last observed node epoch and whether it is known. +func (v *View) GetNodeEpoch(id node.ID) (uint64, bool) { + v.mu.RLock() + defer v.mu.RUnlock() + + r := v.data[id] + if r == nil || !r.everSeenHeartbeat { + return 0, false + } + return r.nodeEpoch, true +} + +// FilterSchedulableDestNodes filters nodes that are eligible to be used as scheduling destinations. +// +// Only ALIVE nodes are eligible. DRAINING/STOPPING/UNKNOWN nodes are excluded. +// Nodes that have never sent node liveness messages are treated as ALIVE. +func (v *View) FilterSchedulableDestNodes(nodes map[node.ID]*node.Info, now time.Time) map[node.ID]*node.Info { + if len(nodes) == 0 { + return map[node.ID]*node.Info{} + } + out := make(map[node.ID]*node.Info, len(nodes)) + for id, info := range nodes { + if v.GetState(id, now) == StateAlive { + out[id] = info + } + } + return out +} + +// FilterSchedulableDestNodeIDs filters node IDs that are eligible to be used as scheduling destinations. +func (v *View) FilterSchedulableDestNodeIDs(ids []node.ID, now time.Time) []node.ID { + if len(ids) == 0 { + return nil + } + out := make([]node.ID, 0, len(ids)) + for _, id := range ids { + if v.GetState(id, now) == StateAlive { + out = append(out, id) + } + } + return out +} + +// GetNodesByState returns node IDs whose derived state equals state. +func (v *View) GetNodesByState(state State, now time.Time) []node.ID { + v.mu.RLock() + defer v.mu.RUnlock() + + out := make([]node.ID, 0) + for id, r := range v.data { + if r == nil || !r.everSeenHeartbeat { + continue + } + s := StateAlive + if v.ttl > 0 && now.Sub(r.lastSeen) > v.ttl { + s = StateUnknown + } else { + switch r.liveness { + case heartbeatpb.NodeLiveness_DRAINING: + s = StateDraining + case heartbeatpb.NodeLiveness_STOPPING: + s = StateStopping + default: + s = StateAlive + } + } + if s == state { + out = append(out, id) + } + } + return out +} diff --git a/coordinator/nodeliveness/view_test.go b/coordinator/nodeliveness/view_test.go new file mode 100644 index 0000000000..ecc8e22ed0 --- /dev/null +++ b/coordinator/nodeliveness/view_test.go @@ -0,0 +1,59 @@ +package nodeliveness + +import ( + "testing" + "time" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/node" + "github.com/stretchr/testify/require" +) + +func TestViewUnknownTTLGuard(t *testing.T) { + v := NewView(30 * time.Second) + n1 := node.ID("n1") + n2 := node.ID("n2") + + now := time.Unix(0, 0) + require.Equal(t, StateAlive, v.GetState(n1, now)) + + v.HandleNodeHeartbeat(n1, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_ALIVE, + NodeEpoch: 1, + }, now) + + require.Equal(t, StateAlive, v.GetState(n1, now.Add(29*time.Second))) + require.Equal(t, StateUnknown, v.GetState(n1, now.Add(31*time.Second))) + + // Never-seen nodes must not become UNKNOWN. + require.Equal(t, StateAlive, v.GetState(n2, now.Add(31*time.Second))) +} + +func TestFilterSchedulableDestNodes(t *testing.T) { + v := NewView(30 * time.Second) + now := time.Unix(0, 0) + + draining := node.ID("draining") + alive := node.ID("alive") + unseen := node.ID("unseen") + + v.HandleNodeHeartbeat(draining, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_DRAINING, + NodeEpoch: 1, + }, now) + v.HandleNodeHeartbeat(alive, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_ALIVE, + NodeEpoch: 1, + }, now) + + nodes := map[node.ID]*node.Info{ + draining: node.NewInfo("127.0.0.1:1", ""), + alive: node.NewInfo("127.0.0.1:2", ""), + unseen: node.NewInfo("127.0.0.1:3", ""), + } + + filtered := v.FilterSchedulableDestNodes(nodes, now) + require.Contains(t, filtered, alive) + require.Contains(t, filtered, unseen) + require.NotContains(t, filtered, draining) +} diff --git a/coordinator/operator/operator_controller.go b/coordinator/operator/operator_controller.go index 2d1842f3a2..df583b83c5 100644 --- a/coordinator/operator/operator_controller.go +++ b/coordinator/operator/operator_controller.go @@ -225,6 +225,31 @@ func (oc *Controller) HasOperator(dispName common.ChangeFeedDisplayName) bool { return false } +// HasOperatorByID returns true if there is an in-flight operator for the changefeed. +func (oc *Controller) HasOperatorByID(id common.ChangeFeedID) bool { + oc.mu.RLock() + defer oc.mu.RUnlock() + _, ok := oc.operators[id] + return ok +} + +// CountOperatorsInvolvingNode counts in-flight operators that have affected nodes containing target. +func (oc *Controller) CountOperatorsInvolvingNode(target node.ID) int { + oc.mu.RLock() + defer oc.mu.RUnlock() + + count := 0 + for _, op := range oc.operators { + for _, n := range op.OP.AffectedNodes() { + if n == target { + count++ + break + } + } + } + return count +} + // OperatorSize returns the number of operators in the controller. func (oc *Controller) OperatorSize() int { oc.mu.RLock() diff --git a/coordinator/scheduler/balance.go b/coordinator/scheduler/balance.go index 4593a95729..c9ec52bad2 100644 --- a/coordinator/scheduler/balance.go +++ b/coordinator/scheduler/balance.go @@ -18,6 +18,7 @@ import ( "time" "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/nodeliveness" "github.com/pingcap/ticdc/coordinator/operator" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/node" @@ -33,6 +34,7 @@ type balanceScheduler struct { operatorController *operator.Controller changefeedDB *changefeed.ChangefeedDB nodeManager *watcher.NodeManager + livenessView *nodeliveness.View random *rand.Rand lastRebalanceTime time.Time @@ -51,6 +53,7 @@ func NewBalanceScheduler( oc *operator.Controller, changefeedDB *changefeed.ChangefeedDB, balanceInterval time.Duration, + livenessView *nodeliveness.View, ) *balanceScheduler { return &balanceScheduler{ id: id, @@ -59,6 +62,7 @@ func NewBalanceScheduler( operatorController: oc, changefeedDB: changefeedDB, nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), + livenessView: livenessView, checkBalanceInterval: balanceInterval, lastRebalanceTime: time.Now(), } @@ -76,13 +80,17 @@ func (s *balanceScheduler) Execute() time.Time { } // check the balance status - moveSize := pkgScheduler.CheckBalanceStatus(s.changefeedDB.GetTaskSizePerNode(), s.nodeManager.GetAliveNodes()) + activeNodes := s.nodeManager.GetAliveNodes() + if s.livenessView != nil { + activeNodes = s.livenessView.FilterSchedulableDestNodes(activeNodes, now) + } + moveSize := pkgScheduler.CheckBalanceStatus(s.changefeedDB.GetTaskSizePerNode(), activeNodes) if moveSize <= 0 { // fast check the balance status, no need to do the balance,skip return now.Add(s.checkBalanceInterval) } // balance changefeeds among the active nodes - movedSize := pkgScheduler.Balance(s.batchSize, s.random, s.nodeManager.GetAliveNodes(), s.changefeedDB.GetReplicating(), + movedSize := pkgScheduler.Balance(s.batchSize, s.random, activeNodes, s.changefeedDB.GetReplicating(), func(cf *changefeed.Changefeed, nodeID node.ID) bool { return s.operatorController.AddOperator(operator.NewMoveMaintainerOperator(s.changefeedDB, cf, cf.GetNodeID(), nodeID)) }) diff --git a/coordinator/scheduler/basic.go b/coordinator/scheduler/basic.go index 301a75f86d..89ddf04fff 100644 --- a/coordinator/scheduler/basic.go +++ b/coordinator/scheduler/basic.go @@ -17,6 +17,7 @@ import ( "time" "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/nodeliveness" "github.com/pingcap/ticdc/coordinator/operator" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/node" @@ -35,12 +36,14 @@ type basicScheduler struct { operatorController *operator.Controller changefeedDB *changefeed.ChangefeedDB nodeManager *watcher.NodeManager + livenessView *nodeliveness.View } func NewBasicScheduler( id string, batchSize int, oc *operator.Controller, changefeedDB *changefeed.ChangefeedDB, + livenessView *nodeliveness.View, ) *basicScheduler { return &basicScheduler{ id: id, @@ -48,6 +51,7 @@ func NewBasicScheduler( operatorController: oc, changefeedDB: changefeedDB, nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), + livenessView: livenessView, } } @@ -74,10 +78,14 @@ func (s *basicScheduler) doBasicSchedule(availableSize int) { absentChangefeeds := s.changefeedDB.GetAbsentByGroup(id, availableSize) nodeTaskSize := s.changefeedDB.GetTaskSizePerNodeByGroup(id) // add the absent node to the node size map + now := time.Now() nodeIDs := s.nodeManager.GetAliveNodeIDs() + if s.livenessView != nil { + nodeIDs = s.livenessView.FilterSchedulableDestNodeIDs(nodeIDs, now) + } nodeSize := make(map[node.ID]int) - for _, id := range nodeIDs { - nodeSize[id] = nodeTaskSize[id] + for _, nodeID := range nodeIDs { + nodeSize[nodeID] = nodeTaskSize[nodeID] } pkgScheduler.BasicSchedule(availableSize, absentChangefeeds, nodeSize, func(cf *changefeed.Changefeed, nodeID node.ID) bool { diff --git a/coordinator/scheduler/drain.go b/coordinator/scheduler/drain.go new file mode 100644 index 0000000000..82108b5b1c --- /dev/null +++ b/coordinator/scheduler/drain.go @@ -0,0 +1,124 @@ +package scheduler + +import ( + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/nodeliveness" + "github.com/pingcap/ticdc/coordinator/operator" + appcontext "github.com/pingcap/ticdc/pkg/common/context" + "github.com/pingcap/ticdc/pkg/node" + "github.com/pingcap/ticdc/server/watcher" + "go.uber.org/zap" +) + +const drainCheckInterval = 200 * time.Millisecond + +// drainScheduler moves maintainers out of nodes in DRAINING state. +type drainScheduler struct { + id string + batchSize int + + operatorController *operator.Controller + changefeedDB *changefeed.ChangefeedDB + nodeManager *watcher.NodeManager + livenessView *nodeliveness.View + + rrIndex int +} + +func NewDrainScheduler( + id string, + batchSize int, + oc *operator.Controller, + changefeedDB *changefeed.ChangefeedDB, + livenessView *nodeliveness.View, +) *drainScheduler { + return &drainScheduler{ + id: id, + batchSize: batchSize, + operatorController: oc, + changefeedDB: changefeedDB, + nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), + livenessView: livenessView, + } +} + +func (s *drainScheduler) Name() string { + return "drain-scheduler" +} + +func (s *drainScheduler) Execute() time.Time { + if s.livenessView == nil { + return time.Now().Add(drainCheckInterval) + } + + now := time.Now() + drainingNodes := s.livenessView.GetNodesByState(nodeliveness.StateDraining, now) + if len(drainingNodes) == 0 { + return now.Add(time.Second) + } + + destNodes := s.livenessView.FilterSchedulableDestNodes(s.nodeManager.GetAliveNodes(), now) + if len(destNodes) == 0 { + return now.Add(time.Second) + } + + nodeTaskSize := s.changefeedDB.GetTaskSizePerNode() + + scheduled := 0 + for i := 0; i < len(drainingNodes) && scheduled < s.batchSize; i++ { + origin := drainingNodes[(s.rrIndex+i)%len(drainingNodes)] + changefeeds := s.changefeedDB.GetByNodeID(origin) + if len(changefeeds) == 0 { + continue + } + + for _, cf := range changefeeds { + if scheduled >= s.batchSize { + break + } + if s.operatorController.HasOperatorByID(cf.ID) { + continue + } + + dest := pickLeastLoadedNode(destNodes, nodeTaskSize) + if dest == "" { + log.Info("no schedulable destination node for drain", + zap.Stringer("origin", origin)) + return now.Add(time.Second) + } + + if !s.operatorController.AddOperator(operator.NewMoveMaintainerOperator(s.changefeedDB, cf, origin, dest)) { + continue + } + nodeTaskSize[dest]++ + scheduled++ + } + } + + s.rrIndex++ + if len(drainingNodes) > 0 { + s.rrIndex %= len(drainingNodes) + } + return now.Add(drainCheckInterval) +} + +func pickLeastLoadedNode(nodes map[node.ID]*node.Info, nodeTaskSize map[node.ID]int) node.ID { + var ( + minNode node.ID + minLoad int + ) + for id := range nodes { + load, ok := nodeTaskSize[id] + if !ok { + load = 0 + } + if minNode == "" || load < minLoad { + minNode = id + minLoad = load + } + } + return minNode +} diff --git a/coordinator/scheduler/drain_test.go b/coordinator/scheduler/drain_test.go new file mode 100644 index 0000000000..66cbcb79b4 --- /dev/null +++ b/coordinator/scheduler/drain_test.go @@ -0,0 +1,68 @@ +package scheduler + +import ( + "testing" + "time" + + "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/nodeliveness" + "github.com/pingcap/ticdc/coordinator/operator" + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/common" + appcontext "github.com/pingcap/ticdc/pkg/common/context" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/messaging" + "github.com/pingcap/ticdc/pkg/node" + "github.com/pingcap/ticdc/server/watcher" + "github.com/stretchr/testify/require" +) + +func TestDrainSchedulerSkipsChangefeedsWithInFlightOperators(t *testing.T) { + changefeedDB := changefeed.NewChangefeedDB(1216) + self := node.NewInfo("127.0.0.1:1", "") + origin := node.ID("origin") + dest := node.ID("dest") + + nodeManager := watcher.NewNodeManager(nil, nil) + nodeManager.GetAliveNodes()[origin] = node.NewInfo("127.0.0.1:2", "") + nodeManager.GetAliveNodes()[dest] = node.NewInfo("127.0.0.1:3", "") + + mc := messaging.NewMockMessageCenter() + appcontext.SetService(appcontext.MessageCenter, mc) + appcontext.SetService(watcher.NodeManagerName, nodeManager) + + oc := operator.NewOperatorController(self, changefeedDB, nil, 10) + + cf1ID := common.NewChangeFeedIDWithName("cf1", common.DefaultKeyspaceName) + cf1 := changefeed.NewChangefeed(cf1ID, &config.ChangeFeedInfo{ + ChangefeedID: cf1ID, + Config: config.GetDefaultReplicaConfig(), + SinkURI: "mysql://127.0.0.1:3306", + }, 1, true) + changefeedDB.AddReplicatingMaintainer(cf1, origin) + + cf2ID := common.NewChangeFeedIDWithName("cf2", common.DefaultKeyspaceName) + cf2 := changefeed.NewChangefeed(cf2ID, &config.ChangeFeedInfo{ + ChangefeedID: cf2ID, + Config: config.GetDefaultReplicaConfig(), + SinkURI: "mysql://127.0.0.1:3306", + }, 1, true) + changefeedDB.AddReplicatingMaintainer(cf2, origin) + + require.True(t, oc.AddOperator(operator.NewMoveMaintainerOperator(changefeedDB, cf1, origin, dest))) + + view := nodeliveness.NewView(30 * time.Second) + view.HandleNodeHeartbeat(origin, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_DRAINING, + NodeEpoch: 1, + }, time.Now()) + + s := NewDrainScheduler("test", 10, oc, changefeedDB, view) + s.Execute() + + require.Equal(t, 2, oc.OperatorSize()) + op := oc.GetOperator(cf2ID) + require.NotNil(t, op) + require.Equal(t, "move", op.Type()) + require.ElementsMatch(t, []node.ID{origin, dest}, op.AffectedNodes()) +} diff --git a/heartbeatpb/heartbeat.pb.go b/heartbeatpb/heartbeat.pb.go index 10f1a63dd5..e1901f738e 100644 --- a/heartbeatpb/heartbeat.pb.go +++ b/heartbeatpb/heartbeat.pb.go @@ -73,6 +73,40 @@ func (ScheduleAction) EnumDescriptor() ([]byte, []int) { return fileDescriptor_6d584080fdadb670, []int{1} } +// NodeLiveness is the node-level liveness state reported by a NodeAgent. +// +// It is used by the coordinator to: +// - filter destination candidates for scheduling +// - drive node drain progress +// - avoid campaigning/residing leadership on nodes preparing to go offline +type NodeLiveness int32 + +const ( + NodeLiveness_ALIVE NodeLiveness = 0 + NodeLiveness_DRAINING NodeLiveness = 1 + NodeLiveness_STOPPING NodeLiveness = 2 +) + +var NodeLiveness_name = map[int32]string{ + 0: "ALIVE", + 1: "DRAINING", + 2: "STOPPING", +} + +var NodeLiveness_value = map[string]int32{ + "ALIVE": 0, + "DRAINING": 1, + "STOPPING": 2, +} + +func (x NodeLiveness) String() string { + return proto.EnumName(NodeLiveness_name, int32(x)) +} + +func (NodeLiveness) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{2} +} + type BlockStage int32 const ( @@ -101,7 +135,7 @@ func (x BlockStage) String() string { } func (BlockStage) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{2} + return fileDescriptor_6d584080fdadb670, []int{3} } type InfluenceType int32 @@ -129,7 +163,7 @@ func (x InfluenceType) String() string { } func (InfluenceType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{3} + return fileDescriptor_6d584080fdadb670, []int{4} } type ComponentState int32 @@ -169,7 +203,7 @@ func (x ComponentState) String() string { } func (ComponentState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{4} + return fileDescriptor_6d584080fdadb670, []int{5} } type ChecksumState int32 @@ -201,7 +235,7 @@ func (x ChecksumState) String() string { } func (ChecksumState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{5} + return fileDescriptor_6d584080fdadb670, []int{6} } type TableSpan struct { @@ -1209,6 +1243,165 @@ func (m *MaintainerHeartbeat) GetStatuses() []*MaintainerStatus { return nil } +// NodeHeartbeat is a node-scoped heartbeat, independent of maintainer heartbeats. +type NodeHeartbeat struct { + Liveness NodeLiveness `protobuf:"varint,1,opt,name=liveness,proto3,enum=heartbeatpb.NodeLiveness" json:"liveness,omitempty"` + NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"` +} + +func (m *NodeHeartbeat) Reset() { *m = NodeHeartbeat{} } +func (m *NodeHeartbeat) String() string { return proto.CompactTextString(m) } +func (*NodeHeartbeat) ProtoMessage() {} +func (*NodeHeartbeat) Descriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{16} +} +func (m *NodeHeartbeat) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NodeHeartbeat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NodeHeartbeat.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NodeHeartbeat) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeHeartbeat.Merge(m, src) +} +func (m *NodeHeartbeat) XXX_Size() int { + return m.Size() +} +func (m *NodeHeartbeat) XXX_DiscardUnknown() { + xxx_messageInfo_NodeHeartbeat.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeHeartbeat proto.InternalMessageInfo + +func (m *NodeHeartbeat) GetLiveness() NodeLiveness { + if m != nil { + return m.Liveness + } + return NodeLiveness_ALIVE +} + +func (m *NodeHeartbeat) GetNodeEpoch() uint64 { + if m != nil { + return m.NodeEpoch + } + return 0 +} + +// SetNodeLivenessRequest asks a node to upgrade its local liveness monotonically. +type SetNodeLivenessRequest struct { + Target NodeLiveness `protobuf:"varint,1,opt,name=target,proto3,enum=heartbeatpb.NodeLiveness" json:"target,omitempty"` + NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"` +} + +func (m *SetNodeLivenessRequest) Reset() { *m = SetNodeLivenessRequest{} } +func (m *SetNodeLivenessRequest) String() string { return proto.CompactTextString(m) } +func (*SetNodeLivenessRequest) ProtoMessage() {} +func (*SetNodeLivenessRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{17} +} +func (m *SetNodeLivenessRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SetNodeLivenessRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SetNodeLivenessRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SetNodeLivenessRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetNodeLivenessRequest.Merge(m, src) +} +func (m *SetNodeLivenessRequest) XXX_Size() int { + return m.Size() +} +func (m *SetNodeLivenessRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SetNodeLivenessRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SetNodeLivenessRequest proto.InternalMessageInfo + +func (m *SetNodeLivenessRequest) GetTarget() NodeLiveness { + if m != nil { + return m.Target + } + return NodeLiveness_ALIVE +} + +func (m *SetNodeLivenessRequest) GetNodeEpoch() uint64 { + if m != nil { + return m.NodeEpoch + } + return 0 +} + +// SetNodeLivenessResponse reports the liveness applied by the node and its current epoch. +type SetNodeLivenessResponse struct { + Applied NodeLiveness `protobuf:"varint,1,opt,name=applied,proto3,enum=heartbeatpb.NodeLiveness" json:"applied,omitempty"` + NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"` +} + +func (m *SetNodeLivenessResponse) Reset() { *m = SetNodeLivenessResponse{} } +func (m *SetNodeLivenessResponse) String() string { return proto.CompactTextString(m) } +func (*SetNodeLivenessResponse) ProtoMessage() {} +func (*SetNodeLivenessResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{18} +} +func (m *SetNodeLivenessResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SetNodeLivenessResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SetNodeLivenessResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SetNodeLivenessResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetNodeLivenessResponse.Merge(m, src) +} +func (m *SetNodeLivenessResponse) XXX_Size() int { + return m.Size() +} +func (m *SetNodeLivenessResponse) XXX_DiscardUnknown() { + xxx_messageInfo_SetNodeLivenessResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_SetNodeLivenessResponse proto.InternalMessageInfo + +func (m *SetNodeLivenessResponse) GetApplied() NodeLiveness { + if m != nil { + return m.Applied + } + return NodeLiveness_ALIVE +} + +func (m *SetNodeLivenessResponse) GetNodeEpoch() uint64 { + if m != nil { + return m.NodeEpoch + } + return 0 +} + type MaintainerStatus struct { ChangefeedID *ChangefeedID `protobuf:"bytes,1,opt,name=changefeedID,proto3" json:"changefeedID,omitempty"` FeedState string `protobuf:"bytes,2,opt,name=feed_state,json=feedState,proto3" json:"feed_state,omitempty"` @@ -1223,7 +1416,7 @@ func (m *MaintainerStatus) Reset() { *m = MaintainerStatus{} } func (m *MaintainerStatus) String() string { return proto.CompactTextString(m) } func (*MaintainerStatus) ProtoMessage() {} func (*MaintainerStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{16} + return fileDescriptor_6d584080fdadb670, []int{19} } func (m *MaintainerStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1309,7 +1502,7 @@ func (m *CoordinatorBootstrapRequest) Reset() { *m = CoordinatorBootstra func (m *CoordinatorBootstrapRequest) String() string { return proto.CompactTextString(m) } func (*CoordinatorBootstrapRequest) ProtoMessage() {} func (*CoordinatorBootstrapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{17} + return fileDescriptor_6d584080fdadb670, []int{20} } func (m *CoordinatorBootstrapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1353,7 +1546,7 @@ func (m *CoordinatorBootstrapResponse) Reset() { *m = CoordinatorBootstr func (m *CoordinatorBootstrapResponse) String() string { return proto.CompactTextString(m) } func (*CoordinatorBootstrapResponse) ProtoMessage() {} func (*CoordinatorBootstrapResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{18} + return fileDescriptor_6d584080fdadb670, []int{21} } func (m *CoordinatorBootstrapResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1401,7 +1594,7 @@ func (m *AddMaintainerRequest) Reset() { *m = AddMaintainerRequest{} } func (m *AddMaintainerRequest) String() string { return proto.CompactTextString(m) } func (*AddMaintainerRequest) ProtoMessage() {} func (*AddMaintainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{19} + return fileDescriptor_6d584080fdadb670, []int{22} } func (m *AddMaintainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1476,7 +1669,7 @@ func (m *RemoveMaintainerRequest) Reset() { *m = RemoveMaintainerRequest func (m *RemoveMaintainerRequest) String() string { return proto.CompactTextString(m) } func (*RemoveMaintainerRequest) ProtoMessage() {} func (*RemoveMaintainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{20} + return fileDescriptor_6d584080fdadb670, []int{23} } func (m *RemoveMaintainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1547,7 +1740,7 @@ func (m *MaintainerBootstrapRequest) Reset() { *m = MaintainerBootstrapR func (m *MaintainerBootstrapRequest) String() string { return proto.CompactTextString(m) } func (*MaintainerBootstrapRequest) ProtoMessage() {} func (*MaintainerBootstrapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{21} + return fileDescriptor_6d584080fdadb670, []int{24} } func (m *MaintainerBootstrapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1647,7 +1840,7 @@ func (m *MaintainerBootstrapResponse) Reset() { *m = MaintainerBootstrap func (m *MaintainerBootstrapResponse) String() string { return proto.CompactTextString(m) } func (*MaintainerBootstrapResponse) ProtoMessage() {} func (*MaintainerBootstrapResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{22} + return fileDescriptor_6d584080fdadb670, []int{25} } func (m *MaintainerBootstrapResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1722,7 +1915,7 @@ func (m *MaintainerPostBootstrapRequest) Reset() { *m = MaintainerPostBo func (m *MaintainerPostBootstrapRequest) String() string { return proto.CompactTextString(m) } func (*MaintainerPostBootstrapRequest) ProtoMessage() {} func (*MaintainerPostBootstrapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{23} + return fileDescriptor_6d584080fdadb670, []int{26} } func (m *MaintainerPostBootstrapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1789,7 +1982,7 @@ func (m *MaintainerPostBootstrapResponse) Reset() { *m = MaintainerPostB func (m *MaintainerPostBootstrapResponse) String() string { return proto.CompactTextString(m) } func (*MaintainerPostBootstrapResponse) ProtoMessage() {} func (*MaintainerPostBootstrapResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{24} + return fileDescriptor_6d584080fdadb670, []int{27} } func (m *MaintainerPostBootstrapResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1849,7 +2042,7 @@ func (m *SchemaInfo) Reset() { *m = SchemaInfo{} } func (m *SchemaInfo) String() string { return proto.CompactTextString(m) } func (*SchemaInfo) ProtoMessage() {} func (*SchemaInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{25} + return fileDescriptor_6d584080fdadb670, []int{28} } func (m *SchemaInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1908,7 +2101,7 @@ func (m *TableInfo) Reset() { *m = TableInfo{} } func (m *TableInfo) String() string { return proto.CompactTextString(m) } func (*TableInfo) ProtoMessage() {} func (*TableInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{26} + return fileDescriptor_6d584080fdadb670, []int{29} } func (m *TableInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1965,7 +2158,7 @@ func (m *BootstrapTableSpan) Reset() { *m = BootstrapTableSpan{} } func (m *BootstrapTableSpan) String() string { return proto.CompactTextString(m) } func (*BootstrapTableSpan) ProtoMessage() {} func (*BootstrapTableSpan) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{27} + return fileDescriptor_6d584080fdadb670, []int{30} } func (m *BootstrapTableSpan) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2053,7 +2246,7 @@ func (m *MaintainerCloseRequest) Reset() { *m = MaintainerCloseRequest{} func (m *MaintainerCloseRequest) String() string { return proto.CompactTextString(m) } func (*MaintainerCloseRequest) ProtoMessage() {} func (*MaintainerCloseRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{28} + return fileDescriptor_6d584080fdadb670, []int{31} } func (m *MaintainerCloseRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2105,7 +2298,7 @@ func (m *MaintainerCloseResponse) Reset() { *m = MaintainerCloseResponse func (m *MaintainerCloseResponse) String() string { return proto.CompactTextString(m) } func (*MaintainerCloseResponse) ProtoMessage() {} func (*MaintainerCloseResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{29} + return fileDescriptor_6d584080fdadb670, []int{32} } func (m *MaintainerCloseResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2160,7 +2353,7 @@ func (m *InfluencedTables) Reset() { *m = InfluencedTables{} } func (m *InfluencedTables) String() string { return proto.CompactTextString(m) } func (*InfluencedTables) ProtoMessage() {} func (*InfluencedTables) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{30} + return fileDescriptor_6d584080fdadb670, []int{33} } func (m *InfluencedTables) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2220,7 +2413,7 @@ func (m *Table) Reset() { *m = Table{} } func (m *Table) String() string { return proto.CompactTextString(m) } func (*Table) ProtoMessage() {} func (*Table) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{31} + return fileDescriptor_6d584080fdadb670, []int{34} } func (m *Table) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2280,7 +2473,7 @@ func (m *SchemaIDChange) Reset() { *m = SchemaIDChange{} } func (m *SchemaIDChange) String() string { return proto.CompactTextString(m) } func (*SchemaIDChange) ProtoMessage() {} func (*SchemaIDChange) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{32} + return fileDescriptor_6d584080fdadb670, []int{35} } func (m *SchemaIDChange) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2345,7 +2538,7 @@ func (m *State) Reset() { *m = State{} } func (m *State) String() string { return proto.CompactTextString(m) } func (*State) ProtoMessage() {} func (*State) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{33} + return fileDescriptor_6d584080fdadb670, []int{36} } func (m *State) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2440,7 +2633,7 @@ func (m *TableSpanBlockStatus) Reset() { *m = TableSpanBlockStatus{} } func (m *TableSpanBlockStatus) String() string { return proto.CompactTextString(m) } func (*TableSpanBlockStatus) ProtoMessage() {} func (*TableSpanBlockStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{34} + return fileDescriptor_6d584080fdadb670, []int{37} } func (m *TableSpanBlockStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2502,7 +2695,7 @@ func (m *TableSpanStatus) Reset() { *m = TableSpanStatus{} } func (m *TableSpanStatus) String() string { return proto.CompactTextString(m) } func (*TableSpanStatus) ProtoMessage() {} func (*TableSpanStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{35} + return fileDescriptor_6d584080fdadb670, []int{38} } func (m *TableSpanStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2576,7 +2769,7 @@ func (m *BlockStatusRequest) Reset() { *m = BlockStatusRequest{} } func (m *BlockStatusRequest) String() string { return proto.CompactTextString(m) } func (*BlockStatusRequest) ProtoMessage() {} func (*BlockStatusRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{36} + return fileDescriptor_6d584080fdadb670, []int{39} } func (m *BlockStatusRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2637,7 +2830,7 @@ func (m *RunningError) Reset() { *m = RunningError{} } func (m *RunningError) String() string { return proto.CompactTextString(m) } func (*RunningError) ProtoMessage() {} func (*RunningError) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{37} + return fileDescriptor_6d584080fdadb670, []int{40} } func (m *RunningError) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2703,7 +2896,7 @@ func (m *DispatcherID) Reset() { *m = DispatcherID{} } func (m *DispatcherID) String() string { return proto.CompactTextString(m) } func (*DispatcherID) ProtoMessage() {} func (*DispatcherID) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{38} + return fileDescriptor_6d584080fdadb670, []int{41} } func (m *DispatcherID) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2757,7 +2950,7 @@ func (m *ChangefeedID) Reset() { *m = ChangefeedID{} } func (m *ChangefeedID) String() string { return proto.CompactTextString(m) } func (*ChangefeedID) ProtoMessage() {} func (*ChangefeedID) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{39} + return fileDescriptor_6d584080fdadb670, []int{42} } func (m *ChangefeedID) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2822,7 +3015,7 @@ func (m *LogCoordinatorResolvedTsRequest) Reset() { *m = LogCoordinatorR func (m *LogCoordinatorResolvedTsRequest) String() string { return proto.CompactTextString(m) } func (*LogCoordinatorResolvedTsRequest) ProtoMessage() {} func (*LogCoordinatorResolvedTsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{40} + return fileDescriptor_6d584080fdadb670, []int{43} } func (m *LogCoordinatorResolvedTsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2867,7 +3060,7 @@ func (m *LogCoordinatorResolvedTsResponse) Reset() { *m = LogCoordinator func (m *LogCoordinatorResolvedTsResponse) String() string { return proto.CompactTextString(m) } func (*LogCoordinatorResolvedTsResponse) ProtoMessage() {} func (*LogCoordinatorResolvedTsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{41} + return fileDescriptor_6d584080fdadb670, []int{44} } func (m *LogCoordinatorResolvedTsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2921,7 +3114,7 @@ func (m *ChecksumMeta) Reset() { *m = ChecksumMeta{} } func (m *ChecksumMeta) String() string { return proto.CompactTextString(m) } func (*ChecksumMeta) ProtoMessage() {} func (*ChecksumMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{42} + return fileDescriptor_6d584080fdadb670, []int{45} } func (m *ChecksumMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2976,7 +3169,7 @@ func (m *DispatcherSetChecksum) Reset() { *m = DispatcherSetChecksum{} } func (m *DispatcherSetChecksum) String() string { return proto.CompactTextString(m) } func (*DispatcherSetChecksum) ProtoMessage() {} func (*DispatcherSetChecksum) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{43} + return fileDescriptor_6d584080fdadb670, []int{46} } func (m *DispatcherSetChecksum) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3053,7 +3246,7 @@ func (m *DispatcherSetChecksumAckResponse) Reset() { *m = DispatcherSetC func (m *DispatcherSetChecksumAckResponse) String() string { return proto.CompactTextString(m) } func (*DispatcherSetChecksumAckResponse) ProtoMessage() {} func (*DispatcherSetChecksumAckResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{44} + return fileDescriptor_6d584080fdadb670, []int{47} } func (m *DispatcherSetChecksumAckResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3124,7 +3317,7 @@ func (m *DispatcherSetChecksumUpdateRequest) Reset() { *m = DispatcherSe func (m *DispatcherSetChecksumUpdateRequest) String() string { return proto.CompactTextString(m) } func (*DispatcherSetChecksumUpdateRequest) ProtoMessage() {} func (*DispatcherSetChecksumUpdateRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{45} + return fileDescriptor_6d584080fdadb670, []int{48} } func (m *DispatcherSetChecksumUpdateRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3191,6 +3384,7 @@ func (m *DispatcherSetChecksumUpdateRequest) GetChecksum() *DispatcherSetChecksu func init() { proto.RegisterEnum("heartbeatpb.Action", Action_name, Action_value) proto.RegisterEnum("heartbeatpb.ScheduleAction", ScheduleAction_name, ScheduleAction_value) + proto.RegisterEnum("heartbeatpb.NodeLiveness", NodeLiveness_name, NodeLiveness_value) proto.RegisterEnum("heartbeatpb.BlockStage", BlockStage_name, BlockStage_value) proto.RegisterEnum("heartbeatpb.InfluenceType", InfluenceType_name, InfluenceType_value) proto.RegisterEnum("heartbeatpb.ComponentState", ComponentState_name, ComponentState_value) @@ -3211,6 +3405,9 @@ func init() { proto.RegisterType((*ScheduleDispatcherRequest)(nil), "heartbeatpb.ScheduleDispatcherRequest") proto.RegisterType((*MergeDispatcherRequest)(nil), "heartbeatpb.MergeDispatcherRequest") proto.RegisterType((*MaintainerHeartbeat)(nil), "heartbeatpb.MaintainerHeartbeat") + proto.RegisterType((*NodeHeartbeat)(nil), "heartbeatpb.NodeHeartbeat") + proto.RegisterType((*SetNodeLivenessRequest)(nil), "heartbeatpb.SetNodeLivenessRequest") + proto.RegisterType((*SetNodeLivenessResponse)(nil), "heartbeatpb.SetNodeLivenessResponse") proto.RegisterType((*MaintainerStatus)(nil), "heartbeatpb.MaintainerStatus") proto.RegisterType((*CoordinatorBootstrapRequest)(nil), "heartbeatpb.CoordinatorBootstrapRequest") proto.RegisterType((*CoordinatorBootstrapResponse)(nil), "heartbeatpb.CoordinatorBootstrapResponse") @@ -3246,156 +3443,163 @@ func init() { func init() { proto.RegisterFile("heartbeatpb/heartbeat.proto", fileDescriptor_6d584080fdadb670) } var fileDescriptor_6d584080fdadb670 = []byte{ - // 2376 bytes of a gzipped FileDescriptorProto + // 2493 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x1a, 0x4d, 0x6f, 0x1c, 0x49, - 0xd5, 0xdd, 0x3d, 0x33, 0x9e, 0x79, 0xe3, 0xb1, 0x3b, 0x95, 0x8f, 0x75, 0x12, 0xc7, 0xf1, 0x36, - 0x20, 0x19, 0xef, 0x92, 0x90, 0xec, 0x46, 0xc0, 0xb2, 0x6c, 0xb0, 0x67, 0xb2, 0x9b, 0x91, 0x63, - 0xaf, 0x55, 0xe3, 0x55, 0x60, 0x39, 0x0c, 0xed, 0xee, 0xca, 0xb8, 0xe5, 0x99, 0xae, 0x4e, 0x57, - 0x4f, 0x1c, 0x47, 0x5a, 0x10, 0x42, 0xdc, 0x38, 0x80, 0xe0, 0xc2, 0x81, 0x03, 0xe2, 0x0f, 0x20, - 0x21, 0x8e, 0x48, 0xdc, 0x40, 0xe2, 0xb2, 0xa7, 0x15, 0x37, 0x50, 0xf2, 0x07, 0x10, 0x1c, 0xb8, - 0xa2, 0xaa, 0xae, 0xea, 0xaf, 0x69, 0xdb, 0x89, 0x3c, 0xca, 0xad, 0x5f, 0xd5, 0x7b, 0xaf, 0x5e, - 0xbd, 0xef, 0x57, 0x33, 0x70, 0x75, 0x9f, 0xd8, 0x61, 0xb4, 0x47, 0xec, 0x28, 0xd8, 0xbb, 0x99, - 0x7c, 0xdf, 0x08, 0x42, 0x1a, 0x51, 0xd4, 0xcc, 0x6c, 0x5a, 0x47, 0xd0, 0xd8, 0xb5, 0xf7, 0x86, - 0xa4, 0x17, 0xd8, 0x3e, 0x5a, 0x84, 0x59, 0x01, 0x74, 0x3b, 0x8b, 0xda, 0x8a, 0xb6, 0x6a, 0x60, - 0x05, 0xa2, 0x2b, 0x50, 0xef, 0x45, 0x76, 0x18, 0x6d, 0x92, 0xa3, 0x45, 0x7d, 0x45, 0x5b, 0x9d, - 0xc3, 0x09, 0x8c, 0x2e, 0x41, 0xed, 0x9e, 0xef, 0xf2, 0x1d, 0x43, 0xec, 0x48, 0x08, 0x2d, 0x03, - 0x6c, 0x92, 0x23, 0x16, 0xd8, 0x0e, 0x67, 0x58, 0x59, 0xd1, 0x56, 0x5b, 0x38, 0xb3, 0x62, 0x7d, - 0xa1, 0x83, 0x79, 0x9f, 0x8b, 0xb2, 0x41, 0xec, 0x08, 0x93, 0xc7, 0x63, 0xc2, 0x22, 0xf4, 0x1d, - 0x98, 0x73, 0xf6, 0x6d, 0x7f, 0x40, 0x1e, 0x11, 0xe2, 0x4a, 0x39, 0x9a, 0xb7, 0x2f, 0xdf, 0xc8, - 0xc8, 0x7c, 0xa3, 0x9d, 0x41, 0xc0, 0x39, 0x74, 0xf4, 0x2e, 0x34, 0x0e, 0xed, 0x88, 0x84, 0x23, - 0x3b, 0x3c, 0x10, 0x82, 0x36, 0x6f, 0x5f, 0xca, 0xd1, 0x3e, 0x54, 0xbb, 0x38, 0x45, 0x44, 0xef, - 0x43, 0x2b, 0x24, 0x2e, 0x4d, 0xf6, 0xc4, 0x45, 0x8e, 0xa7, 0xcc, 0x23, 0xa3, 0x6f, 0x42, 0x9d, - 0x45, 0x76, 0x34, 0x66, 0x84, 0x2d, 0x56, 0x56, 0x8c, 0xd5, 0xe6, 0xed, 0xa5, 0x1c, 0x61, 0xa2, - 0xdf, 0x9e, 0xc0, 0xc2, 0x09, 0x36, 0x5a, 0x85, 0x05, 0x87, 0x8e, 0x02, 0x32, 0x24, 0x11, 0x89, - 0x37, 0x17, 0xab, 0x2b, 0xda, 0x6a, 0x1d, 0x17, 0x97, 0xd1, 0x5b, 0x60, 0x90, 0x30, 0x5c, 0xac, - 0x95, 0x68, 0x03, 0x8f, 0x7d, 0xdf, 0xf3, 0x07, 0xf7, 0xc2, 0x90, 0x86, 0x98, 0x63, 0x59, 0x3f, - 0xd3, 0xa0, 0x91, 0x8a, 0x67, 0x71, 0x8d, 0x12, 0xe7, 0x20, 0xa0, 0x9e, 0x1f, 0xed, 0x32, 0xa1, - 0xd1, 0x0a, 0xce, 0xad, 0x71, 0x53, 0x85, 0x84, 0xd1, 0xe1, 0x13, 0xe2, 0xee, 0x32, 0xa1, 0xb7, - 0x0a, 0xce, 0xac, 0x20, 0x13, 0x0c, 0x46, 0x1e, 0x0b, 0xb5, 0x54, 0x30, 0xff, 0xe4, 0x5c, 0x87, - 0x36, 0x8b, 0x7a, 0x47, 0xbe, 0x23, 0x68, 0x2a, 0x31, 0xd7, 0xec, 0x9a, 0xf5, 0x19, 0x98, 0x1d, - 0x8f, 0x05, 0x76, 0xe4, 0xec, 0x93, 0x70, 0xdd, 0x89, 0x3c, 0xea, 0xa3, 0xb7, 0xa0, 0x66, 0x8b, - 0x2f, 0x21, 0xc7, 0xfc, 0xed, 0xf3, 0xb9, 0xbb, 0xc4, 0x48, 0x58, 0xa2, 0x70, 0xaf, 0x6b, 0xd3, - 0xd1, 0xc8, 0x8b, 0x12, 0xa1, 0x12, 0x18, 0xad, 0x40, 0xb3, 0xcb, 0xf8, 0x51, 0x3b, 0xfc, 0x0e, - 0x42, 0xb4, 0x3a, 0xce, 0x2e, 0x59, 0x6d, 0x30, 0xd6, 0xdb, 0x9b, 0x39, 0x26, 0xda, 0xc9, 0x4c, - 0xf4, 0x49, 0x26, 0x3f, 0xd5, 0xe1, 0x62, 0xd7, 0x7f, 0x34, 0x1c, 0x13, 0x7e, 0xa9, 0xf4, 0x3a, - 0x0c, 0x7d, 0x17, 0x5a, 0xc9, 0xc6, 0xee, 0x51, 0x40, 0xe4, 0x85, 0xae, 0xe4, 0x2e, 0x94, 0xc3, - 0xc0, 0x79, 0x02, 0x74, 0x17, 0x5a, 0x29, 0xc3, 0x6e, 0x87, 0xdf, 0xd1, 0x98, 0x30, 0x6f, 0x16, - 0x03, 0xe7, 0xf1, 0x45, 0x54, 0x3a, 0xfb, 0x64, 0x64, 0x77, 0x3b, 0x42, 0x01, 0x06, 0x4e, 0x60, - 0xb4, 0x09, 0xe7, 0xc9, 0x53, 0x67, 0x38, 0x76, 0x49, 0x86, 0xc6, 0x15, 0x76, 0x3a, 0xf1, 0x88, - 0x32, 0x2a, 0xeb, 0xaf, 0x5a, 0xd6, 0x94, 0xd2, 0x27, 0xbf, 0x07, 0x17, 0xbd, 0x32, 0xcd, 0xc8, - 0x98, 0xb5, 0xca, 0x15, 0x91, 0xc5, 0xc4, 0xe5, 0x0c, 0xd0, 0x9d, 0xc4, 0x49, 0xe2, 0x10, 0xbe, - 0x76, 0x8c, 0xb8, 0x05, 0x77, 0xb1, 0xc0, 0xb0, 0x1d, 0x15, 0xbc, 0x66, 0xde, 0xb1, 0xda, 0x9b, - 0x98, 0x6f, 0x5a, 0x7f, 0xd2, 0xe0, 0x5c, 0x26, 0xe9, 0xb0, 0x80, 0xfa, 0x8c, 0x9c, 0x35, 0xeb, - 0x6c, 0x01, 0x72, 0x0b, 0xda, 0x21, 0xca, 0x9a, 0xc7, 0xc9, 0x2e, 0x93, 0x41, 0x09, 0x21, 0x42, - 0x50, 0x19, 0x51, 0x97, 0x48, 0x93, 0x8a, 0x6f, 0xeb, 0x29, 0x9c, 0x6f, 0x67, 0x22, 0x76, 0x8b, - 0x30, 0x66, 0x0f, 0xce, 0x2c, 0x78, 0x31, 0x37, 0xe8, 0x93, 0xb9, 0xc1, 0xfa, 0xb5, 0x06, 0x0b, - 0x98, 0xb8, 0x74, 0x8b, 0x44, 0xf6, 0x94, 0x8e, 0x3d, 0x2d, 0xdd, 0x14, 0xc5, 0x32, 0x4a, 0xc4, - 0xfa, 0x11, 0x5c, 0xe3, 0x52, 0xe1, 0x84, 0x6a, 0x27, 0xa4, 0x83, 0x90, 0x30, 0xf6, 0x7a, 0x64, - 0xb4, 0x3e, 0x83, 0xa5, 0xfc, 0xf9, 0x1f, 0xd2, 0xf0, 0xd0, 0x0e, 0xdd, 0xd7, 0x74, 0xfc, 0x7f, - 0x73, 0x11, 0xd9, 0xa6, 0xfe, 0x23, 0x6f, 0x80, 0xd6, 0xa0, 0xc2, 0x02, 0xdb, 0x97, 0x67, 0x5d, - 0x2a, 0xaf, 0x42, 0x58, 0xe0, 0xf0, 0x5a, 0xcf, 0x78, 0x05, 0x4f, 0xb8, 0x2b, 0x90, 0x4b, 0xee, - 0x66, 0x32, 0x82, 0x8c, 0xa7, 0x13, 0x52, 0x46, 0x0e, 0x9d, 0x27, 0x25, 0xa6, 0x92, 0x52, 0x25, - 0x4e, 0x4a, 0x0a, 0x4e, 0x3c, 0xbb, 0x9a, 0x7a, 0x36, 0x5a, 0x03, 0x93, 0x1d, 0x78, 0x41, 0x67, - 0xeb, 0xc1, 0x3a, 0xeb, 0x49, 0x89, 0x6a, 0x22, 0x11, 0x4f, 0xac, 0x5b, 0x5f, 0x68, 0x70, 0x99, - 0x67, 0x38, 0x77, 0x3c, 0xcc, 0x24, 0xa8, 0x29, 0xf5, 0x0e, 0x77, 0xa0, 0xe6, 0x08, 0x3d, 0x9e, - 0x92, 0x75, 0x62, 0x65, 0x63, 0x89, 0x8c, 0xda, 0x30, 0xcf, 0xa4, 0x48, 0x71, 0x3e, 0x12, 0x0a, - 0x9b, 0xbf, 0x7d, 0x35, 0x47, 0xde, 0xcb, 0xa1, 0xe0, 0x02, 0x89, 0xf5, 0x3f, 0x0d, 0x2e, 0x6d, - 0x91, 0x70, 0x30, 0xfd, 0x5b, 0xdd, 0x85, 0x96, 0xfb, 0x8a, 0x45, 0x26, 0x87, 0x8f, 0xba, 0x80, - 0x46, 0x5c, 0x32, 0xb7, 0xf3, 0x4a, 0x4e, 0x51, 0x42, 0x94, 0x98, 0xbf, 0x92, 0x49, 0x6c, 0x3b, - 0x70, 0x7e, 0xcb, 0xf6, 0xfc, 0xc8, 0xf6, 0x7c, 0x12, 0xde, 0x57, 0xdc, 0xd0, 0xb7, 0x32, 0x4d, - 0x95, 0x56, 0x92, 0x48, 0x53, 0x9a, 0x62, 0x57, 0x65, 0xfd, 0x45, 0x07, 0xb3, 0xb8, 0x7d, 0x56, - 0x2d, 0x5e, 0x03, 0xe0, 0x5f, 0x7d, 0x7e, 0x08, 0x11, 0xfe, 0xd1, 0xc0, 0x0d, 0xbe, 0xc2, 0xd9, - 0x13, 0x74, 0x0b, 0xaa, 0xf1, 0x4e, 0x99, 0xe9, 0xdb, 0x74, 0x14, 0x50, 0x9f, 0xf8, 0x91, 0xc0, - 0xc5, 0x31, 0x26, 0xfa, 0x12, 0xb4, 0xd2, 0x7c, 0xd6, 0x8f, 0x92, 0x0e, 0x2a, 0xd7, 0x97, 0xc9, - 0xb6, 0xaf, 0x5a, 0x62, 0xb2, 0x89, 0xb6, 0x0f, 0x7d, 0x05, 0xe6, 0xf7, 0x28, 0x8d, 0x58, 0x14, - 0xda, 0x41, 0xdf, 0xa5, 0x3e, 0x91, 0x61, 0xd4, 0x4a, 0x56, 0x3b, 0xd4, 0x27, 0x13, 0x9d, 0xdb, - 0x6c, 0x49, 0xe7, 0xf6, 0x0d, 0xb8, 0xda, 0xa6, 0x34, 0x74, 0x3d, 0xdf, 0x8e, 0x68, 0xb8, 0xa1, - 0xe8, 0x95, 0x4b, 0x2e, 0xc2, 0xec, 0x13, 0x12, 0x32, 0xd5, 0xc5, 0x19, 0x58, 0x81, 0xd6, 0xf7, - 0x61, 0xa9, 0x9c, 0x50, 0x16, 0xda, 0x33, 0x98, 0xf5, 0xef, 0x1a, 0x5c, 0x58, 0x77, 0xdd, 0x14, - 0x43, 0x49, 0xf3, 0x55, 0xd0, 0x3d, 0xf7, 0x74, 0x83, 0xea, 0x9e, 0xcb, 0x47, 0x95, 0x4c, 0x88, - 0xcf, 0x25, 0x31, 0x3c, 0x61, 0x8c, 0x92, 0x8a, 0x83, 0xd6, 0xe0, 0x9c, 0xc7, 0xfa, 0x3e, 0x39, - 0xec, 0xa7, 0xae, 0x21, 0xac, 0x56, 0xc7, 0x0b, 0x1e, 0xdb, 0x26, 0x87, 0xe9, 0x71, 0xe8, 0x3a, - 0x34, 0x0f, 0xe4, 0xa4, 0xd3, 0xf7, 0x5c, 0x91, 0xef, 0x5a, 0x18, 0xd4, 0x52, 0xd7, 0xb5, 0x7e, - 0xa3, 0xc1, 0x1b, 0x98, 0x8c, 0xe8, 0x13, 0x72, 0xa6, 0x0b, 0x2d, 0xc2, 0xac, 0x63, 0x33, 0xc7, - 0x76, 0x89, 0x6c, 0x5e, 0x15, 0xc8, 0x77, 0x42, 0xc1, 0xdf, 0x95, 0xbd, 0xb1, 0x02, 0x8b, 0xb2, - 0x55, 0x26, 0x64, 0xfb, 0xbd, 0x01, 0x57, 0x52, 0xa9, 0x26, 0xac, 0x7f, 0xc6, 0x50, 0x3a, 0xce, - 0x06, 0x97, 0x85, 0x6b, 0x84, 0x19, 0xf5, 0x27, 0x15, 0xc9, 0x81, 0x37, 0x23, 0x5e, 0xbe, 0xfa, - 0x51, 0xe8, 0x0d, 0x06, 0x24, 0xec, 0x93, 0x27, 0xc4, 0x8f, 0xfa, 0x69, 0x9e, 0x52, 0xf7, 0x38, - 0x31, 0x23, 0x5d, 0x13, 0x3c, 0x76, 0x63, 0x16, 0xf7, 0x38, 0x87, 0x6c, 0x8f, 0x5b, 0x6e, 0xde, - 0x6a, 0xb9, 0x79, 0x6d, 0x58, 0xc9, 0x0b, 0xc4, 0x27, 0xc2, 0x82, 0x3c, 0xb5, 0xd3, 0xe4, 0x59, - 0xca, 0xca, 0xc3, 0x7b, 0x89, 0x9c, 0x38, 0x05, 0x2b, 0xcd, 0x4e, 0x58, 0xe9, 0x57, 0x3a, 0x5c, - 0x2d, 0xb5, 0xd2, 0x74, 0x7a, 0xda, 0x3b, 0x50, 0xe5, 0x7d, 0x82, 0xaa, 0x17, 0xd7, 0x73, 0x74, - 0xc9, 0x69, 0x69, 0x57, 0x11, 0x63, 0xab, 0x8c, 0x65, 0xbc, 0xcc, 0xa0, 0xfa, 0x72, 0x39, 0xf0, - 0x6d, 0x40, 0x42, 0xbb, 0x79, 0xcc, 0xaa, 0xc0, 0x34, 0xf9, 0x4e, 0xb6, 0x2f, 0xb6, 0xfe, 0xa8, - 0xc3, 0x72, 0xaa, 0x95, 0x1d, 0xca, 0xa2, 0x69, 0xfb, 0xef, 0x4b, 0x39, 0xa3, 0x7e, 0x46, 0x67, - 0xbc, 0x05, 0xb3, 0x71, 0xd3, 0xc4, 0x63, 0x81, 0xeb, 0xff, 0x8d, 0x89, 0x6e, 0x62, 0x64, 0x77, - 0xfd, 0x47, 0x14, 0x2b, 0x3c, 0xf4, 0x1e, 0xcc, 0x09, 0x3d, 0x29, 0xba, 0xca, 0xc9, 0x74, 0x4d, - 0x8e, 0x1c, 0xc3, 0xcc, 0xfa, 0x8f, 0x06, 0xd7, 0x8f, 0xd5, 0xda, 0x74, 0xfc, 0xe9, 0xb5, 0xa8, - 0xed, 0x55, 0xbc, 0xcf, 0x7a, 0x0a, 0x90, 0xea, 0x23, 0x37, 0x4b, 0x6b, 0x85, 0x59, 0x7a, 0x59, - 0x61, 0x6e, 0xdb, 0x23, 0x55, 0xfd, 0x33, 0x2b, 0xe8, 0x06, 0xd4, 0x44, 0x20, 0x28, 0x63, 0x95, - 0x74, 0xde, 0x42, 0xe7, 0x12, 0xcb, 0x6a, 0xcb, 0x47, 0x37, 0x71, 0xf0, 0xf1, 0x8f, 0x6e, 0x4b, - 0x12, 0x2d, 0x73, 0x6a, 0xba, 0x60, 0xfd, 0x59, 0x07, 0x34, 0x19, 0x87, 0xbc, 0x78, 0x1c, 0x63, - 0x9c, 0x9c, 0x22, 0x75, 0xf9, 0xa8, 0xa7, 0xae, 0xac, 0x17, 0xae, 0xac, 0x46, 0x09, 0xe3, 0x25, - 0x46, 0x89, 0x0f, 0xc1, 0x74, 0x54, 0x8f, 0xd3, 0x8f, 0xeb, 0xb5, 0x88, 0xe4, 0x53, 0x1a, 0xa1, - 0x05, 0x27, 0x0b, 0x8f, 0xd9, 0x64, 0x3a, 0xa8, 0x96, 0xa4, 0x83, 0x77, 0xa0, 0xb9, 0x37, 0xa4, - 0xce, 0x81, 0x6c, 0xc5, 0xe2, 0x2c, 0x8b, 0xf2, 0x5e, 0x2e, 0xd8, 0x83, 0x40, 0x8b, 0xfb, 0x33, - 0xd5, 0x78, 0xce, 0x66, 0x1a, 0xcf, 0xc7, 0x70, 0x29, 0x75, 0xf9, 0xf6, 0x90, 0x32, 0x32, 0xa5, - 0x04, 0x91, 0xa9, 0xbc, 0x7a, 0xae, 0xf2, 0x5a, 0x21, 0xbc, 0x31, 0x71, 0xe4, 0x74, 0xa2, 0x8b, - 0x4f, 0x73, 0x63, 0xc7, 0x21, 0x8c, 0xa9, 0x33, 0x25, 0x68, 0xfd, 0x5c, 0x03, 0x33, 0x7d, 0x7c, - 0x89, 0x1d, 0x70, 0x0a, 0x6f, 0x57, 0x57, 0xa0, 0x2e, 0xdd, 0x34, 0xae, 0x10, 0x06, 0x4e, 0xe0, - 0x93, 0x9e, 0xa5, 0xac, 0x1f, 0x40, 0x55, 0xe0, 0x9d, 0xf2, 0xd6, 0x7c, 0x9c, 0x5b, 0x2e, 0x41, - 0xa3, 0x17, 0x0c, 0x3d, 0x91, 0x05, 0x64, 0x5f, 0x93, 0x2e, 0x58, 0x3e, 0xcc, 0x2b, 0xcc, 0x58, - 0x57, 0x27, 0x9c, 0xb2, 0x02, 0xcd, 0x8f, 0x87, 0x6e, 0xe1, 0xa0, 0xec, 0x12, 0xc7, 0xd8, 0x26, - 0x87, 0x85, 0x9b, 0x64, 0x97, 0xac, 0xdf, 0x19, 0x50, 0x8d, 0x1d, 0x6c, 0x09, 0x1a, 0x5d, 0xb6, - 0xc1, 0x1d, 0x8e, 0xc4, 0x9d, 0x5b, 0x1d, 0xa7, 0x0b, 0x5c, 0x0a, 0xf1, 0x99, 0xce, 0xda, 0x12, - 0x44, 0x77, 0xa1, 0x19, 0x7f, 0xaa, 0xf4, 0x31, 0x39, 0x78, 0x16, 0x8d, 0x87, 0xb3, 0x14, 0x68, - 0x13, 0xce, 0x6d, 0x13, 0xe2, 0x76, 0x42, 0x1a, 0x04, 0x0a, 0x43, 0xb6, 0x42, 0xa7, 0xb0, 0x99, - 0xa4, 0x43, 0xef, 0xc3, 0x02, 0x5f, 0x5c, 0x77, 0xdd, 0x84, 0x55, 0x3c, 0x7a, 0xa0, 0xc9, 0xf8, - 0xc7, 0x45, 0x54, 0x3e, 0x08, 0x7f, 0x12, 0xb8, 0x76, 0x44, 0xa4, 0x0a, 0xf9, 0x18, 0xcf, 0x89, - 0xaf, 0x96, 0x95, 0x20, 0x69, 0x20, 0x5c, 0x20, 0x29, 0xbe, 0xc8, 0xce, 0x4e, 0xbc, 0xc8, 0xa2, - 0xaf, 0x89, 0x59, 0x6b, 0x40, 0x16, 0xeb, 0xc2, 0x67, 0xf3, 0x05, 0x6e, 0x43, 0xc6, 0xfc, 0x20, - 0x9e, 0xb3, 0x06, 0xc4, 0xfa, 0x31, 0x5c, 0x48, 0xf2, 0x95, 0xda, 0xe5, 0xc9, 0xe6, 0x15, 0xf2, - 0xe4, 0xaa, 0x9a, 0xee, 0xf4, 0x63, 0x93, 0x8d, 0x1c, 0xea, 0xca, 0x5e, 0xee, 0xfe, 0xad, 0xc1, - 0x42, 0xe1, 0x27, 0x80, 0x57, 0x39, 0xbc, 0x2c, 0xb9, 0xea, 0xd3, 0x48, 0xae, 0x65, 0x23, 0xce, - 0x2d, 0xb8, 0x18, 0x97, 0x65, 0xe6, 0x3d, 0x23, 0xfd, 0x80, 0x84, 0x7d, 0x46, 0x1c, 0xea, 0xc7, - 0xcd, 0xb5, 0x8e, 0x91, 0xd8, 0xec, 0x79, 0xcf, 0xc8, 0x0e, 0x09, 0x7b, 0x62, 0xa7, 0xec, 0x49, - 0xc7, 0xfa, 0x83, 0x06, 0x28, 0xa3, 0xeb, 0x29, 0xe5, 0xd5, 0x8f, 0xa0, 0xb5, 0x97, 0x32, 0x4d, - 0x1e, 0x58, 0xdf, 0x2c, 0xaf, 0x4d, 0xd9, 0xf3, 0xf3, 0x74, 0xa5, 0x56, 0x72, 0x61, 0x2e, 0xdb, - 0x21, 0x70, 0x9c, 0xc8, 0x1b, 0xc5, 0x89, 0xb1, 0x81, 0xc5, 0x37, 0x5f, 0xf3, 0xa9, 0xab, 0x4a, - 0xb1, 0xf8, 0xe6, 0x6b, 0x8e, 0xe2, 0xd5, 0xc0, 0xe2, 0x9b, 0x87, 0xfb, 0x28, 0x7e, 0x05, 0x14, - 0x7a, 0x6b, 0x60, 0x05, 0x5a, 0xef, 0xc2, 0x5c, 0xf1, 0x41, 0x64, 0xdf, 0x1b, 0xec, 0xcb, 0xdf, - 0x25, 0xc4, 0x37, 0x32, 0xc1, 0x18, 0xd2, 0x43, 0x99, 0x28, 0xf8, 0x27, 0x97, 0x2d, 0xab, 0x96, - 0x97, 0xa3, 0x12, 0xd2, 0xf2, 0xc6, 0x41, 0x4a, 0xc6, 0xbf, 0x79, 0x6a, 0x55, 0x13, 0x84, 0x14, - 0x2d, 0x81, 0xad, 0x1f, 0xc2, 0xf5, 0x07, 0x74, 0x90, 0x99, 0xde, 0xd3, 0xa7, 0xcd, 0xe9, 0x18, - 0xd0, 0xfa, 0x89, 0x06, 0x2b, 0xc7, 0x1f, 0x31, 0x9d, 0x42, 0x78, 0xda, 0xbb, 0xe9, 0x90, 0xeb, - 0x92, 0x38, 0x07, 0x6c, 0x3c, 0xda, 0x22, 0x91, 0x8d, 0xbe, 0xae, 0x62, 0xbb, 0xac, 0x02, 0x2a, - 0xcc, 0x5c, 0x8c, 0xaf, 0x81, 0xe9, 0x64, 0xd7, 0x7b, 0xe4, 0xb1, 0x3c, 0x67, 0x62, 0xdd, 0xfa, - 0xa5, 0x06, 0x17, 0x33, 0x4f, 0xfe, 0x24, 0x52, 0x1c, 0xd1, 0x05, 0xa8, 0x3a, 0x74, 0xec, 0x47, - 0xd2, 0x88, 0x31, 0xc0, 0x3d, 0xe7, 0x29, 0x0d, 0xef, 0x73, 0xe3, 0xca, 0x42, 0x21, 0x41, 0x3e, - 0x35, 0x3f, 0xa5, 0xe1, 0x03, 0x7a, 0x28, 0xe3, 0x56, 0x42, 0x71, 0xe1, 0x1f, 0x09, 0x8a, 0x8a, - 0x1c, 0x9a, 0x63, 0x90, 0x53, 0xb0, 0xf1, 0x88, 0x53, 0xc4, 0x6d, 0x94, 0x84, 0xac, 0xdf, 0x6a, - 0xb0, 0x52, 0x2a, 0xd3, 0xba, 0x73, 0x30, 0x2d, 0x2b, 0x5c, 0x80, 0x2a, 0x09, 0xa8, 0xa3, 0x6e, - 0x11, 0x03, 0x65, 0x71, 0xa7, 0x7e, 0x59, 0xac, 0x24, 0xbf, 0x2c, 0x5a, 0xff, 0xd4, 0xc0, 0x2a, - 0x95, 0x2f, 0xae, 0x14, 0x53, 0x4a, 0x26, 0x67, 0x90, 0x10, 0x7d, 0x00, 0x75, 0x65, 0x69, 0xa1, - 0xdb, 0xe2, 0x6f, 0x5d, 0xa5, 0xd2, 0xe3, 0x84, 0x66, 0xed, 0x1a, 0xd4, 0xe4, 0xaf, 0xa1, 0x0d, - 0xa8, 0x3e, 0x0c, 0xbd, 0x88, 0x98, 0x33, 0xa8, 0x0e, 0x95, 0x1d, 0x9b, 0x31, 0x53, 0x5b, 0x5b, - 0x8d, 0xbb, 0x98, 0xf4, 0x75, 0x18, 0x01, 0xd4, 0xda, 0x21, 0xb1, 0x05, 0x1e, 0x40, 0x2d, 0x7e, - 0x37, 0x32, 0xb5, 0xb5, 0xf7, 0x00, 0xd2, 0x82, 0xc7, 0x39, 0x6c, 0x7f, 0xbc, 0x7d, 0xcf, 0x9c, - 0x41, 0x4d, 0x98, 0x7d, 0xb8, 0xde, 0xdd, 0xed, 0x6e, 0x7f, 0x64, 0x6a, 0x02, 0xc0, 0x31, 0xa0, - 0x73, 0x9c, 0x0e, 0xc7, 0x31, 0xd6, 0xde, 0x2e, 0xb4, 0x80, 0x68, 0x16, 0x8c, 0xf5, 0xe1, 0xd0, - 0x9c, 0x41, 0x35, 0xd0, 0x3b, 0x1b, 0xa6, 0xc6, 0x4f, 0xda, 0xa6, 0xe1, 0xc8, 0x1e, 0x9a, 0xfa, - 0xda, 0x33, 0x98, 0xcf, 0x17, 0x18, 0xc1, 0x96, 0x86, 0x07, 0x9e, 0x3f, 0x88, 0x0f, 0xec, 0x45, - 0xa2, 0x93, 0x88, 0x0f, 0x8c, 0x25, 0x74, 0x4d, 0x1d, 0x99, 0x30, 0xd7, 0xf5, 0xbd, 0xc8, 0xb3, - 0x87, 0xde, 0x33, 0x8e, 0x6b, 0xa0, 0x16, 0x34, 0x76, 0x42, 0x12, 0xd8, 0x21, 0x07, 0x2b, 0x68, - 0x1e, 0x40, 0x3c, 0x7c, 0x63, 0x62, 0xbb, 0x47, 0x66, 0x95, 0x13, 0x3c, 0xb4, 0xbd, 0xc8, 0xf3, - 0x07, 0x62, 0xd9, 0xac, 0xad, 0x7d, 0x1b, 0x5a, 0xb9, 0x40, 0x44, 0xe7, 0xa0, 0xf5, 0xc9, 0x76, - 0x77, 0xbb, 0xbb, 0xdb, 0x5d, 0x7f, 0xd0, 0xfd, 0xf4, 0x5e, 0xc7, 0x9c, 0x41, 0x73, 0x50, 0xdf, - 0xea, 0xf6, 0xb6, 0xd6, 0x77, 0xdb, 0xf7, 0x4d, 0x8d, 0xab, 0x35, 0xfe, 0xd4, 0x37, 0x3e, 0xf8, - 0xdb, 0xf3, 0x65, 0xed, 0xf3, 0xe7, 0xcb, 0xda, 0xbf, 0x9e, 0x2f, 0x6b, 0xbf, 0x78, 0xb1, 0x3c, - 0xf3, 0xf9, 0x8b, 0xe5, 0x99, 0x7f, 0xbc, 0x58, 0x9e, 0xf9, 0xf4, 0xcb, 0x03, 0x2f, 0xda, 0x1f, - 0xef, 0xdd, 0x70, 0xe8, 0xe8, 0x66, 0xe0, 0xf9, 0x03, 0xc7, 0x0e, 0x6e, 0x46, 0x9e, 0xe3, 0x3a, - 0x37, 0x33, 0xb6, 0xdc, 0xab, 0x89, 0xff, 0x4c, 0xbc, 0xf3, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x48, 0x01, 0xd7, 0x2e, 0x52, 0x21, 0x00, 0x00, + 0xd5, 0xdd, 0xf3, 0xe1, 0x99, 0x67, 0x8f, 0xdd, 0xa9, 0x24, 0x8e, 0x93, 0xd8, 0x8e, 0xb7, 0x01, + 0xc9, 0xcc, 0x2e, 0x09, 0x49, 0x36, 0x02, 0x96, 0x65, 0xc3, 0x78, 0xc6, 0xbb, 0x19, 0xd9, 0x9e, + 0x58, 0x35, 0x5e, 0x02, 0xcb, 0x61, 0x68, 0x77, 0x57, 0xc6, 0x8d, 0x67, 0xba, 0x3a, 0x5d, 0x3d, + 0x71, 0x1c, 0x69, 0x41, 0x08, 0x71, 0xe3, 0x00, 0x82, 0x0b, 0x07, 0x0e, 0x88, 0x3f, 0x80, 0x84, + 0x38, 0x22, 0x71, 0x03, 0x89, 0xcb, 0x9e, 0x56, 0xdc, 0x40, 0xc9, 0x1f, 0x40, 0x70, 0xe0, 0x8a, + 0xaa, 0xba, 0xfa, 0x73, 0xda, 0x9e, 0x44, 0x1e, 0xe5, 0xd6, 0xaf, 0xea, 0xbd, 0x7a, 0xaf, 0xde, + 0xf7, 0xab, 0x19, 0xb8, 0x7e, 0x48, 0x0c, 0xcf, 0x3f, 0x20, 0x86, 0xef, 0x1e, 0xdc, 0x8a, 0xbe, + 0x6f, 0xba, 0x1e, 0xf5, 0x29, 0x9a, 0x4b, 0x6c, 0xea, 0x27, 0x50, 0xdd, 0x37, 0x0e, 0x06, 0xa4, + 0xeb, 0x1a, 0x0e, 0x5a, 0x86, 0x59, 0x01, 0xb4, 0x5b, 0xcb, 0xca, 0xba, 0xb2, 0x51, 0xc0, 0x21, + 0x88, 0xae, 0x41, 0xa5, 0xeb, 0x1b, 0x9e, 0xbf, 0x4d, 0x4e, 0x96, 0xd5, 0x75, 0x65, 0x63, 0x1e, + 0x47, 0x30, 0x5a, 0x82, 0xf2, 0x96, 0x63, 0xf1, 0x9d, 0x82, 0xd8, 0x91, 0x10, 0x5a, 0x03, 0xd8, + 0x26, 0x27, 0xcc, 0x35, 0x4c, 0x7e, 0x60, 0x71, 0x5d, 0xd9, 0xa8, 0xe1, 0xc4, 0x8a, 0xfe, 0xb9, + 0x0a, 0xda, 0x03, 0x2e, 0xca, 0x26, 0x31, 0x7c, 0x4c, 0x9e, 0x8c, 0x08, 0xf3, 0xd1, 0xb7, 0x60, + 0xde, 0x3c, 0x34, 0x9c, 0x3e, 0x79, 0x4c, 0x88, 0x25, 0xe5, 0x98, 0xbb, 0x73, 0xf5, 0x66, 0x42, + 0xe6, 0x9b, 0xcd, 0x04, 0x02, 0x4e, 0xa1, 0xa3, 0x77, 0xa1, 0x7a, 0x6c, 0xf8, 0xc4, 0x1b, 0x1a, + 0xde, 0x91, 0x10, 0x74, 0xee, 0xce, 0x52, 0x8a, 0xf6, 0x51, 0xb8, 0x8b, 0x63, 0x44, 0xf4, 0x3e, + 0xd4, 0x3c, 0x62, 0xd1, 0x68, 0x4f, 0x5c, 0xe4, 0x74, 0xca, 0x34, 0x32, 0xfa, 0x3a, 0x54, 0x98, + 0x6f, 0xf8, 0x23, 0x46, 0xd8, 0x72, 0x71, 0xbd, 0xb0, 0x31, 0x77, 0x67, 0x25, 0x45, 0x18, 0xe9, + 0xb7, 0x2b, 0xb0, 0x70, 0x84, 0x8d, 0x36, 0x60, 0xd1, 0xa4, 0x43, 0x97, 0x0c, 0x88, 0x4f, 0x82, + 0xcd, 0xe5, 0xd2, 0xba, 0xb2, 0x51, 0xc1, 0xd9, 0x65, 0xf4, 0x36, 0x14, 0x88, 0xe7, 0x2d, 0x97, + 0x73, 0xb4, 0x81, 0x47, 0x8e, 0x63, 0x3b, 0xfd, 0x2d, 0xcf, 0xa3, 0x1e, 0xe6, 0x58, 0xfa, 0xcf, + 0x14, 0xa8, 0xc6, 0xe2, 0xe9, 0x5c, 0xa3, 0xc4, 0x3c, 0x72, 0xa9, 0xed, 0xf8, 0xfb, 0x4c, 0x68, + 0xb4, 0x88, 0x53, 0x6b, 0xdc, 0x54, 0x1e, 0x61, 0x74, 0xf0, 0x94, 0x58, 0xfb, 0x4c, 0xe8, 0xad, + 0x88, 0x13, 0x2b, 0x48, 0x83, 0x02, 0x23, 0x4f, 0x84, 0x5a, 0x8a, 0x98, 0x7f, 0xf2, 0x53, 0x07, + 0x06, 0xf3, 0xbb, 0x27, 0x8e, 0x29, 0x68, 0x8a, 0xc1, 0xa9, 0xc9, 0x35, 0xfd, 0x53, 0xd0, 0x5a, + 0x36, 0x73, 0x0d, 0xdf, 0x3c, 0x24, 0x5e, 0xc3, 0xf4, 0x6d, 0xea, 0xa0, 0xb7, 0xa1, 0x6c, 0x88, + 0x2f, 0x21, 0xc7, 0xc2, 0x9d, 0x8b, 0xa9, 0xbb, 0x04, 0x48, 0x58, 0xa2, 0x70, 0xaf, 0x6b, 0xd2, + 0xe1, 0xd0, 0xf6, 0x23, 0xa1, 0x22, 0x18, 0xad, 0xc3, 0x5c, 0x9b, 0x71, 0x56, 0x7b, 0xfc, 0x0e, + 0x42, 0xb4, 0x0a, 0x4e, 0x2e, 0xe9, 0x4d, 0x28, 0x34, 0x9a, 0xdb, 0xa9, 0x43, 0x94, 0xb3, 0x0f, + 0x51, 0xc7, 0x0f, 0xf9, 0xa9, 0x0a, 0x97, 0xdb, 0xce, 0xe3, 0xc1, 0x88, 0xf0, 0x4b, 0xc5, 0xd7, + 0x61, 0xe8, 0xdb, 0x50, 0x8b, 0x36, 0xf6, 0x4f, 0x5c, 0x22, 0x2f, 0x74, 0x2d, 0x75, 0xa1, 0x14, + 0x06, 0x4e, 0x13, 0xa0, 0xfb, 0x50, 0x8b, 0x0f, 0x6c, 0xb7, 0xf8, 0x1d, 0x0b, 0x63, 0xe6, 0x4d, + 0x62, 0xe0, 0x34, 0xbe, 0x88, 0x4a, 0xf3, 0x90, 0x0c, 0x8d, 0x76, 0x4b, 0x28, 0xa0, 0x80, 0x23, + 0x18, 0x6d, 0xc3, 0x45, 0xf2, 0xcc, 0x1c, 0x8c, 0x2c, 0x92, 0xa0, 0xb1, 0x84, 0x9d, 0xce, 0x64, + 0x91, 0x47, 0xa5, 0xff, 0x55, 0x49, 0x9a, 0x52, 0xfa, 0xe4, 0x77, 0xe1, 0xb2, 0x9d, 0xa7, 0x19, + 0x19, 0xb3, 0x7a, 0xbe, 0x22, 0x92, 0x98, 0x38, 0xff, 0x00, 0x74, 0x2f, 0x72, 0x92, 0x20, 0x84, + 0x57, 0x4f, 0x11, 0x37, 0xe3, 0x2e, 0x3a, 0x14, 0x0c, 0x33, 0x0c, 0x5e, 0x2d, 0xed, 0x58, 0xcd, + 0x6d, 0xcc, 0x37, 0xf5, 0x3f, 0x29, 0x70, 0x21, 0x91, 0x74, 0x98, 0x4b, 0x1d, 0x46, 0xce, 0x9b, + 0x75, 0x76, 0x01, 0x59, 0x19, 0xed, 0x90, 0xd0, 0x9a, 0xa7, 0xc9, 0x2e, 0x93, 0x41, 0x0e, 0x21, + 0x42, 0x50, 0x1c, 0x52, 0x8b, 0x48, 0x93, 0x8a, 0x6f, 0xfd, 0x19, 0x5c, 0x6c, 0x26, 0x22, 0x76, + 0x97, 0x30, 0x66, 0xf4, 0xcf, 0x2d, 0x78, 0x36, 0x37, 0xa8, 0xe3, 0xb9, 0x41, 0xff, 0xb5, 0x02, + 0x8b, 0x98, 0x58, 0x74, 0x97, 0xf8, 0xc6, 0x94, 0xd8, 0x4e, 0x4a, 0x37, 0x59, 0xb1, 0x0a, 0x39, + 0x62, 0xfd, 0x08, 0x56, 0xb9, 0x54, 0x38, 0xa2, 0xda, 0xf3, 0x68, 0xdf, 0x23, 0x8c, 0xbd, 0x19, + 0x19, 0xf5, 0x4f, 0x61, 0x25, 0xcd, 0xff, 0x43, 0xea, 0x1d, 0x1b, 0x9e, 0xf5, 0x86, 0xd8, 0xff, + 0x37, 0x15, 0x91, 0x4d, 0xea, 0x3c, 0xb6, 0xfb, 0xa8, 0x0e, 0x45, 0xe6, 0x1a, 0x8e, 0xe4, 0xb5, + 0x94, 0x5f, 0x85, 0xb0, 0xc0, 0xe1, 0xb5, 0x9e, 0xf1, 0x0a, 0x1e, 0x9d, 0x1e, 0x82, 0x5c, 0x72, + 0x2b, 0x91, 0x11, 0x64, 0x3c, 0x9d, 0x91, 0x32, 0x52, 0xe8, 0x3c, 0x29, 0xb1, 0x30, 0x29, 0x15, + 0x83, 0xa4, 0x14, 0xc2, 0x91, 0x67, 0x97, 0x62, 0xcf, 0x46, 0x75, 0xd0, 0xd8, 0x91, 0xed, 0xb6, + 0x76, 0x77, 0x1a, 0xac, 0x2b, 0x25, 0x2a, 0x8b, 0x44, 0x3c, 0xb6, 0xae, 0x7f, 0xae, 0xc0, 0x55, + 0x9e, 0xe1, 0xac, 0xd1, 0x20, 0x91, 0xa0, 0xa6, 0xd4, 0x3b, 0xdc, 0x83, 0xb2, 0x29, 0xf4, 0x38, + 0x21, 0xeb, 0x04, 0xca, 0xc6, 0x12, 0x19, 0x35, 0x61, 0x81, 0x49, 0x91, 0x82, 0x7c, 0x24, 0x14, + 0xb6, 0x70, 0xe7, 0x7a, 0x8a, 0xbc, 0x9b, 0x42, 0xc1, 0x19, 0x12, 0xfd, 0x7f, 0x0a, 0x2c, 0xed, + 0x12, 0xaf, 0x3f, 0xfd, 0x5b, 0xdd, 0x87, 0x9a, 0xf5, 0x9a, 0x45, 0x26, 0x85, 0x8f, 0xda, 0x80, + 0x86, 0x5c, 0x32, 0xab, 0xf5, 0x5a, 0x4e, 0x91, 0x43, 0x14, 0x99, 0xbf, 0x98, 0x48, 0x6c, 0x7b, + 0x70, 0x71, 0xd7, 0xb0, 0x1d, 0xdf, 0xb0, 0x1d, 0xe2, 0x3d, 0x08, 0x4f, 0x43, 0xdf, 0x48, 0x34, + 0x55, 0x4a, 0x4e, 0x22, 0x8d, 0x69, 0xb2, 0x5d, 0x95, 0x4e, 0xa0, 0xd6, 0xa1, 0x16, 0x89, 0xcf, + 0xba, 0x07, 0x95, 0x81, 0xfd, 0x94, 0x38, 0x84, 0x31, 0x59, 0xa4, 0xd3, 0x72, 0x73, 0xec, 0x1d, + 0x89, 0x80, 0x23, 0x54, 0xb4, 0x0a, 0xe0, 0x50, 0x8b, 0xf4, 0x88, 0x4b, 0xcd, 0x43, 0x19, 0x24, + 0x55, 0xbe, 0xb2, 0xc5, 0x17, 0xf4, 0x1f, 0xc2, 0x52, 0x97, 0xf8, 0x29, 0x5a, 0x69, 0xb1, 0xdb, + 0x50, 0xf6, 0x0d, 0xaf, 0x4f, 0xfc, 0xc9, 0xdc, 0x24, 0xe2, 0x24, 0x5e, 0x43, 0xb8, 0x32, 0xc6, + 0x4b, 0x96, 0xae, 0xbb, 0x30, 0x6b, 0xb8, 0xee, 0xc0, 0x26, 0xd6, 0x64, 0x6e, 0x21, 0xe6, 0x24, + 0x76, 0x7f, 0x51, 0x41, 0xcb, 0x2a, 0xf8, 0xbc, 0x7e, 0xb8, 0x0a, 0xc0, 0xbf, 0x7a, 0xdc, 0x4c, + 0x44, 0xb0, 0xac, 0xe2, 0x2a, 0x5f, 0xe1, 0xc7, 0x13, 0x74, 0x1b, 0x4a, 0xc1, 0x4e, 0x5e, 0xf0, + 0x34, 0xe9, 0xd0, 0xa5, 0x0e, 0x71, 0x7c, 0x81, 0x8b, 0x03, 0x4c, 0xf4, 0x05, 0xa8, 0xc5, 0x15, + 0xa1, 0xe7, 0x47, 0x3d, 0x68, 0xaa, 0xb3, 0x95, 0x8d, 0x73, 0x29, 0xc7, 0xe9, 0xc7, 0x1a, 0x67, + 0xf4, 0x25, 0x58, 0x38, 0xa0, 0xd4, 0x67, 0xbe, 0x67, 0xb8, 0x3d, 0x8b, 0x3a, 0x44, 0x26, 0xa2, + 0x5a, 0xb4, 0xda, 0xa2, 0x0e, 0x19, 0xeb, 0x7d, 0x67, 0x73, 0x7a, 0xdf, 0xaf, 0xc1, 0xf5, 0x26, + 0xa5, 0x9e, 0x65, 0x3b, 0x86, 0x4f, 0xbd, 0xcd, 0x90, 0x3e, 0x74, 0x91, 0x65, 0x98, 0x7d, 0x4a, + 0x3c, 0x16, 0xf6, 0xc1, 0x05, 0x1c, 0x82, 0xfa, 0xf7, 0x60, 0x25, 0x9f, 0x50, 0xda, 0xfb, 0x1c, + 0x81, 0xf1, 0x77, 0x05, 0x2e, 0x35, 0x2c, 0x2b, 0xc6, 0x08, 0xa5, 0xf9, 0x32, 0xa8, 0xb6, 0x35, + 0xd9, 0xa0, 0xaa, 0x6d, 0xf1, 0x61, 0x2f, 0x91, 0x24, 0xe7, 0xa3, 0x2c, 0x38, 0x66, 0x8c, 0x9c, + 0x9a, 0x8d, 0xea, 0x70, 0xc1, 0x66, 0x3d, 0x87, 0x1c, 0xf7, 0x62, 0xd7, 0x10, 0x56, 0xab, 0xe0, + 0x45, 0x9b, 0x75, 0xc8, 0x71, 0xcc, 0x0e, 0xdd, 0x80, 0xb9, 0x23, 0x39, 0x2b, 0xf6, 0x6c, 0x4b, + 0x54, 0x8c, 0x1a, 0x86, 0x70, 0xa9, 0x6d, 0xe9, 0xbf, 0x51, 0xe0, 0x0a, 0x26, 0x43, 0xfa, 0x94, + 0x9c, 0xeb, 0x42, 0xcb, 0x30, 0x6b, 0x1a, 0xcc, 0x34, 0x2c, 0x22, 0xdb, 0xff, 0x10, 0xe4, 0x3b, + 0x9e, 0x38, 0xdf, 0x92, 0xd3, 0x45, 0x08, 0x66, 0x65, 0x2b, 0x8e, 0xc9, 0xf6, 0xfb, 0x02, 0x5c, + 0x8b, 0xa5, 0x1a, 0xb3, 0xfe, 0x39, 0x43, 0xe9, 0x34, 0x1b, 0x5c, 0x15, 0xae, 0xe1, 0x25, 0xd4, + 0x1f, 0xd5, 0x74, 0x13, 0xde, 0xf2, 0x79, 0x03, 0xd0, 0xf3, 0x3d, 0xbb, 0xdf, 0x27, 0x5e, 0x8f, + 0x3c, 0x25, 0x8e, 0xdf, 0x8b, 0x33, 0x7d, 0x78, 0x8f, 0x33, 0x73, 0xfa, 0xaa, 0x38, 0x63, 0x3f, + 0x38, 0x62, 0x8b, 0x9f, 0x90, 0x9c, 0x12, 0xf2, 0xcd, 0x5b, 0xca, 0x37, 0xaf, 0x01, 0xeb, 0x69, + 0x81, 0xf8, 0x4c, 0x9d, 0x91, 0xa7, 0x3c, 0x49, 0x9e, 0x95, 0xa4, 0x3c, 0xbc, 0x1b, 0x4b, 0x89, + 0x93, 0xb1, 0xd2, 0xec, 0x98, 0x95, 0x7e, 0xa5, 0xc2, 0xf5, 0x5c, 0x2b, 0x4d, 0x67, 0x2a, 0xb8, + 0x07, 0x25, 0xde, 0x69, 0x85, 0x15, 0xf7, 0x46, 0x8a, 0x2e, 0xe2, 0x16, 0xf7, 0x65, 0x01, 0x76, + 0x98, 0xb1, 0x0a, 0xaf, 0x32, 0xea, 0xbf, 0x5a, 0x0e, 0x7c, 0x07, 0x90, 0xd0, 0x6e, 0x1a, 0xb3, + 0x24, 0x30, 0x35, 0xbe, 0x93, 0x9c, 0x2c, 0xf4, 0x3f, 0xaa, 0xb0, 0x16, 0x6b, 0x65, 0x8f, 0x32, + 0x7f, 0xda, 0xfe, 0xfb, 0x4a, 0xce, 0xa8, 0x9e, 0xd3, 0x19, 0x6f, 0xc3, 0x6c, 0xd0, 0x76, 0xf2, + 0x58, 0xe0, 0xfa, 0xbf, 0x32, 0xd6, 0x8f, 0x0d, 0x8d, 0xb6, 0xf3, 0x98, 0xe2, 0x10, 0x0f, 0xbd, + 0x07, 0xf3, 0x42, 0x4f, 0x21, 0x5d, 0xf1, 0x6c, 0xba, 0x39, 0x8e, 0x1c, 0xc0, 0x4c, 0xff, 0x8f, + 0x02, 0x37, 0x4e, 0xd5, 0xda, 0x74, 0xfc, 0xe9, 0x8d, 0xa8, 0xed, 0x75, 0xbc, 0x4f, 0x7f, 0x06, + 0x10, 0xeb, 0x23, 0xf5, 0x1a, 0xa1, 0x64, 0x5e, 0x23, 0xd6, 0x42, 0xcc, 0x8e, 0x31, 0x0c, 0xab, + 0x7f, 0x62, 0x05, 0xdd, 0x84, 0xb2, 0x08, 0x84, 0xd0, 0x58, 0x39, 0xb3, 0x8b, 0xd0, 0xb9, 0xc4, + 0xd2, 0x9b, 0xf2, 0xd9, 0x52, 0x30, 0x3e, 0xfd, 0xd9, 0x72, 0x45, 0xa2, 0x25, 0xb8, 0xc6, 0x0b, + 0xfa, 0x9f, 0x55, 0x40, 0xe3, 0x71, 0xc8, 0x8b, 0xc7, 0x29, 0xc6, 0x49, 0x29, 0x52, 0x95, 0xcf, + 0xa2, 0xe1, 0x95, 0xd5, 0xcc, 0x95, 0xc3, 0x61, 0xac, 0xf0, 0x0a, 0xc3, 0xd8, 0x87, 0xa0, 0x99, + 0x61, 0x8f, 0xd3, 0x0b, 0xea, 0xb5, 0x88, 0xe4, 0x09, 0x8d, 0xd0, 0xa2, 0x99, 0x84, 0x47, 0x6c, + 0x3c, 0x1d, 0x94, 0x72, 0xd2, 0xc1, 0x5d, 0x98, 0x3b, 0x18, 0x50, 0xf3, 0x48, 0xb6, 0x62, 0x41, + 0x96, 0x45, 0x69, 0x2f, 0x17, 0xc7, 0x83, 0x40, 0x0b, 0xfa, 0xb3, 0xb0, 0x75, 0x9f, 0x4d, 0xb4, + 0xee, 0x4f, 0x60, 0x29, 0x76, 0xf9, 0xe6, 0x80, 0x32, 0x32, 0xa5, 0x04, 0x91, 0xa8, 0xbc, 0x6a, + 0xaa, 0xf2, 0xea, 0x1e, 0x5c, 0x19, 0x63, 0x39, 0x9d, 0xe8, 0xe2, 0xf3, 0xf0, 0xc8, 0x34, 0xf9, + 0x8c, 0x20, 0x79, 0x4a, 0x50, 0xff, 0xb9, 0x02, 0x5a, 0xfc, 0x7c, 0x15, 0x38, 0xe0, 0x14, 0x5e, + 0xff, 0xae, 0x41, 0x45, 0xba, 0x69, 0x50, 0x21, 0x0a, 0x38, 0x82, 0xcf, 0x7a, 0xd8, 0xd3, 0xbf, + 0x0f, 0x25, 0x81, 0x37, 0xe1, 0xb5, 0xfe, 0x34, 0xb7, 0x5c, 0x81, 0x6a, 0xd7, 0x1d, 0xd8, 0x22, + 0x0b, 0xc8, 0xbe, 0x26, 0x5e, 0xd0, 0x1d, 0x58, 0x08, 0x31, 0x03, 0x5d, 0x9d, 0xc1, 0x65, 0x1d, + 0xe6, 0x1e, 0x0e, 0xac, 0x0c, 0xa3, 0xe4, 0x12, 0xc7, 0xe8, 0x90, 0xe3, 0xcc, 0x4d, 0x92, 0x4b, + 0xfa, 0xef, 0x0a, 0x50, 0x0a, 0x1c, 0x6c, 0x05, 0xaa, 0x6d, 0xb6, 0xc9, 0x1d, 0x4e, 0x4e, 0x32, + 0x15, 0x1c, 0x2f, 0x70, 0x29, 0xc4, 0x67, 0xfc, 0x5a, 0x21, 0x41, 0x74, 0x1f, 0xe6, 0x82, 0xcf, + 0x30, 0x7d, 0x8c, 0x8f, 0xee, 0x59, 0xe3, 0xe1, 0x24, 0x05, 0xda, 0x86, 0x0b, 0x1d, 0x42, 0xac, + 0x96, 0x47, 0x5d, 0x37, 0xc4, 0x90, 0xad, 0xd0, 0x84, 0x63, 0xc6, 0xe9, 0xd0, 0xfb, 0xb0, 0xc8, + 0x17, 0x1b, 0x96, 0x15, 0x1d, 0x15, 0x8c, 0x1e, 0x68, 0x3c, 0xfe, 0x71, 0x16, 0x15, 0x35, 0x61, + 0xe1, 0x63, 0xd7, 0x32, 0x7c, 0x22, 0x55, 0xc8, 0x96, 0xcb, 0x82, 0xf8, 0x7a, 0x5e, 0x09, 0x92, + 0x06, 0xc2, 0x19, 0x92, 0xec, 0x9b, 0xf6, 0xec, 0xd8, 0x9b, 0x36, 0xfa, 0x8a, 0x98, 0xb5, 0xfa, + 0x64, 0xb9, 0x22, 0x7c, 0x36, 0x5d, 0xe0, 0x36, 0x65, 0xcc, 0xf7, 0x83, 0x39, 0xab, 0x4f, 0xf4, + 0x1f, 0xc3, 0xa5, 0x28, 0x5f, 0x85, 0xbb, 0x3c, 0xd9, 0xbc, 0x46, 0x9e, 0xdc, 0x08, 0xa7, 0x3b, + 0xf5, 0xd4, 0x64, 0x23, 0x87, 0xba, 0xbc, 0xb7, 0xcf, 0x7f, 0x2b, 0xb0, 0x98, 0xf9, 0x11, 0xe5, + 0x75, 0x98, 0xe7, 0x25, 0x57, 0x75, 0x1a, 0xc9, 0x35, 0x6f, 0xc4, 0xb9, 0x0d, 0x97, 0x83, 0xb2, + 0xcc, 0xec, 0xe7, 0xa4, 0xe7, 0x12, 0xaf, 0xc7, 0x88, 0x49, 0x9d, 0xa0, 0xb9, 0x56, 0x31, 0x12, + 0x9b, 0x5d, 0xfb, 0x39, 0xd9, 0x23, 0x5e, 0x57, 0xec, 0xe4, 0x3d, 0x8a, 0xe9, 0x7f, 0x50, 0x00, + 0x25, 0x74, 0x3d, 0xa5, 0xbc, 0xfa, 0x11, 0xd4, 0x0e, 0xe2, 0x43, 0xa3, 0x27, 0xea, 0xb7, 0xf2, + 0x6b, 0x53, 0x92, 0x7f, 0x9a, 0x2e, 0xd7, 0x4a, 0x16, 0xcc, 0x27, 0x3b, 0x04, 0x8e, 0xe3, 0xdb, + 0xc3, 0x20, 0x31, 0x56, 0xb1, 0xf8, 0xe6, 0x6b, 0x0e, 0xb5, 0xc2, 0x52, 0x2c, 0xbe, 0xf9, 0x9a, + 0x19, 0x9e, 0x55, 0xc5, 0xe2, 0x9b, 0x87, 0xfb, 0x30, 0x78, 0x47, 0x15, 0x7a, 0xab, 0xe2, 0x10, + 0xd4, 0xdf, 0x85, 0xf9, 0xec, 0x93, 0xd2, 0xa1, 0xdd, 0x3f, 0x94, 0xbf, 0xec, 0x88, 0x6f, 0xa4, + 0x41, 0x61, 0x40, 0x8f, 0x65, 0xa2, 0xe0, 0x9f, 0x5c, 0xb6, 0xa4, 0x5a, 0x5e, 0x8d, 0x4a, 0x48, + 0xcb, 0x1b, 0x07, 0x29, 0x19, 0xff, 0xe6, 0xa9, 0x35, 0x9c, 0x20, 0xa4, 0x68, 0x11, 0xac, 0xff, + 0x00, 0x6e, 0xec, 0xd0, 0x7e, 0x62, 0x7a, 0x8f, 0x1f, 0x87, 0xa7, 0x63, 0x40, 0xfd, 0x27, 0x0a, + 0xac, 0x9f, 0xce, 0x62, 0x3a, 0x85, 0x70, 0xd2, 0xcb, 0xf3, 0x80, 0xeb, 0x92, 0x98, 0x47, 0x6c, + 0x34, 0xdc, 0x25, 0xbe, 0x81, 0xbe, 0x1a, 0xc6, 0x76, 0x5e, 0x05, 0x0c, 0x31, 0x53, 0x31, 0x5e, + 0x07, 0xcd, 0x4c, 0xae, 0x77, 0xc9, 0x13, 0xc9, 0x67, 0x6c, 0x5d, 0xff, 0xa5, 0x02, 0x97, 0x13, + 0x3f, 0x9a, 0x10, 0x3f, 0x3c, 0x11, 0x5d, 0x82, 0x92, 0x49, 0x47, 0x8e, 0x2f, 0x8d, 0x18, 0x00, + 0xdc, 0x73, 0x9e, 0x51, 0xef, 0x01, 0x37, 0xae, 0x2c, 0x14, 0x12, 0xe4, 0x53, 0xf3, 0x33, 0xea, + 0xed, 0xd0, 0x63, 0x19, 0xb7, 0x12, 0x0a, 0x0a, 0xff, 0x50, 0x50, 0x14, 0xe5, 0xd0, 0x1c, 0x80, + 0x9c, 0x82, 0x8d, 0x86, 0x9c, 0x22, 0x68, 0xa3, 0x24, 0xa4, 0xff, 0x56, 0x81, 0xf5, 0x5c, 0x99, + 0x1a, 0xe6, 0xd1, 0xb4, 0xac, 0x70, 0x09, 0x4a, 0xc9, 0xc7, 0xb9, 0x00, 0xc8, 0x8b, 0xbb, 0xf0, + 0xb7, 0xd9, 0x62, 0xf4, 0xdb, 0xac, 0xfe, 0x4f, 0x05, 0xf4, 0x5c, 0xf9, 0x82, 0x4a, 0x31, 0xa5, + 0x64, 0x72, 0x0e, 0x09, 0xd1, 0x07, 0x50, 0x09, 0x2d, 0x2d, 0x74, 0x9b, 0xfd, 0xb5, 0x30, 0x57, + 0x7a, 0x1c, 0xd1, 0xd4, 0x57, 0xa1, 0x2c, 0x7f, 0x4f, 0xae, 0x42, 0xe9, 0x91, 0x67, 0xfb, 0x44, + 0x9b, 0x41, 0x15, 0x28, 0xee, 0x19, 0x8c, 0x69, 0x4a, 0x7d, 0x23, 0xe8, 0x62, 0xe2, 0xf7, 0x75, + 0x04, 0x50, 0x6e, 0x7a, 0xc4, 0x10, 0x78, 0x00, 0xe5, 0xe0, 0xdd, 0x48, 0x53, 0xea, 0xf7, 0x60, + 0x3e, 0xf9, 0x42, 0xca, 0x8f, 0x6b, 0xec, 0xb4, 0xbf, 0xb3, 0xa5, 0xcd, 0xa0, 0x79, 0xa8, 0xb4, + 0x70, 0xa3, 0xdd, 0x69, 0x77, 0x3e, 0xd2, 0x14, 0x0e, 0x75, 0xf7, 0x1f, 0xee, 0xed, 0x71, 0x48, + 0xad, 0xbf, 0x07, 0x10, 0xd7, 0x49, 0xce, 0xb8, 0xf3, 0xb0, 0xc3, 0x69, 0xe6, 0x60, 0xf6, 0x51, + 0xa3, 0xbd, 0x1f, 0x90, 0x70, 0x00, 0x07, 0x80, 0xca, 0x71, 0x5a, 0x1c, 0xa7, 0x50, 0x7f, 0x27, + 0xd3, 0x39, 0xa2, 0x59, 0x28, 0x34, 0x06, 0x03, 0x6d, 0x06, 0x95, 0x41, 0x6d, 0x6d, 0x6a, 0x0a, + 0x17, 0xb0, 0x43, 0xbd, 0xa1, 0x31, 0xd0, 0xd4, 0xfa, 0x73, 0x58, 0x48, 0xd7, 0x25, 0x71, 0x2c, + 0xf5, 0x8e, 0x6c, 0xa7, 0x1f, 0x30, 0xec, 0xfa, 0xa2, 0x01, 0x09, 0x18, 0x06, 0x17, 0xb3, 0x34, + 0x15, 0x69, 0x30, 0xdf, 0x76, 0x6c, 0xdf, 0x36, 0x06, 0xf6, 0x73, 0x8e, 0x5b, 0x40, 0x35, 0xa8, + 0xee, 0x79, 0xc4, 0x35, 0x3c, 0x0e, 0x16, 0xd1, 0x02, 0x80, 0xf8, 0xc5, 0x01, 0x13, 0xc3, 0x3a, + 0xd1, 0x4a, 0x9c, 0xe0, 0x91, 0x61, 0xfb, 0xb6, 0xd3, 0x17, 0xcb, 0x5a, 0xb9, 0xfe, 0x4d, 0xa8, + 0xa5, 0xe2, 0x17, 0x5d, 0x80, 0xda, 0xc7, 0x9d, 0x76, 0xa7, 0xbd, 0xdf, 0x6e, 0xec, 0xb4, 0x3f, + 0xd9, 0x6a, 0x05, 0x5a, 0xda, 0x6d, 0x77, 0x77, 0x1b, 0xfb, 0xcd, 0x07, 0x9a, 0xc2, 0xd5, 0x17, + 0x7c, 0xaa, 0x9b, 0x1f, 0xfc, 0xed, 0xc5, 0x9a, 0xf2, 0xd9, 0x8b, 0x35, 0xe5, 0x5f, 0x2f, 0xd6, + 0x94, 0x5f, 0xbc, 0x5c, 0x9b, 0xf9, 0xec, 0xe5, 0xda, 0xcc, 0x3f, 0x5e, 0xae, 0xcd, 0x7c, 0xf2, + 0xc5, 0xbe, 0xed, 0x1f, 0x8e, 0x0e, 0x6e, 0x9a, 0x74, 0x78, 0xcb, 0xb5, 0x9d, 0xbe, 0x69, 0xb8, + 0xb7, 0x7c, 0xdb, 0xb4, 0xcc, 0x5b, 0x09, 0x17, 0x38, 0x28, 0x8b, 0x3f, 0xab, 0xdc, 0xfd, 0x7f, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xcf, 0x03, 0xb6, 0xbb, 0xcb, 0x22, 0x00, 0x00, } func (m *TableSpan) Marshal() (dAtA []byte, err error) { @@ -4228,6 +4432,105 @@ func (m *MaintainerHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *NodeHeartbeat) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeHeartbeat) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NodeHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NodeEpoch != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch)) + i-- + dAtA[i] = 0x10 + } + if m.Liveness != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.Liveness)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *SetNodeLivenessRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SetNodeLivenessRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SetNodeLivenessRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NodeEpoch != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch)) + i-- + dAtA[i] = 0x10 + } + if m.Target != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.Target)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *SetNodeLivenessResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SetNodeLivenessResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SetNodeLivenessResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NodeEpoch != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch)) + i-- + dAtA[i] = 0x10 + } + if m.Applied != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.Applied)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *MaintainerStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6186,19 +6489,64 @@ func (m *MaintainerHeartbeat) Size() (n int) { return n } -func (m *MaintainerStatus) Size() (n int) { +func (m *NodeHeartbeat) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.ChangefeedID != nil { - l = m.ChangefeedID.Size() - n += 1 + l + sovHeartbeat(uint64(l)) + if m.Liveness != 0 { + n += 1 + sovHeartbeat(uint64(m.Liveness)) } - l = len(m.FeedState) - if l > 0 { - n += 1 + l + sovHeartbeat(uint64(l)) + if m.NodeEpoch != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeEpoch)) + } + return n +} + +func (m *SetNodeLivenessRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Target != 0 { + n += 1 + sovHeartbeat(uint64(m.Target)) + } + if m.NodeEpoch != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeEpoch)) + } + return n +} + +func (m *SetNodeLivenessResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Applied != 0 { + n += 1 + sovHeartbeat(uint64(m.Applied)) + } + if m.NodeEpoch != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeEpoch)) + } + return n +} + +func (m *MaintainerStatus) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ChangefeedID != nil { + l = m.ChangefeedID.Size() + n += 1 + l + sovHeartbeat(uint64(l)) + } + l = len(m.FeedState) + if l > 0 { + n += 1 + l + sovHeartbeat(uint64(l)) } if m.State != 0 { n += 1 + sovHeartbeat(uint64(m.State)) @@ -9083,6 +9431,270 @@ func (m *MaintainerHeartbeat) Unmarshal(dAtA []byte) error { } return nil } +func (m *NodeHeartbeat) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeHeartbeat: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeHeartbeat: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Liveness", wireType) + } + m.Liveness = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Liveness |= NodeLiveness(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType) + } + m.NodeEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeEpoch |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHeartbeat(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHeartbeat + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SetNodeLivenessRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SetNodeLivenessRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SetNodeLivenessRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Target", wireType) + } + m.Target = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Target |= NodeLiveness(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType) + } + m.NodeEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeEpoch |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHeartbeat(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHeartbeat + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SetNodeLivenessResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SetNodeLivenessResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SetNodeLivenessResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Applied", wireType) + } + m.Applied = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Applied |= NodeLiveness(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType) + } + m.NodeEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeEpoch |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHeartbeat(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHeartbeat + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *MaintainerStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/heartbeatpb/heartbeat.proto b/heartbeatpb/heartbeat.proto index 0e3fba2cd8..faf86507d4 100644 --- a/heartbeatpb/heartbeat.proto +++ b/heartbeatpb/heartbeat.proto @@ -126,6 +126,36 @@ message MaintainerHeartbeat { repeated MaintainerStatus statuses = 1; } +// NodeLiveness is the node-level liveness state reported by a NodeAgent. +// +// It is used by the coordinator to: +// - filter destination candidates for scheduling +// - drive node drain progress +// - avoid campaigning/residing leadership on nodes preparing to go offline +enum NodeLiveness { + ALIVE = 0; + DRAINING = 1; + STOPPING = 2; +} + +// NodeHeartbeat is a node-scoped heartbeat, independent of maintainer heartbeats. +message NodeHeartbeat { + NodeLiveness liveness = 1; + uint64 node_epoch = 2; +} + +// SetNodeLivenessRequest asks a node to upgrade its local liveness monotonically. +message SetNodeLivenessRequest { + NodeLiveness target = 1; + uint64 node_epoch = 2; +} + +// SetNodeLivenessResponse reports the liveness applied by the node and its current epoch. +message SetNodeLivenessResponse { + NodeLiveness applied = 1; + uint64 node_epoch = 2; +} + message MaintainerStatus { ChangefeedID changefeedID = 1; string feed_state = 2; @@ -376,4 +406,4 @@ message DispatcherSetChecksumUpdateRequest { int64 mode = 3; uint64 seq = 4; DispatcherSetChecksum checksum = 5; -} \ No newline at end of file +} diff --git a/maintainer/maintainer_manager.go b/maintainer/maintainer_manager.go index 469dc31fc1..36b1800f61 100644 --- a/maintainer/maintainer_manager.go +++ b/maintainer/maintainer_manager.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/api" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/config" @@ -36,8 +37,13 @@ import ( // 2. Handle other commands from coordinator: like add or remove changefeed maintainer // 3. Manage maintainers lifetime type Manager struct { - mc messaging.MessageCenter - conf *config.SchedulerConfig + mc messaging.MessageCenter + conf *config.SchedulerConfig + liveness *api.Liveness + + nodeEpoch uint64 + lastNodeHeartbeatTime time.Time + nodeHeartbeatInterval time.Duration // changefeedID -> maintainer maintainers sync.Map @@ -58,15 +64,20 @@ type Manager struct { func NewMaintainerManager( nodeInfo *node.Info, conf *config.SchedulerConfig, + liveness *api.Liveness, ) *Manager { mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) m := &Manager{ mc: mc, conf: conf, + liveness: liveness, maintainers: sync.Map{}, nodeInfo: nodeInfo, msgCh: make(chan *messaging.TargetMessage, 1024), taskScheduler: threadpool.NewThreadPoolDefault(), + + nodeEpoch: getNodeEpoch(), + nodeHeartbeatInterval: 5 * time.Second, } mc.RegisterHandler(messaging.MaintainerManagerTopic, m.recvMessages) @@ -84,7 +95,8 @@ func (m *Manager) recvMessages(ctx context.Context, msg *messaging.TargetMessage // Coordinator related messages case messaging.TypeAddMaintainerRequest, messaging.TypeRemoveMaintainerRequest, - messaging.TypeCoordinatorBootstrapRequest: + messaging.TypeCoordinatorBootstrapRequest, + messaging.TypeSetNodeLivenessRequest: select { case <-ctx.Done(): return ctx.Err() @@ -135,6 +147,7 @@ func (m *Manager) Run(ctx context.Context) error { case <-ticker.C: // 1. try to send heartbeat to coordinator m.sendHeartbeat() + m.sendNodeHeartbeat(false) // 2. cleanup removed maintainers m.maintainers.Range(func(key, value interface{}) bool { cf := value.(*Maintainer) @@ -214,9 +227,19 @@ func (m *Manager) onCoordinatorBootstrapRequest(msg *messaging.TargetMessage) { log.Info("new coordinator online, bootstrap response already sent", zap.Stringer("coordinatorID", m.coordinatorID), zap.Int64("version", m.coordinatorVersion)) + + // Report node-scoped liveness state eagerly after coordinator bootstrap. + m.sendNodeHeartbeat(true) } func (m *Manager) onAddMaintainerRequest(req *heartbeatpb.AddMaintainerRequest) *heartbeatpb.MaintainerStatus { + if m.liveness != nil && m.liveness.Load() == api.LivenessCaptureStopping { + log.Info("ignore add maintainer request, node is stopping", + zap.Stringer("changefeedID", common.NewChangefeedIDFromPB(req.Id)), + zap.Stringer("nodeID", m.nodeInfo.ID)) + return nil + } + changefeedID := common.NewChangefeedIDFromPB(req.Id) _, ok := m.maintainers.Load(changefeedID) if ok { @@ -330,10 +353,125 @@ func (m *Manager) handleMessage(msg *messaging.TargetMessage) { } m.sendMessages(response) } + case messaging.TypeSetNodeLivenessRequest: + if m.isBootstrap() { + m.onSetNodeLivenessRequest(msg) + } default: } } +func (m *Manager) onSetNodeLivenessRequest(msg *messaging.TargetMessage) { + req := msg.Message[0].(*heartbeatpb.SetNodeLivenessRequest) + + if m.coordinatorID != msg.From { + log.Warn("ignore set node liveness request from non-coordinator", + zap.Stringer("from", msg.From), + zap.Stringer("coordinatorID", m.coordinatorID)) + return + } + + // Reject stale control messages from a previous process incarnation. + if req.NodeEpoch != m.nodeEpoch { + m.sendSetNodeLivenessResponse() + return + } + + if m.liveness == nil { + m.sendSetNodeLivenessResponse() + return + } + + before := m.liveness.Load() + switch req.Target { + case heartbeatpb.NodeLiveness_DRAINING: + _ = m.liveness.Store(api.LivenessCaptureDraining) + case heartbeatpb.NodeLiveness_STOPPING: + _ = m.liveness.Store(api.LivenessCaptureStopping) + default: + // ALIVE or unknown values are ignored. + } + after := m.liveness.Load() + if before != after { + log.Info("node liveness updated", + zap.Stringer("nodeID", m.nodeInfo.ID), + zap.Uint64("nodeEpoch", m.nodeEpoch), + zap.String("before", before.String()), + zap.String("after", after.String())) + } + m.sendSetNodeLivenessResponse() + m.sendNodeHeartbeat(true) +} + +func (m *Manager) sendSetNodeLivenessResponse() { + applied := heartbeatpb.NodeLiveness_ALIVE + if m.liveness != nil { + applied = toNodeLivenessPB(m.liveness.Load()) + } + resp := &heartbeatpb.SetNodeLivenessResponse{ + Applied: applied, + NodeEpoch: m.nodeEpoch, + } + target := m.newCoordinatorTopicMessage(resp) + if err := m.mc.SendCommand(target); err != nil { + log.Warn("send set node liveness response failed", + zap.Stringer("from", m.nodeInfo.ID), + zap.Stringer("target", target.To), + zap.Error(err)) + } +} + +func (m *Manager) sendNodeHeartbeat(force bool) { + if !m.isBootstrap() { + return + } + now := time.Now() + if !force && !m.lastNodeHeartbeatTime.IsZero() && + now.Sub(m.lastNodeHeartbeatTime) < m.nodeHeartbeatInterval { + return + } + + liveness := heartbeatpb.NodeLiveness_ALIVE + if m.liveness != nil { + liveness = toNodeLivenessPB(m.liveness.Load()) + } + hb := &heartbeatpb.NodeHeartbeat{ + Liveness: liveness, + NodeEpoch: m.nodeEpoch, + } + target := m.newCoordinatorTopicMessage(hb) + if err := m.mc.SendCommand(target); err != nil { + log.Warn("send node heartbeat failed", + zap.Stringer("from", m.nodeInfo.ID), + zap.Stringer("target", target.To), + zap.Error(err)) + return + } + m.lastNodeHeartbeatTime = now +} + +func toNodeLivenessPB(l api.Liveness) heartbeatpb.NodeLiveness { + switch l { + case api.LivenessCaptureAlive: + return heartbeatpb.NodeLiveness_ALIVE + case api.LivenessCaptureDraining: + return heartbeatpb.NodeLiveness_DRAINING + case api.LivenessCaptureStopping: + return heartbeatpb.NodeLiveness_STOPPING + default: + return heartbeatpb.NodeLiveness_ALIVE + } +} + +func getNodeEpoch() uint64 { + // Use UnixNano to ensure uniqueness across restarts. + epoch := uint64(time.Now().UnixNano()) + if epoch == 0 { + epoch = 1 + } + return epoch +} + func (m *Manager) dispatcherMaintainerMessage( ctx context.Context, changefeed common.ChangeFeedID, msg *messaging.TargetMessage, ) error { diff --git a/maintainer/maintainer_manager_test.go b/maintainer/maintainer_manager_test.go index 1f11f8b9e6..ab7a0c696b 100644 --- a/maintainer/maintainer_manager_test.go +++ b/maintainer/maintainer_manager_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/maintainer/testutil" + "github.com/pingcap/ticdc/pkg/api" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" commonEvent "github.com/pingcap/ticdc/pkg/common/event" @@ -99,7 +100,8 @@ func TestMaintainerSchedulesNodeChanges(t *testing.T) { AddTableBatchSize: 1000, CheckBalanceInterval: 0, } - manager := NewMaintainerManager(selfNode, schedulerConf) + var liveness api.Liveness + manager := NewMaintainerManager(selfNode, schedulerConf, &liveness) msg := messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}) @@ -330,7 +332,8 @@ func TestMaintainerBootstrapWithTablesReported(t *testing.T) { mc.RegisterHandler(messaging.CoordinatorTopic, func(ctx context.Context, msg *messaging.TargetMessage) error { return nil }) - manager := NewMaintainerManager(selfNode, config.GetGlobalServerConfig().Debug.Scheduler) + var liveness api.Liveness + manager := NewMaintainerManager(selfNode, config.GetGlobalServerConfig().Debug.Scheduler, &liveness) msg := messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}) @@ -477,7 +480,8 @@ func TestStopNotExistsMaintainer(t *testing.T) { return nil }) schedulerConf := &config.SchedulerConfig{AddTableBatchSize: 1000} - manager := NewMaintainerManager(selfNode, schedulerConf) + var liveness api.Liveness + manager := NewMaintainerManager(selfNode, schedulerConf, &liveness) msg := messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}) diff --git a/pkg/api/util.go b/pkg/api/util.go index a10e4152e4..cd11e59962 100644 --- a/pkg/api/util.go +++ b/pkg/api/util.go @@ -145,20 +145,52 @@ func WriteData(w http.ResponseWriter, data interface{}) { } } -// Liveness can only be changed from alive to stopping, and no way back. +// Liveness is a node-level liveness state used for graceful shutdown and drain. +// +// It can only be upgraded monotonically: +// - ALIVE -> DRAINING -> STOPPING +// - ALIVE -> STOPPING (signal-driven graceful shutdown) type Liveness int32 const ( // LivenessCaptureAlive means the capture is alive, and ready to serve. LivenessCaptureAlive Liveness = 0 + // LivenessCaptureDraining means the capture is preparing to go offline. + // It should not be selected as a scheduling destination and should not campaign leadership. + LivenessCaptureDraining Liveness = 2 // LivenessCaptureStopping means the capture is in the process of graceful shutdown. LivenessCaptureStopping Liveness = 1 ) // Store the given liveness. Returns true if it success. func (l *Liveness) Store(v Liveness) bool { - return atomic.CompareAndSwapInt32( - (*int32)(l), int32(LivenessCaptureAlive), int32(v)) + for { + old := l.Load() + if old == v { + return false + } + + switch old { + case LivenessCaptureAlive: + // Allow direct ALIVE->STOPPING for the SIGTERM path, and ALIVE->DRAINING for manual drain. + if v != LivenessCaptureDraining && v != LivenessCaptureStopping { + return false + } + case LivenessCaptureDraining: + if v != LivenessCaptureStopping { + return false + } + case LivenessCaptureStopping: + return false + default: + // Unknown old value, refuse to update. + return false + } + + if atomic.CompareAndSwapInt32((*int32)(l), int32(old), int32(v)) { + return true + } + } } // Load the liveness. @@ -170,6 +202,8 @@ func (l *Liveness) String() string { switch *l { case LivenessCaptureAlive: return "Alive" + case LivenessCaptureDraining: + return "Draining" case LivenessCaptureStopping: return "Stopping" default: diff --git a/pkg/api/util_test.go b/pkg/api/util_test.go new file mode 100644 index 0000000000..943edae9d8 --- /dev/null +++ b/pkg/api/util_test.go @@ -0,0 +1,35 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLivenessStoreMonotonic(t *testing.T) { + var l Liveness + require.Equal(t, LivenessCaptureAlive, l.Load()) + + require.True(t, l.Store(LivenessCaptureDraining)) + require.Equal(t, LivenessCaptureDraining, l.Load()) + + require.True(t, l.Store(LivenessCaptureStopping)) + require.Equal(t, LivenessCaptureStopping, l.Load()) + + // Reject downgrade and further updates. + require.False(t, l.Store(LivenessCaptureDraining)) + require.False(t, l.Store(LivenessCaptureAlive)) + require.False(t, l.Store(LivenessCaptureStopping)) +} + +func TestLivenessStoreDirectToStopping(t *testing.T) { + var l Liveness + require.Equal(t, LivenessCaptureAlive, l.Load()) + + require.True(t, l.Store(LivenessCaptureStopping)) + require.Equal(t, LivenessCaptureStopping, l.Load()) + + // Reject any updates after STOPPING. + require.False(t, l.Store(LivenessCaptureDraining)) + require.False(t, l.Store(LivenessCaptureAlive)) +} diff --git a/pkg/messaging/message.go b/pkg/messaging/message.go index 61f3559414..10f198fbdc 100644 --- a/pkg/messaging/message.go +++ b/pkg/messaging/message.go @@ -104,6 +104,11 @@ const ( TypeRedoResolvedTsForwardMessage IOType = 39 TypeDispatcherSetChecksumUpdateRequest IOType = 40 TypeDispatcherSetChecksumAckResponse IOType = 41 + + // Node liveness related + TypeNodeHeartbeatRequest IOType = 42 + TypeSetNodeLivenessRequest IOType = 43 + TypeSetNodeLivenessResponse IOType = 44 ) func (t IOType) String() string { @@ -182,6 +187,12 @@ func (t IOType) String() string { return "DispatcherSetChecksumUpdateRequest" case TypeDispatcherSetChecksumAckResponse: return "DispatcherSetChecksumAckResponse" + case TypeNodeHeartbeatRequest: + return "NodeHeartbeatRequest" + case TypeSetNodeLivenessRequest: + return "SetNodeLivenessRequest" + case TypeSetNodeLivenessResponse: + return "SetNodeLivenessResponse" case TypeDispatcherHeartbeatResponse: return "DispatcherHeartbeatResponse" case TypeCongestionControl: @@ -389,6 +400,12 @@ func decodeIOType(ioType IOType, value []byte) (IOTypeT, error) { m = &heartbeatpb.LogCoordinatorResolvedTsRequest{} case TypeLogCoordinatorResolvedTsResponse: m = &heartbeatpb.LogCoordinatorResolvedTsResponse{} + case TypeNodeHeartbeatRequest: + m = &heartbeatpb.NodeHeartbeat{} + case TypeSetNodeLivenessRequest: + m = &heartbeatpb.SetNodeLivenessRequest{} + case TypeSetNodeLivenessResponse: + m = &heartbeatpb.SetNodeLivenessResponse{} default: log.Debug("Unimplemented IOType, ignore the message", zap.Stringer("Type", ioType)) return nil, errors.ErrUnimplementedIOType.GenWithStackByArgs(int(ioType)) @@ -501,6 +518,12 @@ func NewSingleTargetMessage(To node.ID, Topic string, Message IOTypeT, Group ... ioType = TypeLogCoordinatorResolvedTsRequest case *heartbeatpb.LogCoordinatorResolvedTsResponse: ioType = TypeLogCoordinatorResolvedTsResponse + case *heartbeatpb.NodeHeartbeat: + ioType = TypeNodeHeartbeatRequest + case *heartbeatpb.SetNodeLivenessRequest: + ioType = TypeSetNodeLivenessRequest + case *heartbeatpb.SetNodeLivenessResponse: + ioType = TypeSetNodeLivenessResponse default: panic("unknown io type") } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 2d6baabb9a..586933a3e7 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -24,6 +24,7 @@ const DefaultCheckInterval = time.Second * 120 const ( BasicScheduler = "basic-scheduler" BalanceScheduler = "balance-scheduler" + DrainScheduler = "drain-scheduler" BalanceSplitScheduler = "balance-split-scheduler" RedoBasicScheduler = "redo-basic-scheduler" RedoBalanceScheduler = "redo-balance-scheduler" diff --git a/server/module_election.go b/server/module_election.go index 9924428aa8..e335e96dd1 100644 --- a/server/module_election.go +++ b/server/module_election.go @@ -83,8 +83,11 @@ func (e *elector) campaignCoordinator(ctx context.Context) error { return errors.Trace(err) } // Before campaign check liveness - if e.svr.liveness.Load() == api.LivenessCaptureStopping { - log.Info("do not campaign coordinator, liveness is stopping", zap.String("nodeID", nodeID)) + liveness := e.svr.liveness.Load() + if liveness != api.LivenessCaptureAlive { + log.Info("do not campaign coordinator, liveness is not alive", + zap.String("nodeID", nodeID), + zap.String("liveness", liveness.String())) return nil } log.Info("start to campaign coordinator", zap.String("nodeID", nodeID)) @@ -110,9 +113,10 @@ func (e *elector) campaignCoordinator(ctx context.Context) error { } // After campaign check liveness again. // It is possible it becomes the coordinator right after receiving SIGTERM. - if e.svr.liveness.Load() == api.LivenessCaptureStopping { + liveness = e.svr.liveness.Load() + if liveness != api.LivenessCaptureAlive { // If the server is stopping, resign actively. - log.Info("resign coordinator actively, liveness is stopping") + log.Info("resign coordinator actively, liveness is not alive", zap.String("liveness", liveness.String())) if resignErr := e.resign(ctx); resignErr != nil { log.Warn("resign coordinator actively failed", zap.String("nodeID", nodeID), zap.Error(resignErr)) return errors.Trace(err) @@ -128,6 +132,29 @@ func (e *elector) campaignCoordinator(ctx context.Context) error { log.Info("campaign coordinator successfully", zap.String("nodeID", nodeID), zap.Int64("coordinatorVersion", coordinatorVersion)) + leaderCtx, cancelLeader := context.WithCancel(ctx) + leaderStopped := make(chan struct{}) + go func() { + defer close(leaderStopped) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-leaderCtx.Done(): + return + case <-ticker.C: + if e.svr.liveness.Load() != api.LivenessCaptureStopping { + continue + } + log.Info("resign coordinator actively, liveness is stopping", zap.String("nodeID", nodeID)) + resignCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = e.resign(resignCtx) + cancel() + return + } + } + }() + co := coordinator.New( e.svr.info, e.svr.pdClient, @@ -142,6 +169,8 @@ func (e *elector) campaignCoordinator(ctx context.Context) error { // When coordinator exits, we need to stop it. e.svr.coordinator.Stop() e.svr.setCoordinator(nil) + cancelLeader() + <-leaderStopped log.Info("coordinator stop", zap.String("nodeID", nodeID), zap.Int64("coordinatorVersion", coordinatorVersion), zap.Error(err)) @@ -203,8 +232,11 @@ func (e *elector) campaignLogCoordinator(ctx context.Context) error { return errors.Trace(err) } // Before campaign check liveness - if e.svr.liveness.Load() == api.LivenessCaptureStopping { - log.Info("do not campaign log coordinator, liveness is stopping", zap.String("nodeID", nodeID)) + liveness := e.svr.liveness.Load() + if liveness != api.LivenessCaptureAlive { + log.Info("do not campaign log coordinator, liveness is not alive", + zap.String("nodeID", nodeID), + zap.String("liveness", liveness.String())) return nil } // Campaign to be the log coordinator, it blocks until it been elected. @@ -224,10 +256,11 @@ func (e *elector) campaignLogCoordinator(ctx context.Context) error { } // After campaign check liveness again. // It is possible it becomes the coordinator right after receiving SIGTERM. - if e.svr.liveness.Load() == api.LivenessCaptureStopping { + liveness = e.svr.liveness.Load() + if liveness != api.LivenessCaptureAlive { // If the server is stopping, resign actively. - log.Info("resign log coordinator actively, liveness is stopping") - if resignErr := e.resign(ctx); resignErr != nil { + log.Info("resign log coordinator actively, liveness is not alive", zap.String("liveness", liveness.String())) + if resignErr := e.resignLogCoordinator(); resignErr != nil { log.Warn("resign log coordinator actively failed", zap.String("nodeID", nodeID), zap.Error(resignErr)) return errors.Trace(err) @@ -238,8 +271,31 @@ func (e *elector) campaignLogCoordinator(ctx context.Context) error { // FIXME: get log coordinator version from etcd and add it to log log.Info("campaign log coordinator successfully", zap.String("nodeID", nodeID)) + leaderCtx, cancelLeader := context.WithCancel(ctx) + leaderStopped := make(chan struct{}) + go func() { + defer close(leaderStopped) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-leaderCtx.Done(): + return + case <-ticker.C: + if e.svr.liveness.Load() != api.LivenessCaptureStopping { + continue + } + log.Info("resign log coordinator actively, liveness is stopping", zap.String("nodeID", nodeID)) + _ = e.resignLogCoordinator() + return + } + } + }() + co := logcoordinator.New() err = co.Run(ctx) + cancelLeader() + <-leaderStopped if err != nil && !errors.Is(err, context.Canceled) { if !errors.ErrNotOwner.Equal(err) { diff --git a/server/server.go b/server/server.go index 1573fc3eb0..e82e3e8d41 100644 --- a/server/server.go +++ b/server/server.go @@ -219,7 +219,7 @@ func (c *server) initialize(ctx context.Context) error { subscriptionClient, schemaStore, eventStore, - maintainer.NewMaintainerManager(c.info, conf.Debug.Scheduler), + maintainer.NewMaintainerManager(c.info, conf.Debug.Scheduler, &c.liveness), eventService, } // register it into global var