diff --git a/api/v1/api.go b/api/v1/api.go index 7b2981d4b0..a7f5614be3 100644 --- a/api/v1/api.go +++ b/api/v1/api.go @@ -19,7 +19,6 @@ import ( "io" "net/http" "strconv" - "sync/atomic" "github.com/gin-gonic/gin" "github.com/pingcap/log" @@ -189,28 +188,41 @@ func (o *OpenAPIV1) rebalanceTables(c *gin.Context) { c.Status(http.StatusAccepted) } -// drainCapture drains all tables from a capture. +// drainCapture triggers node drain for the given capture ID. // Usage: // curl -X PUT http://127.0.0.1:8300/api/v1/captures/drain -// TODO: Implement this API in the future, currently it is a no-op. +// The response field name keeps API compatibility with old architecture and +// represents remaining drain work instead of a literal table count. func (o *OpenAPIV1) drainCapture(c *gin.Context) { var req drainCaptureRequest if err := c.ShouldBindJSON(&req); err != nil { _ = c.Error(errors.ErrAPIInvalidParam.Wrap(err)) return } - drainCaptureCounter.Add(1) - if drainCaptureCounter.Load()%10 == 0 { - log.Info("api v1 drainCapture", zap.Any("captureID", req.CaptureID), zap.Int64("currentTableCount", drainCaptureCounter.Load())) - c.JSON(http.StatusAccepted, &drainCaptureResp{ - CurrentTableCount: 10, - }) - } else { - log.Info("api v1 drainCapture done", zap.Any("captureID", req.CaptureID), zap.Int64("currentTableCount", drainCaptureCounter.Load())) - c.JSON(http.StatusAccepted, &drainCaptureResp{ - CurrentTableCount: 0, - }) + if req.CaptureID == "" { + _ = c.Error(errors.ErrAPIInvalidParam.GenWithStackByArgs("capture_id is required")) + return + } + + co, err := o.server.GetCoordinator() + if err != nil { + _ = c.Error(err) + return + } + if co == nil || !co.Initialized() { + _ = c.Error(errors.New("coordinator is not fully initialized, wait a moment")) + return + } + + remaining, err := co.DrainNode(c.Request.Context(), req.CaptureID) + if err != nil { + _ = c.Error(err) + return } + log.Info("api v1 drainCapture", + zap.String("captureID", req.CaptureID), + zap.Int("currentTableCount", remaining)) + c.JSON(http.StatusAccepted, &drainCaptureResp{CurrentTableCount: remaining}) } func getV2ChangefeedConfig(changefeedConfig changefeedConfig) *v2.ChangefeedConfig { @@ -258,8 +270,6 @@ type drainCaptureRequest struct { CaptureID string `json:"capture_id"` } -var drainCaptureCounter atomic.Int64 - // drainCaptureResp is response for manual `DrainCapture` type drainCaptureResp struct { CurrentTableCount int `json:"current_table_count"` diff --git a/coordinator/controller.go b/coordinator/controller.go index 4285e3624d..837c7f2edb 100644 --- a/coordinator/controller.go +++ b/coordinator/controller.go @@ -64,6 +64,7 @@ type Controller struct { pdClock pdutil.Clock scheduler *scheduler.Controller operatorController *operator.Controller + drainController *drainController changefeedDB *changefeed.ChangefeedDB backend changefeed.Backend eventCh *chann.DrainableChann[*Event] @@ -119,16 +120,29 @@ func NewController( changefeedDB := changefeed.NewChangefeedDB(version) oc := operator.NewOperatorController(selfNode, changefeedDB, backend, batchSize) + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + nodeLivenessView := newNodeLivenessView(nodeHeartbeatTTL) + messageCenter := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) + drainCtl := newDrainController(selfNode.ID, nodeLivenessView, nodeManager, messageCenter, oc, changefeedDB) c := &Controller{ version: version, selfNode: selfNode, initialized: atomic.NewBool(false), scheduler: scheduler.NewController(map[string]scheduler.Scheduler{ + scheduler.DrainScheduler: coscheduler.NewDrainScheduler( + batchSize, + oc, + changefeedDB, + newControllerDrainView(drainCtl), + ), scheduler.BasicScheduler: coscheduler.NewBasicScheduler( selfNode.ID.String(), batchSize, oc, changefeedDB, + func() []node.ID { + return drainCtl.getSchedulableDestNodeIDs(nodeManager.GetAliveNodes(), time.Now()) + }, ), scheduler.BalanceScheduler: coscheduler.NewBalanceScheduler( selfNode.ID.String(), @@ -136,13 +150,17 @@ func NewController( oc, changefeedDB, balanceInterval, + func() map[node.ID]*node.Info { + return drainCtl.getSchedulableDestNodes(nodeManager.GetAliveNodes(), time.Now()) + }, ), }), eventCh: eventCh, operatorController: oc, - messageCenter: appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter), + drainController: drainCtl, + messageCenter: messageCenter, changefeedDB: changefeedDB, - nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), + nodeManager: nodeManager, taskScheduler: threadpool.NewThreadPoolDefault(), backend: backend, changefeedChangeCh: changefeedChangeCh, @@ -254,6 +272,7 @@ func (c *Controller) checkOnNodeChanged(ctx context.Context) { c.onNodeChanged(ctx) c.nodeChanged.changed = false } + c.handleDrainOnPeriodTask() } func (c *Controller) onPeriodTask() { @@ -264,6 +283,20 @@ func (c *Controller) onPeriodTask() { } } +func (c *Controller) handleDrainOnPeriodTask() { + now := time.Now() + drainNodes := c.drainController.listDrainNodes() + for _, targetNode := range drainNodes { + if c.drainController.shouldSendDrainRequest(targetNode, now) { + c.drainController.sendSetNodeLivenessRequest(targetNode, heartbeatpb.NodeLiveness_DRAINING) + } + if c.drainController.canPromoteToStopping(targetNode, now) && + c.drainController.shouldSendStopRequest(targetNode, now) { + c.drainController.sendSetNodeLivenessRequest(targetNode, heartbeatpb.NodeLiveness_STOPPING) + } + } +} + func (c *Controller) onMessage(ctx context.Context, msg *messaging.TargetMessage) { switch msg.Type { case messaging.TypeCoordinatorBootstrapResponse: @@ -273,6 +306,16 @@ func (c *Controller) onMessage(ctx context.Context, msg *messaging.TargetMessage req := msg.Message[0].(*heartbeatpb.MaintainerHeartbeat) c.handleMaintainerStatus(msg.From, req.Statuses) } + case messaging.TypeNodeHeartbeatRequest: + if c.bootstrapper.AllNodesReady() { + req := msg.Message[0].(*heartbeatpb.NodeHeartbeat) + c.drainController.handleNodeHeartbeat(msg.From, req, time.Now()) + } + case messaging.TypeSetNodeLivenessResponse: + if c.bootstrapper.AllNodesReady() { + resp := msg.Message[0].(*heartbeatpb.SetNodeLivenessResponse) + c.drainController.handleSetNodeLivenessResponse(msg.From, resp) + } case messaging.TypeLogCoordinatorResolvedTsResponse: c.onLogCoordinatorReportResolvedTs(msg) default: @@ -871,6 +914,14 @@ func (c *Controller) calculateKeyspaceGCBarrier() map[common.KeyspaceMeta]uint64 return c.changefeedDB.CalculateKeyspaceGCBarrier() } +func (c *Controller) RequestDrain(targetNode node.ID) { + c.drainController.requestDrain(targetNode) +} + +func (c *Controller) DrainSummary(targetNode node.ID) drainSummary { + return c.drainController.summarizeDrain(targetNode, time.Now()) +} + func shouldRunChangefeed(state config.FeedState) bool { switch state { case config.StateStopped, config.StateFailed, config.StateFinished: diff --git a/coordinator/coordinator.go b/coordinator/coordinator.go index b3b7751255..d7dcc1fef8 100644 --- a/coordinator/coordinator.go +++ b/coordinator/coordinator.go @@ -416,6 +416,28 @@ func (c *coordinator) RequestResolvedTsFromLogCoordinator(ctx context.Context, c c.controller.RequestResolvedTsFromLogCoordinator(ctx, changefeedDisplayName) } +func (c *coordinator) DrainNode(_ context.Context, targetNodeID string) (int, error) { + target := node.ID(targetNodeID) + if target.IsEmpty() { + return 0, errors.ErrAPIInvalidParam.GenWithStackByArgs("empty capture_id") + } + if c.nodeInfo != nil && c.nodeInfo.ID == target { + return 0, errors.ErrSchedulerRequestFailed.GenWithStackByArgs("can not drain coordinator node") + } + + c.controller.RequestDrain(target) + summary := c.controller.DrainSummary(target) + log.Info("drain node requested", + zap.Stringer("targetNode", target), + zap.Int("remaining", summary.remaining), + zap.Int("maintainersOnTarget", summary.maintainersOnTarget), + zap.Int("inflightOperatorCount", summary.inflightOperatorCount), + zap.Bool("drainingObserved", summary.drainingObserved), + zap.Bool("stoppingObserved", summary.stoppingObserved), + zap.String("nodeLiveness", summary.nodeLiveness.String())) + return summary.remaining, nil +} + func (c *coordinator) sendMessages(msgs []*messaging.TargetMessage) { for _, msg := range msgs { err := c.mc.SendCommand(msg) diff --git a/coordinator/drain_controller.go b/coordinator/drain_controller.go new file mode 100644 index 0000000000..b3ad6cd34e --- /dev/null +++ b/coordinator/drain_controller.go @@ -0,0 +1,496 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package coordinator + +import ( + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/operator" + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/messaging" + "github.com/pingcap/ticdc/pkg/node" + "github.com/pingcap/ticdc/server/watcher" + "go.uber.org/zap" +) + +const ( + nodeHeartbeatTTL = 30 * time.Second + nodeLivenessResendInterval = time.Second +) + +type nodeLiveness int32 + +const ( + nodeLivenessAlive nodeLiveness = iota + nodeLivenessDraining + nodeLivenessStopping + nodeLivenessUnknown +) + +func (n nodeLiveness) String() string { + switch n { + case nodeLivenessAlive: + return "alive" + case nodeLivenessDraining: + return "draining" + case nodeLivenessStopping: + return "stopping" + case nodeLivenessUnknown: + return "unknown" + default: + return "unknown" + } +} + +type nodeLivenessStatus struct { + nodeID node.ID + nodeEpoch uint64 + liveness nodeLiveness + lastSeen time.Time + everSeenHeartbeat bool +} + +type nodeLivenessView struct { + mu sync.RWMutex + nodes map[node.ID]*nodeLivenessStatus + ttl time.Duration +} + +func newNodeLivenessView(ttl time.Duration) *nodeLivenessView { + if ttl <= 0 { + ttl = nodeHeartbeatTTL + } + return &nodeLivenessView{ + nodes: make(map[node.ID]*nodeLivenessStatus), + ttl: ttl, + } +} + +func (v *nodeLivenessView) getStatus(nodeID node.ID, now time.Time) nodeLivenessStatus { + v.mu.RLock() + defer v.mu.RUnlock() + + status := nodeLivenessStatus{nodeID: nodeID, liveness: nodeLivenessAlive} + if s, ok := v.nodes[nodeID]; ok { + status = *s + status.liveness = deriveLivenessWithTTL(s, now, v.ttl) + } + return status +} + +func (v *nodeLivenessView) getSchedulableDestNodes(aliveNodes map[node.ID]*node.Info, now time.Time) map[node.ID]*node.Info { + ret := make(map[node.ID]*node.Info, len(aliveNodes)) + for id, info := range aliveNodes { + if v.getStatus(id, now).liveness == nodeLivenessAlive { + ret[id] = info + } + } + return ret +} + +func (v *nodeLivenessView) getSchedulableDestNodeIDs(aliveNodes map[node.ID]*node.Info, now time.Time) []node.ID { + ids := make([]node.ID, 0, len(aliveNodes)) + for id := range aliveNodes { + if v.getStatus(id, now).liveness == nodeLivenessAlive { + ids = append(ids, id) + } + } + return ids +} + +func (v *nodeLivenessView) getNodesByLiveness(target nodeLiveness, now time.Time) []node.ID { + v.mu.RLock() + defer v.mu.RUnlock() + + ids := make([]node.ID, 0) + for id, status := range v.nodes { + if deriveLivenessWithTTL(status, now, v.ttl) == target { + ids = append(ids, id) + } + } + return ids +} + +func (v *nodeLivenessView) applyHeartbeat(nodeID node.ID, heartbeat *heartbeatpb.NodeHeartbeat, now time.Time) nodeLivenessStatus { + liveness, ok := heartbeatNodeLivenessToInternal(heartbeat.Liveness) + if !ok { + log.Warn("ignore node heartbeat with invalid liveness", + zap.Stringer("nodeID", nodeID), + zap.Int32("liveness", int32(heartbeat.Liveness))) + return v.getStatus(nodeID, now) + } + + v.mu.Lock() + defer v.mu.Unlock() + + s := v.nodes[nodeID] + if s == nil || heartbeat.NodeEpoch != s.nodeEpoch { + s = &nodeLivenessStatus{nodeID: nodeID, nodeEpoch: heartbeat.NodeEpoch} + v.nodes[nodeID] = s + } + s.lastSeen = now + s.everSeenHeartbeat = true + if liveness < s.liveness { + log.Warn("ignore stale node liveness downgrade from heartbeat", + zap.Stringer("nodeID", nodeID), + zap.String("current", s.liveness.String()), + zap.String("reported", liveness.String()), + zap.Uint64("nodeEpoch", heartbeat.NodeEpoch)) + } else if liveness > s.liveness { + log.Info("observed node liveness upgrade from heartbeat", + zap.Stringer("nodeID", nodeID), + zap.String("from", s.liveness.String()), + zap.String("to", liveness.String()), + zap.Uint64("nodeEpoch", heartbeat.NodeEpoch)) + s.liveness = liveness + } + + ret := *s + ret.liveness = deriveLivenessWithTTL(s, now, v.ttl) + return ret +} + +func deriveLivenessWithTTL(s *nodeLivenessStatus, now time.Time, ttl time.Duration) nodeLiveness { + if s == nil { + return nodeLivenessAlive + } + if s.everSeenHeartbeat && now.Sub(s.lastSeen) > ttl { + return nodeLivenessUnknown + } + return s.liveness +} + +type drainNodeStatus struct { + drainRequested bool + drainingObserved bool + stoppingObserved bool + lastDrainRequestAt time.Time + lastStopRequestAt time.Time +} + +type drainRemaining struct { + targetNode node.ID + maintainersOnTarget int + inflightOpsInvolvingTarget int + drainingObserved bool + stoppingObserved bool + nodeLiveness nodeLiveness +} + +func (r drainRemaining) remaining() int { + remaining := max(r.maintainersOnTarget, r.inflightOpsInvolvingTarget) + if !r.drainingObserved { + if remaining == 0 { + return 1 + } + return remaining + } + if r.nodeLiveness == nodeLivenessUnknown { + if remaining == 0 { + return 1 + } + return remaining + } + if r.stoppingObserved && remaining == 0 { + return 0 + } + if remaining == 0 { + return 1 + } + return remaining +} + +type drainSummary struct { + remaining int + maintainersOnTarget int + inflightOperatorCount int + drainingObserved bool + stoppingObserved bool + nodeLiveness nodeLiveness +} + +func (c *drainController) summarizeDrain(targetNode node.ID, now time.Time) drainSummary { + r := c.remaining(targetNode, now) + return drainSummary{ + remaining: r.remaining(), + maintainersOnTarget: r.maintainersOnTarget, + inflightOperatorCount: r.inflightOpsInvolvingTarget, + drainingObserved: r.drainingObserved, + stoppingObserved: r.stoppingObserved, + nodeLiveness: r.nodeLiveness, + } +} + +type drainController struct { + mu sync.Mutex + + nodeView *nodeLivenessView + nodeManager *watcher.NodeManager + messageCenter messaging.MessageCenter + operatorController *operator.Controller + changefeedDB *changefeed.ChangefeedDB + selfNodeID node.ID + + drainNodes map[node.ID]*drainNodeStatus +} + +// controllerDrainView exposes only drain-related scheduler view methods. +type controllerDrainView struct { + controller *drainController +} + +func (v *controllerDrainView) GetDrainingNodes(now time.Time) []node.ID { + return v.controller.getDrainingNodes(now) +} + +func (v *controllerDrainView) GetSchedulableDestNodes(now time.Time) map[node.ID]*node.Info { + if v == nil || v.controller == nil { + return nil + } + return v.controller.getSchedulableDestNodes(v.controller.nodeManager.GetAliveNodes(), now) +} + +func newControllerDrainView(controller *drainController) *controllerDrainView { + return &controllerDrainView{controller: controller} +} + +func newDrainController( + selfNodeID node.ID, + nodeView *nodeLivenessView, + nodeManager *watcher.NodeManager, + messageCenter messaging.MessageCenter, + operatorController *operator.Controller, + changefeedDB *changefeed.ChangefeedDB, +) *drainController { + return &drainController{ + nodeView: nodeView, + nodeManager: nodeManager, + messageCenter: messageCenter, + operatorController: operatorController, + changefeedDB: changefeedDB, + selfNodeID: selfNodeID, + drainNodes: make(map[node.ID]*drainNodeStatus), + } +} + +func (c *drainController) requestDrain(nodeID node.ID) { + c.mu.Lock() + defer c.mu.Unlock() + + status := c.ensureDrainNodeLocked(nodeID) + status.drainRequested = true + observed := c.nodeView.getStatus(nodeID, time.Now()) + if observed.liveness == nodeLivenessDraining { + status.drainingObserved = true + } + if observed.liveness == nodeLivenessStopping { + status.drainingObserved = true + status.stoppingObserved = true + } +} + +func (c *drainController) markDrainingObserved(nodeID node.ID) { + c.mu.Lock() + defer c.mu.Unlock() + + status := c.ensureDrainNodeLocked(nodeID) + status.drainingObserved = true +} + +func (c *drainController) markStoppingObserved(nodeID node.ID) { + c.mu.Lock() + defer c.mu.Unlock() + + status := c.ensureDrainNodeLocked(nodeID) + status.drainingObserved = true + status.stoppingObserved = true +} + +func (c *drainController) ensureDrainNodeLocked(nodeID node.ID) *drainNodeStatus { + status := c.drainNodes[nodeID] + if status == nil { + status = &drainNodeStatus{} + c.drainNodes[nodeID] = status + } + return status +} + +func (c *drainController) handleNodeHeartbeat(nodeID node.ID, heartbeat *heartbeatpb.NodeHeartbeat, now time.Time) { + status := c.nodeView.applyHeartbeat(nodeID, heartbeat, now) + if status.liveness == nodeLivenessDraining { + c.markDrainingObserved(nodeID) + } + if status.liveness == nodeLivenessStopping { + c.markStoppingObserved(nodeID) + } +} + +func (c *drainController) handleSetNodeLivenessResponse(nodeID node.ID, resp *heartbeatpb.SetNodeLivenessResponse) { + liveness, ok := heartbeatNodeLivenessToInternal(resp.Applied) + if !ok { + return + } + if liveness == nodeLivenessDraining { + c.markDrainingObserved(nodeID) + } + if liveness == nodeLivenessStopping { + c.markStoppingObserved(nodeID) + } +} + +func (c *drainController) getDrainingNodes(now time.Time) []node.ID { + c.mu.Lock() + defer c.mu.Unlock() + + viewDraining := c.nodeView.getNodesByLiveness(nodeLivenessDraining, now) + viewStopping := c.nodeView.getNodesByLiveness(nodeLivenessStopping, now) + for _, nodeID := range viewDraining { + status := c.ensureDrainNodeLocked(nodeID) + status.drainRequested = true + status.drainingObserved = true + } + for _, nodeID := range viewStopping { + status := c.ensureDrainNodeLocked(nodeID) + status.drainRequested = true + status.drainingObserved = true + status.stoppingObserved = true + } + + ret := make([]node.ID, 0) + for nodeID, status := range c.drainNodes { + if !status.drainRequested { + continue + } + nodeStatus := c.nodeView.getStatus(nodeID, now) + if nodeStatus.liveness == nodeLivenessDraining || nodeStatus.liveness == nodeLivenessStopping { + ret = append(ret, nodeID) + } + } + return ret +} + +func (c *drainController) listDrainNodes() []node.ID { + c.mu.Lock() + defer c.mu.Unlock() + + ret := make([]node.ID, 0, len(c.drainNodes)) + for nodeID, status := range c.drainNodes { + if status.drainRequested { + ret = append(ret, nodeID) + } + } + return ret +} + +func (c *drainController) shouldSendDrainRequest(nodeID node.ID, now time.Time) bool { + c.mu.Lock() + defer c.mu.Unlock() + + status := c.ensureDrainNodeLocked(nodeID) + if !status.drainRequested || status.drainingObserved { + return false + } + if status.lastDrainRequestAt.IsZero() || now.Sub(status.lastDrainRequestAt) >= nodeLivenessResendInterval { + status.lastDrainRequestAt = now + return true + } + return false +} + +func (c *drainController) sendSetNodeLivenessRequest(targetNode node.ID, liveness heartbeatpb.NodeLiveness) { + nodeStatus := c.nodeView.getStatus(targetNode, time.Now()) + err := c.messageCenter.SendCommand(messaging.NewSingleTargetMessage( + targetNode, + messaging.MaintainerManagerTopic, + &heartbeatpb.SetNodeLivenessRequest{Target: liveness, NodeEpoch: nodeStatus.nodeEpoch}, + )) + if err != nil { + log.Warn("failed to send set node liveness request", + zap.Stringer("targetNode", targetNode), + zap.Uint64("nodeEpoch", nodeStatus.nodeEpoch), + zap.Int32("targetLiveness", int32(liveness)), + zap.Error(err)) + } +} + +func (c *drainController) shouldSendStopRequest(nodeID node.ID, now time.Time) bool { + c.mu.Lock() + defer c.mu.Unlock() + + status := c.ensureDrainNodeLocked(nodeID) + if !status.drainRequested || !status.drainingObserved || status.stoppingObserved { + return false + } + if status.lastStopRequestAt.IsZero() || now.Sub(status.lastStopRequestAt) >= nodeLivenessResendInterval { + status.lastStopRequestAt = now + return true + } + return false +} + +func (c *drainController) remaining(nodeID node.ID, now time.Time) drainRemaining { + c.mu.Lock() + status := c.ensureDrainNodeLocked(nodeID) + drainingObserved := status.drainingObserved + stoppingObserved := status.stoppingObserved + c.mu.Unlock() + + maintainersOnTarget := len(c.changefeedDB.GetByNodeID(nodeID)) + inflightOpsInvolvingTarget := c.operatorController.CountOperatorsByNode(nodeID) + nodeStatus := c.nodeView.getStatus(nodeID, now) + return drainRemaining{ + targetNode: nodeID, + maintainersOnTarget: maintainersOnTarget, + inflightOpsInvolvingTarget: inflightOpsInvolvingTarget, + drainingObserved: drainingObserved, + stoppingObserved: stoppingObserved, + nodeLiveness: nodeStatus.liveness, + } +} + +func (c *drainController) canPromoteToStopping(nodeID node.ID, now time.Time) bool { + r := c.remaining(nodeID, now) + return r.drainingObserved && r.maintainersOnTarget == 0 && r.inflightOpsInvolvingTarget == 0 +} + +func (c *drainController) getSchedulableDestNodes(aliveNodes map[node.ID]*node.Info, now time.Time) map[node.ID]*node.Info { + return c.nodeView.getSchedulableDestNodes(aliveNodes, now) +} + +func (c *drainController) getSchedulableDestNodeIDs(aliveNodes map[node.ID]*node.Info, now time.Time) []node.ID { + return c.nodeView.getSchedulableDestNodeIDs(aliveNodes, now) +} + +func heartbeatNodeLivenessToInternal(liveness heartbeatpb.NodeLiveness) (nodeLiveness, bool) { + switch liveness { + case heartbeatpb.NodeLiveness_ALIVE: + return nodeLivenessAlive, true + case heartbeatpb.NodeLiveness_DRAINING: + return nodeLivenessDraining, true + case heartbeatpb.NodeLiveness_STOPPING: + return nodeLivenessStopping, true + default: + return nodeLivenessAlive, false + } +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/coordinator/drain_controller_test.go b/coordinator/drain_controller_test.go new file mode 100644 index 0000000000..91dc3d7d06 --- /dev/null +++ b/coordinator/drain_controller_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package coordinator + +import ( + "testing" + "time" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/node" + "github.com/stretchr/testify/require" +) + +func TestNodeLivenessViewTTL(t *testing.T) { + view := newNodeLivenessView(30 * time.Second) + now := time.Now() + nodeID := node.ID("node-1") + + status := view.getStatus(nodeID, now) + require.Equal(t, nodeLivenessAlive, status.liveness) + + view.applyHeartbeat(nodeID, &heartbeatpb.NodeHeartbeat{ + Liveness: heartbeatpb.NodeLiveness_DRAINING, + NodeEpoch: 11, + }, now) + + status = view.getStatus(nodeID, now.Add(10*time.Second)) + require.Equal(t, nodeLivenessDraining, status.liveness) + + status = view.getStatus(nodeID, now.Add(31*time.Second)) + require.Equal(t, nodeLivenessUnknown, status.liveness) +} + +func TestDrainRemainingGuard(t *testing.T) { + nodeID := node.ID("node-1") + now := time.Now() + remaining := drainRemaining{ + targetNode: nodeID, + maintainersOnTarget: 0, + inflightOpsInvolvingTarget: 0, + drainingObserved: false, + stoppingObserved: false, + nodeLiveness: nodeLivenessAlive, + } + require.Equal(t, 1, remaining.remaining()) + + remaining.drainingObserved = true + remaining.nodeLiveness = nodeLivenessUnknown + require.Equal(t, 1, remaining.remaining()) + + remaining.nodeLiveness = nodeLivenessStopping + require.Equal(t, 1, remaining.remaining()) + + remaining.stoppingObserved = true + require.Equal(t, 0, remaining.remaining()) + + _ = now +} diff --git a/coordinator/operator/operator_controller.go b/coordinator/operator/operator_controller.go index 2d1842f3a2..6e6ce0108f 100644 --- a/coordinator/operator/operator_controller.go +++ b/coordinator/operator/operator_controller.go @@ -301,3 +301,20 @@ func (oc *Controller) NewAddMaintainerOperator(cf *changefeed.Changefeed, dest n func (oc *Controller) NewMoveMaintainerOperator(cf *changefeed.Changefeed, origin, dest node.ID) operator.Operator[common.ChangeFeedID, *heartbeatpb.MaintainerStatus] { return NewMoveMaintainerOperator(oc.changefeedDB, cf, origin, dest) } + +// CountOperatorsByNode counts in-flight operators that affect the specified node. +func (oc *Controller) CountOperatorsByNode(target node.ID) int { + oc.mu.RLock() + defer oc.mu.RUnlock() + + count := 0 + for _, op := range oc.operators { + for _, affected := range op.OP.AffectedNodes() { + if affected == target { + count++ + break + } + } + } + return count +} diff --git a/coordinator/operator/operator_controller_test.go b/coordinator/operator/operator_controller_test.go index cf9f1ad12d..f6fa3a72e5 100644 --- a/coordinator/operator/operator_controller_test.go +++ b/coordinator/operator/operator_controller_test.go @@ -69,6 +69,11 @@ func TestController_AddOperator(t *testing.T) { self := node.NewInfo("localhost:8300", "") nodeManager := watcher.NewNodeManager(nil, nil) nodeManager.GetAliveNodes()[self.ID] = self + + mc := messaging.NewMockMessageCenter() + appcontext.SetService(appcontext.MessageCenter, mc) + appcontext.SetService(watcher.NodeManagerName, nodeManager) + oc := NewOperatorController(self, changefeedDB, backend, 10) cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) cf := changefeed.NewChangefeed(cfID, &config.ChangeFeedInfo{ @@ -105,6 +110,11 @@ func TestController_StopChangefeedDuringAddOperator(t *testing.T) { self := node.NewInfo("localhost:8300", "") nodeManager := watcher.NewNodeManager(nil, nil) nodeManager.GetAliveNodes()[self.ID] = self + + mc := messaging.NewMockMessageCenter() + appcontext.SetService(appcontext.MessageCenter, mc) + appcontext.SetService(watcher.NodeManagerName, nodeManager) + oc := NewOperatorController(self, changefeedDB, backend, 10) // Create changefeed and add it to absent state (simulating a newly created changefeed) diff --git a/coordinator/scheduler/balance.go b/coordinator/scheduler/balance.go index 4593a95729..421beda481 100644 --- a/coordinator/scheduler/balance.go +++ b/coordinator/scheduler/balance.go @@ -19,10 +19,8 @@ import ( "github.com/pingcap/ticdc/coordinator/changefeed" "github.com/pingcap/ticdc/coordinator/operator" - appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/node" pkgScheduler "github.com/pingcap/ticdc/pkg/scheduler" - "github.com/pingcap/ticdc/server/watcher" ) // balanceScheduler is used to check the balance status of all spans among all nodes @@ -30,9 +28,9 @@ type balanceScheduler struct { id string batchSize int - operatorController *operator.Controller - changefeedDB *changefeed.ChangefeedDB - nodeManager *watcher.NodeManager + operatorController *operator.Controller + changefeedDB *changefeed.ChangefeedDB + getSchedulableNodes func() map[node.ID]*node.Info random *rand.Rand lastRebalanceTime time.Time @@ -51,6 +49,7 @@ func NewBalanceScheduler( oc *operator.Controller, changefeedDB *changefeed.ChangefeedDB, balanceInterval time.Duration, + getSchedulableNodes func() map[node.ID]*node.Info, ) *balanceScheduler { return &balanceScheduler{ id: id, @@ -58,7 +57,7 @@ func NewBalanceScheduler( random: rand.New(rand.NewSource(time.Now().UnixNano())), operatorController: oc, changefeedDB: changefeedDB, - nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), + getSchedulableNodes: getSchedulableNodes, checkBalanceInterval: balanceInterval, lastRebalanceTime: time.Now(), } @@ -74,15 +73,19 @@ func (s *balanceScheduler) Execute() time.Time { // not in stable schedule state, skip balance return now.Add(s.checkBalanceInterval) } + schedulableNodes := s.getSchedulableNodes() + if len(schedulableNodes) <= 1 { + return now.Add(s.checkBalanceInterval) + } // check the balance status - moveSize := pkgScheduler.CheckBalanceStatus(s.changefeedDB.GetTaskSizePerNode(), s.nodeManager.GetAliveNodes()) + moveSize := pkgScheduler.CheckBalanceStatus(s.changefeedDB.GetTaskSizePerNode(), schedulableNodes) if moveSize <= 0 { // fast check the balance status, no need to do the balance,skip return now.Add(s.checkBalanceInterval) } // balance changefeeds among the active nodes - movedSize := pkgScheduler.Balance(s.batchSize, s.random, s.nodeManager.GetAliveNodes(), s.changefeedDB.GetReplicating(), + movedSize := pkgScheduler.Balance(s.batchSize, s.random, schedulableNodes, s.changefeedDB.GetReplicating(), func(cf *changefeed.Changefeed, nodeID node.ID) bool { return s.operatorController.AddOperator(operator.NewMoveMaintainerOperator(s.changefeedDB, cf, cf.GetNodeID(), nodeID)) }) diff --git a/coordinator/scheduler/basic.go b/coordinator/scheduler/basic.go index 301a75f86d..e677fbf89f 100644 --- a/coordinator/scheduler/basic.go +++ b/coordinator/scheduler/basic.go @@ -18,11 +18,9 @@ import ( "github.com/pingcap/ticdc/coordinator/changefeed" "github.com/pingcap/ticdc/coordinator/operator" - appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/node" pkgScheduler "github.com/pingcap/ticdc/pkg/scheduler" "github.com/pingcap/ticdc/pkg/scheduler/replica" - "github.com/pingcap/ticdc/server/watcher" ) // basicScheduler generates operators for the spans, and push them to the operator controller @@ -32,22 +30,23 @@ type basicScheduler struct { id string batchSize int - operatorController *operator.Controller - changefeedDB *changefeed.ChangefeedDB - nodeManager *watcher.NodeManager + operatorController *operator.Controller + changefeedDB *changefeed.ChangefeedDB + getSchedulableNodeIDs func() []node.ID } func NewBasicScheduler( id string, batchSize int, oc *operator.Controller, changefeedDB *changefeed.ChangefeedDB, + getSchedulableNodeIDs func() []node.ID, ) *basicScheduler { return &basicScheduler{ - id: id, - batchSize: batchSize, - operatorController: oc, - changefeedDB: changefeedDB, - nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), + id: id, + batchSize: batchSize, + operatorController: oc, + changefeedDB: changefeedDB, + getSchedulableNodeIDs: getSchedulableNodeIDs, } } @@ -74,7 +73,10 @@ func (s *basicScheduler) doBasicSchedule(availableSize int) { absentChangefeeds := s.changefeedDB.GetAbsentByGroup(id, availableSize) nodeTaskSize := s.changefeedDB.GetTaskSizePerNodeByGroup(id) // add the absent node to the node size map - nodeIDs := s.nodeManager.GetAliveNodeIDs() + nodeIDs := s.getSchedulableNodeIDs() + if len(nodeIDs) == 0 { + return + } nodeSize := make(map[node.ID]int) for _, id := range nodeIDs { nodeSize[id] = nodeTaskSize[id] diff --git a/coordinator/scheduler/drain.go b/coordinator/scheduler/drain.go new file mode 100644 index 0000000000..10859e0650 --- /dev/null +++ b/coordinator/scheduler/drain.go @@ -0,0 +1,130 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "time" + + "github.com/pingcap/ticdc/coordinator/changefeed" + "github.com/pingcap/ticdc/coordinator/operator" + "github.com/pingcap/ticdc/pkg/node" +) + +const drainScheduleInterval = 200 * time.Millisecond + +// DrainController provides the minimal drain state and command operations +// required by drainScheduler. +type DrainController interface { + GetDrainingNodes(now time.Time) []node.ID + GetSchedulableDestNodes(now time.Time) map[node.ID]*node.Info +} + +// drainScheduler migrates maintainers out of draining nodes. +type drainScheduler struct { + batchSize int + + operatorController *operator.Controller + changefeedDB *changefeed.ChangefeedDB + drainController DrainController + + lastNodeCursor int +} + +func NewDrainScheduler( + batchSize int, + oc *operator.Controller, + changefeedDB *changefeed.ChangefeedDB, + drainController DrainController, +) *drainScheduler { + return &drainScheduler{ + batchSize: batchSize, + operatorController: oc, + changefeedDB: changefeedDB, + drainController: drainController, + } +} + +func (s *drainScheduler) Name() string { + return "drain-scheduler" +} + +func (s *drainScheduler) Execute() time.Time { + if s.batchSize <= 0 { + return time.Now().Add(drainScheduleInterval) + } + availableSize := s.batchSize - s.operatorController.OperatorSize() + if availableSize <= 0 { + return time.Now().Add(drainScheduleInterval) + } + + now := time.Now() + drainingNodes := s.drainController.GetDrainingNodes(now) + if len(drainingNodes) == 0 { + return now.Add(drainScheduleInterval) + } + schedulableNodes := s.drainController.GetSchedulableDestNodes(now) + if len(schedulableNodes) == 0 { + return now.Add(drainScheduleInterval) + } + + nodeLoads := s.changefeedDB.GetTaskSizePerNode() + created := 0 + for i := 0; i < len(drainingNodes) && created < availableSize; i++ { + nodeIdx := (s.lastNodeCursor + i) % len(drainingNodes) + targetNode := drainingNodes[nodeIdx] + + for _, cf := range s.changefeedDB.GetByNodeID(targetNode) { + if created >= availableSize { + break + } + if s.operatorController.GetOperator(cf.ID) != nil { + continue + } + dest, ok := chooseLeastLoadedNode(schedulableNodes, targetNode, nodeLoads) + if !ok { + break + } + if s.operatorController.AddOperator(operator.NewMoveMaintainerOperator(s.changefeedDB, cf, targetNode, dest)) { + nodeLoads[dest]++ + created++ + } + } + } + s.lastNodeCursor = (s.lastNodeCursor + 1) % len(drainingNodes) + return now.Add(drainScheduleInterval) +} + +func chooseLeastLoadedNode( + schedulableNodes map[node.ID]*node.Info, + excluded node.ID, + nodeLoads map[node.ID]int, +) (node.ID, bool) { + var ( + selected node.ID + found bool + minLoad int + ) + for id := range schedulableNodes { + if id == excluded { + continue + } + load := nodeLoads[id] + if !found || load < minLoad { + selected = id + minLoad = load + found = true + } + } + return selected, found +} diff --git a/coordinator/scheduler/drain_test.go b/coordinator/scheduler/drain_test.go new file mode 100644 index 0000000000..cfee684fed --- /dev/null +++ b/coordinator/scheduler/drain_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/pingcap/ticdc/coordinator/changefeed" + mock_changefeed "github.com/pingcap/ticdc/coordinator/changefeed/mock" + "github.com/pingcap/ticdc/coordinator/operator" + "github.com/pingcap/ticdc/pkg/common" + appcontext "github.com/pingcap/ticdc/pkg/common/context" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/messaging" + "github.com/pingcap/ticdc/pkg/node" + "github.com/pingcap/ticdc/server/watcher" + "github.com/stretchr/testify/require" +) + +type mockDrainController struct { + drainingNodes []node.ID + destNodes map[node.ID]*node.Info +} + +func (m *mockDrainController) GetDrainingNodes(_ time.Time) []node.ID { + return m.drainingNodes +} + +func (m *mockDrainController) GetSchedulableDestNodes(_ time.Time) map[node.ID]*node.Info { + return m.destNodes +} + +func TestDrainSchedulerSkipsInFlightAndSchedulesMove(t *testing.T) { + changefeedDB := changefeed.NewChangefeedDB(1) + ctrl := gomock.NewController(t) + backend := mock_changefeed.NewMockBackend(ctrl) + t.Cleanup(ctrl.Finish) + mc := messaging.NewMockMessageCenter() + appcontext.SetService(appcontext.MessageCenter, mc) + nodeManager := watcher.NewNodeManager(nil, nil) + appcontext.SetService(watcher.NodeManagerName, nodeManager) + selfNode := node.NewInfo("127.0.0.1:8300", "") + nodeManager.GetAliveNodes()[selfNode.ID] = selfNode + destNode := node.NewInfo("127.0.0.1:8301", "") + nodeManager.GetAliveNodes()[destNode.ID] = destNode + + oc := operator.NewOperatorController(selfNode, changefeedDB, backend, 16) + + cfID1 := common.NewChangeFeedIDWithName("cf-1", common.DefaultKeyspaceName) + cfID2 := common.NewChangeFeedIDWithName("cf-2", common.DefaultKeyspaceName) + cf1 := changefeed.NewChangefeed(cfID1, &config.ChangeFeedInfo{ + ChangefeedID: cfID1, + Config: config.GetDefaultReplicaConfig(), + SinkURI: "mysql://127.0.0.1:3306", + }, 10, false) + cf2 := changefeed.NewChangefeed(cfID2, &config.ChangeFeedInfo{ + ChangefeedID: cfID2, + Config: config.GetDefaultReplicaConfig(), + SinkURI: "mysql://127.0.0.1:3306", + }, 10, false) + + target := node.ID("drain-node") + changefeedDB.AddReplicatingMaintainer(cf1, target) + changefeedDB.AddReplicatingMaintainer(cf2, target) + + // One in-flight operator already exists, drain scheduler should skip it. + require.True(t, oc.AddOperator(oc.NewMoveMaintainerOperator(cf1, target, destNode.ID))) + + s := NewDrainScheduler(16, oc, changefeedDB, &mockDrainController{ + drainingNodes: []node.ID{target}, + destNodes: map[node.ID]*node.Info{ + destNode.ID: destNode, + }, + }) + + s.Execute() + + // We already had one operator for cf1, now scheduler should add one for cf2. + require.NotNil(t, oc.GetOperator(cf1.ID)) + require.NotNil(t, oc.GetOperator(cf2.ID)) +} diff --git a/heartbeatpb/heartbeat.pb.go b/heartbeatpb/heartbeat.pb.go index 10f1a63dd5..e5f54c53ba 100644 --- a/heartbeatpb/heartbeat.pb.go +++ b/heartbeatpb/heartbeat.pb.go @@ -73,6 +73,34 @@ func (ScheduleAction) EnumDescriptor() ([]byte, []int) { return fileDescriptor_6d584080fdadb670, []int{1} } +type NodeLiveness int32 + +const ( + NodeLiveness_ALIVE NodeLiveness = 0 + NodeLiveness_DRAINING NodeLiveness = 1 + NodeLiveness_STOPPING NodeLiveness = 2 +) + +var NodeLiveness_name = map[int32]string{ + 0: "ALIVE", + 1: "DRAINING", + 2: "STOPPING", +} + +var NodeLiveness_value = map[string]int32{ + "ALIVE": 0, + "DRAINING": 1, + "STOPPING": 2, +} + +func (x NodeLiveness) String() string { + return proto.EnumName(NodeLiveness_name, int32(x)) +} + +func (NodeLiveness) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{2} +} + type BlockStage int32 const ( @@ -101,7 +129,7 @@ func (x BlockStage) String() string { } func (BlockStage) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{2} + return fileDescriptor_6d584080fdadb670, []int{3} } type InfluenceType int32 @@ -129,7 +157,7 @@ func (x InfluenceType) String() string { } func (InfluenceType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{3} + return fileDescriptor_6d584080fdadb670, []int{4} } type ComponentState int32 @@ -169,7 +197,7 @@ func (x ComponentState) String() string { } func (ComponentState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{4} + return fileDescriptor_6d584080fdadb670, []int{5} } type ChecksumState int32 @@ -201,7 +229,7 @@ func (x ChecksumState) String() string { } func (ChecksumState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{5} + return fileDescriptor_6d584080fdadb670, []int{6} } type TableSpan struct { @@ -1209,6 +1237,162 @@ func (m *MaintainerHeartbeat) GetStatuses() []*MaintainerStatus { return nil } +type NodeHeartbeat struct { + Liveness NodeLiveness `protobuf:"varint,1,opt,name=liveness,proto3,enum=heartbeatpb.NodeLiveness" json:"liveness,omitempty"` + NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"` +} + +func (m *NodeHeartbeat) Reset() { *m = NodeHeartbeat{} } +func (m *NodeHeartbeat) String() string { return proto.CompactTextString(m) } +func (*NodeHeartbeat) ProtoMessage() {} +func (*NodeHeartbeat) Descriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{16} +} +func (m *NodeHeartbeat) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NodeHeartbeat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NodeHeartbeat.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NodeHeartbeat) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeHeartbeat.Merge(m, src) +} +func (m *NodeHeartbeat) XXX_Size() int { + return m.Size() +} +func (m *NodeHeartbeat) XXX_DiscardUnknown() { + xxx_messageInfo_NodeHeartbeat.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeHeartbeat proto.InternalMessageInfo + +func (m *NodeHeartbeat) GetLiveness() NodeLiveness { + if m != nil { + return m.Liveness + } + return NodeLiveness_ALIVE +} + +func (m *NodeHeartbeat) GetNodeEpoch() uint64 { + if m != nil { + return m.NodeEpoch + } + return 0 +} + +type SetNodeLivenessRequest struct { + Target NodeLiveness `protobuf:"varint,1,opt,name=target,proto3,enum=heartbeatpb.NodeLiveness" json:"target,omitempty"` + NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"` +} + +func (m *SetNodeLivenessRequest) Reset() { *m = SetNodeLivenessRequest{} } +func (m *SetNodeLivenessRequest) String() string { return proto.CompactTextString(m) } +func (*SetNodeLivenessRequest) ProtoMessage() {} +func (*SetNodeLivenessRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{17} +} +func (m *SetNodeLivenessRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SetNodeLivenessRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SetNodeLivenessRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SetNodeLivenessRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetNodeLivenessRequest.Merge(m, src) +} +func (m *SetNodeLivenessRequest) XXX_Size() int { + return m.Size() +} +func (m *SetNodeLivenessRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SetNodeLivenessRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SetNodeLivenessRequest proto.InternalMessageInfo + +func (m *SetNodeLivenessRequest) GetTarget() NodeLiveness { + if m != nil { + return m.Target + } + return NodeLiveness_ALIVE +} + +func (m *SetNodeLivenessRequest) GetNodeEpoch() uint64 { + if m != nil { + return m.NodeEpoch + } + return 0 +} + +type SetNodeLivenessResponse struct { + Applied NodeLiveness `protobuf:"varint,1,opt,name=applied,proto3,enum=heartbeatpb.NodeLiveness" json:"applied,omitempty"` + NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"` +} + +func (m *SetNodeLivenessResponse) Reset() { *m = SetNodeLivenessResponse{} } +func (m *SetNodeLivenessResponse) String() string { return proto.CompactTextString(m) } +func (*SetNodeLivenessResponse) ProtoMessage() {} +func (*SetNodeLivenessResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{18} +} +func (m *SetNodeLivenessResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SetNodeLivenessResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SetNodeLivenessResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SetNodeLivenessResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetNodeLivenessResponse.Merge(m, src) +} +func (m *SetNodeLivenessResponse) XXX_Size() int { + return m.Size() +} +func (m *SetNodeLivenessResponse) XXX_DiscardUnknown() { + xxx_messageInfo_SetNodeLivenessResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_SetNodeLivenessResponse proto.InternalMessageInfo + +func (m *SetNodeLivenessResponse) GetApplied() NodeLiveness { + if m != nil { + return m.Applied + } + return NodeLiveness_ALIVE +} + +func (m *SetNodeLivenessResponse) GetNodeEpoch() uint64 { + if m != nil { + return m.NodeEpoch + } + return 0 +} + type MaintainerStatus struct { ChangefeedID *ChangefeedID `protobuf:"bytes,1,opt,name=changefeedID,proto3" json:"changefeedID,omitempty"` FeedState string `protobuf:"bytes,2,opt,name=feed_state,json=feedState,proto3" json:"feed_state,omitempty"` @@ -1223,7 +1407,7 @@ func (m *MaintainerStatus) Reset() { *m = MaintainerStatus{} } func (m *MaintainerStatus) String() string { return proto.CompactTextString(m) } func (*MaintainerStatus) ProtoMessage() {} func (*MaintainerStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{16} + return fileDescriptor_6d584080fdadb670, []int{19} } func (m *MaintainerStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1309,7 +1493,7 @@ func (m *CoordinatorBootstrapRequest) Reset() { *m = CoordinatorBootstra func (m *CoordinatorBootstrapRequest) String() string { return proto.CompactTextString(m) } func (*CoordinatorBootstrapRequest) ProtoMessage() {} func (*CoordinatorBootstrapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{17} + return fileDescriptor_6d584080fdadb670, []int{20} } func (m *CoordinatorBootstrapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1353,7 +1537,7 @@ func (m *CoordinatorBootstrapResponse) Reset() { *m = CoordinatorBootstr func (m *CoordinatorBootstrapResponse) String() string { return proto.CompactTextString(m) } func (*CoordinatorBootstrapResponse) ProtoMessage() {} func (*CoordinatorBootstrapResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{18} + return fileDescriptor_6d584080fdadb670, []int{21} } func (m *CoordinatorBootstrapResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1401,7 +1585,7 @@ func (m *AddMaintainerRequest) Reset() { *m = AddMaintainerRequest{} } func (m *AddMaintainerRequest) String() string { return proto.CompactTextString(m) } func (*AddMaintainerRequest) ProtoMessage() {} func (*AddMaintainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{19} + return fileDescriptor_6d584080fdadb670, []int{22} } func (m *AddMaintainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1476,7 +1660,7 @@ func (m *RemoveMaintainerRequest) Reset() { *m = RemoveMaintainerRequest func (m *RemoveMaintainerRequest) String() string { return proto.CompactTextString(m) } func (*RemoveMaintainerRequest) ProtoMessage() {} func (*RemoveMaintainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{20} + return fileDescriptor_6d584080fdadb670, []int{23} } func (m *RemoveMaintainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1547,7 +1731,7 @@ func (m *MaintainerBootstrapRequest) Reset() { *m = MaintainerBootstrapR func (m *MaintainerBootstrapRequest) String() string { return proto.CompactTextString(m) } func (*MaintainerBootstrapRequest) ProtoMessage() {} func (*MaintainerBootstrapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{21} + return fileDescriptor_6d584080fdadb670, []int{24} } func (m *MaintainerBootstrapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1647,7 +1831,7 @@ func (m *MaintainerBootstrapResponse) Reset() { *m = MaintainerBootstrap func (m *MaintainerBootstrapResponse) String() string { return proto.CompactTextString(m) } func (*MaintainerBootstrapResponse) ProtoMessage() {} func (*MaintainerBootstrapResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{22} + return fileDescriptor_6d584080fdadb670, []int{25} } func (m *MaintainerBootstrapResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1722,7 +1906,7 @@ func (m *MaintainerPostBootstrapRequest) Reset() { *m = MaintainerPostBo func (m *MaintainerPostBootstrapRequest) String() string { return proto.CompactTextString(m) } func (*MaintainerPostBootstrapRequest) ProtoMessage() {} func (*MaintainerPostBootstrapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{23} + return fileDescriptor_6d584080fdadb670, []int{26} } func (m *MaintainerPostBootstrapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1789,7 +1973,7 @@ func (m *MaintainerPostBootstrapResponse) Reset() { *m = MaintainerPostB func (m *MaintainerPostBootstrapResponse) String() string { return proto.CompactTextString(m) } func (*MaintainerPostBootstrapResponse) ProtoMessage() {} func (*MaintainerPostBootstrapResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{24} + return fileDescriptor_6d584080fdadb670, []int{27} } func (m *MaintainerPostBootstrapResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1849,7 +2033,7 @@ func (m *SchemaInfo) Reset() { *m = SchemaInfo{} } func (m *SchemaInfo) String() string { return proto.CompactTextString(m) } func (*SchemaInfo) ProtoMessage() {} func (*SchemaInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{25} + return fileDescriptor_6d584080fdadb670, []int{28} } func (m *SchemaInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1908,7 +2092,7 @@ func (m *TableInfo) Reset() { *m = TableInfo{} } func (m *TableInfo) String() string { return proto.CompactTextString(m) } func (*TableInfo) ProtoMessage() {} func (*TableInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{26} + return fileDescriptor_6d584080fdadb670, []int{29} } func (m *TableInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1965,7 +2149,7 @@ func (m *BootstrapTableSpan) Reset() { *m = BootstrapTableSpan{} } func (m *BootstrapTableSpan) String() string { return proto.CompactTextString(m) } func (*BootstrapTableSpan) ProtoMessage() {} func (*BootstrapTableSpan) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{27} + return fileDescriptor_6d584080fdadb670, []int{30} } func (m *BootstrapTableSpan) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2053,7 +2237,7 @@ func (m *MaintainerCloseRequest) Reset() { *m = MaintainerCloseRequest{} func (m *MaintainerCloseRequest) String() string { return proto.CompactTextString(m) } func (*MaintainerCloseRequest) ProtoMessage() {} func (*MaintainerCloseRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{28} + return fileDescriptor_6d584080fdadb670, []int{31} } func (m *MaintainerCloseRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2105,7 +2289,7 @@ func (m *MaintainerCloseResponse) Reset() { *m = MaintainerCloseResponse func (m *MaintainerCloseResponse) String() string { return proto.CompactTextString(m) } func (*MaintainerCloseResponse) ProtoMessage() {} func (*MaintainerCloseResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{29} + return fileDescriptor_6d584080fdadb670, []int{32} } func (m *MaintainerCloseResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2160,7 +2344,7 @@ func (m *InfluencedTables) Reset() { *m = InfluencedTables{} } func (m *InfluencedTables) String() string { return proto.CompactTextString(m) } func (*InfluencedTables) ProtoMessage() {} func (*InfluencedTables) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{30} + return fileDescriptor_6d584080fdadb670, []int{33} } func (m *InfluencedTables) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2220,7 +2404,7 @@ func (m *Table) Reset() { *m = Table{} } func (m *Table) String() string { return proto.CompactTextString(m) } func (*Table) ProtoMessage() {} func (*Table) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{31} + return fileDescriptor_6d584080fdadb670, []int{34} } func (m *Table) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2280,7 +2464,7 @@ func (m *SchemaIDChange) Reset() { *m = SchemaIDChange{} } func (m *SchemaIDChange) String() string { return proto.CompactTextString(m) } func (*SchemaIDChange) ProtoMessage() {} func (*SchemaIDChange) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{32} + return fileDescriptor_6d584080fdadb670, []int{35} } func (m *SchemaIDChange) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2345,7 +2529,7 @@ func (m *State) Reset() { *m = State{} } func (m *State) String() string { return proto.CompactTextString(m) } func (*State) ProtoMessage() {} func (*State) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{33} + return fileDescriptor_6d584080fdadb670, []int{36} } func (m *State) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2440,7 +2624,7 @@ func (m *TableSpanBlockStatus) Reset() { *m = TableSpanBlockStatus{} } func (m *TableSpanBlockStatus) String() string { return proto.CompactTextString(m) } func (*TableSpanBlockStatus) ProtoMessage() {} func (*TableSpanBlockStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{34} + return fileDescriptor_6d584080fdadb670, []int{37} } func (m *TableSpanBlockStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2502,7 +2686,7 @@ func (m *TableSpanStatus) Reset() { *m = TableSpanStatus{} } func (m *TableSpanStatus) String() string { return proto.CompactTextString(m) } func (*TableSpanStatus) ProtoMessage() {} func (*TableSpanStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{35} + return fileDescriptor_6d584080fdadb670, []int{38} } func (m *TableSpanStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2576,7 +2760,7 @@ func (m *BlockStatusRequest) Reset() { *m = BlockStatusRequest{} } func (m *BlockStatusRequest) String() string { return proto.CompactTextString(m) } func (*BlockStatusRequest) ProtoMessage() {} func (*BlockStatusRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{36} + return fileDescriptor_6d584080fdadb670, []int{39} } func (m *BlockStatusRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2637,7 +2821,7 @@ func (m *RunningError) Reset() { *m = RunningError{} } func (m *RunningError) String() string { return proto.CompactTextString(m) } func (*RunningError) ProtoMessage() {} func (*RunningError) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{37} + return fileDescriptor_6d584080fdadb670, []int{40} } func (m *RunningError) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2703,7 +2887,7 @@ func (m *DispatcherID) Reset() { *m = DispatcherID{} } func (m *DispatcherID) String() string { return proto.CompactTextString(m) } func (*DispatcherID) ProtoMessage() {} func (*DispatcherID) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{38} + return fileDescriptor_6d584080fdadb670, []int{41} } func (m *DispatcherID) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2757,7 +2941,7 @@ func (m *ChangefeedID) Reset() { *m = ChangefeedID{} } func (m *ChangefeedID) String() string { return proto.CompactTextString(m) } func (*ChangefeedID) ProtoMessage() {} func (*ChangefeedID) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{39} + return fileDescriptor_6d584080fdadb670, []int{42} } func (m *ChangefeedID) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2822,7 +3006,7 @@ func (m *LogCoordinatorResolvedTsRequest) Reset() { *m = LogCoordinatorR func (m *LogCoordinatorResolvedTsRequest) String() string { return proto.CompactTextString(m) } func (*LogCoordinatorResolvedTsRequest) ProtoMessage() {} func (*LogCoordinatorResolvedTsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{40} + return fileDescriptor_6d584080fdadb670, []int{43} } func (m *LogCoordinatorResolvedTsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2867,7 +3051,7 @@ func (m *LogCoordinatorResolvedTsResponse) Reset() { *m = LogCoordinator func (m *LogCoordinatorResolvedTsResponse) String() string { return proto.CompactTextString(m) } func (*LogCoordinatorResolvedTsResponse) ProtoMessage() {} func (*LogCoordinatorResolvedTsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{41} + return fileDescriptor_6d584080fdadb670, []int{44} } func (m *LogCoordinatorResolvedTsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2921,7 +3105,7 @@ func (m *ChecksumMeta) Reset() { *m = ChecksumMeta{} } func (m *ChecksumMeta) String() string { return proto.CompactTextString(m) } func (*ChecksumMeta) ProtoMessage() {} func (*ChecksumMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{42} + return fileDescriptor_6d584080fdadb670, []int{45} } func (m *ChecksumMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2976,7 +3160,7 @@ func (m *DispatcherSetChecksum) Reset() { *m = DispatcherSetChecksum{} } func (m *DispatcherSetChecksum) String() string { return proto.CompactTextString(m) } func (*DispatcherSetChecksum) ProtoMessage() {} func (*DispatcherSetChecksum) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{43} + return fileDescriptor_6d584080fdadb670, []int{46} } func (m *DispatcherSetChecksum) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3053,7 +3237,7 @@ func (m *DispatcherSetChecksumAckResponse) Reset() { *m = DispatcherSetC func (m *DispatcherSetChecksumAckResponse) String() string { return proto.CompactTextString(m) } func (*DispatcherSetChecksumAckResponse) ProtoMessage() {} func (*DispatcherSetChecksumAckResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{44} + return fileDescriptor_6d584080fdadb670, []int{47} } func (m *DispatcherSetChecksumAckResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3124,7 +3308,7 @@ func (m *DispatcherSetChecksumUpdateRequest) Reset() { *m = DispatcherSe func (m *DispatcherSetChecksumUpdateRequest) String() string { return proto.CompactTextString(m) } func (*DispatcherSetChecksumUpdateRequest) ProtoMessage() {} func (*DispatcherSetChecksumUpdateRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{45} + return fileDescriptor_6d584080fdadb670, []int{48} } func (m *DispatcherSetChecksumUpdateRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3191,6 +3375,7 @@ func (m *DispatcherSetChecksumUpdateRequest) GetChecksum() *DispatcherSetChecksu func init() { proto.RegisterEnum("heartbeatpb.Action", Action_name, Action_value) proto.RegisterEnum("heartbeatpb.ScheduleAction", ScheduleAction_name, ScheduleAction_value) + proto.RegisterEnum("heartbeatpb.NodeLiveness", NodeLiveness_name, NodeLiveness_value) proto.RegisterEnum("heartbeatpb.BlockStage", BlockStage_name, BlockStage_value) proto.RegisterEnum("heartbeatpb.InfluenceType", InfluenceType_name, InfluenceType_value) proto.RegisterEnum("heartbeatpb.ComponentState", ComponentState_name, ComponentState_value) @@ -3211,6 +3396,9 @@ func init() { proto.RegisterType((*ScheduleDispatcherRequest)(nil), "heartbeatpb.ScheduleDispatcherRequest") proto.RegisterType((*MergeDispatcherRequest)(nil), "heartbeatpb.MergeDispatcherRequest") proto.RegisterType((*MaintainerHeartbeat)(nil), "heartbeatpb.MaintainerHeartbeat") + proto.RegisterType((*NodeHeartbeat)(nil), "heartbeatpb.NodeHeartbeat") + proto.RegisterType((*SetNodeLivenessRequest)(nil), "heartbeatpb.SetNodeLivenessRequest") + proto.RegisterType((*SetNodeLivenessResponse)(nil), "heartbeatpb.SetNodeLivenessResponse") proto.RegisterType((*MaintainerStatus)(nil), "heartbeatpb.MaintainerStatus") proto.RegisterType((*CoordinatorBootstrapRequest)(nil), "heartbeatpb.CoordinatorBootstrapRequest") proto.RegisterType((*CoordinatorBootstrapResponse)(nil), "heartbeatpb.CoordinatorBootstrapResponse") @@ -3246,156 +3434,163 @@ func init() { func init() { proto.RegisterFile("heartbeatpb/heartbeat.proto", fileDescriptor_6d584080fdadb670) } var fileDescriptor_6d584080fdadb670 = []byte{ - // 2376 bytes of a gzipped FileDescriptorProto + // 2493 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x1a, 0x4d, 0x6f, 0x1c, 0x49, - 0xd5, 0xdd, 0x3d, 0x33, 0x9e, 0x79, 0xe3, 0xb1, 0x3b, 0x95, 0x8f, 0x75, 0x12, 0xc7, 0xf1, 0x36, - 0x20, 0x19, 0xef, 0x92, 0x90, 0xec, 0x46, 0xc0, 0xb2, 0x6c, 0xb0, 0x67, 0xb2, 0x9b, 0x91, 0x63, - 0xaf, 0x55, 0xe3, 0x55, 0x60, 0x39, 0x0c, 0xed, 0xee, 0xca, 0xb8, 0xe5, 0x99, 0xae, 0x4e, 0x57, - 0x4f, 0x1c, 0x47, 0x5a, 0x10, 0x42, 0xdc, 0x38, 0x80, 0xe0, 0xc2, 0x81, 0x03, 0xe2, 0x0f, 0x20, - 0x21, 0x8e, 0x48, 0xdc, 0x40, 0xe2, 0xb2, 0xa7, 0x15, 0x37, 0x50, 0xf2, 0x07, 0x10, 0x1c, 0xb8, - 0xa2, 0xaa, 0xae, 0xea, 0xaf, 0x69, 0xdb, 0x89, 0x3c, 0xca, 0xad, 0x5f, 0xd5, 0x7b, 0xaf, 0x5e, - 0xbd, 0xef, 0x57, 0x33, 0x70, 0x75, 0x9f, 0xd8, 0x61, 0xb4, 0x47, 0xec, 0x28, 0xd8, 0xbb, 0x99, - 0x7c, 0xdf, 0x08, 0x42, 0x1a, 0x51, 0xd4, 0xcc, 0x6c, 0x5a, 0x47, 0xd0, 0xd8, 0xb5, 0xf7, 0x86, - 0xa4, 0x17, 0xd8, 0x3e, 0x5a, 0x84, 0x59, 0x01, 0x74, 0x3b, 0x8b, 0xda, 0x8a, 0xb6, 0x6a, 0x60, - 0x05, 0xa2, 0x2b, 0x50, 0xef, 0x45, 0x76, 0x18, 0x6d, 0x92, 0xa3, 0x45, 0x7d, 0x45, 0x5b, 0x9d, - 0xc3, 0x09, 0x8c, 0x2e, 0x41, 0xed, 0x9e, 0xef, 0xf2, 0x1d, 0x43, 0xec, 0x48, 0x08, 0x2d, 0x03, - 0x6c, 0x92, 0x23, 0x16, 0xd8, 0x0e, 0x67, 0x58, 0x59, 0xd1, 0x56, 0x5b, 0x38, 0xb3, 0x62, 0x7d, - 0xa1, 0x83, 0x79, 0x9f, 0x8b, 0xb2, 0x41, 0xec, 0x08, 0x93, 0xc7, 0x63, 0xc2, 0x22, 0xf4, 0x1d, - 0x98, 0x73, 0xf6, 0x6d, 0x7f, 0x40, 0x1e, 0x11, 0xe2, 0x4a, 0x39, 0x9a, 0xb7, 0x2f, 0xdf, 0xc8, - 0xc8, 0x7c, 0xa3, 0x9d, 0x41, 0xc0, 0x39, 0x74, 0xf4, 0x2e, 0x34, 0x0e, 0xed, 0x88, 0x84, 0x23, - 0x3b, 0x3c, 0x10, 0x82, 0x36, 0x6f, 0x5f, 0xca, 0xd1, 0x3e, 0x54, 0xbb, 0x38, 0x45, 0x44, 0xef, - 0x43, 0x2b, 0x24, 0x2e, 0x4d, 0xf6, 0xc4, 0x45, 0x8e, 0xa7, 0xcc, 0x23, 0xa3, 0x6f, 0x42, 0x9d, - 0x45, 0x76, 0x34, 0x66, 0x84, 0x2d, 0x56, 0x56, 0x8c, 0xd5, 0xe6, 0xed, 0xa5, 0x1c, 0x61, 0xa2, - 0xdf, 0x9e, 0xc0, 0xc2, 0x09, 0x36, 0x5a, 0x85, 0x05, 0x87, 0x8e, 0x02, 0x32, 0x24, 0x11, 0x89, - 0x37, 0x17, 0xab, 0x2b, 0xda, 0x6a, 0x1d, 0x17, 0x97, 0xd1, 0x5b, 0x60, 0x90, 0x30, 0x5c, 0xac, - 0x95, 0x68, 0x03, 0x8f, 0x7d, 0xdf, 0xf3, 0x07, 0xf7, 0xc2, 0x90, 0x86, 0x98, 0x63, 0x59, 0x3f, - 0xd3, 0xa0, 0x91, 0x8a, 0x67, 0x71, 0x8d, 0x12, 0xe7, 0x20, 0xa0, 0x9e, 0x1f, 0xed, 0x32, 0xa1, - 0xd1, 0x0a, 0xce, 0xad, 0x71, 0x53, 0x85, 0x84, 0xd1, 0xe1, 0x13, 0xe2, 0xee, 0x32, 0xa1, 0xb7, - 0x0a, 0xce, 0xac, 0x20, 0x13, 0x0c, 0x46, 0x1e, 0x0b, 0xb5, 0x54, 0x30, 0xff, 0xe4, 0x5c, 0x87, - 0x36, 0x8b, 0x7a, 0x47, 0xbe, 0x23, 0x68, 0x2a, 0x31, 0xd7, 0xec, 0x9a, 0xf5, 0x19, 0x98, 0x1d, - 0x8f, 0x05, 0x76, 0xe4, 0xec, 0x93, 0x70, 0xdd, 0x89, 0x3c, 0xea, 0xa3, 0xb7, 0xa0, 0x66, 0x8b, - 0x2f, 0x21, 0xc7, 0xfc, 0xed, 0xf3, 0xb9, 0xbb, 0xc4, 0x48, 0x58, 0xa2, 0x70, 0xaf, 0x6b, 0xd3, - 0xd1, 0xc8, 0x8b, 0x12, 0xa1, 0x12, 0x18, 0xad, 0x40, 0xb3, 0xcb, 0xf8, 0x51, 0x3b, 0xfc, 0x0e, - 0x42, 0xb4, 0x3a, 0xce, 0x2e, 0x59, 0x6d, 0x30, 0xd6, 0xdb, 0x9b, 0x39, 0x26, 0xda, 0xc9, 0x4c, - 0xf4, 0x49, 0x26, 0x3f, 0xd5, 0xe1, 0x62, 0xd7, 0x7f, 0x34, 0x1c, 0x13, 0x7e, 0xa9, 0xf4, 0x3a, - 0x0c, 0x7d, 0x17, 0x5a, 0xc9, 0xc6, 0xee, 0x51, 0x40, 0xe4, 0x85, 0xae, 0xe4, 0x2e, 0x94, 0xc3, - 0xc0, 0x79, 0x02, 0x74, 0x17, 0x5a, 0x29, 0xc3, 0x6e, 0x87, 0xdf, 0xd1, 0x98, 0x30, 0x6f, 0x16, - 0x03, 0xe7, 0xf1, 0x45, 0x54, 0x3a, 0xfb, 0x64, 0x64, 0x77, 0x3b, 0x42, 0x01, 0x06, 0x4e, 0x60, - 0xb4, 0x09, 0xe7, 0xc9, 0x53, 0x67, 0x38, 0x76, 0x49, 0x86, 0xc6, 0x15, 0x76, 0x3a, 0xf1, 0x88, - 0x32, 0x2a, 0xeb, 0xaf, 0x5a, 0xd6, 0x94, 0xd2, 0x27, 0xbf, 0x07, 0x17, 0xbd, 0x32, 0xcd, 0xc8, - 0x98, 0xb5, 0xca, 0x15, 0x91, 0xc5, 0xc4, 0xe5, 0x0c, 0xd0, 0x9d, 0xc4, 0x49, 0xe2, 0x10, 0xbe, - 0x76, 0x8c, 0xb8, 0x05, 0x77, 0xb1, 0xc0, 0xb0, 0x1d, 0x15, 0xbc, 0x66, 0xde, 0xb1, 0xda, 0x9b, - 0x98, 0x6f, 0x5a, 0x7f, 0xd2, 0xe0, 0x5c, 0x26, 0xe9, 0xb0, 0x80, 0xfa, 0x8c, 0x9c, 0x35, 0xeb, - 0x6c, 0x01, 0x72, 0x0b, 0xda, 0x21, 0xca, 0x9a, 0xc7, 0xc9, 0x2e, 0x93, 0x41, 0x09, 0x21, 0x42, - 0x50, 0x19, 0x51, 0x97, 0x48, 0x93, 0x8a, 0x6f, 0xeb, 0x29, 0x9c, 0x6f, 0x67, 0x22, 0x76, 0x8b, - 0x30, 0x66, 0x0f, 0xce, 0x2c, 0x78, 0x31, 0x37, 0xe8, 0x93, 0xb9, 0xc1, 0xfa, 0xb5, 0x06, 0x0b, - 0x98, 0xb8, 0x74, 0x8b, 0x44, 0xf6, 0x94, 0x8e, 0x3d, 0x2d, 0xdd, 0x14, 0xc5, 0x32, 0x4a, 0xc4, - 0xfa, 0x11, 0x5c, 0xe3, 0x52, 0xe1, 0x84, 0x6a, 0x27, 0xa4, 0x83, 0x90, 0x30, 0xf6, 0x7a, 0x64, - 0xb4, 0x3e, 0x83, 0xa5, 0xfc, 0xf9, 0x1f, 0xd2, 0xf0, 0xd0, 0x0e, 0xdd, 0xd7, 0x74, 0xfc, 0x7f, - 0x73, 0x11, 0xd9, 0xa6, 0xfe, 0x23, 0x6f, 0x80, 0xd6, 0xa0, 0xc2, 0x02, 0xdb, 0x97, 0x67, 0x5d, - 0x2a, 0xaf, 0x42, 0x58, 0xe0, 0xf0, 0x5a, 0xcf, 0x78, 0x05, 0x4f, 0xb8, 0x2b, 0x90, 0x4b, 0xee, - 0x66, 0x32, 0x82, 0x8c, 0xa7, 0x13, 0x52, 0x46, 0x0e, 0x9d, 0x27, 0x25, 0xa6, 0x92, 0x52, 0x25, - 0x4e, 0x4a, 0x0a, 0x4e, 0x3c, 0xbb, 0x9a, 0x7a, 0x36, 0x5a, 0x03, 0x93, 0x1d, 0x78, 0x41, 0x67, - 0xeb, 0xc1, 0x3a, 0xeb, 0x49, 0x89, 0x6a, 0x22, 0x11, 0x4f, 0xac, 0x5b, 0x5f, 0x68, 0x70, 0x99, - 0x67, 0x38, 0x77, 0x3c, 0xcc, 0x24, 0xa8, 0x29, 0xf5, 0x0e, 0x77, 0xa0, 0xe6, 0x08, 0x3d, 0x9e, - 0x92, 0x75, 0x62, 0x65, 0x63, 0x89, 0x8c, 0xda, 0x30, 0xcf, 0xa4, 0x48, 0x71, 0x3e, 0x12, 0x0a, - 0x9b, 0xbf, 0x7d, 0x35, 0x47, 0xde, 0xcb, 0xa1, 0xe0, 0x02, 0x89, 0xf5, 0x3f, 0x0d, 0x2e, 0x6d, - 0x91, 0x70, 0x30, 0xfd, 0x5b, 0xdd, 0x85, 0x96, 0xfb, 0x8a, 0x45, 0x26, 0x87, 0x8f, 0xba, 0x80, - 0x46, 0x5c, 0x32, 0xb7, 0xf3, 0x4a, 0x4e, 0x51, 0x42, 0x94, 0x98, 0xbf, 0x92, 0x49, 0x6c, 0x3b, - 0x70, 0x7e, 0xcb, 0xf6, 0xfc, 0xc8, 0xf6, 0x7c, 0x12, 0xde, 0x57, 0xdc, 0xd0, 0xb7, 0x32, 0x4d, - 0x95, 0x56, 0x92, 0x48, 0x53, 0x9a, 0x62, 0x57, 0x65, 0xfd, 0x45, 0x07, 0xb3, 0xb8, 0x7d, 0x56, - 0x2d, 0x5e, 0x03, 0xe0, 0x5f, 0x7d, 0x7e, 0x08, 0x11, 0xfe, 0xd1, 0xc0, 0x0d, 0xbe, 0xc2, 0xd9, - 0x13, 0x74, 0x0b, 0xaa, 0xf1, 0x4e, 0x99, 0xe9, 0xdb, 0x74, 0x14, 0x50, 0x9f, 0xf8, 0x91, 0xc0, - 0xc5, 0x31, 0x26, 0xfa, 0x12, 0xb4, 0xd2, 0x7c, 0xd6, 0x8f, 0x92, 0x0e, 0x2a, 0xd7, 0x97, 0xc9, - 0xb6, 0xaf, 0x5a, 0x62, 0xb2, 0x89, 0xb6, 0x0f, 0x7d, 0x05, 0xe6, 0xf7, 0x28, 0x8d, 0x58, 0x14, - 0xda, 0x41, 0xdf, 0xa5, 0x3e, 0x91, 0x61, 0xd4, 0x4a, 0x56, 0x3b, 0xd4, 0x27, 0x13, 0x9d, 0xdb, - 0x6c, 0x49, 0xe7, 0xf6, 0x0d, 0xb8, 0xda, 0xa6, 0x34, 0x74, 0x3d, 0xdf, 0x8e, 0x68, 0xb8, 0xa1, - 0xe8, 0x95, 0x4b, 0x2e, 0xc2, 0xec, 0x13, 0x12, 0x32, 0xd5, 0xc5, 0x19, 0x58, 0x81, 0xd6, 0xf7, - 0x61, 0xa9, 0x9c, 0x50, 0x16, 0xda, 0x33, 0x98, 0xf5, 0xef, 0x1a, 0x5c, 0x58, 0x77, 0xdd, 0x14, - 0x43, 0x49, 0xf3, 0x55, 0xd0, 0x3d, 0xf7, 0x74, 0x83, 0xea, 0x9e, 0xcb, 0x47, 0x95, 0x4c, 0x88, - 0xcf, 0x25, 0x31, 0x3c, 0x61, 0x8c, 0x92, 0x8a, 0x83, 0xd6, 0xe0, 0x9c, 0xc7, 0xfa, 0x3e, 0x39, - 0xec, 0xa7, 0xae, 0x21, 0xac, 0x56, 0xc7, 0x0b, 0x1e, 0xdb, 0x26, 0x87, 0xe9, 0x71, 0xe8, 0x3a, - 0x34, 0x0f, 0xe4, 0xa4, 0xd3, 0xf7, 0x5c, 0x91, 0xef, 0x5a, 0x18, 0xd4, 0x52, 0xd7, 0xb5, 0x7e, - 0xa3, 0xc1, 0x1b, 0x98, 0x8c, 0xe8, 0x13, 0x72, 0xa6, 0x0b, 0x2d, 0xc2, 0xac, 0x63, 0x33, 0xc7, - 0x76, 0x89, 0x6c, 0x5e, 0x15, 0xc8, 0x77, 0x42, 0xc1, 0xdf, 0x95, 0xbd, 0xb1, 0x02, 0x8b, 0xb2, - 0x55, 0x26, 0x64, 0xfb, 0xbd, 0x01, 0x57, 0x52, 0xa9, 0x26, 0xac, 0x7f, 0xc6, 0x50, 0x3a, 0xce, - 0x06, 0x97, 0x85, 0x6b, 0x84, 0x19, 0xf5, 0x27, 0x15, 0xc9, 0x81, 0x37, 0x23, 0x5e, 0xbe, 0xfa, - 0x51, 0xe8, 0x0d, 0x06, 0x24, 0xec, 0x93, 0x27, 0xc4, 0x8f, 0xfa, 0x69, 0x9e, 0x52, 0xf7, 0x38, - 0x31, 0x23, 0x5d, 0x13, 0x3c, 0x76, 0x63, 0x16, 0xf7, 0x38, 0x87, 0x6c, 0x8f, 0x5b, 0x6e, 0xde, - 0x6a, 0xb9, 0x79, 0x6d, 0x58, 0xc9, 0x0b, 0xc4, 0x27, 0xc2, 0x82, 0x3c, 0xb5, 0xd3, 0xe4, 0x59, - 0xca, 0xca, 0xc3, 0x7b, 0x89, 0x9c, 0x38, 0x05, 0x2b, 0xcd, 0x4e, 0x58, 0xe9, 0x57, 0x3a, 0x5c, - 0x2d, 0xb5, 0xd2, 0x74, 0x7a, 0xda, 0x3b, 0x50, 0xe5, 0x7d, 0x82, 0xaa, 0x17, 0xd7, 0x73, 0x74, - 0xc9, 0x69, 0x69, 0x57, 0x11, 0x63, 0xab, 0x8c, 0x65, 0xbc, 0xcc, 0xa0, 0xfa, 0x72, 0x39, 0xf0, - 0x6d, 0x40, 0x42, 0xbb, 0x79, 0xcc, 0xaa, 0xc0, 0x34, 0xf9, 0x4e, 0xb6, 0x2f, 0xb6, 0xfe, 0xa8, - 0xc3, 0x72, 0xaa, 0x95, 0x1d, 0xca, 0xa2, 0x69, 0xfb, 0xef, 0x4b, 0x39, 0xa3, 0x7e, 0x46, 0x67, - 0xbc, 0x05, 0xb3, 0x71, 0xd3, 0xc4, 0x63, 0x81, 0xeb, 0xff, 0x8d, 0x89, 0x6e, 0x62, 0x64, 0x77, - 0xfd, 0x47, 0x14, 0x2b, 0x3c, 0xf4, 0x1e, 0xcc, 0x09, 0x3d, 0x29, 0xba, 0xca, 0xc9, 0x74, 0x4d, - 0x8e, 0x1c, 0xc3, 0xcc, 0xfa, 0x8f, 0x06, 0xd7, 0x8f, 0xd5, 0xda, 0x74, 0xfc, 0xe9, 0xb5, 0xa8, - 0xed, 0x55, 0xbc, 0xcf, 0x7a, 0x0a, 0x90, 0xea, 0x23, 0x37, 0x4b, 0x6b, 0x85, 0x59, 0x7a, 0x59, - 0x61, 0x6e, 0xdb, 0x23, 0x55, 0xfd, 0x33, 0x2b, 0xe8, 0x06, 0xd4, 0x44, 0x20, 0x28, 0x63, 0x95, - 0x74, 0xde, 0x42, 0xe7, 0x12, 0xcb, 0x6a, 0xcb, 0x47, 0x37, 0x71, 0xf0, 0xf1, 0x8f, 0x6e, 0x4b, - 0x12, 0x2d, 0x73, 0x6a, 0xba, 0x60, 0xfd, 0x59, 0x07, 0x34, 0x19, 0x87, 0xbc, 0x78, 0x1c, 0x63, - 0x9c, 0x9c, 0x22, 0x75, 0xf9, 0xa8, 0xa7, 0xae, 0xac, 0x17, 0xae, 0xac, 0x46, 0x09, 0xe3, 0x25, - 0x46, 0x89, 0x0f, 0xc1, 0x74, 0x54, 0x8f, 0xd3, 0x8f, 0xeb, 0xb5, 0x88, 0xe4, 0x53, 0x1a, 0xa1, - 0x05, 0x27, 0x0b, 0x8f, 0xd9, 0x64, 0x3a, 0xa8, 0x96, 0xa4, 0x83, 0x77, 0xa0, 0xb9, 0x37, 0xa4, - 0xce, 0x81, 0x6c, 0xc5, 0xe2, 0x2c, 0x8b, 0xf2, 0x5e, 0x2e, 0xd8, 0x83, 0x40, 0x8b, 0xfb, 0x33, - 0xd5, 0x78, 0xce, 0x66, 0x1a, 0xcf, 0xc7, 0x70, 0x29, 0x75, 0xf9, 0xf6, 0x90, 0x32, 0x32, 0xa5, - 0x04, 0x91, 0xa9, 0xbc, 0x7a, 0xae, 0xf2, 0x5a, 0x21, 0xbc, 0x31, 0x71, 0xe4, 0x74, 0xa2, 0x8b, - 0x4f, 0x73, 0x63, 0xc7, 0x21, 0x8c, 0xa9, 0x33, 0x25, 0x68, 0xfd, 0x5c, 0x03, 0x33, 0x7d, 0x7c, - 0x89, 0x1d, 0x70, 0x0a, 0x6f, 0x57, 0x57, 0xa0, 0x2e, 0xdd, 0x34, 0xae, 0x10, 0x06, 0x4e, 0xe0, - 0x93, 0x9e, 0xa5, 0xac, 0x1f, 0x40, 0x55, 0xe0, 0x9d, 0xf2, 0xd6, 0x7c, 0x9c, 0x5b, 0x2e, 0x41, - 0xa3, 0x17, 0x0c, 0x3d, 0x91, 0x05, 0x64, 0x5f, 0x93, 0x2e, 0x58, 0x3e, 0xcc, 0x2b, 0xcc, 0x58, - 0x57, 0x27, 0x9c, 0xb2, 0x02, 0xcd, 0x8f, 0x87, 0x6e, 0xe1, 0xa0, 0xec, 0x12, 0xc7, 0xd8, 0x26, - 0x87, 0x85, 0x9b, 0x64, 0x97, 0xac, 0xdf, 0x19, 0x50, 0x8d, 0x1d, 0x6c, 0x09, 0x1a, 0x5d, 0xb6, - 0xc1, 0x1d, 0x8e, 0xc4, 0x9d, 0x5b, 0x1d, 0xa7, 0x0b, 0x5c, 0x0a, 0xf1, 0x99, 0xce, 0xda, 0x12, - 0x44, 0x77, 0xa1, 0x19, 0x7f, 0xaa, 0xf4, 0x31, 0x39, 0x78, 0x16, 0x8d, 0x87, 0xb3, 0x14, 0x68, - 0x13, 0xce, 0x6d, 0x13, 0xe2, 0x76, 0x42, 0x1a, 0x04, 0x0a, 0x43, 0xb6, 0x42, 0xa7, 0xb0, 0x99, - 0xa4, 0x43, 0xef, 0xc3, 0x02, 0x5f, 0x5c, 0x77, 0xdd, 0x84, 0x55, 0x3c, 0x7a, 0xa0, 0xc9, 0xf8, - 0xc7, 0x45, 0x54, 0x3e, 0x08, 0x7f, 0x12, 0xb8, 0x76, 0x44, 0xa4, 0x0a, 0xf9, 0x18, 0xcf, 0x89, - 0xaf, 0x96, 0x95, 0x20, 0x69, 0x20, 0x5c, 0x20, 0x29, 0xbe, 0xc8, 0xce, 0x4e, 0xbc, 0xc8, 0xa2, - 0xaf, 0x89, 0x59, 0x6b, 0x40, 0x16, 0xeb, 0xc2, 0x67, 0xf3, 0x05, 0x6e, 0x43, 0xc6, 0xfc, 0x20, - 0x9e, 0xb3, 0x06, 0xc4, 0xfa, 0x31, 0x5c, 0x48, 0xf2, 0x95, 0xda, 0xe5, 0xc9, 0xe6, 0x15, 0xf2, - 0xe4, 0xaa, 0x9a, 0xee, 0xf4, 0x63, 0x93, 0x8d, 0x1c, 0xea, 0xca, 0x5e, 0xee, 0xfe, 0xad, 0xc1, - 0x42, 0xe1, 0x27, 0x80, 0x57, 0x39, 0xbc, 0x2c, 0xb9, 0xea, 0xd3, 0x48, 0xae, 0x65, 0x23, 0xce, - 0x2d, 0xb8, 0x18, 0x97, 0x65, 0xe6, 0x3d, 0x23, 0xfd, 0x80, 0x84, 0x7d, 0x46, 0x1c, 0xea, 0xc7, - 0xcd, 0xb5, 0x8e, 0x91, 0xd8, 0xec, 0x79, 0xcf, 0xc8, 0x0e, 0x09, 0x7b, 0x62, 0xa7, 0xec, 0x49, - 0xc7, 0xfa, 0x83, 0x06, 0x28, 0xa3, 0xeb, 0x29, 0xe5, 0xd5, 0x8f, 0xa0, 0xb5, 0x97, 0x32, 0x4d, - 0x1e, 0x58, 0xdf, 0x2c, 0xaf, 0x4d, 0xd9, 0xf3, 0xf3, 0x74, 0xa5, 0x56, 0x72, 0x61, 0x2e, 0xdb, - 0x21, 0x70, 0x9c, 0xc8, 0x1b, 0xc5, 0x89, 0xb1, 0x81, 0xc5, 0x37, 0x5f, 0xf3, 0xa9, 0xab, 0x4a, - 0xb1, 0xf8, 0xe6, 0x6b, 0x8e, 0xe2, 0xd5, 0xc0, 0xe2, 0x9b, 0x87, 0xfb, 0x28, 0x7e, 0x05, 0x14, - 0x7a, 0x6b, 0x60, 0x05, 0x5a, 0xef, 0xc2, 0x5c, 0xf1, 0x41, 0x64, 0xdf, 0x1b, 0xec, 0xcb, 0xdf, - 0x25, 0xc4, 0x37, 0x32, 0xc1, 0x18, 0xd2, 0x43, 0x99, 0x28, 0xf8, 0x27, 0x97, 0x2d, 0xab, 0x96, - 0x97, 0xa3, 0x12, 0xd2, 0xf2, 0xc6, 0x41, 0x4a, 0xc6, 0xbf, 0x79, 0x6a, 0x55, 0x13, 0x84, 0x14, - 0x2d, 0x81, 0xad, 0x1f, 0xc2, 0xf5, 0x07, 0x74, 0x90, 0x99, 0xde, 0xd3, 0xa7, 0xcd, 0xe9, 0x18, - 0xd0, 0xfa, 0x89, 0x06, 0x2b, 0xc7, 0x1f, 0x31, 0x9d, 0x42, 0x78, 0xda, 0xbb, 0xe9, 0x90, 0xeb, - 0x92, 0x38, 0x07, 0x6c, 0x3c, 0xda, 0x22, 0x91, 0x8d, 0xbe, 0xae, 0x62, 0xbb, 0xac, 0x02, 0x2a, - 0xcc, 0x5c, 0x8c, 0xaf, 0x81, 0xe9, 0x64, 0xd7, 0x7b, 0xe4, 0xb1, 0x3c, 0x67, 0x62, 0xdd, 0xfa, - 0xa5, 0x06, 0x17, 0x33, 0x4f, 0xfe, 0x24, 0x52, 0x1c, 0xd1, 0x05, 0xa8, 0x3a, 0x74, 0xec, 0x47, - 0xd2, 0x88, 0x31, 0xc0, 0x3d, 0xe7, 0x29, 0x0d, 0xef, 0x73, 0xe3, 0xca, 0x42, 0x21, 0x41, 0x3e, - 0x35, 0x3f, 0xa5, 0xe1, 0x03, 0x7a, 0x28, 0xe3, 0x56, 0x42, 0x71, 0xe1, 0x1f, 0x09, 0x8a, 0x8a, - 0x1c, 0x9a, 0x63, 0x90, 0x53, 0xb0, 0xf1, 0x88, 0x53, 0xc4, 0x6d, 0x94, 0x84, 0xac, 0xdf, 0x6a, - 0xb0, 0x52, 0x2a, 0xd3, 0xba, 0x73, 0x30, 0x2d, 0x2b, 0x5c, 0x80, 0x2a, 0x09, 0xa8, 0xa3, 0x6e, - 0x11, 0x03, 0x65, 0x71, 0xa7, 0x7e, 0x59, 0xac, 0x24, 0xbf, 0x2c, 0x5a, 0xff, 0xd4, 0xc0, 0x2a, - 0x95, 0x2f, 0xae, 0x14, 0x53, 0x4a, 0x26, 0x67, 0x90, 0x10, 0x7d, 0x00, 0x75, 0x65, 0x69, 0xa1, - 0xdb, 0xe2, 0x6f, 0x5d, 0xa5, 0xd2, 0xe3, 0x84, 0x66, 0xed, 0x1a, 0xd4, 0xe4, 0xaf, 0xa1, 0x0d, - 0xa8, 0x3e, 0x0c, 0xbd, 0x88, 0x98, 0x33, 0xa8, 0x0e, 0x95, 0x1d, 0x9b, 0x31, 0x53, 0x5b, 0x5b, - 0x8d, 0xbb, 0x98, 0xf4, 0x75, 0x18, 0x01, 0xd4, 0xda, 0x21, 0xb1, 0x05, 0x1e, 0x40, 0x2d, 0x7e, - 0x37, 0x32, 0xb5, 0xb5, 0xf7, 0x00, 0xd2, 0x82, 0xc7, 0x39, 0x6c, 0x7f, 0xbc, 0x7d, 0xcf, 0x9c, - 0x41, 0x4d, 0x98, 0x7d, 0xb8, 0xde, 0xdd, 0xed, 0x6e, 0x7f, 0x64, 0x6a, 0x02, 0xc0, 0x31, 0xa0, - 0x73, 0x9c, 0x0e, 0xc7, 0x31, 0xd6, 0xde, 0x2e, 0xb4, 0x80, 0x68, 0x16, 0x8c, 0xf5, 0xe1, 0xd0, - 0x9c, 0x41, 0x35, 0xd0, 0x3b, 0x1b, 0xa6, 0xc6, 0x4f, 0xda, 0xa6, 0xe1, 0xc8, 0x1e, 0x9a, 0xfa, - 0xda, 0x33, 0x98, 0xcf, 0x17, 0x18, 0xc1, 0x96, 0x86, 0x07, 0x9e, 0x3f, 0x88, 0x0f, 0xec, 0x45, - 0xa2, 0x93, 0x88, 0x0f, 0x8c, 0x25, 0x74, 0x4d, 0x1d, 0x99, 0x30, 0xd7, 0xf5, 0xbd, 0xc8, 0xb3, - 0x87, 0xde, 0x33, 0x8e, 0x6b, 0xa0, 0x16, 0x34, 0x76, 0x42, 0x12, 0xd8, 0x21, 0x07, 0x2b, 0x68, - 0x1e, 0x40, 0x3c, 0x7c, 0x63, 0x62, 0xbb, 0x47, 0x66, 0x95, 0x13, 0x3c, 0xb4, 0xbd, 0xc8, 0xf3, - 0x07, 0x62, 0xd9, 0xac, 0xad, 0x7d, 0x1b, 0x5a, 0xb9, 0x40, 0x44, 0xe7, 0xa0, 0xf5, 0xc9, 0x76, - 0x77, 0xbb, 0xbb, 0xdb, 0x5d, 0x7f, 0xd0, 0xfd, 0xf4, 0x5e, 0xc7, 0x9c, 0x41, 0x73, 0x50, 0xdf, - 0xea, 0xf6, 0xb6, 0xd6, 0x77, 0xdb, 0xf7, 0x4d, 0x8d, 0xab, 0x35, 0xfe, 0xd4, 0x37, 0x3e, 0xf8, - 0xdb, 0xf3, 0x65, 0xed, 0xf3, 0xe7, 0xcb, 0xda, 0xbf, 0x9e, 0x2f, 0x6b, 0xbf, 0x78, 0xb1, 0x3c, - 0xf3, 0xf9, 0x8b, 0xe5, 0x99, 0x7f, 0xbc, 0x58, 0x9e, 0xf9, 0xf4, 0xcb, 0x03, 0x2f, 0xda, 0x1f, - 0xef, 0xdd, 0x70, 0xe8, 0xe8, 0x66, 0xe0, 0xf9, 0x03, 0xc7, 0x0e, 0x6e, 0x46, 0x9e, 0xe3, 0x3a, - 0x37, 0x33, 0xb6, 0xdc, 0xab, 0x89, 0xff, 0x4c, 0xbc, 0xf3, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x48, 0x01, 0xd7, 0x2e, 0x52, 0x21, 0x00, 0x00, + 0xd5, 0xdd, 0xf3, 0xe1, 0x99, 0x67, 0x8f, 0xdd, 0xa9, 0x24, 0x8e, 0x93, 0xd8, 0x8e, 0xb7, 0x01, + 0xc9, 0xcc, 0x2e, 0x09, 0x49, 0x36, 0x02, 0x96, 0x65, 0xc3, 0x78, 0xc6, 0xbb, 0x19, 0xd9, 0x9e, + 0x58, 0x35, 0x5e, 0x02, 0xcb, 0x61, 0x68, 0x77, 0x57, 0xc6, 0x8d, 0x67, 0xba, 0x3a, 0x5d, 0x3d, + 0x71, 0x1c, 0x69, 0x41, 0x08, 0x71, 0xe3, 0x00, 0x82, 0x0b, 0x07, 0x0e, 0x88, 0x3f, 0x80, 0x84, + 0x38, 0x22, 0x71, 0x03, 0x89, 0xcb, 0x9e, 0x56, 0xdc, 0x40, 0xc9, 0x1f, 0x40, 0x70, 0xe0, 0x8a, + 0xaa, 0xba, 0xfa, 0x73, 0xda, 0x9e, 0x44, 0x1e, 0xe5, 0xd6, 0xaf, 0xea, 0xbd, 0x7a, 0xaf, 0xde, + 0xf7, 0xab, 0x19, 0xb8, 0x7e, 0x48, 0x0c, 0xcf, 0x3f, 0x20, 0x86, 0xef, 0x1e, 0xdc, 0x8a, 0xbe, + 0x6f, 0xba, 0x1e, 0xf5, 0x29, 0x9a, 0x4b, 0x6c, 0xea, 0x27, 0x50, 0xdd, 0x37, 0x0e, 0x06, 0xa4, + 0xeb, 0x1a, 0x0e, 0x5a, 0x86, 0x59, 0x01, 0xb4, 0x5b, 0xcb, 0xca, 0xba, 0xb2, 0x51, 0xc0, 0x21, + 0x88, 0xae, 0x41, 0xa5, 0xeb, 0x1b, 0x9e, 0xbf, 0x4d, 0x4e, 0x96, 0xd5, 0x75, 0x65, 0x63, 0x1e, + 0x47, 0x30, 0x5a, 0x82, 0xf2, 0x96, 0x63, 0xf1, 0x9d, 0x82, 0xd8, 0x91, 0x10, 0x5a, 0x03, 0xd8, + 0x26, 0x27, 0xcc, 0x35, 0x4c, 0x7e, 0x60, 0x71, 0x5d, 0xd9, 0xa8, 0xe1, 0xc4, 0x8a, 0xfe, 0xb9, + 0x0a, 0xda, 0x03, 0x2e, 0xca, 0x26, 0x31, 0x7c, 0x4c, 0x9e, 0x8c, 0x08, 0xf3, 0xd1, 0xb7, 0x60, + 0xde, 0x3c, 0x34, 0x9c, 0x3e, 0x79, 0x4c, 0x88, 0x25, 0xe5, 0x98, 0xbb, 0x73, 0xf5, 0x66, 0x42, + 0xe6, 0x9b, 0xcd, 0x04, 0x02, 0x4e, 0xa1, 0xa3, 0x77, 0xa1, 0x7a, 0x6c, 0xf8, 0xc4, 0x1b, 0x1a, + 0xde, 0x91, 0x10, 0x74, 0xee, 0xce, 0x52, 0x8a, 0xf6, 0x51, 0xb8, 0x8b, 0x63, 0x44, 0xf4, 0x3e, + 0xd4, 0x3c, 0x62, 0xd1, 0x68, 0x4f, 0x5c, 0xe4, 0x74, 0xca, 0x34, 0x32, 0xfa, 0x3a, 0x54, 0x98, + 0x6f, 0xf8, 0x23, 0x46, 0xd8, 0x72, 0x71, 0xbd, 0xb0, 0x31, 0x77, 0x67, 0x25, 0x45, 0x18, 0xe9, + 0xb7, 0x2b, 0xb0, 0x70, 0x84, 0x8d, 0x36, 0x60, 0xd1, 0xa4, 0x43, 0x97, 0x0c, 0x88, 0x4f, 0x82, + 0xcd, 0xe5, 0xd2, 0xba, 0xb2, 0x51, 0xc1, 0xd9, 0x65, 0xf4, 0x36, 0x14, 0x88, 0xe7, 0x2d, 0x97, + 0x73, 0xb4, 0x81, 0x47, 0x8e, 0x63, 0x3b, 0xfd, 0x2d, 0xcf, 0xa3, 0x1e, 0xe6, 0x58, 0xfa, 0xcf, + 0x14, 0xa8, 0xc6, 0xe2, 0xe9, 0x5c, 0xa3, 0xc4, 0x3c, 0x72, 0xa9, 0xed, 0xf8, 0xfb, 0x4c, 0x68, + 0xb4, 0x88, 0x53, 0x6b, 0xdc, 0x54, 0x1e, 0x61, 0x74, 0xf0, 0x94, 0x58, 0xfb, 0x4c, 0xe8, 0xad, + 0x88, 0x13, 0x2b, 0x48, 0x83, 0x02, 0x23, 0x4f, 0x84, 0x5a, 0x8a, 0x98, 0x7f, 0xf2, 0x53, 0x07, + 0x06, 0xf3, 0xbb, 0x27, 0x8e, 0x29, 0x68, 0x8a, 0xc1, 0xa9, 0xc9, 0x35, 0xfd, 0x53, 0xd0, 0x5a, + 0x36, 0x73, 0x0d, 0xdf, 0x3c, 0x24, 0x5e, 0xc3, 0xf4, 0x6d, 0xea, 0xa0, 0xb7, 0xa1, 0x6c, 0x88, + 0x2f, 0x21, 0xc7, 0xc2, 0x9d, 0x8b, 0xa9, 0xbb, 0x04, 0x48, 0x58, 0xa2, 0x70, 0xaf, 0x6b, 0xd2, + 0xe1, 0xd0, 0xf6, 0x23, 0xa1, 0x22, 0x18, 0xad, 0xc3, 0x5c, 0x9b, 0x71, 0x56, 0x7b, 0xfc, 0x0e, + 0x42, 0xb4, 0x0a, 0x4e, 0x2e, 0xe9, 0x4d, 0x28, 0x34, 0x9a, 0xdb, 0xa9, 0x43, 0x94, 0xb3, 0x0f, + 0x51, 0xc7, 0x0f, 0xf9, 0xa9, 0x0a, 0x97, 0xdb, 0xce, 0xe3, 0xc1, 0x88, 0xf0, 0x4b, 0xc5, 0xd7, + 0x61, 0xe8, 0xdb, 0x50, 0x8b, 0x36, 0xf6, 0x4f, 0x5c, 0x22, 0x2f, 0x74, 0x2d, 0x75, 0xa1, 0x14, + 0x06, 0x4e, 0x13, 0xa0, 0xfb, 0x50, 0x8b, 0x0f, 0x6c, 0xb7, 0xf8, 0x1d, 0x0b, 0x63, 0xe6, 0x4d, + 0x62, 0xe0, 0x34, 0xbe, 0x88, 0x4a, 0xf3, 0x90, 0x0c, 0x8d, 0x76, 0x4b, 0x28, 0xa0, 0x80, 0x23, + 0x18, 0x6d, 0xc3, 0x45, 0xf2, 0xcc, 0x1c, 0x8c, 0x2c, 0x92, 0xa0, 0xb1, 0x84, 0x9d, 0xce, 0x64, + 0x91, 0x47, 0xa5, 0xff, 0x55, 0x49, 0x9a, 0x52, 0xfa, 0xe4, 0x77, 0xe1, 0xb2, 0x9d, 0xa7, 0x19, + 0x19, 0xb3, 0x7a, 0xbe, 0x22, 0x92, 0x98, 0x38, 0xff, 0x00, 0x74, 0x2f, 0x72, 0x92, 0x20, 0x84, + 0x57, 0x4f, 0x11, 0x37, 0xe3, 0x2e, 0x3a, 0x14, 0x0c, 0x33, 0x0c, 0x5e, 0x2d, 0xed, 0x58, 0xcd, + 0x6d, 0xcc, 0x37, 0xf5, 0x3f, 0x29, 0x70, 0x21, 0x91, 0x74, 0x98, 0x4b, 0x1d, 0x46, 0xce, 0x9b, + 0x75, 0x76, 0x01, 0x59, 0x19, 0xed, 0x90, 0xd0, 0x9a, 0xa7, 0xc9, 0x2e, 0x93, 0x41, 0x0e, 0x21, + 0x42, 0x50, 0x1c, 0x52, 0x8b, 0x48, 0x93, 0x8a, 0x6f, 0xfd, 0x19, 0x5c, 0x6c, 0x26, 0x22, 0x76, + 0x97, 0x30, 0x66, 0xf4, 0xcf, 0x2d, 0x78, 0x36, 0x37, 0xa8, 0xe3, 0xb9, 0x41, 0xff, 0xb5, 0x02, + 0x8b, 0x98, 0x58, 0x74, 0x97, 0xf8, 0xc6, 0x94, 0xd8, 0x4e, 0x4a, 0x37, 0x59, 0xb1, 0x0a, 0x39, + 0x62, 0xfd, 0x08, 0x56, 0xb9, 0x54, 0x38, 0xa2, 0xda, 0xf3, 0x68, 0xdf, 0x23, 0x8c, 0xbd, 0x19, + 0x19, 0xf5, 0x4f, 0x61, 0x25, 0xcd, 0xff, 0x43, 0xea, 0x1d, 0x1b, 0x9e, 0xf5, 0x86, 0xd8, 0xff, + 0x37, 0x15, 0x91, 0x4d, 0xea, 0x3c, 0xb6, 0xfb, 0xa8, 0x0e, 0x45, 0xe6, 0x1a, 0x8e, 0xe4, 0xb5, + 0x94, 0x5f, 0x85, 0xb0, 0xc0, 0xe1, 0xb5, 0x9e, 0xf1, 0x0a, 0x1e, 0x9d, 0x1e, 0x82, 0x5c, 0x72, + 0x2b, 0x91, 0x11, 0x64, 0x3c, 0x9d, 0x91, 0x32, 0x52, 0xe8, 0x3c, 0x29, 0xb1, 0x30, 0x29, 0x15, + 0x83, 0xa4, 0x14, 0xc2, 0x91, 0x67, 0x97, 0x62, 0xcf, 0x46, 0x75, 0xd0, 0xd8, 0x91, 0xed, 0xb6, + 0x76, 0x77, 0x1a, 0xac, 0x2b, 0x25, 0x2a, 0x8b, 0x44, 0x3c, 0xb6, 0xae, 0x7f, 0xae, 0xc0, 0x55, + 0x9e, 0xe1, 0xac, 0xd1, 0x20, 0x91, 0xa0, 0xa6, 0xd4, 0x3b, 0xdc, 0x83, 0xb2, 0x29, 0xf4, 0x38, + 0x21, 0xeb, 0x04, 0xca, 0xc6, 0x12, 0x19, 0x35, 0x61, 0x81, 0x49, 0x91, 0x82, 0x7c, 0x24, 0x14, + 0xb6, 0x70, 0xe7, 0x7a, 0x8a, 0xbc, 0x9b, 0x42, 0xc1, 0x19, 0x12, 0xfd, 0x7f, 0x0a, 0x2c, 0xed, + 0x12, 0xaf, 0x3f, 0xfd, 0x5b, 0xdd, 0x87, 0x9a, 0xf5, 0x9a, 0x45, 0x26, 0x85, 0x8f, 0xda, 0x80, + 0x86, 0x5c, 0x32, 0xab, 0xf5, 0x5a, 0x4e, 0x91, 0x43, 0x14, 0x99, 0xbf, 0x98, 0x48, 0x6c, 0x7b, + 0x70, 0x71, 0xd7, 0xb0, 0x1d, 0xdf, 0xb0, 0x1d, 0xe2, 0x3d, 0x08, 0x4f, 0x43, 0xdf, 0x48, 0x34, + 0x55, 0x4a, 0x4e, 0x22, 0x8d, 0x69, 0xb2, 0x5d, 0x95, 0x4e, 0xa0, 0xd6, 0xa1, 0x16, 0x89, 0xcf, + 0xba, 0x07, 0x95, 0x81, 0xfd, 0x94, 0x38, 0x84, 0x31, 0x59, 0xa4, 0xd3, 0x72, 0x73, 0xec, 0x1d, + 0x89, 0x80, 0x23, 0x54, 0xb4, 0x0a, 0xe0, 0x50, 0x8b, 0xf4, 0x88, 0x4b, 0xcd, 0x43, 0x19, 0x24, + 0x55, 0xbe, 0xb2, 0xc5, 0x17, 0xf4, 0x1f, 0xc2, 0x52, 0x97, 0xf8, 0x29, 0x5a, 0x69, 0xb1, 0xdb, + 0x50, 0xf6, 0x0d, 0xaf, 0x4f, 0xfc, 0xc9, 0xdc, 0x24, 0xe2, 0x24, 0x5e, 0x43, 0xb8, 0x32, 0xc6, + 0x4b, 0x96, 0xae, 0xbb, 0x30, 0x6b, 0xb8, 0xee, 0xc0, 0x26, 0xd6, 0x64, 0x6e, 0x21, 0xe6, 0x24, + 0x76, 0x7f, 0x51, 0x41, 0xcb, 0x2a, 0xf8, 0xbc, 0x7e, 0xb8, 0x0a, 0xc0, 0xbf, 0x7a, 0xdc, 0x4c, + 0x44, 0xb0, 0xac, 0xe2, 0x2a, 0x5f, 0xe1, 0xc7, 0x13, 0x74, 0x1b, 0x4a, 0xc1, 0x4e, 0x5e, 0xf0, + 0x34, 0xe9, 0xd0, 0xa5, 0x0e, 0x71, 0x7c, 0x81, 0x8b, 0x03, 0x4c, 0xf4, 0x05, 0xa8, 0xc5, 0x15, + 0xa1, 0xe7, 0x47, 0x3d, 0x68, 0xaa, 0xb3, 0x95, 0x8d, 0x73, 0x29, 0xc7, 0xe9, 0xc7, 0x1a, 0x67, + 0xf4, 0x25, 0x58, 0x38, 0xa0, 0xd4, 0x67, 0xbe, 0x67, 0xb8, 0x3d, 0x8b, 0x3a, 0x44, 0x26, 0xa2, + 0x5a, 0xb4, 0xda, 0xa2, 0x0e, 0x19, 0xeb, 0x7d, 0x67, 0x73, 0x7a, 0xdf, 0xaf, 0xc1, 0xf5, 0x26, + 0xa5, 0x9e, 0x65, 0x3b, 0x86, 0x4f, 0xbd, 0xcd, 0x90, 0x3e, 0x74, 0x91, 0x65, 0x98, 0x7d, 0x4a, + 0x3c, 0x16, 0xf6, 0xc1, 0x05, 0x1c, 0x82, 0xfa, 0xf7, 0x60, 0x25, 0x9f, 0x50, 0xda, 0xfb, 0x1c, + 0x81, 0xf1, 0x77, 0x05, 0x2e, 0x35, 0x2c, 0x2b, 0xc6, 0x08, 0xa5, 0xf9, 0x32, 0xa8, 0xb6, 0x35, + 0xd9, 0xa0, 0xaa, 0x6d, 0xf1, 0x61, 0x2f, 0x91, 0x24, 0xe7, 0xa3, 0x2c, 0x38, 0x66, 0x8c, 0x9c, + 0x9a, 0x8d, 0xea, 0x70, 0xc1, 0x66, 0x3d, 0x87, 0x1c, 0xf7, 0x62, 0xd7, 0x10, 0x56, 0xab, 0xe0, + 0x45, 0x9b, 0x75, 0xc8, 0x71, 0xcc, 0x0e, 0xdd, 0x80, 0xb9, 0x23, 0x39, 0x2b, 0xf6, 0x6c, 0x4b, + 0x54, 0x8c, 0x1a, 0x86, 0x70, 0xa9, 0x6d, 0xe9, 0xbf, 0x51, 0xe0, 0x0a, 0x26, 0x43, 0xfa, 0x94, + 0x9c, 0xeb, 0x42, 0xcb, 0x30, 0x6b, 0x1a, 0xcc, 0x34, 0x2c, 0x22, 0xdb, 0xff, 0x10, 0xe4, 0x3b, + 0x9e, 0x38, 0xdf, 0x92, 0xd3, 0x45, 0x08, 0x66, 0x65, 0x2b, 0x8e, 0xc9, 0xf6, 0xfb, 0x02, 0x5c, + 0x8b, 0xa5, 0x1a, 0xb3, 0xfe, 0x39, 0x43, 0xe9, 0x34, 0x1b, 0x5c, 0x15, 0xae, 0xe1, 0x25, 0xd4, + 0x1f, 0xd5, 0x74, 0x13, 0xde, 0xf2, 0x79, 0x03, 0xd0, 0xf3, 0x3d, 0xbb, 0xdf, 0x27, 0x5e, 0x8f, + 0x3c, 0x25, 0x8e, 0xdf, 0x8b, 0x33, 0x7d, 0x78, 0x8f, 0x33, 0x73, 0xfa, 0xaa, 0x38, 0x63, 0x3f, + 0x38, 0x62, 0x8b, 0x9f, 0x90, 0x9c, 0x12, 0xf2, 0xcd, 0x5b, 0xca, 0x37, 0xaf, 0x01, 0xeb, 0x69, + 0x81, 0xf8, 0x4c, 0x9d, 0x91, 0xa7, 0x3c, 0x49, 0x9e, 0x95, 0xa4, 0x3c, 0xbc, 0x1b, 0x4b, 0x89, + 0x93, 0xb1, 0xd2, 0xec, 0x98, 0x95, 0x7e, 0xa5, 0xc2, 0xf5, 0x5c, 0x2b, 0x4d, 0x67, 0x2a, 0xb8, + 0x07, 0x25, 0xde, 0x69, 0x85, 0x15, 0xf7, 0x46, 0x8a, 0x2e, 0xe2, 0x16, 0xf7, 0x65, 0x01, 0x76, + 0x98, 0xb1, 0x0a, 0xaf, 0x32, 0xea, 0xbf, 0x5a, 0x0e, 0x7c, 0x07, 0x90, 0xd0, 0x6e, 0x1a, 0xb3, + 0x24, 0x30, 0x35, 0xbe, 0x93, 0x9c, 0x2c, 0xf4, 0x3f, 0xaa, 0xb0, 0x16, 0x6b, 0x65, 0x8f, 0x32, + 0x7f, 0xda, 0xfe, 0xfb, 0x4a, 0xce, 0xa8, 0x9e, 0xd3, 0x19, 0x6f, 0xc3, 0x6c, 0xd0, 0x76, 0xf2, + 0x58, 0xe0, 0xfa, 0xbf, 0x32, 0xd6, 0x8f, 0x0d, 0x8d, 0xb6, 0xf3, 0x98, 0xe2, 0x10, 0x0f, 0xbd, + 0x07, 0xf3, 0x42, 0x4f, 0x21, 0x5d, 0xf1, 0x6c, 0xba, 0x39, 0x8e, 0x1c, 0xc0, 0x4c, 0xff, 0x8f, + 0x02, 0x37, 0x4e, 0xd5, 0xda, 0x74, 0xfc, 0xe9, 0x8d, 0xa8, 0xed, 0x75, 0xbc, 0x4f, 0x7f, 0x06, + 0x10, 0xeb, 0x23, 0xf5, 0x1a, 0xa1, 0x64, 0x5e, 0x23, 0xd6, 0x42, 0xcc, 0x8e, 0x31, 0x0c, 0xab, + 0x7f, 0x62, 0x05, 0xdd, 0x84, 0xb2, 0x08, 0x84, 0xd0, 0x58, 0x39, 0xb3, 0x8b, 0xd0, 0xb9, 0xc4, + 0xd2, 0x9b, 0xf2, 0xd9, 0x52, 0x30, 0x3e, 0xfd, 0xd9, 0x72, 0x45, 0xa2, 0x25, 0xb8, 0xc6, 0x0b, + 0xfa, 0x9f, 0x55, 0x40, 0xe3, 0x71, 0xc8, 0x8b, 0xc7, 0x29, 0xc6, 0x49, 0x29, 0x52, 0x95, 0xcf, + 0xa2, 0xe1, 0x95, 0xd5, 0xcc, 0x95, 0xc3, 0x61, 0xac, 0xf0, 0x0a, 0xc3, 0xd8, 0x87, 0xa0, 0x99, + 0x61, 0x8f, 0xd3, 0x0b, 0xea, 0xb5, 0x88, 0xe4, 0x09, 0x8d, 0xd0, 0xa2, 0x99, 0x84, 0x47, 0x6c, + 0x3c, 0x1d, 0x94, 0x72, 0xd2, 0xc1, 0x5d, 0x98, 0x3b, 0x18, 0x50, 0xf3, 0x48, 0xb6, 0x62, 0x41, + 0x96, 0x45, 0x69, 0x2f, 0x17, 0xc7, 0x83, 0x40, 0x0b, 0xfa, 0xb3, 0xb0, 0x75, 0x9f, 0x4d, 0xb4, + 0xee, 0x4f, 0x60, 0x29, 0x76, 0xf9, 0xe6, 0x80, 0x32, 0x32, 0xa5, 0x04, 0x91, 0xa8, 0xbc, 0x6a, + 0xaa, 0xf2, 0xea, 0x1e, 0x5c, 0x19, 0x63, 0x39, 0x9d, 0xe8, 0xe2, 0xf3, 0xf0, 0xc8, 0x34, 0xf9, + 0x8c, 0x20, 0x79, 0x4a, 0x50, 0xff, 0xb9, 0x02, 0x5a, 0xfc, 0x7c, 0x15, 0x38, 0xe0, 0x14, 0x5e, + 0xff, 0xae, 0x41, 0x45, 0xba, 0x69, 0x50, 0x21, 0x0a, 0x38, 0x82, 0xcf, 0x7a, 0xd8, 0xd3, 0xbf, + 0x0f, 0x25, 0x81, 0x37, 0xe1, 0xb5, 0xfe, 0x34, 0xb7, 0x5c, 0x81, 0x6a, 0xd7, 0x1d, 0xd8, 0x22, + 0x0b, 0xc8, 0xbe, 0x26, 0x5e, 0xd0, 0x1d, 0x58, 0x08, 0x31, 0x03, 0x5d, 0x9d, 0xc1, 0x65, 0x1d, + 0xe6, 0x1e, 0x0e, 0xac, 0x0c, 0xa3, 0xe4, 0x12, 0xc7, 0xe8, 0x90, 0xe3, 0xcc, 0x4d, 0x92, 0x4b, + 0xfa, 0xef, 0x0a, 0x50, 0x0a, 0x1c, 0x6c, 0x05, 0xaa, 0x6d, 0xb6, 0xc9, 0x1d, 0x4e, 0x4e, 0x32, + 0x15, 0x1c, 0x2f, 0x70, 0x29, 0xc4, 0x67, 0xfc, 0x5a, 0x21, 0x41, 0x74, 0x1f, 0xe6, 0x82, 0xcf, + 0x30, 0x7d, 0x8c, 0x8f, 0xee, 0x59, 0xe3, 0xe1, 0x24, 0x05, 0xda, 0x86, 0x0b, 0x1d, 0x42, 0xac, + 0x96, 0x47, 0x5d, 0x37, 0xc4, 0x90, 0xad, 0xd0, 0x84, 0x63, 0xc6, 0xe9, 0xd0, 0xfb, 0xb0, 0xc8, + 0x17, 0x1b, 0x96, 0x15, 0x1d, 0x15, 0x8c, 0x1e, 0x68, 0x3c, 0xfe, 0x71, 0x16, 0x15, 0x35, 0x61, + 0xe1, 0x63, 0xd7, 0x32, 0x7c, 0x22, 0x55, 0xc8, 0x96, 0xcb, 0x82, 0xf8, 0x7a, 0x5e, 0x09, 0x92, + 0x06, 0xc2, 0x19, 0x92, 0xec, 0x9b, 0xf6, 0xec, 0xd8, 0x9b, 0x36, 0xfa, 0x8a, 0x98, 0xb5, 0xfa, + 0x64, 0xb9, 0x22, 0x7c, 0x36, 0x5d, 0xe0, 0x36, 0x65, 0xcc, 0xf7, 0x83, 0x39, 0xab, 0x4f, 0xf4, + 0x1f, 0xc3, 0xa5, 0x28, 0x5f, 0x85, 0xbb, 0x3c, 0xd9, 0xbc, 0x46, 0x9e, 0xdc, 0x08, 0xa7, 0x3b, + 0xf5, 0xd4, 0x64, 0x23, 0x87, 0xba, 0xbc, 0xb7, 0xcf, 0x7f, 0x2b, 0xb0, 0x98, 0xf9, 0x11, 0xe5, + 0x75, 0x98, 0xe7, 0x25, 0x57, 0x75, 0x1a, 0xc9, 0x35, 0x6f, 0xc4, 0xb9, 0x0d, 0x97, 0x83, 0xb2, + 0xcc, 0xec, 0xe7, 0xa4, 0xe7, 0x12, 0xaf, 0xc7, 0x88, 0x49, 0x9d, 0xa0, 0xb9, 0x56, 0x31, 0x12, + 0x9b, 0x5d, 0xfb, 0x39, 0xd9, 0x23, 0x5e, 0x57, 0xec, 0xe4, 0x3d, 0x8a, 0xe9, 0x7f, 0x50, 0x00, + 0x25, 0x74, 0x3d, 0xa5, 0xbc, 0xfa, 0x11, 0xd4, 0x0e, 0xe2, 0x43, 0xa3, 0x27, 0xea, 0xb7, 0xf2, + 0x6b, 0x53, 0x92, 0x7f, 0x9a, 0x2e, 0xd7, 0x4a, 0x16, 0xcc, 0x27, 0x3b, 0x04, 0x8e, 0xe3, 0xdb, + 0xc3, 0x20, 0x31, 0x56, 0xb1, 0xf8, 0xe6, 0x6b, 0x0e, 0xb5, 0xc2, 0x52, 0x2c, 0xbe, 0xf9, 0x9a, + 0x19, 0x9e, 0x55, 0xc5, 0xe2, 0x9b, 0x87, 0xfb, 0x30, 0x78, 0x47, 0x15, 0x7a, 0xab, 0xe2, 0x10, + 0xd4, 0xdf, 0x85, 0xf9, 0xec, 0x93, 0xd2, 0xa1, 0xdd, 0x3f, 0x94, 0xbf, 0xec, 0x88, 0x6f, 0xa4, + 0x41, 0x61, 0x40, 0x8f, 0x65, 0xa2, 0xe0, 0x9f, 0x5c, 0xb6, 0xa4, 0x5a, 0x5e, 0x8d, 0x4a, 0x48, + 0xcb, 0x1b, 0x07, 0x29, 0x19, 0xff, 0xe6, 0xa9, 0x35, 0x9c, 0x20, 0xa4, 0x68, 0x11, 0xac, 0xff, + 0x00, 0x6e, 0xec, 0xd0, 0x7e, 0x62, 0x7a, 0x8f, 0x1f, 0x87, 0xa7, 0x63, 0x40, 0xfd, 0x27, 0x0a, + 0xac, 0x9f, 0xce, 0x62, 0x3a, 0x85, 0x70, 0xd2, 0xcb, 0xf3, 0x80, 0xeb, 0x92, 0x98, 0x47, 0x6c, + 0x34, 0xdc, 0x25, 0xbe, 0x81, 0xbe, 0x1a, 0xc6, 0x76, 0x5e, 0x05, 0x0c, 0x31, 0x53, 0x31, 0x5e, + 0x07, 0xcd, 0x4c, 0xae, 0x77, 0xc9, 0x13, 0xc9, 0x67, 0x6c, 0x5d, 0xff, 0xa5, 0x02, 0x97, 0x13, + 0x3f, 0x9a, 0x10, 0x3f, 0x3c, 0x11, 0x5d, 0x82, 0x92, 0x49, 0x47, 0x8e, 0x2f, 0x8d, 0x18, 0x00, + 0xdc, 0x73, 0x9e, 0x51, 0xef, 0x01, 0x37, 0xae, 0x2c, 0x14, 0x12, 0xe4, 0x53, 0xf3, 0x33, 0xea, + 0xed, 0xd0, 0x63, 0x19, 0xb7, 0x12, 0x0a, 0x0a, 0xff, 0x50, 0x50, 0x14, 0xe5, 0xd0, 0x1c, 0x80, + 0x9c, 0x82, 0x8d, 0x86, 0x9c, 0x22, 0x68, 0xa3, 0x24, 0xa4, 0xff, 0x56, 0x81, 0xf5, 0x5c, 0x99, + 0x1a, 0xe6, 0xd1, 0xb4, 0xac, 0x70, 0x09, 0x4a, 0xc9, 0xc7, 0xb9, 0x00, 0xc8, 0x8b, 0xbb, 0xf0, + 0xb7, 0xd9, 0x62, 0xf4, 0xdb, 0xac, 0xfe, 0x4f, 0x05, 0xf4, 0x5c, 0xf9, 0x82, 0x4a, 0x31, 0xa5, + 0x64, 0x72, 0x0e, 0x09, 0xd1, 0x07, 0x50, 0x09, 0x2d, 0x2d, 0x74, 0x9b, 0xfd, 0xb5, 0x30, 0x57, + 0x7a, 0x1c, 0xd1, 0xd4, 0x57, 0xa1, 0x2c, 0x7f, 0x4f, 0xae, 0x42, 0xe9, 0x91, 0x67, 0xfb, 0x44, + 0x9b, 0x41, 0x15, 0x28, 0xee, 0x19, 0x8c, 0x69, 0x4a, 0x7d, 0x23, 0xe8, 0x62, 0xe2, 0xf7, 0x75, + 0x04, 0x50, 0x6e, 0x7a, 0xc4, 0x10, 0x78, 0x00, 0xe5, 0xe0, 0xdd, 0x48, 0x53, 0xea, 0xf7, 0x60, + 0x3e, 0xf9, 0x42, 0xca, 0x8f, 0x6b, 0xec, 0xb4, 0xbf, 0xb3, 0xa5, 0xcd, 0xa0, 0x79, 0xa8, 0xb4, + 0x70, 0xa3, 0xdd, 0x69, 0x77, 0x3e, 0xd2, 0x14, 0x0e, 0x75, 0xf7, 0x1f, 0xee, 0xed, 0x71, 0x48, + 0xad, 0xbf, 0x07, 0x10, 0xd7, 0x49, 0xce, 0xb8, 0xf3, 0xb0, 0xc3, 0x69, 0xe6, 0x60, 0xf6, 0x51, + 0xa3, 0xbd, 0x1f, 0x90, 0x70, 0x00, 0x07, 0x80, 0xca, 0x71, 0x5a, 0x1c, 0xa7, 0x50, 0x7f, 0x27, + 0xd3, 0x39, 0xa2, 0x59, 0x28, 0x34, 0x06, 0x03, 0x6d, 0x06, 0x95, 0x41, 0x6d, 0x6d, 0x6a, 0x0a, + 0x17, 0xb0, 0x43, 0xbd, 0xa1, 0x31, 0xd0, 0xd4, 0xfa, 0x73, 0x58, 0x48, 0xd7, 0x25, 0x71, 0x2c, + 0xf5, 0x8e, 0x6c, 0xa7, 0x1f, 0x30, 0xec, 0xfa, 0xa2, 0x01, 0x09, 0x18, 0x06, 0x17, 0xb3, 0x34, + 0x15, 0x69, 0x30, 0xdf, 0x76, 0x6c, 0xdf, 0x36, 0x06, 0xf6, 0x73, 0x8e, 0x5b, 0x40, 0x35, 0xa8, + 0xee, 0x79, 0xc4, 0x35, 0x3c, 0x0e, 0x16, 0xd1, 0x02, 0x80, 0xf8, 0xc5, 0x01, 0x13, 0xc3, 0x3a, + 0xd1, 0x4a, 0x9c, 0xe0, 0x91, 0x61, 0xfb, 0xb6, 0xd3, 0x17, 0xcb, 0x5a, 0xb9, 0xfe, 0x4d, 0xa8, + 0xa5, 0xe2, 0x17, 0x5d, 0x80, 0xda, 0xc7, 0x9d, 0x76, 0xa7, 0xbd, 0xdf, 0x6e, 0xec, 0xb4, 0x3f, + 0xd9, 0x6a, 0x05, 0x5a, 0xda, 0x6d, 0x77, 0x77, 0x1b, 0xfb, 0xcd, 0x07, 0x9a, 0xc2, 0xd5, 0x17, + 0x7c, 0xaa, 0x9b, 0x1f, 0xfc, 0xed, 0xc5, 0x9a, 0xf2, 0xd9, 0x8b, 0x35, 0xe5, 0x5f, 0x2f, 0xd6, + 0x94, 0x5f, 0xbc, 0x5c, 0x9b, 0xf9, 0xec, 0xe5, 0xda, 0xcc, 0x3f, 0x5e, 0xae, 0xcd, 0x7c, 0xf2, + 0xc5, 0xbe, 0xed, 0x1f, 0x8e, 0x0e, 0x6e, 0x9a, 0x74, 0x78, 0xcb, 0xb5, 0x9d, 0xbe, 0x69, 0xb8, + 0xb7, 0x7c, 0xdb, 0xb4, 0xcc, 0x5b, 0x09, 0x17, 0x38, 0x28, 0x8b, 0x3f, 0xab, 0xdc, 0xfd, 0x7f, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xcf, 0x03, 0xb6, 0xbb, 0xcb, 0x22, 0x00, 0x00, } func (m *TableSpan) Marshal() (dAtA []byte, err error) { @@ -4228,6 +4423,105 @@ func (m *MaintainerHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *NodeHeartbeat) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeHeartbeat) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NodeHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NodeEpoch != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch)) + i-- + dAtA[i] = 0x10 + } + if m.Liveness != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.Liveness)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *SetNodeLivenessRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SetNodeLivenessRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SetNodeLivenessRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NodeEpoch != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch)) + i-- + dAtA[i] = 0x10 + } + if m.Target != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.Target)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *SetNodeLivenessResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SetNodeLivenessResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SetNodeLivenessResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NodeEpoch != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch)) + i-- + dAtA[i] = 0x10 + } + if m.Applied != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.Applied)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *MaintainerStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6186,19 +6480,64 @@ func (m *MaintainerHeartbeat) Size() (n int) { return n } -func (m *MaintainerStatus) Size() (n int) { +func (m *NodeHeartbeat) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.ChangefeedID != nil { - l = m.ChangefeedID.Size() - n += 1 + l + sovHeartbeat(uint64(l)) + if m.Liveness != 0 { + n += 1 + sovHeartbeat(uint64(m.Liveness)) } - l = len(m.FeedState) - if l > 0 { - n += 1 + l + sovHeartbeat(uint64(l)) + if m.NodeEpoch != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeEpoch)) + } + return n +} + +func (m *SetNodeLivenessRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Target != 0 { + n += 1 + sovHeartbeat(uint64(m.Target)) + } + if m.NodeEpoch != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeEpoch)) + } + return n +} + +func (m *SetNodeLivenessResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Applied != 0 { + n += 1 + sovHeartbeat(uint64(m.Applied)) + } + if m.NodeEpoch != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeEpoch)) + } + return n +} + +func (m *MaintainerStatus) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ChangefeedID != nil { + l = m.ChangefeedID.Size() + n += 1 + l + sovHeartbeat(uint64(l)) + } + l = len(m.FeedState) + if l > 0 { + n += 1 + l + sovHeartbeat(uint64(l)) } if m.State != 0 { n += 1 + sovHeartbeat(uint64(m.State)) @@ -9083,6 +9422,270 @@ func (m *MaintainerHeartbeat) Unmarshal(dAtA []byte) error { } return nil } +func (m *NodeHeartbeat) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeHeartbeat: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeHeartbeat: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Liveness", wireType) + } + m.Liveness = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Liveness |= NodeLiveness(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType) + } + m.NodeEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeEpoch |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHeartbeat(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHeartbeat + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SetNodeLivenessRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SetNodeLivenessRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SetNodeLivenessRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Target", wireType) + } + m.Target = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Target |= NodeLiveness(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType) + } + m.NodeEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeEpoch |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHeartbeat(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHeartbeat + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SetNodeLivenessResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SetNodeLivenessResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SetNodeLivenessResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Applied", wireType) + } + m.Applied = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Applied |= NodeLiveness(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType) + } + m.NodeEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeEpoch |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHeartbeat(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHeartbeat + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *MaintainerStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/heartbeatpb/heartbeat.proto b/heartbeatpb/heartbeat.proto index 0e3fba2cd8..d403d3f18f 100644 --- a/heartbeatpb/heartbeat.proto +++ b/heartbeatpb/heartbeat.proto @@ -126,6 +126,27 @@ message MaintainerHeartbeat { repeated MaintainerStatus statuses = 1; } +enum NodeLiveness { + ALIVE = 0; + DRAINING = 1; + STOPPING = 2; +} + +message NodeHeartbeat { + NodeLiveness liveness = 1; + uint64 node_epoch = 2; +} + +message SetNodeLivenessRequest { + NodeLiveness target = 1; + uint64 node_epoch = 2; +} + +message SetNodeLivenessResponse { + NodeLiveness applied = 1; + uint64 node_epoch = 2; +} + message MaintainerStatus { ChangefeedID changefeedID = 1; string feed_state = 2; @@ -376,4 +397,4 @@ message DispatcherSetChecksumUpdateRequest { int64 mode = 3; uint64 seq = 4; DispatcherSetChecksum checksum = 5; -} \ No newline at end of file +} diff --git a/maintainer/maintainer_manager.go b/maintainer/maintainer_manager.go index 469dc31fc1..8eb7241264 100644 --- a/maintainer/maintainer_manager.go +++ b/maintainer/maintainer_manager.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/api" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/config" @@ -46,6 +47,13 @@ type Manager struct { coordinatorVersion int64 nodeInfo *node.Info + // serverLiveness is shared with the server so election and local maintainer admission + // observe a single monotonic liveness state. + serverLiveness *api.Liveness + nodeEpoch uint64 + + nodeHeartbeatInterval time.Duration + lastNodeHeartbeatTime time.Time // msgCh is used to cache messages from coordinator msgCh chan *messaging.TargetMessage @@ -58,15 +66,33 @@ type Manager struct { func NewMaintainerManager( nodeInfo *node.Info, conf *config.SchedulerConfig, +) *Manager { + return NewMaintainerManagerWithLiveness(nodeInfo, conf, nil) +} + +// NewMaintainerManagerWithLiveness creates a changefeed maintainer manager and binds +// node-level liveness to the provided server liveness pointer. +// +// If serverLiveness is nil, a local liveness instance is created for tests. +func NewMaintainerManagerWithLiveness( + nodeInfo *node.Info, + conf *config.SchedulerConfig, + serverLiveness *api.Liveness, ) *Manager { mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) + if serverLiveness == nil { + serverLiveness = new(api.Liveness) + } m := &Manager{ - mc: mc, - conf: conf, - maintainers: sync.Map{}, - nodeInfo: nodeInfo, - msgCh: make(chan *messaging.TargetMessage, 1024), - taskScheduler: threadpool.NewThreadPoolDefault(), + mc: mc, + conf: conf, + maintainers: sync.Map{}, + nodeInfo: nodeInfo, + serverLiveness: serverLiveness, + nodeEpoch: uint64(time.Now().UnixNano()), + nodeHeartbeatInterval: 5 * time.Second, + msgCh: make(chan *messaging.TargetMessage, 1024), + taskScheduler: threadpool.NewThreadPoolDefault(), } mc.RegisterHandler(messaging.MaintainerManagerTopic, m.recvMessages) @@ -84,7 +110,8 @@ func (m *Manager) recvMessages(ctx context.Context, msg *messaging.TargetMessage // Coordinator related messages case messaging.TypeAddMaintainerRequest, messaging.TypeRemoveMaintainerRequest, - messaging.TypeCoordinatorBootstrapRequest: + messaging.TypeCoordinatorBootstrapRequest, + messaging.TypeSetNodeLivenessRequest: select { case <-ctx.Done(): return ctx.Err() @@ -160,7 +187,7 @@ func (m *Manager) newCoordinatorTopicMessage(msg messaging.IOTypeT) *messaging.T ) } -func (m *Manager) sendMessages(msg *heartbeatpb.MaintainerHeartbeat) { +func (m *Manager) sendMessages(msg messaging.IOTypeT) { target := m.newCoordinatorTopicMessage(msg) err := m.mc.SendCommand(target) if err != nil { @@ -210,6 +237,7 @@ func (m *Manager) onCoordinatorBootstrapRequest(msg *messaging.TargetMessage) { zap.Int64("coordinatorVersion", m.coordinatorVersion), zap.Error(err)) } + m.sendNodeHeartbeat(true) log.Info("new coordinator online, bootstrap response already sent", zap.Stringer("coordinatorID", m.coordinatorID), @@ -217,6 +245,12 @@ func (m *Manager) onCoordinatorBootstrapRequest(msg *messaging.TargetMessage) { } func (m *Manager) onAddMaintainerRequest(req *heartbeatpb.AddMaintainerRequest) *heartbeatpb.MaintainerStatus { + if m.serverLiveness.Load() == api.LivenessCaptureStopping { + log.Warn("reject add maintainer request because node is stopping", + zap.Stringer("changefeedID", common.NewChangefeedIDFromPB(req.Id))) + return nil + } + changefeedID := common.NewChangefeedIDFromPB(req.Id) _, ok := m.maintainers.Load(changefeedID) if ok { @@ -272,6 +306,54 @@ func (m *Manager) onRemoveMaintainerRequest(msg *messaging.TargetMessage) *heart return nil } +func (m *Manager) onSetNodeLivenessRequest(msg *messaging.TargetMessage) { + if m.coordinatorID != msg.From { + log.Warn("ignore set node liveness request from unexpected sender", + zap.Stringer("coordinatorID", m.coordinatorID), + zap.Stringer("from", msg.From)) + return + } + + req := msg.Message[0].(*heartbeatpb.SetNodeLivenessRequest) + applied := m.serverLiveness.Load() + transitioned := false + if req.NodeEpoch != m.nodeEpoch { + log.Warn("reject set node liveness request due to stale node epoch", + zap.Uint64("requestNodeEpoch", req.NodeEpoch), + zap.Uint64("nodeEpoch", m.nodeEpoch)) + } else { + target, ok := heartbeatNodeLivenessToServer(req.Target) + if !ok { + log.Warn("ignore set node liveness request with invalid target", + zap.Int32("target", int32(req.Target))) + } else { + before := m.serverLiveness.Load() + if m.serverLiveness.Store(target) { + after := m.serverLiveness.Load() + transitioned = before != after + if transitioned { + beforeStr := (&before).String() + afterStr := (&after).String() + log.Info("node liveness updated", + zap.String("from", beforeStr), + zap.String("to", afterStr), + zap.Uint64("nodeEpoch", m.nodeEpoch)) + } + } + applied = m.serverLiveness.Load() + } + } + + response := &heartbeatpb.SetNodeLivenessResponse{ + Applied: serverLivenessToHeartbeat(applied), + NodeEpoch: m.nodeEpoch, + } + m.sendMessages(response) + if transitioned { + m.sendNodeHeartbeat(true) + } +} + func (m *Manager) onDispatchMaintainerRequest( msg *messaging.TargetMessage, ) *heartbeatpb.MaintainerStatus { @@ -295,6 +377,8 @@ func (m *Manager) onDispatchMaintainerRequest( } func (m *Manager) sendHeartbeat() { + m.sendNodeHeartbeat(false) + if m.isBootstrap() { response := &heartbeatpb.MaintainerHeartbeat{} m.maintainers.Range(func(key, value interface{}) bool { @@ -330,7 +414,54 @@ func (m *Manager) handleMessage(msg *messaging.TargetMessage) { } m.sendMessages(response) } + case messaging.TypeSetNodeLivenessRequest: + if m.isBootstrap() { + m.onSetNodeLivenessRequest(msg) + } + default: + } +} + +func (m *Manager) sendNodeHeartbeat(force bool) { + if !m.isBootstrap() { + return + } + + now := time.Now() + if !force && now.Sub(m.lastNodeHeartbeatTime) < m.nodeHeartbeatInterval { + return + } + + m.sendMessages(&heartbeatpb.NodeHeartbeat{ + Liveness: serverLivenessToHeartbeat(m.serverLiveness.Load()), + NodeEpoch: m.nodeEpoch, + }) + m.lastNodeHeartbeatTime = now +} + +func serverLivenessToHeartbeat(liveness api.Liveness) heartbeatpb.NodeLiveness { + switch liveness { + case api.LivenessCaptureAlive: + return heartbeatpb.NodeLiveness_ALIVE + case api.LivenessCaptureDraining: + return heartbeatpb.NodeLiveness_DRAINING + case api.LivenessCaptureStopping: + return heartbeatpb.NodeLiveness_STOPPING + default: + return heartbeatpb.NodeLiveness_ALIVE + } +} + +func heartbeatNodeLivenessToServer(liveness heartbeatpb.NodeLiveness) (api.Liveness, bool) { + switch liveness { + case heartbeatpb.NodeLiveness_ALIVE: + return api.LivenessCaptureAlive, true + case heartbeatpb.NodeLiveness_DRAINING: + return api.LivenessCaptureDraining, true + case heartbeatpb.NodeLiveness_STOPPING: + return api.LivenessCaptureStopping, true default: + return api.LivenessCaptureAlive, false } } diff --git a/maintainer/maintainer_manager_liveness_test.go b/maintainer/maintainer_manager_liveness_test.go new file mode 100644 index 0000000000..2dd9796383 --- /dev/null +++ b/maintainer/maintainer_manager_liveness_test.go @@ -0,0 +1,68 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package maintainer + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/api" + appcontext "github.com/pingcap/ticdc/pkg/common/context" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/messaging" + "github.com/pingcap/ticdc/pkg/node" + "github.com/stretchr/testify/require" +) + +func TestSetNodeLivenessRequestMonotonic(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + selfNode := node.NewInfo("127.0.0.1:0", "") + coordinator := node.NewInfo("127.0.0.1:1", "") + mc := messaging.NewMessageCenter(ctx, selfNode.ID, config.NewDefaultMessageCenterConfig(selfNode.AdvertiseAddr), nil) + mc.Run(ctx) + defer mc.Close() + appcontext.SetService(appcontext.MessageCenter, mc) + + var liveness api.Liveness + manager := NewMaintainerManagerWithLiveness(selfNode, config.GetDefaultServerConfig().Debug.Scheduler, &liveness) + manager.coordinatorID = coordinator.ID + manager.coordinatorVersion = 1 + + msg := messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.SetNodeLivenessRequest{ + Target: heartbeatpb.NodeLiveness_DRAINING, + NodeEpoch: manager.nodeEpoch, + }) + msg.From = coordinator.ID + manager.onSetNodeLivenessRequest(msg) + require.Equal(t, api.LivenessCaptureDraining, liveness.Load()) + + msg = messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.SetNodeLivenessRequest{ + Target: heartbeatpb.NodeLiveness_ALIVE, + NodeEpoch: manager.nodeEpoch, + }) + msg.From = coordinator.ID + manager.onSetNodeLivenessRequest(msg) + require.Equal(t, api.LivenessCaptureDraining, liveness.Load()) + + msg = messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.SetNodeLivenessRequest{ + Target: heartbeatpb.NodeLiveness_STOPPING, + NodeEpoch: manager.nodeEpoch, + }) + msg.From = coordinator.ID + manager.onSetNodeLivenessRequest(msg) + require.Equal(t, api.LivenessCaptureStopping, liveness.Load()) +} diff --git a/pkg/api/util.go b/pkg/api/util.go index a10e4152e4..0d5b1228fd 100644 --- a/pkg/api/util.go +++ b/pkg/api/util.go @@ -145,20 +145,36 @@ func WriteData(w http.ResponseWriter, data interface{}) { } } -// Liveness can only be changed from alive to stopping, and no way back. +// Liveness can only move forward in a monotonic order, and no way back. type Liveness int32 const ( // LivenessCaptureAlive means the capture is alive, and ready to serve. LivenessCaptureAlive Liveness = 0 + // LivenessCaptureDraining means the capture is draining and should not receive new workload. + LivenessCaptureDraining Liveness = 1 // LivenessCaptureStopping means the capture is in the process of graceful shutdown. - LivenessCaptureStopping Liveness = 1 + LivenessCaptureStopping Liveness = 2 ) // Store the given liveness. Returns true if it success. func (l *Liveness) Store(v Liveness) bool { - return atomic.CompareAndSwapInt32( - (*int32)(l), int32(LivenessCaptureAlive), int32(v)) + if v < LivenessCaptureAlive || v > LivenessCaptureStopping { + return false + } + + for { + old := Liveness(atomic.LoadInt32((*int32)(l))) + if v < old { + return false + } + if v == old { + return true + } + if atomic.CompareAndSwapInt32((*int32)(l), int32(old), int32(v)) { + return true + } + } } // Load the liveness. @@ -170,6 +186,8 @@ func (l *Liveness) String() string { switch *l { case LivenessCaptureAlive: return "Alive" + case LivenessCaptureDraining: + return "Draining" case LivenessCaptureStopping: return "Stopping" default: diff --git a/pkg/api/util_liveness_test.go b/pkg/api/util_liveness_test.go new file mode 100644 index 0000000000..aa2b80f90b --- /dev/null +++ b/pkg/api/util_liveness_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLivenessMonotonicStore(t *testing.T) { + var l Liveness + require.Equal(t, LivenessCaptureAlive, l.Load()) + + require.True(t, l.Store(LivenessCaptureDraining)) + require.Equal(t, LivenessCaptureDraining, l.Load()) + + require.True(t, l.Store(LivenessCaptureStopping)) + require.Equal(t, LivenessCaptureStopping, l.Load()) + + require.False(t, l.Store(LivenessCaptureAlive)) + require.Equal(t, LivenessCaptureStopping, l.Load()) +} diff --git a/pkg/messaging/message.go b/pkg/messaging/message.go index 61f3559414..2618bae738 100644 --- a/pkg/messaging/message.go +++ b/pkg/messaging/message.go @@ -104,6 +104,9 @@ const ( TypeRedoResolvedTsForwardMessage IOType = 39 TypeDispatcherSetChecksumUpdateRequest IOType = 40 TypeDispatcherSetChecksumAckResponse IOType = 41 + TypeNodeHeartbeatRequest IOType = 42 + TypeSetNodeLivenessRequest IOType = 43 + TypeSetNodeLivenessResponse IOType = 44 ) func (t IOType) String() string { @@ -182,6 +185,12 @@ func (t IOType) String() string { return "DispatcherSetChecksumUpdateRequest" case TypeDispatcherSetChecksumAckResponse: return "DispatcherSetChecksumAckResponse" + case TypeNodeHeartbeatRequest: + return "NodeHeartbeatRequest" + case TypeSetNodeLivenessRequest: + return "SetNodeLivenessRequest" + case TypeSetNodeLivenessResponse: + return "SetNodeLivenessResponse" case TypeDispatcherHeartbeatResponse: return "DispatcherHeartbeatResponse" case TypeCongestionControl: @@ -379,6 +388,12 @@ func decodeIOType(ioType IOType, value []byte) (IOTypeT, error) { m = &heartbeatpb.DispatcherSetChecksumUpdateRequest{} case TypeDispatcherSetChecksumAckResponse: m = &heartbeatpb.DispatcherSetChecksumAckResponse{} + case TypeNodeHeartbeatRequest: + m = &heartbeatpb.NodeHeartbeat{} + case TypeSetNodeLivenessRequest: + m = &heartbeatpb.SetNodeLivenessRequest{} + case TypeSetNodeLivenessResponse: + m = &heartbeatpb.SetNodeLivenessResponse{} case TypeCongestionControl: m = &commonEvent.CongestionControl{} case TypeMergeDispatcherRequest: @@ -491,6 +506,12 @@ func NewSingleTargetMessage(To node.ID, Topic string, Message IOTypeT, Group ... ioType = TypeDispatcherSetChecksumUpdateRequest case *heartbeatpb.DispatcherSetChecksumAckResponse: ioType = TypeDispatcherSetChecksumAckResponse + case *heartbeatpb.NodeHeartbeat: + ioType = TypeNodeHeartbeatRequest + case *heartbeatpb.SetNodeLivenessRequest: + ioType = TypeSetNodeLivenessRequest + case *heartbeatpb.SetNodeLivenessResponse: + ioType = TypeSetNodeLivenessResponse case *commonEvent.CongestionControl: ioType = TypeCongestionControl case *heartbeatpb.MergeDispatcherRequest: diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 2d6baabb9a..8865332c33 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -23,6 +23,7 @@ const DefaultCheckInterval = time.Second * 120 const ( BasicScheduler = "basic-scheduler" + DrainScheduler = "drain-scheduler" BalanceScheduler = "balance-scheduler" BalanceSplitScheduler = "balance-split-scheduler" RedoBasicScheduler = "redo-basic-scheduler" diff --git a/pkg/server/coordinator.go b/pkg/server/coordinator.go index 299dfe2938..37dc621b0a 100644 --- a/pkg/server/coordinator.go +++ b/pkg/server/coordinator.go @@ -47,6 +47,8 @@ type Coordinator interface { // RequestResolvedTsFromLogCoordinator requests the log coordinator to report the resolved ts of the changefeed, // and coordinator will update the changefeed status after receiving the resolved ts from log coordinator. RequestResolvedTsFromLogCoordinator(ctx context.Context, changefeedDisplayName common.ChangeFeedDisplayName) + // DrainNode triggers drain for the target node and returns current remaining work. + DrainNode(ctx context.Context, targetNodeID string) (int, error) Initialized() bool } diff --git a/server/module_election.go b/server/module_election.go index 9924428aa8..25b1d833a8 100644 --- a/server/module_election.go +++ b/server/module_election.go @@ -15,6 +15,7 @@ package server import ( "context" + "sync" "time" "github.com/pingcap/failpoint" @@ -38,8 +39,10 @@ type elector struct { // election used for coordinator election *concurrency.Election // election used for log coordinator - logElection *concurrency.Election - svr *server + logElection *concurrency.Election + svr *server + coordinatorMu sync.Mutex + logMu sync.Mutex } func NewElector(server *server) common.SubModule { @@ -58,6 +61,7 @@ func (e *elector) Run(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return e.campaignCoordinator(ctx) }) g.Go(func() error { return e.campaignLogCoordinator(ctx) }) + g.Go(func() error { return e.watchLivenessAndResign(ctx) }) return g.Wait() } @@ -66,9 +70,14 @@ func (e *elector) Name() string { } func (e *elector) campaignCoordinator(ctx context.Context) error { + e.coordinatorMu.Lock() + defer e.coordinatorMu.Unlock() + // Limit the frequency of elections to avoid putting too much pressure on the etcd server rl := rate.NewLimiter(rate.Every(time.Second), 1 /* burst */) nodeID := string(e.svr.info.ID) + forbiddenLiveness := api.LivenessCaptureDraining + stopLiveness := api.LivenessCaptureStopping for { select { case <-ctx.Done(): @@ -83,8 +92,12 @@ func (e *elector) campaignCoordinator(ctx context.Context) error { return errors.Trace(err) } // Before campaign check liveness - if e.svr.liveness.Load() == api.LivenessCaptureStopping { - log.Info("do not campaign coordinator, liveness is stopping", zap.String("nodeID", nodeID)) + if e.svr.liveness.Load() >= forbiddenLiveness { + liveness := e.svr.liveness.Load() + livenessStr := (&liveness).String() + log.Info("do not campaign coordinator, liveness is not alive", + zap.String("nodeID", nodeID), + zap.String("liveness", livenessStr)) return nil } log.Info("start to campaign coordinator", zap.String("nodeID", nodeID)) @@ -110,9 +123,12 @@ func (e *elector) campaignCoordinator(ctx context.Context) error { } // After campaign check liveness again. // It is possible it becomes the coordinator right after receiving SIGTERM. - if e.svr.liveness.Load() == api.LivenessCaptureStopping { + if e.svr.liveness.Load() >= forbiddenLiveness { + liveness := e.svr.liveness.Load() + livenessStr := (&liveness).String() // If the server is stopping, resign actively. - log.Info("resign coordinator actively, liveness is stopping") + log.Info("resign coordinator actively, liveness is not alive", + zap.String("liveness", livenessStr)) if resignErr := e.resign(ctx); resignErr != nil { log.Warn("resign coordinator actively failed", zap.String("nodeID", nodeID), zap.Error(resignErr)) return errors.Trace(err) @@ -182,13 +198,26 @@ func (e *elector) campaignCoordinator(ctx context.Context) error { log.Info("coordinator exited normally", zap.String("nodeID", nodeID), zap.Int64("coordinatorVersion", coordinatorVersion), zap.Error(err)) + + if e.svr.liveness.Load() >= stopLiveness { + liveness := e.svr.liveness.Load() + livenessStr := (&liveness).String() + log.Info("stop coordinator campaign loop after server enters stopping", + zap.String("nodeID", nodeID), + zap.String("liveness", livenessStr)) + return nil + } } } func (e *elector) campaignLogCoordinator(ctx context.Context) error { + e.logMu.Lock() + defer e.logMu.Unlock() + // Limit the frequency of elections to avoid putting too much pressure on the etcd server rl := rate.NewLimiter(rate.Every(time.Second), 1 /* burst */) nodeID := string(e.svr.info.ID) + forbiddenLiveness := api.LivenessCaptureDraining for { select { case <-ctx.Done(): @@ -203,8 +232,12 @@ func (e *elector) campaignLogCoordinator(ctx context.Context) error { return errors.Trace(err) } // Before campaign check liveness - if e.svr.liveness.Load() == api.LivenessCaptureStopping { - log.Info("do not campaign log coordinator, liveness is stopping", zap.String("nodeID", nodeID)) + if e.svr.liveness.Load() >= forbiddenLiveness { + liveness := e.svr.liveness.Load() + livenessStr := (&liveness).String() + log.Info("do not campaign log coordinator, liveness is not alive", + zap.String("nodeID", nodeID), + zap.String("liveness", livenessStr)) return nil } // Campaign to be the log coordinator, it blocks until it been elected. @@ -224,10 +257,13 @@ func (e *elector) campaignLogCoordinator(ctx context.Context) error { } // After campaign check liveness again. // It is possible it becomes the coordinator right after receiving SIGTERM. - if e.svr.liveness.Load() == api.LivenessCaptureStopping { + if e.svr.liveness.Load() >= forbiddenLiveness { + liveness := e.svr.liveness.Load() + livenessStr := (&liveness).String() // If the server is stopping, resign actively. - log.Info("resign log coordinator actively, liveness is stopping") - if resignErr := e.resign(ctx); resignErr != nil { + log.Info("resign log coordinator actively, liveness is not alive", + zap.String("liveness", livenessStr)) + if resignErr := e.resignLogCoordinator(); resignErr != nil { log.Warn("resign log coordinator actively failed", zap.String("nodeID", nodeID), zap.Error(resignErr)) return errors.Trace(err) @@ -260,6 +296,38 @@ func (e *elector) Close(_ context.Context) error { return nil } +func (e *elector) watchLivenessAndResign(ctx context.Context) error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + nodeID := string(e.svr.info.ID) + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if e.svr.liveness.Load() < api.LivenessCaptureStopping { + continue + } + + if !e.svr.IsCoordinator() { + continue + } + + resignCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := e.resign(resignCtx); err != nil { + liveness := e.svr.liveness.Load() + livenessStr := (&liveness).String() + log.Warn("resign coordinator on stopping liveness failed", + zap.String("nodeID", nodeID), + zap.String("liveness", livenessStr), + zap.Error(err)) + } + cancel() + } + } +} + // resign lets the coordinator start a new election. func (e *elector) resign(ctx context.Context) error { if e.election == nil { diff --git a/server/server.go b/server/server.go index 1573fc3eb0..471b77fbf0 100644 --- a/server/server.go +++ b/server/server.go @@ -219,7 +219,7 @@ func (c *server) initialize(ctx context.Context) error { subscriptionClient, schemaStore, eventStore, - maintainer.NewMaintainerManager(c.info, conf.Debug.Scheduler), + maintainer.NewMaintainerManagerWithLiveness(c.info, conf.Debug.Scheduler, &c.liveness), eventService, } // register it into global var