From 0effc71e1ab0643c31086597022223d3b3e8077d Mon Sep 17 00:00:00 2001 From: dongzhenyangofficial-ctrl Date: Tue, 22 Sep 2026 16:55:29 +0800 Subject: [PATCH 1/2] feat(repository): add upload inventory and official revision management --- docs/node-discovery.md | 13 + docs/node-repositories.md | 33 + docs/upload-inventory-schema.sql | 34 + docs/upload-inventory.md | 33 + go.mod | 8 + go.sum | 19 + internal/authority/service.go | 378 ++++++++ internal/authority/service_test.go | 147 +++ internal/dao/cache_job_dao.go | 52 +- internal/dao/dingospeed_dao.go | 35 +- internal/dao/ingest_dao.go | 312 +++++++ internal/dao/ingest_validation_test.go | 18 + internal/dao/legacy_isolation_test.go | 199 ++++ internal/dao/model_file_process_dao.go | 27 +- internal/dao/model_file_record_dao.go | 114 +-- internal/dao/node_endpoint.go | 26 + internal/dao/node_health_dao.go | 70 ++ internal/dao/node_repository_dao.go | 70 ++ internal/dao/remote_repository_metadata.go | 43 + .../dao/remote_repository_metadata_test.go | 52 ++ internal/dao/repository_dao.go | 92 +- internal/dao/repository_tag_dao.go | 4 +- internal/dao/upload_inventory_test.go | 165 ++++ internal/data/data.go | 9 + internal/handler/authority_handler.go | 40 + internal/handler/cache_job_handler.go | 24 +- internal/handler/ingest_handler.go | 60 ++ internal/handler/node_health.html | 105 +++ internal/handler/node_health_handler.go | 102 ++ internal/model/dto/dto.go | 3 + internal/model/node_endpoint.go | 8 + internal/model/query/legacy_contract_test.go | 23 + internal/model/query/query.go | 3 + internal/model/upload_inventory.go | 47 + internal/router/http_router.go | 8 + internal/service/authority_service.go | 10 + internal/service/cache_job_service.go | 13 +- internal/service/ingest_service.go | 25 + internal/service/node_discovery_test.go | 83 ++ .../service/node_health_integration_test.go | 261 ++++++ internal/service/repository_service.go | 102 +- internal/service/scheduler_service.go | 86 +- model_import/model_import.go | 6 + pkg/nodehealth/health.go | 105 +++ pkg/nodehealth/health_test.go | 84 ++ pkg/proto/manager.proto | 44 +- pkg/proto/manager/manager.pb.go | 884 +++++++++++++----- pkg/proto/manager/manager_grpc.pb.go | 40 +- pkg/repository/key.go | 132 +++ pkg/repository/storage_test.go | 35 + process_import/process_import.go | 6 + repair/data_import.go | 6 + 52 files changed, 3847 insertions(+), 451 deletions(-) create mode 100644 docs/node-discovery.md create mode 100644 docs/node-repositories.md create mode 100644 docs/upload-inventory-schema.sql create mode 100644 docs/upload-inventory.md create mode 100644 internal/authority/service.go create mode 100644 internal/authority/service_test.go create mode 100644 internal/dao/ingest_dao.go create mode 100644 internal/dao/ingest_validation_test.go create mode 100644 internal/dao/legacy_isolation_test.go create mode 100644 internal/dao/node_endpoint.go create mode 100644 internal/dao/node_health_dao.go create mode 100644 internal/dao/node_repository_dao.go create mode 100644 internal/dao/remote_repository_metadata.go create mode 100644 internal/dao/remote_repository_metadata_test.go create mode 100644 internal/dao/upload_inventory_test.go create mode 100644 internal/handler/authority_handler.go create mode 100644 internal/handler/ingest_handler.go create mode 100644 internal/handler/node_health.html create mode 100644 internal/handler/node_health_handler.go create mode 100644 internal/model/node_endpoint.go create mode 100644 internal/model/query/legacy_contract_test.go create mode 100644 internal/model/upload_inventory.go create mode 100644 internal/service/authority_service.go create mode 100644 internal/service/ingest_service.go create mode 100644 internal/service/node_discovery_test.go create mode 100644 internal/service/node_health_integration_test.go create mode 100644 pkg/nodehealth/health.go create mode 100644 pkg/nodehealth/health_test.go create mode 100644 pkg/repository/key.go create mode 100644 pkg/repository/storage_test.go diff --git a/docs/node-discovery.md b/docs/node-discovery.md new file mode 100644 index 0000000..b00c95e --- /dev/null +++ b/docs/node-discovery.md @@ -0,0 +1,13 @@ +# 节点自动发现接口 + +2026-09-22：Register 请求新增 managementUrl（字段 5）与 downloadUrl（字段 6）。 +可选 HTTP(S) URL 随注册保存至附加表 node_endpoints，以 Scheduler 数字节点 ID 为主键。 +服务启动自动创建该表，不修改模型文件、远端库存或官方版本。旧客户端不发送字段仍可注册。 + +`GET /api/v1/nodes/health?endpoints=true&after=0&limit=200` 在原健康视图上附加 +managementUrl/downloadUrl，继续使用 nextAfter 游标。端点尚未上报的旧节点返回空字符串。 +communication 表示注册通信有效性；onlineMode 表示节点运行模式，不能当成在线状态。 +端点存储或健康查询失败返回 503，不返回假成功的空列表。 + +ModelFleet 使用唯一 Scheduler 的 HTTP 地址发现节点，用 gRPC 地址配置 Speed。 +只读发现无需管理令牌，不返回任何节点凭据。 diff --git a/docs/node-repositories.md b/docs/node-repositories.md new file mode 100644 index 0000000..ea5b0b3 --- /dev/null +++ b/docs/node-repositories.md @@ -0,0 +1,33 @@ +# 节点仓库观察 + +打开 `/node-health`，在节点卡片点击“查看仓库”。仓库区域展示选中实例的仓库总数、已记录容量以及仓库来源、namespace、名称、类型、commit、挂载状态和源仓库修改时间；挂载错误文本随记录展示。 + +当前页每 10 秒从数据库重新读取，刷新保留选中节点和页码。查询失败会清除旧表格并显示错误，可手动重试。每页 20 条，支持上一页/下一页。大表格在自身区域内滚动，小屏可横向滚动查看其余列。 + +## 数据口径 + +只读原有 `dingospeed` 和 `repository` 表，以节点 ID 查询 instance_id，再严格限定 `repository.instance_id`。不调用 Speed、不触发持久化或回源下载，不修改任何表结构或旧记录。 + +这里展示数据库中的仓库投影，不是磁盘扫描结果；未发布或未入库的缓存不在列表中，上传仓库沿用 main 投影。容量为 `used_storage` 的逻辑大小合计，不代表物理磁盘用量或去重后大小。节点断连时，仍能读取数据库记录，但这不证明当前磁盘内容完整。若在线/离线节点共用同一个 instance_id，现有表只能提供该实例的共同记录。 + +旧 HF `org=Qwen,repo=demo` 显示为 `namespace=huggingface,repo=Qwen/demo`；上传 `org=dingo-local/alice,repo=team/demo` 显示为 `namespace=alice,repo=team/demo`。无法按新身份规则解码的历史记录仍展示原始值，并标为“未识别身份”。 + +## 只读接口 + +`GET /api/v1/nodes/:id/repositories?after=0&limit=20` + +- id：已注册节点的正整数 ID。 +- after：非负 int64 仓库 ID 游标，默认 0。 +- limit：1..100，默认 20。按仓库 ID 升序，用额外一行判断下一页。 +- 返回：`nodeId`、`instanceId`、`items`、`total`、`usedStorage`、`nextAfter`、`observedAt`。 +- 每条 item：`id`、`namespace`、`repo`、`datatype`、`identityValid`、`commit`、`usedStorage`、`mountStatus`、`errorMessage`、`lastModified`、`updatedAt`。 +- 仓库 `id` 和 `nextAfter` 是十进制字符串，避免浏览器损失 int64 精度；`nextAfter="0"` 表示无下一页。 +- 参数错误 400，节点不存在 404，数据库不可用 503。成功结果禁用 HTTP 缓存，数据库查询超时为 5 秒。 + +总数与列表分别读取，持续入库或删除期间可能出现短暂差异，下一次轮询会更新。查询严格使用绑定参数，数据库文本通过 textContent 渲染。 + +## 本地验证 + +全量 Go 测试与编译通过。隔离 MySQL + 正式 Scheduler + Chromium 验证脚本为工作空间 `integration/verify_node_repositories.py`，覆盖身份解码、大整数游标、跨节点隔离、分页、参数校验、数据库变更自动同步、空数据、浏览器错误恢复、文本转义与移动端布局。该脚本仅对专用本地夹具插入和删除测试行，不可用于生产库。 + +本次没有部署到 10.220.70.213，也没有向该环境写入数据。 diff --git a/docs/upload-inventory-schema.sql b/docs/upload-inventory-schema.sql new file mode 100644 index 0000000..87ba0ca --- /dev/null +++ b/docs/upload-inventory-schema.sql @@ -0,0 +1,34 @@ +-- First-phase uploaded inventory only. These additive tables intentionally do +-- not alter repository/model_file_record/model_file_process (remote business). +CREATE TABLE IF NOT EXISTS upload_inventory_state ( + instance_id VARCHAR(191) PRIMARY KEY, + epoch VARCHAR(64) NOT NULL, + epoch_started_at DATETIME(3) NOT NULL, + last_sequence BIGINT UNSIGNED NOT NULL, + inventory_complete BOOLEAN NOT NULL, + last_attempt_at DATETIME(3) NOT NULL, + last_confirmed_at DATETIME(3) NULL, + error_message TEXT NULL +); +CREATE TABLE IF NOT EXISTS upload_inventory_file ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + identity_hash CHAR(64) NOT NULL UNIQUE, + namespace VARCHAR(255) NOT NULL, + repo_type VARCHAR(32) NOT NULL, + repo VARCHAR(1024) NOT NULL, + path VARCHAR(1000) NOT NULL, + sha256 CHAR(64) NOT NULL, + size BIGINT NOT NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + INDEX idx_upload_repo(namespace, repo_type), INDEX idx_upload_sha(sha256) +); +CREATE TABLE IF NOT EXISTS upload_inventory_holding ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + file_id BIGINT NOT NULL, + instance_id VARCHAR(191) NOT NULL, + sequence BIGINT UNSIGNED NOT NULL, + confirmed_at DATETIME(3) NOT NULL, + UNIQUE KEY uk_upload_holding(file_id, instance_id), + INDEX idx_upload_holding_instance(instance_id), INDEX idx_upload_holding_file(file_id) +); diff --git a/docs/upload-inventory.md b/docs/upload-inventory.md new file mode 100644 index 0000000..4663de7 --- /dev/null +++ b/docs/upload-inventory.md @@ -0,0 +1,33 @@ +# 上传库存（第一阶段） + +本功能只记录各 DingoSpeed 节点已经生效且可读取的本地上传文件，不参与远端仓库下载，也不提供权威 revision、集群发布事务或自动同步。 + +## 持久化边界 + +Scheduler 使用三个独立表： + +- `upload_inventory_file`：由 `namespace + repo_type + repo + path + sha256` 唯一确定的文件；`size` 用于描述和一致性校验。 +- `upload_inventory_holding`：文件与 Speed `instance_id` 的多对多持有关系。 +- `upload_inventory_state`：每个节点最后接收的 epoch、单调序号、完整性、确认时间和错误信息。 + +表结构见 `upload-inventory-schema.sql`。Scheduler 启动时也会以 GORM `AutoMigrate` 创建这些新增表。原有 `repository`、`model_file_record`、`model_file_process` 等远端业务表不会被上传库存的写入、删除、恢复或对账修改。 + +## 收敛机制 + +DingoSpeed 每 30 秒以及本地上传、发布、删除、回收或恢复发生变化后扫描一次有效本地 revision，并将完整快照原子写入 `/.upload-inventory/snapshot.json`。快照 epoch 和 sequence 持久化，因此服务重启后仍能继续单调报告；本地操作已经成功但进程在通知前退出时，下一次启动扫描也会补报。 + +Speed 向 Scheduler 的 `IngestRepository` RPC 只发送一次性读取令牌。Scheduler 从已注册节点的内部 HTTP 端点读取对应完整快照。读取或校验失败不会改变现有持有关系；扫描不完整时只记录失败状态，也不会把未出现的文件解释为删除。只有更新的完整快照才能替换该节点的持有集合。重复、旧序号或旧 epoch 快照会被忽略。 + +节点失去心跳只影响查询结果中的 `nodeAvailable`，不会删除最后确认的持有关系。只有完整快照确认缺失，或本地明确操作后产生的完整快照，才移除该节点持有关系；某文件不再有任何节点持有后才删除全局文件记录。 + +## 查询接口 + +- `GET /api/v1/upload-inventory/repositories` +- `GET /api/v1/upload-inventory/files`，可选 `namespace`、`repoType`、`repo`、`instanceId` +- `GET /api/v1/upload-inventory/nodes/:instanceId/files`（响应的 `state` 即使文件列表为空也会给出节点最后库存确认状态） + +文件结果同时包含仓库身份、相对路径、SHA256、大小、节点、最后确认时间、节点当前可用性，以及该节点最近库存是否完整确认。Speed 的 `/api/upload-inventory` 是 Scheduler 使用的令牌保护内部端点，不是面向用户的查询接口。 + +## 配置和启动 + +沿用现有 Scheduler 数据库配置和 Speed 注册/心跳配置，无需新增配置项。先启动 Scheduler，再按原方式启动 Speed；Scheduler 暂时不可用不影响 Speed 的本地上传、发布、删除、回收或恢复。连接恢复后，周期完整快照会自动对账。 diff --git a/go.mod b/go.mod index 8fcda79..664b6ce 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/andybalholm/brotli v1.1.1 github.com/avast/retry-go v3.0.0+incompatible github.com/bytedance/sonic v1.13.2 + github.com/glebarez/sqlite v1.11.0 github.com/go-playground/validator/v10 v10.26.0 github.com/go-sql-driver/mysql v1.9.3 github.com/gocolly/colly v1.2.0 @@ -43,7 +44,9 @@ require ( github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/gobwas/glob v0.2.3 // indirect @@ -62,6 +65,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -76,4 +80,8 @@ require ( golang.org/x/time v0.8.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect + modernc.org/libc v1.22.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.23.1 // indirect ) diff --git a/go.sum b/go.sum index 980041e..e40e52c 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,14 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -69,6 +75,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= @@ -127,6 +135,9 @@ github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -295,4 +306,12 @@ gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/internal/authority/service.go b/internal/authority/service.go new file mode 100644 index 0000000..83bdbae --- /dev/null +++ b/internal/authority/service.go @@ -0,0 +1,378 @@ +// Package authority owns explicitly confirmed official definitions, independently +// of inventory lifetime. It never invokes a Speed mutation. +package authority + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "dingoscheduler/internal/model" + "dingoscheduler/pkg/repository" + dbmysql "github.com/go-sql-driver/mysql" + "gorm.io/gorm" +) + +type File struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} +type Definition struct { + ID string `gorm:"primaryKey;size:64" json:"-"` + Repository string `gorm:"size:64;index;not null" json:"-"` + Revision string `gorm:"size:255;not null" json:"revision"` + Version uint64 `gorm:"not null" json:"version"` + Manifest string `gorm:"type:longtext;not null" json:"-"` + Actor string `gorm:"size:255" json:"actor"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (Definition) TableName() string { return "official_revision" } + +type Receipt struct { + ID string `gorm:"primaryKey;size:64"` + Digest string `gorm:"size:64;not null"` + Result string `gorm:"type:longtext;not null"` +} + +func (Receipt) TableName() string { return "official_revision_receipt" } + +type View struct { + Definition + Files []File `json:"files"` + Available map[string]bool `json:"available"` +} +type Source struct { + Node string `json:"node"` + Revision string `json:"revision"` + Commit string `json:"commit"` +} +type Change struct { + Path string `json:"path"` + File *File `json:"file"` + Source *Source `json:"source"` +} +type Request struct { + repository.Key + Action string `json:"action"` + Revision string `json:"revision"` + Node string `json:"node"` + BaseVersion uint64 `json:"baseVersion"` + RequestID string `json:"requestId"` + Actor string `json:"actor"` + Changes []Change `json:"changes"` +} +type Failure struct { + Status int + Message string +} + +func (e *Failure) Error() string { return e.Message } +func fail(status int, message string) error { return &Failure{status, message} } + +type Service struct { + DB *gorm.DB + Client *http.Client +} + +func New(db *gorm.DB) *Service { + return &Service{db, &http.Client{Timeout: 90 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}} +} +func hash(v any) string { + b, _ := json.Marshal(v) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} +func (s *Service) Do(ctx context.Context, r Request) (any, error) { + if err := r.Key.Validate(); err != nil { + return nil, fail(400, err.Error()) + } + if r.Namespace == "huggingface" || r.Namespace == "modelscope" { + return nil, fail(403, "remote repositories are read only") + } + switch r.Action { + case "list": + rows := []Definition{} + if err := s.DB.WithContext(ctx).Where("repository = ?", hash(r.Key)).Order("revision").Find(&rows).Error; err != nil { + return nil, err + } + return map[string]any{"items": rows}, nil + case "get": + return s.get(ctx, r) + case "nodes": + var nodes []model.Dingospeed + if err := s.DB.WithContext(ctx).Where("instance_id IN (?)", s.DB.Model(&model.UploadInventoryHolding{}).Select("instance_id").Where("file_id IN (?)", s.DB.Model(&model.UploadInventoryFile{}).Select("id").Where("namespace = ? AND repo_type = ? AND repo = ?", r.Namespace, r.RepoType, r.Repo))).Find(&nodes).Error; err != nil { + return nil, err + } + out := []map[string]any{} + for _, n := range nodes { + out = append(out, map[string]any{"node": n.InstanceID, "online": time.Since(n.UpdatedAt) < 5*time.Minute}) + } + return map[string]any{"items": out}, nil + case "revisions": + var out []struct { + Name string `json:"name"` + Commit string `json:"commit"` + } + if err := s.read(ctx, r.Key, r.Node, "revisions", "", &out); err != nil { + return nil, err + } + if out == nil { + return nil, fail(502, "incomplete revision listing") + } + return map[string]any{"items": out}, nil + case "snapshot": + return s.snapshot(ctx, r.Key, r.Node, r.Revision) + case "commit": + return s.commit(ctx, r) + default: + return nil, fail(400, "unknown official operation") + } +} +func (s *Service) get(ctx context.Context, r Request) (View, error) { + v := View{Files: []File{}, Available: map[string]bool{}} + err := s.DB.WithContext(ctx).Where("id = ?", hash([]string{hash(r.Key), r.Revision})).Take(&v.Definition).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return v, fail(404, "official revision not found") + } + if err != nil { + return v, err + } + if err = json.Unmarshal([]byte(v.Manifest), &v.Files); err != nil { + return v, err + } + for _, f := range v.Files { + var count int64 + err = s.DB.WithContext(ctx).Table("upload_inventory_holding h").Joins("JOIN upload_inventory_file f ON f.id=h.file_id").Joins("JOIN dingospeed n ON n.instance_id=h.instance_id").Where("f.namespace=? AND f.repo_type=? AND f.repo=? AND f.path=? AND f.sha256=? AND n.updated_at>?", r.Namespace, r.RepoType, r.Repo, f.Path, f.SHA256, time.Now().Add(-5*time.Minute)).Count(&count).Error + if err != nil { + return v, err + } + v.Available[f.Path] = count > 0 + } + return v, nil +} +func (s *Service) read(ctx context.Context, k repository.Key, node, op, revision string, out any) error { + var n model.Dingospeed + if err := s.DB.WithContext(ctx).Where("instance_id = ?", node).Take(&n).Error; err != nil { + return fail(503, "source node unavailable") + } + if time.Since(n.UpdatedAt) > 5*time.Minute { + return fail(503, "source node offline; comparison unavailable") + } + q := url.Values{"repo": {k.Repo}} + if revision != "" { + q.Set("revision", revision) + q.Set("verify", "sha256") + } + endpoint := "http://" + net.JoinHostPort(n.Host, strconv.Itoa(int(n.Port))) + "/api/repositories/" + url.PathEscape(k.RepoType) + "/" + url.PathEscape(k.Namespace) + "/" + op + "?" + q.Encode() + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return err + } + resp, err := s.Client.Do(req) + if err != nil { + return fail(503, "source content unreadable: "+err.Error()) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return fail(503, fmt.Sprintf("source content unreadable (HTTP %d); cannot compare", resp.StatusCode)) + } + dec := json.NewDecoder(io.LimitReader(resp.Body, 32<<20)) + if err = dec.Decode(out); err != nil { + return fail(502, "incomplete source response") + } + if dec.Decode(&struct{}{}) != io.EOF { + return fail(502, "incomplete or oversized source response") + } + return nil +} + +type Snapshot struct { + repository.Key + Revision string `json:"revision"` + Commit string `json:"commit"` + Files []File `json:"files"` + Complete bool `json:"complete"` + Verified bool `json:"verified"` + ContentVerified bool `json:"contentVerified"` + FileCount int `json:"fileCount"` +} + +func validateFiles(files []File) error { + seen := map[string]bool{} + for _, f := range files { + k := repository.Key{Namespace: "local", RepoType: "models", Repo: f.Path} + if k.Validate() != nil || len(f.SHA256) != 64 || f.Size < 0 { + return fail(400, "invalid file identity") + } + if _, err := hex.DecodeString(f.SHA256); err != nil || f.SHA256 != strings.ToLower(f.SHA256) { + return fail(400, "invalid SHA256") + } + if seen[f.Path] { + return fail(400, "duplicate file path") + } + seen[f.Path] = true + } + for p := range seen { + parts := strings.Split(p, "/") + for i := 1; i < len(parts); i++ { + if seen[strings.Join(parts[:i], "/")] { + return fail(400, "file/folder path conflict") + } + } + } + return nil +} +func (s *Service) snapshot(ctx context.Context, k repository.Key, node, revision string) (Snapshot, error) { + var snap Snapshot + if revision == "" { + return snap, fail(400, "source revision required") + } + if err := s.read(ctx, k, node, "snapshot", revision, &snap); err != nil { + return snap, err + } + if snap.Key != k || snap.Revision != revision || !snap.Complete || !snap.Verified || !snap.ContentVerified || snap.Commit == "" || snap.Files == nil || snap.FileCount != len(snap.Files) { + return snap, fail(502, "source manifest is incomplete or unverified; cannot compare") + } + if err := validateFiles(snap.Files); err != nil { + return snap, fail(502, err.Error()) + } + return snap, nil +} +func (s *Service) commit(ctx context.Context, r Request) (any, error) { + // Revision names use the same single-segment rules as repository names. + k := r.Key + k.Repo = r.Revision + if k.Validate() != nil || strings.Contains(r.Revision, "/") || r.RequestID == "" || len(r.RequestID) > 128 { + return nil, fail(400, "revision and request ID are required") + } + receiptID := hash([]string{hash(r.Key), r.Revision, r.Actor, r.RequestID}) + digest := hash(r) + lookup := func(db *gorm.DB) (any, error) { + var row Receipt + err := db.Where("id = ?", receiptID).Take(&row).Error + if err != nil { + return nil, err + } + if row.Digest != digest { + return nil, fail(409, "request ID already used for a different preview") + } + var out any + err = json.Unmarshal([]byte(row.Result), &out) + return out, err + } + if out, err := lookup(s.DB.WithContext(ctx)); err == nil { + return out, nil + } else if !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, err + } + current, err := s.get(ctx, r) + if err != nil { + var e *Failure + if !errors.As(err, &e) || e.Status != 404 { + return nil, err + } + if r.BaseVersion != 0 { + return nil, fail(409, "official revision changed; keep draft and compare again") + } + } + if current.Version != r.BaseVersion { + return nil, fail(409, "official revision changed; keep draft and compare again") + } + files := map[string]File{} + for _, f := range current.Files { + files[f.Path] = f + } + seen := map[string]bool{} + sources := map[string]Snapshot{} + for _, c := range r.Changes { + if seen[c.Path] { + return nil, fail(400, "duplicate change path") + } + seen[c.Path] = true + if c.File == nil { + if _, ok := files[c.Path]; !ok { + return nil, fail(400, "delete path not in official definition") + } + delete(files, c.Path) + continue + } + if c.Path != c.File.Path || c.Source == nil { + return nil, fail(400, "selected file requires an explicit source") + } + sk := hash(c.Source) + snap, ok := sources[sk] + if !ok { + snap, err = s.snapshot(ctx, r.Key, c.Source.Node, c.Source.Commit) + if err != nil { + return nil, err + } + if snap.Commit != c.Source.Commit { + return nil, fail(409, "source changed; compare again") + } + sources[sk] = snap + } + found := false + for _, f := range snap.Files { + if f == *c.File { + found = true + break + } + } + if !found { + return nil, fail(409, "selected source content changed; compare again") + } + files[c.Path] = *c.File + } + final := []File{} + for _, f := range files { + final = append(final, f) + } + sort.Slice(final, func(i, j int) bool { return final[i].Path < final[j].Path }) + if err = validateFiles(final); err != nil { + return nil, err + } + manifest, _ := json.Marshal(final) + next := Definition{ID: hash([]string{hash(r.Key), r.Revision}), Repository: hash(r.Key), Revision: r.Revision, Version: r.BaseVersion + 1, Manifest: string(manifest), Actor: r.Actor, UpdatedAt: time.Now().UTC()} + out := View{Definition: next, Files: final} + encoded, _ := json.Marshal(out) + err = s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if r.BaseVersion == 0 { + if err := tx.Create(&next).Error; err != nil { + var duplicate *dbmysql.MySQLError + if errors.Is(err, gorm.ErrDuplicatedKey) || errors.As(err, &duplicate) && duplicate.Number == 1062 { + return fail(409, "official revision already created; keep draft and compare again") + } + return err + } + } else { + res := tx.Model(&Definition{}).Where("id = ? AND version = ?", next.ID, r.BaseVersion).Updates(map[string]any{"version": next.Version, "manifest": next.Manifest, "actor": next.Actor, "updated_at": next.UpdatedAt}) + if res.Error != nil { + return res.Error + } + if res.RowsAffected != 1 { + return fail(409, "official revision changed; keep draft and compare again") + } + } + return tx.Create(&Receipt{ID: receiptID, Digest: digest, Result: string(encoded)}).Error + }) + if err != nil { + if prior, e := lookup(s.DB.WithContext(ctx)); e == nil { + return prior, nil + } + return nil, err + } + return out, nil +} diff --git a/internal/authority/service_test.go b/internal/authority/service_test.go new file mode 100644 index 0000000..6b062ec --- /dev/null +++ b/internal/authority/service_test.go @@ -0,0 +1,147 @@ +package authority + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "dingoscheduler/internal/model" + "dingoscheduler/pkg/repository" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func TestAtomicConfirmedDefinitions(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err = db.AutoMigrate(&Definition{}, &Receipt{}, &model.Dingospeed{}, &model.UploadInventoryFile{}, &model.UploadInventoryHolding{}); err != nil { + t.Fatal(err) + } + key := repository.Key{Namespace: "datacanvas", RepoType: "models", Repo: "demo"} + a := File{Path: "folder/a", SHA256: strings.Repeat("a", 64), Size: 1} + b := File{Path: "folder/b", SHA256: strings.Repeat("b", 64), Size: 2} + snap := Snapshot{Key: key, Revision: "commit", Commit: "commit", Files: []File{a, b}, Complete: true, Verified: true, ContentVerified: true, FileCount: 2} + broken := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if broken { + w.WriteHeader(503) + return + } + _ = json.NewEncoder(w).Encode(snap) + })) + defer server.Close() + u, _ := url.Parse(server.URL) + port, _ := strconv.Atoi(u.Port()) + n := model.Dingospeed{InstanceID: "A", Host: u.Hostname(), Port: int32(port), UpdatedAt: time.Now()} + db.Create(&n) + s := New(db) + ctx := context.Background() + source := &Source{Node: "A", Revision: "main", Commit: "commit"} + r := Request{Key: key, Action: "commit", Revision: "release", Actor: "admin", RequestID: "create", Changes: []Change{{Path: a.Path, File: &a, Source: source}, {Path: b.Path, File: &b, Source: source}}} + if _, err = s.Do(ctx, r); err != nil { + t.Fatal(err) + } + if _, err = s.Do(ctx, r); err != nil { + t.Fatal("idempotent retry", err) + } + get := r + get.Action = "get" + v, err := s.get(ctx, get) + if err != nil || v.Version != 1 || len(v.Files) != 2 { + t.Fatalf("%+v %v", v, err) + } + r.RequestID = "stale" + if _, err = s.Do(ctx, r); err == nil { + t.Fatal("stale revision overwrote") + } + emptyRequest := r + emptyRequest.Revision = "empty-revision" + emptyRequest.BaseVersion = 0 + emptyRequest.RequestID = "create-empty" + emptyRequest.Changes = nil + if _, err = s.Do(ctx, emptyRequest); err != nil { + t.Fatal("create empty revision", err) + } + emptyView, err := s.get(ctx, emptyRequest) + if err != nil || emptyView.Version != 1 || len(emptyView.Files) != 0 { + t.Fatalf("empty revision readback: %+v %v", emptyView, err) + } + deleteRequest := r + deleteRequest.Revision = "delete-all" + deleteRequest.RequestID = "create-before-delete" + if _, err = s.Do(ctx, deleteRequest); err != nil { + t.Fatal(err) + } + deleteRequest.BaseVersion = 1 + deleteRequest.RequestID = "delete-all" + deleteRequest.Changes = []Change{{Path: a.Path}, {Path: b.Path}} + if _, err = s.Do(ctx, deleteRequest); err != nil { + t.Fatal("delete all files", err) + } + emptyView, err = s.get(ctx, deleteRequest) + if err != nil || emptyView.Version != 2 || len(emptyView.Files) != 0 { + t.Fatalf("delete all readback: %+v %v", emptyView, err) + } + r.BaseVersion = 1 + c := File{Path: "c", SHA256: strings.Repeat("c", 64), Size: 3} + r.RequestID = "bad-source" + r.Changes = []Change{{Path: a.Path}, {Path: c.Path, File: &c, Source: source}} + if _, err = s.Do(ctx, r); err == nil { + t.Fatal("unverified content accepted") + } + v, _ = s.get(ctx, get) + if v.Version != 1 || len(v.Files) != 2 { + t.Fatal("partial commit") + } + snap.Complete = false + if _, err = s.snapshot(ctx, key, "A", "commit"); err == nil { + t.Fatal("incomplete accepted") + } + snap.Complete = true + snap.ContentVerified = false + if _, err = s.snapshot(ctx, key, "A", "commit"); err == nil { + t.Fatal("lightweight completion check accepted as SHA256 verification") + } + snap.ContentVerified = true + snap.FileCount = 3 + if _, err = s.snapshot(ctx, key, "A", "commit"); err == nil { + t.Fatal("truncated accepted") + } + snap.FileCount = 2 + broken = true + r.RequestID = "unreadable" + r.Changes = []Change{{Path: a.Path, File: &a, Source: source}} + if _, err = s.Do(ctx, r); err == nil { + t.Fatal("unreadable accepted") + } + // A retained official file needs no source. Deletion changes only definition. + r.RequestID = "delete-one" + r.Changes = []Change{{Path: a.Path}} + if _, err = s.Do(ctx, r); err != nil { + t.Fatal(err) + } + v, _ = s.get(ctx, get) + if v.Version != 2 || len(v.Files) != 1 || v.Files[0] != b { + t.Fatal("retained unavailable file lost") + } + // A successful receipt survives a new service instance and source outage. + if _, err = New(db).Do(ctx, r); err != nil { + t.Fatal("restart retry", err) + } + r.Changes = []Change{{Path: b.Path}} + if _, err = s.Do(ctx, r); err == nil { + t.Fatal("idempotency identity reused") + } + db.Model(&n).Update("updated_at", time.Now().Add(-time.Hour)) + if _, err = s.snapshot(ctx, key, "A", "commit"); err == nil { + t.Fatal("offline accepted") + } +} diff --git a/internal/dao/cache_job_dao.go b/internal/dao/cache_job_dao.go index e731120..c6bd5d5 100644 --- a/internal/dao/cache_job_dao.go +++ b/internal/dao/cache_job_dao.go @@ -51,22 +51,22 @@ func (c *CacheJobDao) GetCacheJob(condition *query.CacheJobQuery) (*model.CacheJ var preheatJobs []*model.CacheJob db := c.baseData.BizDB.Model(&model.CacheJob{}) if condition.Id != 0 { - db.Where("id = ?", condition.Id) + db = db.Where("id = ?", condition.Id) } if condition.Type != 0 { - db.Where("type = ?", condition.Type) + db = db.Where("type = ?", condition.Type) } if condition.InstanceId != "" { - db.Where("instance_id = ?", condition.InstanceId) + db = db.Where("instance_id = ?", condition.InstanceId) } if condition.Datatype != "" { - db.Where("datatype = ?", condition.Datatype) + db = db.Where("datatype = ?", condition.Datatype) } if condition.Org != "" { - db.Where("org = ?", condition.Org) + db = db.Where("org = ?", condition.Org) } if condition.Repo != "" { - db.Where("repo = ?", condition.Repo) + db = db.Where("repo = ?", condition.Repo) } if err := db.Find(&preheatJobs).Error; err != nil { return nil, err @@ -105,12 +105,26 @@ func (c *CacheJobDao) UpdateCacheStatus(statusReq *query.UpdateJobStatusReq) err } func (c *CacheJobDao) UpdateStatusAndRepo(jobStatusReq *query.UpdateJobStatusReq) error { - err := c.UpdateCacheStatus(jobStatusReq) + if jobStatusReq.Id <= 0 { + return fmt.Errorf("cache job ID is required") + } + job, err := c.GetCacheJob(&query.CacheJobQuery{Id: jobStatusReq.Id}) + if err != nil { + return err + } + if job == nil || jobStatusReq.InstanceId == "" || job.InstanceId != jobStatusReq.InstanceId { + return fmt.Errorf("cache job does not match instance") + } + if (jobStatusReq.Org != "" && jobStatusReq.Org != job.Org) || (jobStatusReq.Repo != "" && jobStatusReq.Repo != job.Repo) { + return fmt.Errorf("cache job does not match repository identity") + } + jobStatusReq.Org, jobStatusReq.Repo = job.Org, job.Repo + err = c.UpdateCacheStatus(jobStatusReq) if err != nil { return err } if jobStatusReq.Status == consts.RunningStatusJobComplete { - err = c.repositoryDao.PersistRepo(&query.PersistRepoReq{InstanceIds: []string{jobStatusReq.InstanceId}, + err = c.repositoryDao.PersistRepo(&query.PersistRepoReq{Datatype: job.Datatype, InstanceIds: []string{jobStatusReq.InstanceId}, Org: jobStatusReq.Org, Repo: jobStatusReq.Repo, OffVerify: true}) if err != nil { return err @@ -129,22 +143,22 @@ func (c *CacheJobDao) ListCacheJob(condition *query.CacheJobQuery) ([]*model.Cac var cacheJobs []*model.CacheJob db := c.baseData.BizDB.Model(&model.CacheJob{}) if condition.Id != 0 { - db.Where("id = ?", condition.Id) + db = db.Where("id = ?", condition.Id) } if condition.Type != 0 { - db.Where("type = ?", condition.Type) + db = db.Where("type = ?", condition.Type) } if condition.InstanceId != "" { - db.Where("instance_id = ?", condition.InstanceId) + db = db.Where("instance_id = ?", condition.InstanceId) } if condition.Datatype != "" { - db.Where("datatype = ?", condition.Datatype) + db = db.Where("datatype = ?", condition.Datatype) } if condition.Org != "" { - db.Where("org = ?", condition.Org) + db = db.Where("org = ?", condition.Org) } if condition.Repo != "" { - db.Where("repo = ?", condition.Repo) + db = db.Where("repo = ?", condition.Repo) } var count int64 if err := db.Count(&count).Error; err != nil { @@ -152,7 +166,7 @@ func (c *CacheJobDao) ListCacheJob(condition *query.CacheJobQuery) ([]*model.Cac return nil, 0, err } offset, pageSize := paginate(condition.Page, condition.PageSize) - db.Order(fmt.Sprintf("created_at desc offset %d limit %d", offset, pageSize)) + db = db.Order(fmt.Sprintf("created_at desc offset %d limit %d", offset, pageSize)) if err := db.Find(&cacheJobs).Error; err != nil { return nil, 0, err } @@ -163,16 +177,16 @@ func (c *CacheJobDao) GetUnCacheJob(instanceId string, ids []int, runningStatus cacheJobs := make([]*model.CacheJob, 0) db := c.baseData.BizDB.Table("cache_job t1") if instanceId != "" { - db.Where("t1.instance_id = ?", instanceId) + db = db.Where("t1.instance_id = ?", instanceId) } if len(ids) > 0 { - db.Where("t1.id in (?)", ids) + db = db.Where("t1.id in (?)", ids) } if len(runningStatus) > 0 { - db.Where("t1.status in (?)", runningStatus) + db = db.Where("t1.status in (?)", runningStatus) } if limit > 0 { - db.Limit(limit) + db = db.Limit(limit) } err := db.Find(&cacheJobs).Error // 中断或等待中的 return cacheJobs, err diff --git a/internal/dao/dingospeed_dao.go b/internal/dao/dingospeed_dao.go index cf16567..19a7ced 100644 --- a/internal/dao/dingospeed_dao.go +++ b/internal/dao/dingospeed_dao.go @@ -16,18 +16,23 @@ package dao import ( "fmt" + "net/url" + "strings" "sync" "dingoscheduler/internal/data" "dingoscheduler/internal/model" "dingoscheduler/pkg/common" "dingoscheduler/pkg/config" + "dingoscheduler/pkg/repository" "dingoscheduler/pkg/util" ) type DingospeedDao struct { baseData *data.BaseData mu sync.Mutex + healthMu sync.Mutex + health map[int32]*string } func NewDingospeedDao(data *data.BaseData) *DingospeedDao { @@ -37,12 +42,12 @@ func NewDingospeedDao(data *data.BaseData) *DingospeedDao { } func (d *DingospeedDao) Save(speed *model.Dingospeed) (int64, error) { - insertSql := fmt.Sprintf("INSERT INTO dingospeed(instance_id, host, port, online) VALUES('%s','%s',%d,%v)", speed.InstanceID, speed.Host, speed.Port, speed.Online) + insertSql := "INSERT INTO dingospeed(instance_id, host, port, online) VALUES(?,?,?,?)" db, err := d.baseData.BizDB.DB() if err != nil { return 0, err } - result, err := db.Exec(insertSql) + result, err := db.Exec(insertSql, speed.InstanceID, speed.Host, speed.Port, speed.Online) if err != nil { return 0, err } @@ -50,10 +55,12 @@ func (d *DingospeedDao) Save(speed *model.Dingospeed) (int64, error) { } func (d *DingospeedDao) RegisterUpdate(speed *model.Dingospeed) error { - sql := fmt.Sprintf("UPDATE dingospeed SET host='%s', port=%d, updated_at = '%s' WHERE id = %d", speed.Host, speed.Port, util.GetCurrentTimeStr(), speed.ID) - if err := d.baseData.BizDB.Exec(sql).Error; err != nil { + d.healthMu.Lock() + defer d.healthMu.Unlock() + if err := d.baseData.BizDB.Exec("UPDATE dingospeed SET host = ?, port = ?, updated_at = ? WHERE id = ?", speed.Host, speed.Port, util.GetCurrentTimeStr(), speed.ID).Error; err != nil { return err } + delete(d.health, speed.ID) return nil } @@ -97,12 +104,20 @@ func (d *DingospeedDao) GetEntity(instanceId string, online bool) (*model.Dingos return nil, nil } -func (d *DingospeedDao) RemoteRequestMeta(domain, repoType, orgRepo, commit string, headers map[string]string) (*common.Response, error) { - var reqUri string - if commit == "" { - reqUri = fmt.Sprintf("/api/%s/%s", repoType, orgRepo) - } else { - reqUri = fmt.Sprintf("/api/%s/%s/revision/%s", repoType, orgRepo, commit) +func (d *DingospeedDao) RemoteRequestMeta(domain string, key repository.Key, commit string, headers map[string]string) (*common.Response, error) { + reqUri, err := key.OperationURI("metadata", commit, "") + if key.Namespace == "huggingface" { + if commit == "" { + commit = "main" + } + parts := strings.Split(key.Repo, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + reqUri = "/api/" + key.RepoType + "/" + strings.Join(parts, "/") + "/revision/" + url.PathEscape(commit) + } + if err != nil { + return nil, err } return util.RetryRequest(func() (*common.Response, error) { return util.GetForDomain(domain, reqUri, headers) diff --git a/internal/dao/ingest_dao.go b/internal/dao/ingest_dao.go new file mode 100644 index 0000000..3582c79 --- /dev/null +++ b/internal/dao/ingest_dao.go @@ -0,0 +1,312 @@ +package dao + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/url" + "strings" + "time" + + "dingoscheduler/internal/model" + pb "dingoscheduler/pkg/proto/manager" + "dingoscheduler/pkg/repository" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type UploadedInventoryItem struct { + Namespace string `json:"namespace"` + RepoType string `json:"repoType"` + Repo string `json:"repo"` + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +type UploadedInventorySnapshot struct { + Version int `json:"version"` + InstanceID string `json:"instanceId"` + Epoch string `json:"epoch"` + EpochStartedAt time.Time `json:"epochStartedAt"` + Sequence uint64 `json:"sequence"` + GeneratedAt time.Time `json:"generatedAt"` + Complete bool `json:"complete"` + Error string `json:"error,omitempty"` + Items []UploadedInventoryItem `json:"items"` +} + +// IngestPublished retains the new RPC's wire name but now pulls one durable +// complete inventory. revision/commit are wake-up hints, never inventory facts. +func (r *RepositoryDao) IngestPublished(ctx context.Context, req *pb.IngestRepositoryRequest) (*pb.IngestRepositoryResponse, error) { + if req.InstanceId == "" || req.Commit == "" { + return nil, fmt.Errorf("instance and inventory report token are required") + } + speed, err := r.dingospeedDao.GetEntity(req.InstanceId, req.Online) + if err != nil { + return nil, err + } + if speed == nil { + return nil, fmt.Errorf("registered Speed not found") + } + endpoint := fmt.Sprintf("http://%s:%d/api/upload-inventory?token=%s", speed.Host, speed.Port, url.QueryEscape(req.Commit)) + hreq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + client := &http.Client{Timeout: 45 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := client.Do(hreq) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("upload inventory returned %d", resp.StatusCode) + } + var snap UploadedInventorySnapshot + decoder := json.NewDecoder(resp.Body) + if err = decoder.Decode(&snap); err != nil { + return nil, err + } + if err = decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("upload inventory contains trailing data") + } + if snap.InstanceID != req.InstanceId { + return nil, fmt.Errorf("upload inventory instance mismatch") + } + if err = validateUploadedInventory(&snap); err != nil { + return nil, err + } + accepted, count, total, err := r.ApplyUploadedInventory(ctx, &snap) + if err != nil { + return nil, err + } + commit := fmt.Sprintf("inventory:%s:%d", snap.Epoch, snap.Sequence) + if !accepted { + commit = "ignored:" + commit + } + return &pb.IngestRepositoryResponse{Commit: commit, FileCount: count, UsedStorage: total}, nil +} + +func validateUploadedInventory(s *UploadedInventorySnapshot) error { + if s.Version != 1 || s.InstanceID == "" || s.Epoch == "" || s.Sequence == 0 || s.EpochStartedAt.IsZero() || s.GeneratedAt.IsZero() { + return fmt.Errorf("invalid upload inventory envelope") + } + seen := make(map[string]struct{}, len(s.Items)) + for i := range s.Items { + item := &s.Items[i] + key := repository.Key{Namespace: item.Namespace, RepoType: item.RepoType, Repo: item.Repo} + if err := key.Validate(); err != nil { + return err + } + if key.Namespace == "huggingface" || key.Namespace == "modelscope" { + return fmt.Errorf("remote namespaces are not uploaded inventory") + } + if err := repository.ValidatePath(item.Path, true); err != nil || len(item.Path) > 1000 { + return fmt.Errorf("invalid uploaded path %q", item.Path) + } + hash, err := hex.DecodeString(item.SHA256) + if err != nil || len(hash) != 32 || strings.ToLower(item.SHA256) != item.SHA256 || item.Size < 0 { + return fmt.Errorf("invalid uploaded content for %q", item.Path) + } + identity := uploadIdentity(item) + if _, ok := seen[identity]; ok { + return fmt.Errorf("duplicate uploaded inventory item") + } + seen[identity] = struct{}{} + } + return nil +} + +func uploadIdentity(item *UploadedInventoryItem) string { + raw := strings.Join([]string{item.Namespace, item.RepoType, item.Repo, item.Path, item.SHA256}, "\x00") + sum := sha256.Sum256([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +// ApplyUploadedInventory is transactional and monotonic. Incomplete scans only +// mark confirmation state and never mutate holdings. Complete snapshots replace +// exactly one node's upload holdings and cannot touch remote-domain tables. +func (r *RepositoryDao) ApplyUploadedInventory(ctx context.Context, snap *UploadedInventorySnapshot) (accepted bool, count, total int64, err error) { + err = r.baseData.BizDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var state model.UploadInventoryState + findErr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("instance_id = ?", snap.InstanceID).Take(&state).Error + if findErr != nil && findErr != gorm.ErrRecordNotFound { + return findErr + } + if findErr == nil { + if snap.Epoch == state.Epoch && snap.Sequence <= state.LastSequence { + return nil + } + if snap.Epoch != state.Epoch && !snap.EpochStartedAt.After(state.EpochStartedAt) { + return nil + } + } + accepted = true + previousConfirmedAt := state.LastConfirmedAt + now := time.Now().UTC() + state = model.UploadInventoryState{InstanceID: snap.InstanceID, Epoch: snap.Epoch, EpochStartedAt: snap.EpochStartedAt.UTC(), LastSequence: snap.Sequence, InventoryComplete: snap.Complete, LastAttemptAt: now, ErrorMessage: snap.Error} + if snap.Complete { + confirmed := snap.GeneratedAt.UTC() + state.LastConfirmedAt = &confirmed + } else { + state.LastConfirmedAt = previousConfirmedAt + } + if err := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "instance_id"}}, DoUpdates: clause.AssignmentColumns([]string{"epoch", "epoch_started_at", "last_sequence", "inventory_complete", "last_attempt_at", "last_confirmed_at", "error_message"})}).Create(&state).Error; err != nil { + return err + } + if !snap.Complete { + return nil + } + fileIDs := make([]int64, 0, len(snap.Items)) + for i := range snap.Items { + item := &snap.Items[i] + identity := uploadIdentity(item) + file := model.UploadInventoryFile{IdentityHash: identity, Namespace: item.Namespace, RepoType: item.RepoType, Repo: item.Repo, Path: item.Path, SHA256: item.SHA256, Size: item.Size} + if err := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "identity_hash"}}, DoNothing: true}).Create(&file).Error; err != nil { + return err + } + if err := tx.Where("identity_hash = ?", identity).Take(&file).Error; err != nil { + return err + } + if file.Size != item.Size || file.Namespace != item.Namespace || file.RepoType != item.RepoType || file.Repo != item.Repo || file.Path != item.Path || file.SHA256 != item.SHA256 { + return fmt.Errorf("uploaded inventory identity collision or size conflict") + } + fileIDs = append(fileIDs, file.ID) + holding := model.UploadInventoryHolding{FileID: file.ID, InstanceID: snap.InstanceID, Sequence: snap.Sequence, ConfirmedAt: snap.GeneratedAt.UTC()} + if err := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "file_id"}, {Name: "instance_id"}}, DoUpdates: clause.AssignmentColumns([]string{"sequence", "confirmed_at"})}).Create(&holding).Error; err != nil { + return err + } + count++ + if total > math.MaxInt64-item.Size { + return fmt.Errorf("uploaded inventory size overflow") + } + total += item.Size + } + remove := tx.Where("instance_id = ?", snap.InstanceID) + if len(fileIDs) > 0 { + remove = remove.Where("file_id NOT IN ?", fileIDs) + } + if err := remove.Delete(&model.UploadInventoryHolding{}).Error; err != nil { + return err + } + return tx.Where("NOT EXISTS (SELECT 1 FROM upload_inventory_holding h WHERE h.file_id = upload_inventory_file.id)").Delete(&model.UploadInventoryFile{}).Error + }) + return +} + +type UploadedHoldingView struct { + FileID int64 `json:"fileId,string"` + Namespace string `json:"namespace"` + RepoType string `json:"repoType"` + Repo string `json:"repo"` + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + InstanceID string `json:"instanceId"` + ConfirmedAt time.Time `json:"confirmedAt"` + NodeOnline bool `json:"nodeOnline"` + NodeAvailable bool `json:"nodeAvailable"` + HeartbeatAt time.Time `json:"heartbeatAt"` + InventoryComplete bool `json:"inventoryComplete"` + LastConfirmedAt *time.Time `json:"lastInventoryConfirmedAt,omitempty"` +} + +type UploadedNodeInventoryStateView struct { + InstanceID string `json:"instanceId"` + Epoch string `json:"epoch"` + LastSequence uint64 `json:"lastSequence"` + InventoryComplete bool `json:"inventoryComplete"` + LastAttemptAt time.Time `json:"lastAttemptAt"` + LastConfirmedAt *time.Time `json:"lastConfirmedAt,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + NodeAvailable bool `json:"nodeAvailable"` + HeartbeatAt time.Time `json:"heartbeatAt"` +} + +func (r *RepositoryDao) GetUploadedNodeInventoryState(ctx context.Context, instanceID string) (*UploadedNodeInventoryStateView, error) { + var result UploadedNodeInventoryStateView + err := r.baseData.BizDB.WithContext(ctx).Table("upload_inventory_state s"). + Select("s.instance_id, s.epoch, s.last_sequence, s.inventory_complete, s.last_attempt_at, s.last_confirmed_at, s.error_message, d.updated_at AS heartbeat_at"). + Joins("LEFT JOIN dingospeed d ON d.id = (SELECT MAX(d2.id) FROM dingospeed d2 WHERE d2.instance_id = s.instance_id)"). + Where("s.instance_id = ?", instanceID).Take(&result).Error + if err != nil { + return nil, err + } + result.NodeAvailable = !result.HeartbeatAt.IsZero() && result.HeartbeatAt.After(time.Now().Add(-5*time.Minute)) + return &result, nil +} + +func (r *RepositoryDao) ListUploadedHoldings(ctx context.Context, instanceID, namespace, repoType, repo, path, sha string) ([]UploadedHoldingView, error) { + var rows []UploadedHoldingView + db := r.baseData.BizDB.WithContext(ctx).Table("upload_inventory_holding h").Select("f.id AS file_id, f.namespace, f.repo_type, f.repo, f.path, f.sha256, f.size, h.instance_id, h.confirmed_at, d.online AS node_online, d.updated_at AS heartbeat_at, s.inventory_complete, s.last_confirmed_at").Joins("JOIN upload_inventory_file f ON f.id = h.file_id").Joins("LEFT JOIN dingospeed d ON d.id = (SELECT MAX(d2.id) FROM dingospeed d2 WHERE d2.instance_id = h.instance_id)").Joins("LEFT JOIN upload_inventory_state s ON s.instance_id = h.instance_id") + filters := [][2]string{{"h.instance_id", instanceID}, {"f.namespace", namespace}, {"f.repo_type", repoType}, {"f.repo", repo}, {"f.path", path}, {"f.sha256", sha}} + for _, filter := range filters { + if filter[1] != "" { + db = db.Where(filter[0]+" = ?", filter[1]) + } + } + if err := db.Order("f.namespace, f.repo_type, f.repo, f.path, f.sha256, h.instance_id").Find(&rows).Error; err != nil { + return nil, err + } + cutoff := time.Now().Add(-5 * time.Minute) + for i := range rows { + // Current availability is a heartbeat observation. The legacy `online` + // flag is retained in the response for compatibility but Speed currently + // also uses it for its upstream-network mode, so it is not an availability + // gate for uploaded local files. + rows[i].NodeAvailable = !rows[i].HeartbeatAt.IsZero() && rows[i].HeartbeatAt.After(cutoff) + } + return rows, nil +} + +type UploadedRepositoryView struct { + Namespace string `json:"namespace"` + RepoType string `json:"repoType"` + Repo string `json:"repo"` + FileCount int64 `json:"fileCount"` + HoldingCount int64 `json:"holdingCount"` +} + +func (r *RepositoryDao) ListUploadedRepositories(ctx context.Context) ([]UploadedRepositoryView, error) { + var rows []UploadedRepositoryView + err := r.baseData.BizDB.WithContext(ctx).Table("upload_inventory_file f").Select("f.namespace, f.repo_type, f.repo, COUNT(DISTINCT f.id) AS file_count, COUNT(h.id) AS holding_count").Joins("JOIN upload_inventory_holding h ON h.file_id = f.id").Group("f.namespace, f.repo_type, f.repo").Order("f.namespace, f.repo_type, f.repo").Find(&rows).Error + return rows, err +} + +// Legacy validation names are retained for the existing publication contract +// tests. They are no longer persisted or used as Scheduler inventory facts. +type PublishedFile struct { + Path string `json:"path"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} +type PublishedSnapshot struct { + Commit string `json:"commit"` + Files []PublishedFile `json:"files"` +} + +func validateSnapshot(s PublishedSnapshot) (int64, error) { + seen := map[string]struct{}{} + var total int64 + for _, file := range s.Files { + if err := repository.ValidatePath(file.Path, true); err != nil || len(file.Path) > 1000 || file.Size < 0 || total > math.MaxInt64-file.Size { + return 0, fmt.Errorf("invalid published file") + } + hash, err := hex.DecodeString(file.SHA256) + if err != nil || len(hash) != 32 || strings.ToLower(file.SHA256) != file.SHA256 { + return 0, fmt.Errorf("invalid sha256") + } + if _, ok := seen[file.Path]; ok { + return 0, fmt.Errorf("duplicate manifest path") + } + seen[file.Path] = struct{}{} + total += file.Size + } + return total, nil +} diff --git a/internal/dao/ingest_validation_test.go b/internal/dao/ingest_validation_test.go new file mode 100644 index 0000000..f8f6ed5 --- /dev/null +++ b/internal/dao/ingest_validation_test.go @@ -0,0 +1,18 @@ +package dao + +import ( + "strings" + "testing" +) + +func TestPublishedManifestRejectsInvalidContent(t *testing.T) { + good := PublishedFile{Path: "weights/a.bin", Size: 9, SHA256: strings.Repeat("a", 64)} + for _, files := range [][]PublishedFile{{good, good}, {{Path: "../outside", Size: 9, SHA256: good.SHA256}}, {{Path: "a", Size: -1, SHA256: good.SHA256}}, {{Path: "a", Size: 9, SHA256: "not-a-hash"}}} { + if _, e := validateSnapshot(PublishedSnapshot{Commit: "c", Files: files}); e == nil { + t.Fatalf("accepted %+v", files) + } + } + if n, e := validateSnapshot(PublishedSnapshot{Commit: "c", Files: []PublishedFile{good}}); e != nil || n != 9 { + t.Fatal(n, e) + } +} diff --git a/internal/dao/legacy_isolation_test.go b/internal/dao/legacy_isolation_test.go new file mode 100644 index 0000000..3cbf9ab --- /dev/null +++ b/internal/dao/legacy_isolation_test.go @@ -0,0 +1,199 @@ +package dao + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "dingoscheduler/internal/data" + "dingoscheduler/internal/model/query" + "dingoscheduler/pkg/config" + pb "dingoscheduler/pkg/proto/manager" + "dingoscheduler/pkg/repository" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +// A deterministic SQL fixture executes selection over otherwise identical +// records. No local database service or configured production DSN is accessed. +type fixtureRow struct { + id int64 + repoType, namespace, repo, name, etag string +} +type fixtureConnector struct { + records []fixtureRow + queries *[]string +} + +func (c fixtureConnector) Connect(context.Context) (driver.Conn, error) { return fixtureConn{c}, nil } +func (c fixtureConnector) Driver() driver.Driver { return fixtureDriver{} } + +type fixtureDriver struct{} + +func (fixtureDriver) Open(string) (driver.Conn, error) { return nil, fmt.Errorf("use connector") } + +type fixtureConn struct{ fixtureConnector } + +func (fixtureConn) Prepare(string) (driver.Stmt, error) { return nil, fmt.Errorf("unexpected prepare") } +func (fixtureConn) Close() error { return nil } +func (fixtureConn) Begin() (driver.Tx, error) { return nil, fmt.Errorf("unexpected write transaction") } + +type fixtureRows struct { + ids []int64 + pos int +} + +func (fixtureRows) Columns() []string { return []string{"id"} } +func (fixtureRows) Close() error { return nil } +func (r *fixtureRows) Next(dest []driver.Value) error { + if r.pos == len(r.ids) { + return io.EOF + } + dest[0] = r.ids[r.pos] + r.pos++ + return nil +} +func (c fixtureConn) QueryContext(_ context.Context, statement string, args []driver.NamedValue) (driver.Rows, error) { + *c.queries = append(*c.queries, statement) + if strings.Contains(statement, "model_file_process p") { + var count int64 + if len(args) != 7 { + return nil, fmt.Errorf("incomplete process identity") + } + for _, r := range c.records { + if r.id == args[0].Value && args[1].Value == "node" && r.repoType == args[2].Value && r.namespace == args[3].Value && r.repo == args[4].Value && r.name == args[5].Value && r.etag == args[6].Value { + count++ + } + } + return &fixtureRows{ids: []int64{count}}, nil + } + if !strings.Contains(statement, "datatype = ? AND org = ? AND repo = ?") || strings.Contains(statement, " OR ") { + return nil, fmt.Errorf("query did not scope content selection to a complete identity: %s", statement) + } + if len(args) < 3 { + return nil, fmt.Errorf("missing identity arguments") + } + ids := []int64{} + for _, r := range c.records { + if r.repoType != args[0].Value || r.namespace != args[1].Value || r.repo != args[2].Value { + continue + } + index := 3 + if strings.Contains(statement, "etag = ?") { + if r.etag != args[index].Value { + continue + } + index++ + } + if strings.Contains(statement, "name = ?") && r.name != args[index].Value { + continue + } + ids = append(ids, r.id) + } + return &fixtureRows{ids: ids}, nil +} + +func TestQueuedProgressCannotReuseAnotherNamespaceProcessID(t *testing.T) { + db, _ := fixtureDB(t, []fixtureRow{{1, "models", "dingo-local/alice", "team/model", "nested/a.bin", "same"}}) + d := NewModelFileProcessDao(&data.BaseData{BizDB: db}) + entry := &pb.FileProcessEntry{ProcessId: 1, InstanceId: "node", DataType: "models", Org: "dingo-local/alice", Repo: "team/model", Name: "nested/a.bin", Etag: "same"} + if err := d.ValidateProcessIdentity(entry); err != nil { + t.Fatal(err) + } + entry.Org = "dingo-local/bob" + if err := d.ValidateProcessIdentity(entry); err == nil { + t.Fatal("accepted progress ID belonging to alice for bob") + } + entry.Org = "dingo-local/alice" + entry.Repo = "team/other" + if err := d.ValidateProcessIdentity(entry); err == nil { + t.Fatal("accepted progress ID for another repository") + } +} +func fixtureDB(t *testing.T, rows []fixtureRow) (*gorm.DB, *[]string) { + t.Helper() + statements := []string{} + sqldb := sql.OpenDB(fixtureConnector{rows, &statements}) + t.Cleanup(func() { _ = sqldb.Close() }) + db, err := gorm.Open(mysql.New(mysql.Config{Conn: sqldb, SkipInitializeWithVersion: true}), &gorm.Config{DisableAutomaticPing: true}) + if err != nil { + t.Fatal(err) + } + return db, &statements +} + +func TestContentDeletionNeverCrossesNamespaceRepoOrType(t *testing.T) { + db, _ := fixtureDB(t, []fixtureRow{ + {1, "models", "dingo-local/alice", "team/model", "weights/a.bin", "same"}, + {2, "models", "dingo-local/bob", "team/model", "weights/a.bin", "same"}, + {3, "models", "dingo-local/alice", "other/model", "weights/a.bin", "same"}, + {4, "datasets", "dingo-local/alice", "team/model", "weights/a.bin", "same"}, + {5, "models", "dingo-local/alice", "team/model", "weights/a.bin", "different"}, + }) + d := NewModelFileRecordDao(&data.BaseData{BizDB: db}) + for _, name := range []string{"", "weights/a.bin"} { + ids, err := d.GetIDsByEtagsOrFields("same", "models", "dingo-local/alice", "team/model", name) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(ids, []int64{1}) { + t.Fatalf("deleted other identity: %v", ids) + } + } + if _, err := d.GetIDsByEtagsOrFields("same", "", "", "", ""); err == nil { + t.Fatal("accepted unscoped etag deletion") + } +} + +func TestRepositoryEnumerationUsesWholeIdentity(t *testing.T) { + db, _ := fixtureDB(t, nil) + statement := db.ToSQL(func(tx *gorm.DB) *gorm.DB { + d := NewRepositoryDaoForTest(tx) + _, _ = d.GetFreeRepository("node", "models", "dingo-local/alice", "team/model") + return tx + }) + for _, predicate := range []string{"r.datatype = t1.datatype", "r.org = t1.org", "r.repo = t1.repo", "NOT EXISTS"} { + if !strings.Contains(statement, predicate) { + t.Fatalf("missing %s: %s", predicate, statement) + } + } +} +func NewRepositoryDaoForTest(db *gorm.DB) *RepositoryDao { + return &RepositoryDao{baseData: &data.BaseData{BizDB: db}} +} + +func TestSpeedMetadataUsesIndependentIdentityAndCredentials(t *testing.T) { + previous := config.SysConfig + config.SysConfig = &config.Config{Retry: config.Retry{Attempts: 1}} + t.Cleanup(func() { config.SysConfig = previous }) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/models/Qwen/Qwen3/revision/v1" { + t.Errorf("wrong metadata address: %s", r.URL) + } + if r.Header.Get("X-Dingo-Service-Token") != "" || r.Header.Get("Authorization") != "Bearer upstream" { + t.Error("credentials lost or merged") + } + _, _ = io.WriteString(w, `{"sha":"commit","siblings":[{"rfilename":"nested/model.bin"}],"usedStorage":9}`) + })) + defer server.Close() + _, err := (&DingospeedDao{}).RemoteRequestMeta(server.URL, repository.Key{"huggingface", "models", "Qwen/Qwen3"}, "v1", map[string]string{"Authorization": "Bearer upstream"}) + if err != nil { + t.Fatal(err) + } +} + +func TestRecordLookupAcceptsEmptyLegacyOwner(t *testing.T) { + db, _ := fixtureDB(t, []fixtureRow{{1, "models", "", "gpt2", "config.json", "sha"}, {2, "models", "other", "gpt2", "config.json", "sha"}}) + d := NewModelFileRecordDao(&data.BaseData{BizDB: db}) + r, e := d.FirstModelFileRecord(&query.ModelFileRecordQuery{Datatype: "models", Repo: "gpt2", FileName: "config.json", Etag: "sha"}) + if e != nil || r == nil || r.ID != 1 { + t.Fatalf("legacy unowned: %+v %v", r, e) + } +} diff --git a/internal/dao/model_file_process_dao.go b/internal/dao/model_file_process_dao.go index 6b2528d..2e0b50c 100644 --- a/internal/dao/model_file_process_dao.go +++ b/internal/dao/model_file_process_dao.go @@ -44,16 +44,10 @@ func (d *ModelFileProcessDao) Save(process *model.ModelFileProcess) (int64, erro } func SaveProcessBySql(tx *gorm.DB, process *model.ModelFileProcess) (int64, error) { - recordSql := fmt.Sprintf("INSERT INTO model_file_process(record_id, instance_id, offset_num, status, master_instance_id) VALUES (%d, '%s',%d,%d,'%s')", process.RecordID, process.InstanceID, process.OffsetNum, process.Status, process.MasterInstanceID) - db, err := tx.DB() - if err != nil { - return 0, err - } - result, err := db.Exec(recordSql) - if err != nil { + if err := tx.Omit("CreatedAt", "UpdatedAt").Create(process).Error; err != nil { return 0, err } - return result.LastInsertId() + return process.ID, nil } func (d *ModelFileProcessDao) BatchSave(processes []model.ModelFileProcess) error { @@ -190,3 +184,20 @@ func (d *ModelFileProcessDao) DeleteByRecordIDAndInstanceID(recordID []int64, in return result.RowsAffected, nil } + +// A queued report carrying a process ID must still refer to that exact +// namespace/repository/file and node; stale IDs cannot update another tenant. +func (d *ModelFileProcessDao) ValidateProcessIdentity(entry *pb.FileProcessEntry) error { + var count int64 + err := d.baseData.BizDB.Table("model_file_process p"). + Joins("JOIN model_file_record r ON r.id = p.record_id"). + Where("p.id = ? AND p.instance_id = ? AND r.datatype = ? AND r.org = ? AND r.repo = ? AND r.name = ? AND r.etag = ?", + entry.ProcessId, entry.InstanceId, entry.DataType, entry.Org, entry.Repo, entry.Name, entry.Etag).Count(&count).Error + if err != nil { + return err + } + if count != 1 { + return fmt.Errorf("process does not match repository identity") + } + return nil +} diff --git a/internal/dao/model_file_record_dao.go b/internal/dao/model_file_record_dao.go index de27b7d..2418cb1 100644 --- a/internal/dao/model_file_record_dao.go +++ b/internal/dao/model_file_record_dao.go @@ -21,6 +21,7 @@ import ( "dingoscheduler/internal/model" "dingoscheduler/internal/model/query" pb "dingoscheduler/pkg/proto/manager" + "dingoscheduler/pkg/repository" "go.uber.org/zap" "gorm.io/gorm" @@ -37,74 +38,38 @@ func NewModelFileRecordDao(data *data.BaseData) *ModelFileRecordDao { } func (d *ModelFileRecordDao) BatchSave(records []model.ModelFileRecord) error { - tx := d.baseData.BizDB.Begin() - if tx.Error != nil { - zap.S().Error("开启事务失败: %v", tx.Error) - return tx.Error - } - db, err := tx.DB() - if err != nil { - tx.Rollback() - zap.S().Error("从事务获取 DB 实例失败: %v", err) - return err - } - for _, record := range records { - sql := fmt.Sprintf( - "INSERT INTO model_file_record(datatype, org, repo, name, etag, file_size) VALUES ('%s','%s','%s','%s','%s',%d)", - record.Datatype, - record.Org, - record.Repo, - record.Name, - record.Etag, - record.FileSize, - ) - result, err := db.Exec(sql) - if err != nil { - tx.Rollback() - zap.S().Error("批量插入失败: %v, SQL: %s", err, sql) - return err + return d.baseData.BizDB.Transaction(func(tx *gorm.DB) error { + for i := range records { + if _, err := SaveRecordBySql(tx, &records[i]); err != nil { + return err + } } - _, _ = result.LastInsertId() - } - - if err := tx.Commit().Error; err != nil { - tx.Rollback() - zap.S().Fatalf("事务提交失败: %v", err) - return err - } - return nil + return nil + }) } func SaveRecordBySql(tx *gorm.DB, record *model.ModelFileRecord) (int64, error) { - recordSql := fmt.Sprintf("INSERT INTO model_file_record(datatype, org, repo, name, etag, file_size) VALUES ('%s','%s','%s','%s','%s',%d)", record.Datatype, record.Org, record.Repo, record.Name, record.Etag, record.FileSize) - db, err := tx.DB() - if err != nil { + if _, err := repository.FromWire(record.Datatype, record.Org, record.Repo); err != nil { return 0, err } - result, err := db.Exec(recordSql) - if err != nil { + if err := tx.Omit("CreatedAt", "UpdatedAt").Create(record).Error; err != nil { return 0, err } - return result.LastInsertId() + return record.ID, nil } func (d *ModelFileRecordDao) FirstModelFileRecord(condition *query.ModelFileRecordQuery) (*model.ModelFileRecord, error) { + if _, err := repository.FromWire(condition.Datatype, condition.Org, condition.Repo); err != nil { + return nil, err + } var records []*model.ModelFileRecord db := d.baseData.BizDB.Model(&model.ModelFileRecord{}).Select("id") - if condition.Datatype != "" { - db.Where("datatype = ?", condition.Datatype) - } - if condition.Org != "" { - db.Where("org = ?", condition.Org) - } - if condition.Repo != "" { - db.Where("repo = ?", condition.Repo) + db = db.Where("datatype = ? AND org = ? AND repo = ?", condition.Datatype, condition.Org, condition.Repo) + if condition.Etag != "" { + db = db.Where("etag = ?", condition.Etag) } if condition.FileName != "" { - db.Where("name = ?", condition.FileName) - } - if condition.Etag != "" { - db.Where("etag = ?", condition.Etag) + db = db.Where("name = ?", condition.FileName) } if err := db.Find(&records).Error; err != nil { return nil, err @@ -198,33 +163,26 @@ func (d *ModelFileRecordDao) FindDistinctOrgs() ([]string, error) { // GetIDsByEtagsOrFields 根据Etag列表查询，或者根据Datatype、Org、Repo、Name四者都匹配的条件查询对应的ID func (d *ModelFileRecordDao) GetIDsByEtagsOrFields(etag, datatype, org, repo, name string) ([]int64, error) { - var ids []int64 - query := d.baseData.BizDB.Model(&model.ModelFileRecord{}) - - hasEtagCondition := etag != "" - hasFieldCondition := datatype != "" && org != "" && repo != "" && name != "" - - if !hasEtagCondition && !hasFieldCondition { - return []int64{}, nil + // Content IDs are repository-scoped. An etag alone must never select another + // namespace, nor may a filename OR broaden an exact content-ID deletion. + if _, err := repository.FromWire(datatype, org, repo); err != nil { + return nil, err } - - if hasEtagCondition { - query = query.Where("etag = ?", etag) + if etag == "" && name == "" { + return nil, fmt.Errorf("etag or file path is required") } - - if hasFieldCondition { - condition := "datatype = ? AND org = ? AND repo = ? AND name = ?" - if hasEtagCondition { - query = query.Or(condition, datatype, org, repo, name) - } else { - query = query.Where(condition, datatype, org, repo, name) - } + var ids []int64 + db := d.baseData.BizDB.Model(&model.ModelFileRecord{}). + Where("datatype = ? AND org = ? AND repo = ?", datatype, org, repo) + if etag != "" { + db = db.Where("etag = ?", etag) } - - if err := query.Pluck("id", &ids).Error; err != nil { - return nil, fmt.Errorf("查询ID失败: %w", err) + if name != "" { + db = db.Where("name = ?", name) + } + if err := db.Pluck("id", &ids).Error; err != nil { + return nil, err } - return ids, nil } @@ -270,8 +228,8 @@ func (d *ModelFileRecordDao) ExistRecords(records []model.ModelFileRecord) ([]mo db := d.baseData.BizDB.Model(&model.ModelFileRecord{}).Where("1 = 0") for _, r := range records { - db = db.Or("etag = ? AND name = ? AND org = ? AND repo = ?", - r.Etag, r.Name, r.Org, r.Repo) + db = db.Or("datatype = ? AND etag = ? AND name = ? AND org = ? AND repo = ?", + r.Datatype, r.Etag, r.Name, r.Org, r.Repo) } if err := db.Find(&existing).Error; err != nil { diff --git a/internal/dao/node_endpoint.go b/internal/dao/node_endpoint.go new file mode 100644 index 0000000..7d44024 --- /dev/null +++ b/internal/dao/node_endpoint.go @@ -0,0 +1,26 @@ +package dao + +import ( + "context" + "dingoscheduler/internal/model" + "gorm.io/gorm/clause" +) + +func (d *DingospeedDao) SaveNodeEndpoint(ctx context.Context, id int32, management, download string) error { + return d.baseData.BizDB.WithContext(ctx).Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "node_id"}}, DoUpdates: clause.AssignmentColumns([]string{"management_url", "download_url"})}).Create(&model.NodeEndpoint{NodeID: id, ManagementURL: management, DownloadURL: download}).Error +} + +func (d *DingospeedDao) NodeEndpoints(ctx context.Context, ids []int32) (map[int32]model.NodeEndpoint, error) { + out := map[int32]model.NodeEndpoint{} + if len(ids) == 0 { + return out, nil + } + var rows []model.NodeEndpoint + if err := d.baseData.BizDB.WithContext(ctx).Where("node_id IN ?", ids).Find(&rows).Error; err != nil { + return nil, err + } + for _, row := range rows { + out[row.NodeID] = row + } + return out, nil +} diff --git a/internal/dao/node_health_dao.go b/internal/dao/node_health_dao.go new file mode 100644 index 0000000..a4366b0 --- /dev/null +++ b/internal/dao/node_health_dao.go @@ -0,0 +1,70 @@ +package dao + +import ( + "context" + "time" + + "dingoscheduler/internal/model" + "dingoscheduler/pkg/nodehealth" + pb "dingoscheduler/pkg/proto/manager" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Update the report and communication timestamp atomically. A missing report +// clears the previous report, so downgraded clients never inherit old health. +func (d *DingospeedDao) HeartbeatHealth(ctx context.Context, req *pb.HeartbeatRequest, encoded *string, now time.Time) error { + d.healthMu.Lock() + defer d.healthMu.Unlock() + result := d.baseData.BizDB.WithContext(ctx).Exec( + "UPDATE dingospeed SET updated_at = ? WHERE id = ? AND instance_id = ? AND online = ?", + now, req.Id, req.InstanceId, req.Online) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + // MySQL may report zero for an identical update in the same timestamp tick. + var count int64 + err := d.baseData.BizDB.WithContext(ctx).Model(&model.Dingospeed{}).Where("id = ? AND instance_id = ? AND online = ?", req.Id, req.InstanceId, req.Online).Count(&count).Error + if err != nil { + return err + } + if count == 0 { + return status.Error(codes.NotFound, "registered node not found") + } + } + if d.health == nil { + d.health = make(map[int32]*string) + } + d.health[req.Id] = encoded + return nil +} + +type nodeHealthRow struct { + ID int32 + InstanceID string + Online bool + UpdatedAt time.Time +} + +type NodeHealthView struct { + ID int32 `json:"id"` + InstanceID string `json:"instanceId"` + OnlineMode bool `json:"onlineMode"` + nodehealth.View +} + +func (d *DingospeedDao) ListNodeHealth(ctx context.Context, after int32, limit int, now time.Time) ([]NodeHealthView, error) { + d.healthMu.Lock() + defer d.healthMu.Unlock() + var rows []nodeHealthRow + err := d.baseData.BizDB.WithContext(ctx).Table("dingospeed").Select("id, instance_id, online, updated_at").Where("id > ?", after).Order("id ASC").Limit(limit).Find(&rows).Error + if err != nil { + return nil, err + } + result := make([]NodeHealthView, 0, len(rows)) + for _, r := range rows { + result = append(result, NodeHealthView{r.ID, r.InstanceID, r.Online, nodehealth.Present(d.health[r.ID], r.UpdatedAt, now)}) + } + return result, nil +} diff --git a/internal/dao/node_repository_dao.go b/internal/dao/node_repository_dao.go new file mode 100644 index 0000000..26ce3f0 --- /dev/null +++ b/internal/dao/node_repository_dao.go @@ -0,0 +1,70 @@ +package dao + +import ( + "context" + "time" + + "dingoscheduler/internal/model" + "dingoscheduler/pkg/repository" +) + +type NodeRepositoryView struct { + ID int64 `json:"id,string"` + Namespace string `json:"namespace"` + Repo string `json:"repo"` + Datatype string `json:"datatype"` + IdentityValid bool `json:"identityValid"` + Commit string `json:"commit"` + UsedStorage int64 `json:"usedStorage"` + MountStatus int32 `json:"mountStatus"` + ErrorMessage string `json:"errorMessage"` + LastModified string `json:"lastModified"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type NodeRepositoryPage struct { + NodeID int32 `json:"nodeId"` + InstanceID string `json:"instanceId"` + Items []NodeRepositoryView `json:"items"` + Total int64 `json:"total"` + UsedStorage int64 `json:"usedStorage"` + NextAfter int64 `json:"nextAfter,string"` + ObservedAt time.Time `json:"observedAt"` +} + +// Read the existing instance projection only. This is not a disk inventory and +// does not trigger persistence, contact Speed, or alter existing records. +func (d *DingospeedDao) ListNodeRepositories(ctx context.Context, nodeID int32, after int64, limit int) (*NodeRepositoryPage, error) { + db := d.baseData.BizDB.WithContext(ctx) + var node model.Dingospeed + if err := db.Select("id, instance_id").Where("id = ?", nodeID).Take(&node).Error; err != nil { + return nil, err + } + page := &NodeRepositoryPage{NodeID: node.ID, InstanceID: node.InstanceID, Items: make([]NodeRepositoryView, 0)} + var totals struct { + Total int64 + UsedStorage int64 + } + if err := db.Model(&model.Repository{}).Select("COUNT(*) AS total, COALESCE(SUM(used_storage), 0) AS used_storage").Where("instance_id = ?", node.InstanceID).Scan(&totals).Error; err != nil { + return nil, err + } + page.Total, page.UsedStorage = totals.Total, totals.UsedStorage + var rows []model.Repository + if err := db.Select("id, datatype, org, repo, sha, used_storage, status, error_msg, last_modified, updated_at").Where("instance_id = ? AND id > ?", node.InstanceID, after).Order("id ASC").Limit(limit + 1).Find(&rows).Error; err != nil { + return nil, err + } + if len(rows) > limit { + rows = rows[:limit] + page.NextAfter = rows[len(rows)-1].ID + } + for _, row := range rows { + key, err := repository.FromWire(row.Datatype, row.Org, row.Repo) + item := NodeRepositoryView{ID: row.ID, Namespace: key.Namespace, Repo: key.Repo, Datatype: row.Datatype, IdentityValid: err == nil, Commit: row.Sha, UsedStorage: row.UsedStorage, MountStatus: row.Status, ErrorMessage: row.ErrorMsg, LastModified: row.LastModified, UpdatedAt: row.UpdatedAt} + if err != nil { + item.Namespace, item.Repo = row.Org, row.Repo + } + page.Items = append(page.Items, item) + } + page.ObservedAt = time.Now().UTC() + return page, nil +} diff --git a/internal/dao/remote_repository_metadata.go b/internal/dao/remote_repository_metadata.go new file mode 100644 index 0000000..deeea3d --- /dev/null +++ b/internal/dao/remote_repository_metadata.go @@ -0,0 +1,43 @@ +package dao + +import ( + "errors" + "strings" + "unicode/utf8" + + "dingoscheduler/internal/model" + "github.com/go-sql-driver/mysql" + "go.uber.org/zap" +) + +// Preserve the normal legacy insert on databases that support the metadata. +// Some existing schemas have VARCHAR(10) tag IDs and INT repository_tag.repo_id. +// Those optional fields must not roll back a verified, complete repository. +// Retry only explicit column-capacity errors, after the transaction rolls back. +func saveRemoteRepositoryMetadata(repo *model.Repository, tags []*model.RepositoryTag, save func(*model.Repository, []*model.RepositoryTag) error) error { + for { + err := save(repo, tags) + if err == nil { + return nil + } + var sqlErr *mysql.MySQLError + if !errors.As(err, &sqlErr) { + return err + } + switch { + case sqlErr.Number == 1406 && strings.Contains(sqlErr.Message, "'pipeline_tag_id'") && repo.PipelineTagId != "": + if repo.PipelineTag == "" && utf8.RuneCountInString(repo.PipelineTagId) <= 100 { + repo.PipelineTag = repo.PipelineTagId + } + repo.PipelineTagId = "" + case len(tags) > 0 && ((sqlErr.Number == 1406 && strings.Contains(sqlErr.Message, "'tag_id'")) || + (sqlErr.Number == 1264 && strings.Contains(sqlErr.Message, "'repo_id'"))): + tags = nil + default: + return err + } + zap.S().Warnf("remote repository %s optional tags exceed existing schema; retrying without unsupported tag fields: %v", repo.OrgRepo, err) + // A rolled-back Create can still assign the generated ID to the Go value. + repo.ID = 0 + } +} diff --git a/internal/dao/remote_repository_metadata_test.go b/internal/dao/remote_repository_metadata_test.go new file mode 100644 index 0000000..f43b320 --- /dev/null +++ b/internal/dao/remote_repository_metadata_test.go @@ -0,0 +1,52 @@ +package dao + +import ( + "dingoscheduler/internal/model" + "errors" + "github.com/go-sql-driver/mysql" + "testing" +) + +func TestRemoteMetadataCapacityFallback(t *testing.T) { + repo := &model.Repository{OrgRepo: "sshleifer/tiny-gpt2", PipelineTagId: "text-generation"} + tags := []*model.RepositoryTag{{TagId: "pytorch"}} + calls := 0 + err := saveRemoteRepositoryMetadata(repo, tags, func(r *model.Repository, ts []*model.RepositoryTag) error { + calls++ + if r.ID != 0 { + t.Fatal("rolled-back ID reused") + } + switch calls { + case 1: + r.ID = 9007199254740993 + return &mysql.MySQLError{Number: 1406, Message: "Data too long for column 'pipeline_tag_id' at row 1"} + case 2: + if r.PipelineTagId != "" || r.PipelineTag != "text-generation" { + t.Fatal("pipeline label lost") + } + r.ID = 9007199254740994 + return &mysql.MySQLError{Number: 1264, Message: "Out of range value for column 'repo_id' at row 1"} + case 3: + if len(ts) != 0 { + t.Fatal("unsupported associations retained") + } + return nil + default: + t.Fatal("unbounded retry") + return nil + } + }) + if err != nil || calls != 3 { + t.Fatalf("calls=%d error=%v", calls, err) + } +} + +func TestRemoteMetadataDoesNotHideOtherFailures(t *testing.T) { + for _, failure := range []error{errors.New("connection failed"), &mysql.MySQLError{Number: 1406, Message: "Data too long for column 'repo' at row 1"}} { + calls := 0 + err := saveRemoteRepositoryMetadata(&model.Repository{}, nil, func(*model.Repository, []*model.RepositoryTag) error { calls++; return failure }) + if err != failure || calls != 1 { + t.Fatalf("failure masked: %v", err) + } + } +} diff --git a/internal/dao/repository_dao.go b/internal/dao/repository_dao.go index c65d915..e41a629 100644 --- a/internal/dao/repository_dao.go +++ b/internal/dao/repository_dao.go @@ -25,6 +25,7 @@ import ( "dingoscheduler/internal/model/dto" "dingoscheduler/internal/model/query" myerr "dingoscheduler/pkg/error" + "dingoscheduler/pkg/repository" "dingoscheduler/pkg/util" "github.com/bytedance/sonic" @@ -55,6 +56,20 @@ func NewRepositoryDao(data *data.BaseData, repositoryTagDao *RepositoryTagDao, t } func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error { + if persistRepoReq.Datatype == "" { + persistRepoReq.Datatype = "models" + } + if persistRepoReq.Namespace != "" { + if persistRepoReq.Org != "" { + return fmt.Errorf("org and namespace cannot both be supplied") + } + org, repo, err := (repository.Key{Namespace: persistRepoReq.Namespace, RepoType: persistRepoReq.Datatype, Repo: persistRepoReq.Repo}).Storage() + if err != nil { + return err + } + persistRepoReq.Org, persistRepoReq.Repo = org, repo + } + zap.S().Debugf("PersistRepo start instanceId:%s， org:%s, repo:%s", persistRepoReq.InstanceIds, persistRepoReq.Org, persistRepoReq.Repo) var ( pipelineMap map[string]string @@ -71,7 +86,7 @@ func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error continue } // 存在下载记录和进度，但【模型】在仓库不存在，没有数据集。 - freeRepositories, err := r.GetFreeRepository(instanceId, persistRepoReq.Org, persistRepoReq.Repo) + freeRepositories, err := r.GetFreeRepository(instanceId, persistRepoReq.Datatype, persistRepoReq.Org, persistRepoReq.Repo) if err != nil { return err } @@ -88,6 +103,9 @@ func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error } speedDomain := fmt.Sprintf("http://%s:%d", speed.Host, speed.Port) for _, repository := range freeRepositories { + if repositoryKey(repository).Namespace != "huggingface" && repositoryKey(repository).Namespace != "modelscope" { + continue + } if err = r.singleRepositoryPersist(repository, instanceId, speedDomain, pipelineMap, persistRepoReq.OffVerify); err != nil { zap.S().Errorf("singleRepositoryPersist err.%v", err) continue @@ -100,7 +118,7 @@ func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error func (r *RepositoryDao) singleRepositoryPersist(repository *model.Repository, instanceId, speedDomain string, pipelineMap map[string]string, offVerify bool) error { orgRepo := util.GetOrgRepo(repository.Org, repository.Repo) - metaResp, err := r.dingospeedDao.RemoteRequestMeta(speedDomain, repository.Datatype, orgRepo, "main", r.hfTokenDao.GetHeaders()) + metaResp, err := r.dingospeedDao.RemoteRequestMeta(speedDomain, repositoryKey(repository), "main", r.hfTokenDao.GetHeaders()) if err != nil { return err } @@ -124,7 +142,9 @@ func (r *RepositoryDao) singleRepositoryPersist(repository *model.Repository, in } } // 保存组织图片 - err = r.organizationDao.PersistOrgLogo(repository.Org) + if repositoryKey(repository).Namespace == "huggingface" && repository.Org != "" { + err = r.organizationDao.PersistOrgLogo(repository.Org) + } if err != nil { zap.S().Errorf("PersistOrgLogo err.org:%s, %v", repository.Org, err) } @@ -148,7 +168,7 @@ func (r *RepositoryDao) singleRepositoryPersist(repository *model.Repository, in TagId: tag, }) } - err = r.RepoAndTagSave(repo, tags) + err = saveRemoteRepositoryMetadata(repo, tags, r.RepoAndTagSave) if err != nil { zap.S().Errorf("repository save err.orgRepo:%s,%v", orgRepo, err) return err @@ -183,18 +203,13 @@ func (r *RepositoryDao) verifyRepoComplete(metaData *dto.CommitHfSha, instanceId } func (r *RepositoryDao) SaveBySql(tx *gorm.DB, repo *model.Repository) (int64, error) { - recordSql := fmt.Sprintf("INSERT INTO repository (instance_id, datatype, org, repo, org_repo, like_num, download_num, pipeline_tag_id, pipeline_tag, last_modified, used_storage, sha)"+ - " VALUES( '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', '%s', '%s', %d, '%s')", - repo.InstanceId, repo.Datatype, repo.Org, repo.Repo, repo.OrgRepo, repo.LikeNum, repo.DownloadNum, repo.PipelineTagId, repo.PipelineTag, repo.LastModified, repo.UsedStorage, repo.Sha) - db, err := tx.DB() - if err != nil { + if err := repositoryKey(repo).Validate(); err != nil { return 0, err } - result, err := db.Exec(recordSql) - if err != nil { + if err := tx.Omit("CreatedAt", "UpdatedAt").Create(repo).Error; err != nil { return 0, err } - return result.LastInsertId() + return repo.ID, nil } func (r *RepositoryDao) Get(id int64) (*model.Repository, error) { @@ -228,42 +243,48 @@ func (r *RepositoryDao) RepoAndTagSave(repository *model.Repository, tags []*mod return nil } -func (r *RepositoryDao) GetFreeRepository(instanceId, org, repo string) ([]*model.Repository, error) { +func (r *RepositoryDao) GetFreeRepository(instanceId, repoType, org, repo string) ([]*model.Repository, error) { var repositories []*model.Repository - tx := r.baseData.BizDB.Table("model_file_record t1").Select("distinct t1.datatype, t1.org, t1.repo ") - if org != "" && repo != "" { - tx.Where(fmt.Sprintf(" t1.org = '%s' and t1.repo= '%s'", org, repo)) + tx := r.baseData.BizDB.Table("model_file_record t1").Select("distinct t1.datatype, t1.org, t1.repo") + if repoType != "" { + tx = tx.Where("t1.datatype = ?", repoType) } - err := tx.Where("t1.id in (SELECT x.record_id FROM dingo.model_file_process x where x.instance_id = ?) "+ - "and t1.repo not in (select repo from repository where instance_id = ?)", instanceId, instanceId).Find(&repositories).Error + if org != "" { + tx = tx.Where("t1.org = ?", org) + } + if repo != "" { + tx = tx.Where("t1.repo = ?", repo) + } + err := tx.Where("EXISTS (SELECT 1 FROM model_file_process x WHERE x.record_id = t1.id AND x.instance_id = ?)", instanceId). + Where("NOT EXISTS (SELECT 1 FROM repository r WHERE r.instance_id = ? AND r.datatype = t1.datatype AND r.org = t1.org AND r.repo = t1.repo)", instanceId).Find(&repositories).Error return repositories, err } func (r *RepositoryDao) VerifyRepoComplete(instanceId, datatype, org, repo string) (int64, error) { var recordCount int64 - err := r.baseData.BizDB.Table("model_file_record t1").Select("t1.id").InnerJoins(", model_file_process t2"). + err := r.baseData.BizDB.Table("model_file_record t1").Select("t1.id").Joins("JOIN model_file_process t2 ON t1.id = t2.record_id"). Where("t1.datatype = ? and t1.org=? and t1.repo= ? and t1.id = t2.record_id and t2.instance_id = ? and t1.file_size = t2.offset_num", datatype, org, repo, instanceId).Count(&recordCount).Error return recordCount, err } func (r *RepositoryDao) ModelList(query *query.ModelQuery) ([]*model.Repository, int64, error) { repositories := make([]*model.Repository, 0) - db := r.baseData.BizDB.Table("repository t1").Select("t1.id, t1.org, t1.org_repo, t1.like_num, t1.download_num, t1.sha, t1.pipeline_tag, t1.last_modified, t1.used_storage, t1.status") + db := r.baseData.BizDB.Table("repository t1").Select("t1.id, t1.datatype, t1.org, t1.repo, t1.org_repo, t1.like_num, t1.download_num, t1.sha, t1.pipeline_tag, t1.last_modified, t1.used_storage, t1.status") if query.InstanceId != "" { - db.Where("t1.instance_id = ?", query.InstanceId) + db = db.Where("t1.instance_id = ?", query.InstanceId) } if query.Name != "" { - db.Where(fmt.Sprintf("t1.org_repo like '%s'", "%"+query.Name+"%")) + db = db.Where("t1.org_repo LIKE ?", "%"+query.Name+"%") } if query.PipelineTag != "" { - db.Where("t1.pipeline_tag_id = ?", query.PipelineTag) + db = db.Where("t1.pipeline_tag_id = ?", query.PipelineTag) } if query.Datatype != "" { - db.Where("t1.datatype = ?", query.Datatype) + db = db.Where("t1.datatype = ?", query.Datatype) } if query.Status != "" { - db.Where("t1.status = ?", util.Atoi(query.Status)) + db = db.Where("t1.status = ?", util.Atoi(query.Status)) } tags := make([]string, 0) @@ -286,7 +307,7 @@ func (r *RepositoryDao) ModelList(query *query.ModelQuery) ([]*model.Repository, tags = append(tags, strings.Split(query.Other, ",")...) } if len(tags) > 0 { - db.Where(" t1.id in (select repo_id from repository_tag where tag_id in (?))", tags) + db = db.Where(" t1.id in (select repo_id from repository_tag where tag_id in (?))", tags) } var count int64 if err := db.Count(&count).Error; err != nil { @@ -295,11 +316,11 @@ func (r *RepositoryDao) ModelList(query *query.ModelQuery) ([]*model.Repository, } offset, pageSize := paginate(query.Page, query.PageSize) if query.Sort != "" && query.Order != "" { - db.Order(fmt.Sprintf("%s %s offset %d limit %d", query.Sort, query.Order, offset, pageSize)) + db = db.Order(fmt.Sprintf("%s %s", query.Sort, query.Order)) } else { - db.Order(fmt.Sprintf("offset %d limit %d", offset, pageSize)) + db = db.Order("t1.id ASC") } - err := db.Find(&repositories).Error + err := db.Offset(offset).Limit(pageSize).Find(&repositories).Error return repositories, count, err } @@ -359,17 +380,22 @@ func (r *RepositoryDao) GetUnmountRepository(instanceId string, ids []int, runni repositories := make([]*model.Repository, 0) db := r.baseData.BizDB.Table("repository t1").Select("t1.id, t1.datatype, t1.org, t1.repo, t1.org_repo, t1.status") if instanceId != "" { - db.Where("t1.instance_id = ?", instanceId) + db = db.Where("t1.instance_id = ?", instanceId) } if len(ids) > 0 { - db.Where("t1.id in (?)", ids) + db = db.Where("t1.id in (?)", ids) } if len(runningStatus) > 0 { - db.Where("t1.status in (?)", runningStatus) + db = db.Where("t1.status in (?)", runningStatus) } if limit > 0 { - db.Limit(limit) + db = db.Limit(limit) } err := db.Find(&repositories).Error // 中断或等待中的 return repositories, err } + +func repositoryKey(repo *model.Repository) repository.Key { + k, _ := repository.FromWire(repo.Datatype, repo.Org, repo.Repo) + return k +} diff --git a/internal/dao/repository_tag_dao.go b/internal/dao/repository_tag_dao.go index 9027c30..87b929f 100644 --- a/internal/dao/repository_tag_dao.go +++ b/internal/dao/repository_tag_dao.go @@ -36,9 +36,7 @@ func (r *RepositoryTagDao) SaveBySql(tx *gorm.DB, repo *model.RepositoryTag) (in func (r *RepositoryTagDao) BatchSave(tx *gorm.DB, repositoryTags []*model.RepositoryTag) error { for _, repositoryTag := range repositoryTags { - recordSql := fmt.Sprintf("INSERT INTO repository_tag (repo_id, tag_id) VALUES(%d, '%s')", - repositoryTag.RepoId, repositoryTag.TagId) - result := tx.Exec(recordSql) + result := tx.Exec("INSERT INTO repository_tag (repo_id, tag_id) VALUES(?, ?)", repositoryTag.RepoId, repositoryTag.TagId) if result.Error != nil { // 出错回滚事务 zap.S().Error("批量插入失败: %v", result.Error) diff --git a/internal/dao/upload_inventory_test.go b/internal/dao/upload_inventory_test.go new file mode 100644 index 0000000..d71d4eb --- /dev/null +++ b/internal/dao/upload_inventory_test.go @@ -0,0 +1,165 @@ +package dao + +import ( + "context" + "testing" + "time" + + "dingoscheduler/internal/data" + "dingoscheduler/internal/model" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func inventoryTestDAO(t *testing.T) (*RepositoryDao, *gorm.DB) { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err = db.AutoMigrate(&model.UploadInventoryState{}, &model.UploadInventoryFile{}, &model.UploadInventoryHolding{}, &model.Dingospeed{}, &model.ModelFileRecord{}, &model.ModelFileProcess{}, &model.Repository{}); err != nil { + t.Fatal(err) + } + return &RepositoryDao{baseData: &data.BaseData{BizDB: db}}, db +} + +func inventory(epoch string, started time.Time, sequence uint64, instance string, complete bool, items ...UploadedInventoryItem) *UploadedInventorySnapshot { + return &UploadedInventorySnapshot{Version: 1, InstanceID: instance, Epoch: epoch, EpochStartedAt: started, Sequence: sequence, GeneratedAt: started.Add(time.Duration(sequence) * time.Second), Complete: complete, Items: items} +} + +func item(path, hash string, size int64) UploadedInventoryItem { + return UploadedInventoryItem{Namespace: "team", RepoType: "models", Repo: "demo/full", Path: path, SHA256: hash, Size: size} +} + +func TestUploadedInventoryMultiNodeIdentityOrderingAndCompleteness(t *testing.T) { + dao, db := inventoryTestDAO(t) + ctx := context.Background() + started := time.Now().Add(-time.Hour).UTC() + h1 := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + h2 := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + base := item("same/path.bin", h1, 10) + if accepted, _, _, err := dao.ApplyUploadedInventory(ctx, inventory("e1", started, 1, "A", true, base, item("other/path.bin", h1, 10), item("same/path.bin", h2, 11))); err != nil || !accepted { + t.Fatalf("A apply: accepted=%v err=%v", accepted, err) + } + if accepted, _, _, err := dao.ApplyUploadedInventory(ctx, inventory("e2", started, 1, "B", true, base)); err != nil || !accepted { + t.Fatalf("B apply: accepted=%v err=%v", accepted, err) + } + var holdings, files int64 + db.Model(&model.UploadInventoryHolding{}).Count(&holdings) + db.Model(&model.UploadInventoryFile{}).Count(&files) + if holdings != 4 || files != 3 { + t.Fatalf("holdings=%d files=%d, want 4/3", holdings, files) + } + + // A's complete empty inventory removes only A; B keeps the shared file. + if _, _, _, err := dao.ApplyUploadedInventory(ctx, inventory("e1", started, 2, "A", true)); err != nil { + t.Fatal(err) + } + db.Model(&model.UploadInventoryHolding{}).Count(&holdings) + db.Model(&model.UploadInventoryFile{}).Count(&files) + if holdings != 1 || files != 1 { + t.Fatalf("after A delete holdings=%d files=%d, want 1/1", holdings, files) + } + + // Late pre-delete inventory cannot revive A. + accepted, _, _, err := dao.ApplyUploadedInventory(ctx, inventory("e1", started, 1, "A", true, base)) + if err != nil || accepted { + t.Fatalf("late snapshot accepted=%v err=%v", accepted, err) + } + db.Model(&model.UploadInventoryHolding{}).Count(&holdings) + if holdings != 1 { + t.Fatalf("late snapshot revived holding: %d", holdings) + } + + // Incomplete higher sequence marks uncertainty but never interprets absence. + if _, _, _, err = dao.ApplyUploadedInventory(ctx, inventory("e2", started, 2, "B", false)); err != nil { + t.Fatal(err) + } + db.Model(&model.UploadInventoryHolding{}).Count(&holdings) + if holdings != 1 { + t.Fatalf("incomplete scan removed holding: %d", holdings) + } + var state model.UploadInventoryState + if err = db.Where("instance_id = ?", "B").Take(&state).Error; err != nil || state.InventoryComplete { + t.Fatalf("state=%+v err=%v", state, err) + } + if state.LastConfirmedAt == nil { + t.Fatal("incomplete scan erased the last complete confirmation time") + } + + // A genuinely newer epoch may restore the file; an older epoch cannot. + accepted, _, _, err = dao.ApplyUploadedInventory(ctx, inventory("old", started.Add(-time.Minute), 99, "A", true, base)) + if err != nil || accepted { + t.Fatalf("older epoch accepted=%v err=%v", accepted, err) + } + accepted, _, _, err = dao.ApplyUploadedInventory(ctx, inventory("new", started.Add(time.Minute), 1, "A", true, base)) + if err != nil || !accepted { + t.Fatalf("new epoch accepted=%v err=%v", accepted, err) + } + db.Model(&model.UploadInventoryHolding{}).Count(&holdings) + if holdings != 2 { + t.Fatalf("restore holdings=%d, want 2", holdings) + } + if err = db.Create(&model.Dingospeed{InstanceID: "A", Host: "127.0.0.1", Port: 1, Online: true, UpdatedAt: time.Now().Add(-10 * time.Minute)}).Error; err != nil { + t.Fatal(err) + } + views, err := dao.ListUploadedHoldings(ctx, "A", "", "", "", "", "") + if err != nil || len(views) != 1 || views[0].NodeAvailable { + t.Fatalf("offline view=%+v err=%v", views, err) + } + nodeState, err := dao.GetUploadedNodeInventoryState(ctx, "A") + if err != nil || nodeState.NodeAvailable || !nodeState.InventoryComplete || nodeState.LastConfirmedAt == nil { + t.Fatalf("node state=%+v err=%v", nodeState, err) + } + db.Model(&model.UploadInventoryHolding{}).Count(&holdings) + if holdings != 2 { + t.Fatalf("offline observation deleted holdings: %d", holdings) + } + + // Removing both confirmed copies finally removes the global uploaded-file row. + if _, _, _, err = dao.ApplyUploadedInventory(ctx, inventory("new", started.Add(time.Minute), 2, "A", true)); err != nil { + t.Fatal(err) + } + if _, _, _, err = dao.ApplyUploadedInventory(ctx, inventory("e2", started, 3, "B", true)); err != nil { + t.Fatal(err) + } + db.Model(&model.UploadInventoryHolding{}).Count(&holdings) + db.Model(&model.UploadInventoryFile{}).Count(&files) + if holdings != 0 || files != 0 { + t.Fatalf("last confirmed removal left holdings=%d files=%d", holdings, files) + } +} + +func TestUploadedInventoryDoesNotTouchRemoteTables(t *testing.T) { + dao, db := inventoryTestDAO(t) + legacyRecord := model.ModelFileRecord{Datatype: "models", Org: "remote-org", Repo: "remote-repo", Name: "x", Etag: "etag", FileSize: 7} + if err := db.Create(&legacyRecord).Error; err != nil { + t.Fatal(err) + } + legacyProcess := model.ModelFileProcess{RecordID: legacyRecord.ID, InstanceID: "A", OffsetNum: 7, Status: 3} + if err := db.Create(&legacyProcess).Error; err != nil { + t.Fatal(err) + } + legacyRepo := model.Repository{InstanceId: "A", Datatype: "models", Org: "remote-org", Repo: "remote-repo", OrgRepo: "remote-org/remote-repo", Sha: "remote"} + if err := db.Create(&legacyRepo).Error; err != nil { + t.Fatal(err) + } + started := time.Now().UTC() + if _, _, _, err := dao.ApplyUploadedInventory(context.Background(), inventory("e", started, 1, "A", true, item("x", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 7))); err != nil { + t.Fatal(err) + } + if _, _, _, err := dao.ApplyUploadedInventory(context.Background(), inventory("e", started, 2, "A", true)); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + model interface{} + }{ + {"record", &model.ModelFileRecord{}}, {"process", &model.ModelFileProcess{}}, {"repository", &model.Repository{}}, + } { + var count int64 + if err := db.Model(tc.model).Count(&count).Error; err != nil || count != 1 { + t.Fatalf("legacy %s count=%d err=%v", tc.name, count, err) + } + } +} diff --git a/internal/data/data.go b/internal/data/data.go index 596de1c..d138cb5 100644 --- a/internal/data/data.go +++ b/internal/data/data.go @@ -14,9 +14,11 @@ package data import ( + "dingoscheduler/internal/authority" "errors" "fmt" + "dingoscheduler/internal/model" "dingoscheduler/pkg/config" "dingoscheduler/pkg/consts" myorm "dingoscheduler/pkg/gorm" @@ -61,6 +63,13 @@ func NewBaseData(conf *config.Config) (*BaseData, func(), error) { if debug { bizClient = bizClient.Debug() } + // First-phase uploaded inventory is an additive schema. AutoMigrate only + // creates/extends these dedicated tables and never migrates legacy remote + // repository or download records. + if err = bizClient.AutoMigrate(&model.NodeEndpoint{}, &model.UploadInventoryState{}, &model.UploadInventoryFile{}, &model.UploadInventoryHolding{}, &authority.Definition{}, &authority.Receipt{}); err != nil { + cleanup() + return nil, nil, fmt.Errorf("migrate uploaded inventory schema: %w", err) + } return &BaseData{ BizDB: bizClient, Cache: gCache, diff --git a/internal/handler/authority_handler.go b/internal/handler/authority_handler.go new file mode 100644 index 0000000..c561a84 --- /dev/null +++ b/internal/handler/authority_handler.go @@ -0,0 +1,40 @@ +package handler + +import ( + "crypto/subtle" + "encoding/json" + "errors" + "io" + "net/http" + "os" + + "dingoscheduler/internal/authority" + "github.com/labstack/echo/v4" +) + +// Only the authenticated Spinfield business gateway may call this internal API. +// End-user authorization and repository scope are checked there on every call. +func (h *ManagerHandler) OfficialRevision(c echo.Context) error { + key := os.Getenv("DINGO_AUTHORITY_KEY") + if key == "" || subtle.ConstantTimeCompare([]byte(c.Request().Header.Get("Authorization")), []byte("Bearer "+key)) != 1 { + return c.JSON(http.StatusForbidden, map[string]string{"message": "official gateway authentication required"}) + } + var r authority.Request + dec := json.NewDecoder(http.MaxBytesReader(c.Response(), c.Request().Body, 8<<20)) + if dec.Decode(&r) != nil || dec.Decode(&struct{}{}) != io.EOF { + return c.JSON(400, map[string]string{"message": "invalid request"}) + } + if r.Actor == "" { + return c.JSON(403, map[string]string{"message": "authenticated actor required"}) + } + result, err := h.schedulerService.OfficialRevision(c.Request().Context(), r) + if err != nil { + status := 500 + var e *authority.Failure + if errors.As(err, &e) { + status = e.Status + } + return c.JSON(status, map[string]string{"message": err.Error()}) + } + return c.JSON(200, result) +} diff --git a/internal/handler/cache_job_handler.go b/internal/handler/cache_job_handler.go index 7c8f4b0..78452ba 100644 --- a/internal/handler/cache_job_handler.go +++ b/internal/handler/cache_job_handler.go @@ -7,6 +7,7 @@ import ( "dingoscheduler/internal/model/query" "dingoscheduler/internal/service" "dingoscheduler/pkg/consts" + "dingoscheduler/pkg/repository" "dingoscheduler/pkg/util" "github.com/labstack/echo/v4" @@ -37,13 +38,24 @@ func (handler *CacheJobHandler) CreateCacheJobHandler(c echo.Context) error { zap.S().Errorf("MetaProxyCommon repoType:%s is not exist RepoTypesMapping", createCacheJobReq.Datatype) return util.ErrorRequestParamCN(c) } - org, repo := util.SplitOrgRepo(createCacheJobReq.OrgRepo) - if org == "" || repo == "" { - zap.S().Errorf("MetaProxyCommon org and repo is null") - return util.ErrorRepoNotFoundCN(c) + if createCacheJobReq.Namespace != "" { + if createCacheJobReq.Org != "" || createCacheJobReq.OrgRepo != "" { + return util.ErrorRequestParamCN(c) + } + org, repo, err := (repository.Key{Namespace: createCacheJobReq.Namespace, RepoType: createCacheJobReq.Datatype, Repo: createCacheJobReq.Repo}).Storage() + if err != nil { + return util.ErrorRequestParamCN(c) + } + createCacheJobReq.Org, createCacheJobReq.Repo = org, repo + createCacheJobReq.OrgRepo = util.GetOrgRepo(org, repo) + createCacheJobReq.Namespace = "" + } else { + org, repo := util.SplitOrgRepo(createCacheJobReq.OrgRepo) + if org == "" || repo == "" { + return util.ErrorRepoNotFoundCN(c) + } + createCacheJobReq.Org, createCacheJobReq.Repo = org, repo } - createCacheJobReq.Org = org - createCacheJobReq.Repo = repo createCacheJobReq.Type = consts.CacheTypePreheat resp, err := handler.cacheJobService.CreateCacheJob(createCacheJobReq) if err != nil { diff --git a/internal/handler/ingest_handler.go b/internal/handler/ingest_handler.go new file mode 100644 index 0000000..880a98b --- /dev/null +++ b/internal/handler/ingest_handler.go @@ -0,0 +1,60 @@ +package handler + +import ( + pb "dingoscheduler/pkg/proto/manager" + "dingoscheduler/pkg/util" + "github.com/labstack/echo/v4" + "net/http" +) + +func (h *ManagerHandler) IngestRepository(c echo.Context) error { + var req struct { + InstanceID string `json:"instanceId"` + Namespace string `json:"namespace"` + RepoType string `json:"repoType"` + Repo string `json:"repo"` + Revision string `json:"revision"` + Commit string `json:"commit"` + Online *bool `json:"online"` + } + if err := c.Bind(&req); err != nil { + return util.ErrorRequestParamCN(c) + } + online := true + if req.Online != nil { + online = *req.Online + } + result, err := h.schedulerService.IngestRepository(c.Request().Context(), &pb.IngestRepositoryRequest{InstanceId: req.InstanceID, Namespace: req.Namespace, RepoType: req.RepoType, Repo: req.Repo, Revision: req.Revision, Commit: req.Commit, Online: online}) + if err != nil { + return c.JSON(http.StatusConflict, map[string]string{"error": err.Error(), "code": "INGEST_NOT_COMMITTED"}) + } + return util.NormalResponseData(c, map[string]interface{}{"repositoryId": result.RepositoryId, "namespace": req.Namespace, "repoType": req.RepoType, "repo": req.Repo, "revision": req.Revision, "commit": result.Commit, "fileCount": result.FileCount, "usedStorage": result.UsedStorage, "status": "persisted"}) +} + +func (h *ManagerHandler) UploadedHoldings(c echo.Context) error { + rows, err := h.schedulerService.UploadedHoldings(c.Request().Context(), c.QueryParam("instanceId"), c.QueryParam("namespace"), c.QueryParam("repoType"), c.QueryParam("repo"), c.QueryParam("path"), c.QueryParam("sha256")) + if err != nil { + return util.ResponseError(c, err) + } + return util.NormalResponseData(c, map[string]interface{}{"items": rows, "count": len(rows)}) +} + +func (h *ManagerHandler) UploadedNodeHoldings(c echo.Context) error { + rows, err := h.schedulerService.UploadedHoldings(c.Request().Context(), c.Param("instanceId"), c.QueryParam("namespace"), c.QueryParam("repoType"), c.QueryParam("repo"), c.QueryParam("path"), c.QueryParam("sha256")) + if err != nil { + return util.ResponseError(c, err) + } + state, err := h.schedulerService.UploadedNodeInventoryState(c.Request().Context(), c.Param("instanceId")) + if err != nil { + return util.ResponseError(c, err) + } + return util.NormalResponseData(c, map[string]interface{}{"instanceId": c.Param("instanceId"), "state": state, "items": rows, "count": len(rows)}) +} + +func (h *ManagerHandler) UploadedRepositories(c echo.Context) error { + rows, err := h.schedulerService.UploadedRepositories(c.Request().Context()) + if err != nil { + return util.ResponseError(c, err) + } + return util.NormalResponseData(c, map[string]interface{}{"items": rows, "count": len(rows)}) +} diff --git a/internal/handler/node_health.html b/internal/handler/node_health.html new file mode 100644 index 0000000..b8d19cc --- /dev/null +++ b/internal/handler/node_health.html @@ -0,0 +1,105 @@ + + + +dingoscheduler · 节点与仓库 + + +
+

dingospeed 节点与仓库

+

选择节点查看运行状态与已入库仓库。仓库列表每 10 秒同步数据库。

+
每 10 秒刷新当前页。
+
尚未连接。没有收到数据不能视为正常。
+
+

节点仓库

+

数据来自 Scheduler 数据库，按实例展示已入库仓库。容量是已记录的逻辑大小，不是磁盘用量；不包含未发布、未入库的缓存。上传仓库展示 main 投影。

+
选择下方节点的“查看仓库”。
+
+表格可横向滚动，查看版本、容量和挂载状态。 +
+
+
+
+ + diff --git a/internal/handler/node_health_handler.go b/internal/handler/node_health_handler.go new file mode 100644 index 0000000..aafa30f --- /dev/null +++ b/internal/handler/node_health_handler.go @@ -0,0 +1,102 @@ +package handler + +import ( + "context" + "dingoscheduler/internal/dao" + "dingoscheduler/internal/model" + _ "embed" + "errors" + "net/http" + "strconv" + "time" + + "github.com/labstack/echo/v4" + "gorm.io/gorm" +) + +//go:embed node_health.html +var nodeHealthPage string + +// The page fetches node data from the health API. +func NodeHealthPage(c echo.Context) error { + c.Response().Header().Set("Cache-Control", "no-store") + return c.HTML(http.StatusOK, nodeHealthPage) +} + +func (h *ManagerHandler) NodeHealth(c echo.Context) error { + after, err := strconv.ParseInt(defaultQuery(c, "after", "0"), 10, 32) + if err != nil || after < 0 { + return echo.NewHTTPError(http.StatusBadRequest, "invalid after") + } + limit, err := strconv.Atoi(defaultQuery(c, "limit", "100")) + if err != nil || limit < 1 || limit > 200 { + return echo.NewHTTPError(http.StatusBadRequest, "limit must be 1..200") + } + items, err := h.schedulerService.NodeHealth(c.Request().Context(), int32(after), limit) + if err != nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "node status unavailable") + } + var next int32 + if len(items) == limit { + next = items[len(items)-1].ID + } + // Endpoints are optional for old clients; request explicitly for discovery. + var output any = items + if c.QueryParam("endpoints") == "true" { + ids := make([]int32, 0, len(items)) + for _, item := range items { + ids = append(ids, item.ID) + } + endpoints, err := h.schedulerService.NodeEndpoints(c.Request().Context(), ids) + if err != nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "node endpoints unavailable") + } + type discovered struct { + dao.NodeHealthView + model.NodeEndpoint + } + rows := make([]discovered, 0, len(items)) + for _, item := range items { + rows = append(rows, discovered{item, endpoints[item.ID]}) + } + output = rows + } + c.Response().Header().Set("Cache-Control", "no-store") + return c.JSON(http.StatusOK, struct { + Items any `json:"items"` + NextAfter int32 `json:"nextAfter"` + }{output, next}) +} + +func defaultQuery(c echo.Context, key, fallback string) string { + if value := c.QueryParam(key); value != "" { + return value + } + return fallback +} + +func (h *ManagerHandler) NodeRepositories(c echo.Context) error { + id, err := strconv.ParseInt(c.Param("id"), 10, 32) + if err != nil || id < 1 { + return echo.NewHTTPError(http.StatusBadRequest, "invalid node id") + } + after, err := strconv.ParseInt(defaultQuery(c, "after", "0"), 10, 64) + if err != nil || after < 0 { + return echo.NewHTTPError(http.StatusBadRequest, "invalid after") + } + limit, err := strconv.Atoi(defaultQuery(c, "limit", "20")) + if err != nil || limit < 1 || limit > 100 { + return echo.NewHTTPError(http.StatusBadRequest, "limit must be 1..100") + } + ctx, cancel := context.WithTimeout(c.Request().Context(), 5*time.Second) + defer cancel() + page, err := h.schedulerService.NodeRepositories(ctx, int32(id), after, limit) + if errors.Is(err, gorm.ErrRecordNotFound) { + return echo.NewHTTPError(http.StatusNotFound, "node not found") + } + if err != nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "repository records unavailable") + } + c.Response().Header().Set("Cache-Control", "no-store") + return c.JSON(http.StatusOK, page) +} diff --git a/internal/model/dto/dto.go b/internal/model/dto/dto.go index a25e2f7..2723a61 100644 --- a/internal/model/dto/dto.go +++ b/internal/model/dto/dto.go @@ -1,6 +1,9 @@ package dto type Repository struct { + Namespace string `json:"namespace,omitempty"` + Repo string `json:"repo,omitempty"` + Datatype string `json:"repoType,omitempty"` ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"id"` Org string `gorm:"column:org;not null" json:"org"` OrgRepo string `gorm:"column:org_repo;not null" json:"orgRepo"` diff --git a/internal/model/node_endpoint.go b/internal/model/node_endpoint.go new file mode 100644 index 0000000..931521f --- /dev/null +++ b/internal/model/node_endpoint.go @@ -0,0 +1,8 @@ +package model + +// NodeEndpoint extends registration without changing legacy inventory tables. +type NodeEndpoint struct { + NodeID int32 `gorm:"primaryKey" json:"-"` + ManagementURL string `gorm:"size:2048" json:"managementUrl"` + DownloadURL string `gorm:"size:2048" json:"downloadUrl"` +} diff --git a/internal/model/query/legacy_contract_test.go b/internal/model/query/legacy_contract_test.go new file mode 100644 index 0000000..392281a --- /dev/null +++ b/internal/model/query/legacy_contract_test.go @@ -0,0 +1,23 @@ +package query + +import ( + "encoding/json" + "testing" +) + +func TestLegacyPersistRequest(t *testing.T) { + var r PersistRepoReq + if e := json.Unmarshal([]byte(`{"instanceIds":["node-1"],"org":"Qwen","repo":"Qwen2.5-3B-Instruct","offVerify":false}`), &r); e != nil { + t.Fatal(e) + } + if r.Org != "Qwen" || r.Repo != "Qwen2.5-3B-Instruct" || len(r.InstanceIds) != 1 || r.OffVerify { + t.Fatalf("legacy request changed: %+v", r) + } +} +func TestLegacyTaskRequest(t *testing.T) { + var r CreateCacheJobReq + json.Unmarshal([]byte(`{"instanceId":"node-1","datatype":"models","orgRepo":"Qwen/demo","org":"Qwen","repo":"demo","type":1}`), &r) + if r.Org != "Qwen" || r.Repo != "demo" || r.OrgRepo != "Qwen/demo" { + t.Fatalf("legacy task changed: %+v", r) + } +} diff --git a/internal/model/query/query.go b/internal/model/query/query.go index cf01271..76c1c2b 100644 --- a/internal/model/query/query.go +++ b/internal/model/query/query.go @@ -11,6 +11,7 @@ type ModelFileRecordQuery struct { } type CreateCacheJobReq struct { + Namespace string `json:"namespace,omitempty"` Type int32 `json:"type"` AidcCode string `json:"aidcCode"` InstanceId string `json:"instanceId"` @@ -89,6 +90,8 @@ type PathInfoQuery struct { } type PersistRepoReq struct { + Datatype string `json:"datatype"` + Namespace string `json:"namespace"` InstanceIds []string `json:"instanceIds"` Org string `json:"org"` Repo string `json:"repo"` diff --git a/internal/model/upload_inventory.go b/internal/model/upload_inventory.go new file mode 100644 index 0000000..678190c --- /dev/null +++ b/internal/model/upload_inventory.go @@ -0,0 +1,47 @@ +package model + +import "time" + +// UploadInventoryState is the accepted complete-inventory watermark for one +// Speed. It is deliberately separate from dingospeed/repository: heartbeat +// availability and uploaded-file truth have different lifecycles. +type UploadInventoryState struct { + InstanceID string `gorm:"column:instance_id;primaryKey;size:191" json:"instanceId"` + Epoch string `gorm:"column:epoch;size:64;not null" json:"epoch"` + EpochStartedAt time.Time `gorm:"column:epoch_started_at;not null" json:"epochStartedAt"` + LastSequence uint64 `gorm:"column:last_sequence;not null" json:"lastSequence"` + InventoryComplete bool `gorm:"column:inventory_complete;not null" json:"inventoryComplete"` + LastAttemptAt time.Time `gorm:"column:last_attempt_at;not null" json:"lastAttemptAt"` + LastConfirmedAt *time.Time `gorm:"column:last_confirmed_at" json:"lastConfirmedAt,omitempty"` + ErrorMessage string `gorm:"column:error_message;type:text" json:"errorMessage,omitempty"` +} + +func (*UploadInventoryState) TableName() string { return "upload_inventory_state" } + +// UploadInventoryFile is an uploaded-domain identity. IdentityHash avoids a +// MySQL oversized composite index while the individual identity columns remain +// queryable and human-readable. +type UploadInventoryFile struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"id,string"` + IdentityHash string `gorm:"column:identity_hash;size:64;not null;uniqueIndex" json:"-"` + Namespace string `gorm:"column:namespace;size:255;not null;index:idx_upload_repo" json:"namespace"` + RepoType string `gorm:"column:repo_type;size:32;not null;index:idx_upload_repo" json:"repoType"` + Repo string `gorm:"column:repo;size:1024;not null" json:"repo"` + Path string `gorm:"column:path;size:1000;not null" json:"path"` + SHA256 string `gorm:"column:sha256;size:64;not null;index" json:"sha256"` + Size int64 `gorm:"column:size;not null" json:"size"` + CreatedAt time.Time `gorm:"column:created_at;not null;autoCreateTime" json:"createdAt"` + UpdatedAt time.Time `gorm:"column:updated_at;not null;autoUpdateTime" json:"updatedAt"` +} + +func (*UploadInventoryFile) TableName() string { return "upload_inventory_file" } + +type UploadInventoryHolding struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"id,string"` + FileID int64 `gorm:"column:file_id;not null;uniqueIndex:uk_upload_holding;index" json:"fileId,string"` + InstanceID string `gorm:"column:instance_id;size:191;not null;uniqueIndex:uk_upload_holding;index" json:"instanceId"` + Sequence uint64 `gorm:"column:sequence;not null" json:"sequence"` + ConfirmedAt time.Time `gorm:"column:confirmed_at;not null" json:"confirmedAt"` +} + +func (*UploadInventoryHolding) TableName() string { return "upload_inventory_holding" } diff --git a/internal/router/http_router.go b/internal/router/http_router.go index adc90ad..e2847a9 100644 --- a/internal/router/http_router.go +++ b/internal/router/http_router.go @@ -50,11 +50,19 @@ func (r *HttpRouter) GetHandler() *echo.Echo { } func (r *HttpRouter) initRouter() { + r.echo.POST("/internal/v1/official-revisions", r.managerHandler.OfficialRevision) + r.echo.GET("/node-health", handler.NodeHealthPage) + r.echo.GET("/api/v1/nodes/health", r.managerHandler.NodeHealth) + r.echo.GET("/api/v1/nodes/:id/repositories", r.managerHandler.NodeRepositories) // 系统信息 r.echo.GET("/info", r.sysHandler.Info) if config.SysConfig.EnableMetric() { r.echo.GET("/metrics", echo.WrapHandler(promhttp.Handler())) } + r.echo.POST("/api/v1/repositories/ingest", r.managerHandler.IngestRepository) + r.echo.GET("/api/v1/upload-inventory/repositories", r.managerHandler.UploadedRepositories) + r.echo.GET("/api/v1/upload-inventory/files", r.managerHandler.UploadedHoldings) + r.echo.GET("/api/v1/upload-inventory/nodes/:instanceId/files", r.managerHandler.UploadedNodeHoldings) r.echo.POST("/api/persistRepo", r.managerHandler.PersistRepoHandler) // 持久化仓库 r.echo.GET("/api/refreshToken", r.managerHandler.RefreshToken) // 刷新默认token r.echo.POST("/api/execWaitTask", r.managerHandler.ExecWaitTaskHandler) // 执行等待中的缓存下载任务和挂载模型任务 diff --git a/internal/service/authority_service.go b/internal/service/authority_service.go new file mode 100644 index 0000000..71914ec --- /dev/null +++ b/internal/service/authority_service.go @@ -0,0 +1,10 @@ +package service + +import ( + "context" + "dingoscheduler/internal/authority" +) + +func (s *SchedulerService) OfficialRevision(ctx context.Context, r authority.Request) (any, error) { + return authority.New(s.baseData.BizDB).Do(ctx, r) +} diff --git a/internal/service/cache_job_service.go b/internal/service/cache_job_service.go index 20f51f7..e4764db 100644 --- a/internal/service/cache_job_service.go +++ b/internal/service/cache_job_service.go @@ -23,6 +23,7 @@ import ( "dingoscheduler/pkg/common" "dingoscheduler/pkg/consts" myerr "dingoscheduler/pkg/error" + "dingoscheduler/pkg/repository" "dingoscheduler/pkg/util" "github.com/bytedance/sonic" @@ -119,7 +120,11 @@ func (c *CacheJobService) getJobRealtimeStatus(jobIds []int64, instanceId string func (c *CacheJobService) CreateCacheJob(createCacheJobReq *query.CreateCacheJobReq) (*common.Response, error) { zap.S().Debugf("Cache instanceId:%s, %s/%s", createCacheJobReq.InstanceId, createCacheJobReq.Org, createCacheJobReq.Repo) - lock := c.lockDao.GetCacheJobReqLock(createCacheJobReq.OrgRepo) + key, err := repository.FromWire(createCacheJobReq.Datatype, createCacheJobReq.Org, createCacheJobReq.Repo) + if err != nil { + return nil, err + } + lock := c.lockDao.GetCacheJobReqLock(key.LockKey("job", createCacheJobReq.InstanceId, fmt.Sprint(createCacheJobReq.Type))) lock.Lock() defer lock.Unlock() cacheJob, err := c.cacheJobDao.GetCacheJob(&query.CacheJobQuery{InstanceId: createCacheJobReq.InstanceId, Type: createCacheJobReq.Type, @@ -156,6 +161,9 @@ func (c *CacheJobService) StopCacheJob(jobStatusReq *query.JobStatusReq) error { if cacheJob == nil { return myerr.New(fmt.Sprintf("任务不存在。")) } + if cacheJob.InstanceId != jobStatusReq.InstanceId { + return myerr.New("task instance mismatch") + } if cacheJob.Status != consts.RunningStatusJobIng { return myerr.New(fmt.Sprintf("job is not running, Can't be stopped.%d", cacheJob.Status)) } @@ -196,6 +204,9 @@ func (c *CacheJobService) ResumeCacheJob(resumeCacheJobReq *query.ResumeCacheJob if cacheJob == nil { return myerr.New(fmt.Sprintf("job is not exist.jobId:%d", resumeCacheJobReq.Id)) } + if cacheJob.InstanceId != resumeCacheJobReq.InstanceId { + return myerr.New("task instance mismatch") + } if cacheJob.Status != consts.RunningStatusJobBreak && cacheJob.Status != consts.RunningStatusJobStop && cacheJob.Status != consts.RunningStatusJobWait { diff --git a/internal/service/ingest_service.go b/internal/service/ingest_service.go new file mode 100644 index 0000000..b2168a8 --- /dev/null +++ b/internal/service/ingest_service.go @@ -0,0 +1,25 @@ +package service + +import ( + "context" + "dingoscheduler/internal/dao" + pb "dingoscheduler/pkg/proto/manager" +) + +func (s *SchedulerService) IngestRepository(ctx context.Context, req *pb.IngestRepositoryRequest) (*pb.IngestRepositoryResponse, error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + return s.repositoryDao.IngestPublished(ctx, req) +} + +func (s *SchedulerService) UploadedHoldings(ctx context.Context, instanceID, namespace, repoType, repo, path, sha string) ([]dao.UploadedHoldingView, error) { + return s.repositoryDao.ListUploadedHoldings(ctx, instanceID, namespace, repoType, repo, path, sha) +} + +func (s *SchedulerService) UploadedNodeInventoryState(ctx context.Context, instanceID string) (*dao.UploadedNodeInventoryStateView, error) { + return s.repositoryDao.GetUploadedNodeInventoryState(ctx, instanceID) +} + +func (s *SchedulerService) UploadedRepositories(ctx context.Context) ([]dao.UploadedRepositoryView, error) { + return s.repositoryDao.ListUploadedRepositories(ctx) +} diff --git a/internal/service/node_discovery_test.go b/internal/service/node_discovery_test.go new file mode 100644 index 0000000..75af88f --- /dev/null +++ b/internal/service/node_discovery_test.go @@ -0,0 +1,83 @@ +package service_test + +import ( + "context" + "dingoscheduler/internal/dao" + "dingoscheduler/internal/data" + "dingoscheduler/internal/handler" + "dingoscheduler/internal/model" + "dingoscheduler/internal/service" + pb "dingoscheduler/pkg/proto/manager" + "encoding/json" + "github.com/glebarez/sqlite" + "github.com/labstack/echo/v4" + "github.com/patrickmn/go-cache" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "gorm.io/gorm" + "net" + "net/http/httptest" + "testing" + "time" +) + +func TestRegisteredEndpointsSurviveDAORecreationAndAppearInDiscovery(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err = db.AutoMigrate(&model.Dingospeed{}, &model.NodeEndpoint{}); err != nil { + t.Fatal(err) + } + d := &data.BaseData{BizDB: db, Cache: cache.New(time.Minute, time.Minute)} + newService := func() *service.SchedulerService { + return service.NewSchedulerService(d, dao.NewDingospeedDao(d), nil, nil, nil, nil) + } + s := newService() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + rpcServer := grpc.NewServer() + pb.RegisterManagerServer(rpcServer, s) + go func() { _ = rpcServer.Serve(listener) }() + defer rpcServer.Stop() + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + client := pb.NewManagerClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req := &pb.RegisterRequest{InstanceId: "external-node", Host: "speed", Port: 8090, ManagementUrl: "http://speed:8091", DownloadUrl: "http://speed:8090"} + first, err := client.Register(ctx, req) + if err != nil { + t.Fatal(err) + } + req.ManagementUrl = "http://new-entry:18091" + second, err := client.Register(ctx, req) + if err != nil || first.Id != second.Id { + t.Fatalf("idempotent registration: %v %v", second, err) + } + s = newService() + h := handler.NewManagerHandler(s, nil, nil, nil) + rec := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/api/v1/nodes/health?endpoints=true", nil) + if err = h.NodeHealth(echo.New().NewContext(request, rec)); err != nil { + t.Fatal(err) + } + var out struct { + Items []struct { + InstanceID string `json:"instanceId"` + ManagementURL string `json:"managementUrl"` + Communication string `json:"communication"` + } + } + if err = json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if len(out.Items) != 1 || out.Items[0].ManagementURL != req.ManagementUrl || out.Items[0].Communication != "connected" { + t.Fatalf("discovery: %s", rec.Body.String()) + } +} diff --git a/internal/service/node_health_integration_test.go b/internal/service/node_health_integration_test.go new file mode 100644 index 0000000..ebb2970 --- /dev/null +++ b/internal/service/node_health_integration_test.go @@ -0,0 +1,261 @@ +package service_test + +import ( + "context" + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "dingoscheduler/internal/dao" + "dingoscheduler/internal/data" + "dingoscheduler/internal/handler" + "dingoscheduler/internal/model" + "dingoscheduler/internal/router" + "dingoscheduler/internal/service" + "dingoscheduler/pkg/config" + pb "dingoscheduler/pkg/proto/manager" + "dingoscheduler/pkg/util" + "github.com/labstack/echo/v4" + "github.com/patrickmn/go-cache" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// Exercise the real gRPC -> service -> DAO -> SQL and HTTP handler chain using +// an in-memory SQL connector. It cannot open a network/database connection. +type healthStore struct { + sync.Mutex + rows map[int64]*healthRow + fail bool +} +type healthRow struct { + instance string + online bool + received time.Time + report driver.Value +} +type healthConnector struct{ store *healthStore } + +func (c healthConnector) Connect(context.Context) (driver.Conn, error) { + return &healthConn{c.store}, nil +} +func (c healthConnector) Driver() driver.Driver { return healthDriver{} } + +type healthDriver struct{} + +func (healthDriver) Open(string) (driver.Conn, error) { + return nil, errors.New("network databases forbidden in health test") +} + +type healthConn struct{ store *healthStore } + +func (c *healthConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("unexpected prepare") +} +func (c *healthConn) Close() error { return nil } +func (c *healthConn) Begin() (driver.Tx, error) { return nil, errors.New("unexpected transaction") } +func (c *healthConn) ExecContext(_ context.Context, q string, a []driver.NamedValue) (driver.Result, error) { + c.store.Lock() + defer c.store.Unlock() + if c.store.fail { + return nil, errors.New("simulated database unavailable") + } + if q != "UPDATE dingospeed SET updated_at = ? WHERE id = ? AND instance_id = ? AND online = ?" { + return nil, fmt.Errorf("unexpected SQL: %s", q) + } + id := a[1].Value.(int64) + r := c.store.rows[id] + if r == nil || r.instance != a[2].Value || r.online != a[3].Value { + return driver.RowsAffected(0), nil + } + r.received = a[0].Value.(time.Time) + return driver.RowsAffected(1), nil +} + +type healthRows struct { + columns []string + values [][]driver.Value + pos int +} + +func (r *healthRows) Columns() []string { return r.columns } +func (r *healthRows) Close() error { return nil } +func (r *healthRows) Next(v []driver.Value) error { + if r.pos >= len(r.values) { + return io.EOF + } + copy(v, r.values[r.pos]) + r.pos++ + return nil +} +func (c *healthConn) QueryContext(_ context.Context, q string, a []driver.NamedValue) (driver.Rows, error) { + c.store.Lock() + defer c.store.Unlock() + if c.store.fail { + return nil, errors.New("simulated database unavailable") + } + if strings.Contains(q, "count(*)") { + var n int64 + r := c.store.rows[a[0].Value.(int64)] + if r != nil && r.instance == a[1].Value && r.online == a[2].Value { + n = 1 + } + return &healthRows{columns: []string{"count(*)"}, values: [][]driver.Value{{n}}}, nil + } + if strings.Contains(q, "health_snapshot") || !strings.Contains(q, "ORDER BY id ASC LIMIT ?") { + return nil, fmt.Errorf("unexpected SQL: %s", q) + } + rows := &healthRows{columns: []string{"id", "instance_id", "online", "updated_at"}} + after, limit := a[0].Value.(int64), a[1].Value.(int64) + for id := int64(1); id <= 3; id++ { + if r := c.store.rows[id]; r != nil && id > after && int64(len(rows.values)) < limit { + rows.values = append(rows.values, []driver.Value{id, r.instance, r.online, r.received}) + } + } + return rows, nil +} + +func TestNodeHealthHeartbeatToHTTP(t *testing.T) { + old := config.SysConfig + config.SysConfig = &config.Config{} + t.Cleanup(func() { config.SysConfig = old }) + now := time.Now().Truncate(time.Second) + store := &healthStore{rows: map[int64]*healthRow{ + 1: {instance: "hd-05", online: true, received: now}, + 2: {instance: "hd-05", online: false, received: now}, + 3: {instance: "legacy", online: true, received: now.Add(-10 * time.Minute)}, + }} + sqlDB := sql.OpenDB(healthConnector{store}) + defer sqlDB.Close() + db, err := gorm.Open(mysql.New(mysql.Config{Conn: sqlDB, SkipInitializeWithVersion: true}), &gorm.Config{DisableAutomaticPing: true, Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + base := &data.BaseData{BizDB: db, Cache: cache.New(time.Hour, time.Hour)} + for id, r := range store.rows { + base.Cache.Set(util.GetSpeedKey(r.instance, r.online), &model.Dingospeed{ID: int32(id), InstanceID: r.instance, Online: r.online}, cache.NoExpiration) + } + svc := service.NewSchedulerService(base, dao.NewDingospeedDao(base), nil, nil, nil, nil) + lis := bufconn.Listen(64 * 1024) + gs := grpc.NewServer() + pb.RegisterManagerServer(gs, svc) + go gs.Serve(lis) + defer gs.Stop() + conn, err := grpc.NewClient("passthrough:///local-health", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) })) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + client := pb.NewManagerClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + h := &pb.NodeHealthSnapshot{Version: 1, ProcessId: "boot-1", StartedAt: now.Unix() - 10, CollectedAt: now.Unix(), HeartbeatPeriodSeconds: 5, + Capabilities: []*pb.CapabilityObservation{{Capability: "metadata_read", State: 2, Unresolved: true, LastObservation: now.Unix(), LastFailure: now.Unix()}, {Capability: "metadata_write"}}, + Errors: []*pb.StorageErrorCount{{Operation: "read", Kind: "mount_disconnected", Count: 12}}, + } + req := &pb.HeartbeatRequest{Id: 1, InstanceId: "hd-05", Online: true, Health: h} + if _, err = client.Heartbeat(ctx, req); err != nil { + t.Fatal(err) + } + fresh := dao.NewDingospeedDao(base) + views, err := fresh.ListNodeHealth(ctx, 0, 10, time.Now()) + if err != nil { + t.Fatal(err) + } + if views[0].Reported { + t.Fatal("health unexpectedly survived DAO restart") + } + e := echo.New() + manager := handler.NewManagerHandler(svc, nil, nil, nil) + router.NewHttpRouter(e, manager, &handler.SysHandler{}, &handler.RepositoryHandler{}, &handler.TagHandler{}, &handler.CacheJobHandler{}) + get := func(query string) *httptest.ResponseRecorder { + r := httptest.NewRequest(http.MethodGet, "/api/v1/nodes/health"+query, nil) + w := httptest.NewRecorder() + e.ServeHTTP(w, r) + return w + } + if w := get(""); w.Code != 200 { + t.Fatalf("status endpoint without token: %d", w.Code) + } + w := get("?limit=1") + if w.Code != 200 { + t.Fatalf("status=%d body=%s", w.Code, w.Body) + } + var page struct { + Items []dao.NodeHealthView `json:"items"` + NextAfter int32 `json:"nextAfter"` + } + if err = json.Unmarshal(w.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.NextAfter != 1 || page.Items[0].Communication != "connected" || page.Items[0].Capabilities[0].Status != "unresolved" { + t.Fatalf("bad page: %+v", page) + } + if !strings.Contains(string(page.Items[0].Snapshot), `"count":"12"`) { + t.Fatalf("counter not exposed losslessly: %s", page.Items[0].Snapshot) + } + if w = get("?after=1"); w.Code != 200 || strings.Contains(w.Body.String(), "boot-1") { + t.Fatalf("pagination mixed node snapshots: %s", w.Body) + } + if w = get("?limit=201"); w.Code != 400 { + t.Fatal("unbounded list accepted") + } + // Same instance online/offline processes remain independent. + req.Online = false + if _, err = client.Heartbeat(ctx, req); status.Code(err) != codes.NotFound { + t.Fatalf("wrong mode accepted: %v", err) + } + req.Online = true + req.Health.Version = 99 + if _, err = client.Heartbeat(ctx, req); status.Code(err) != codes.InvalidArgument { + t.Fatalf("invalid version accepted: %v", err) + } + req.Health.Version = 1 + // Storage failure does not update communication timestamp or snapshot. + store.Lock() + store.fail = true + before := store.rows[1].received + store.Unlock() + if _, err = client.Heartbeat(ctx, req); err == nil { + t.Fatal("database failure acknowledged") + } + if w = get(""); w.Code != 503 { + t.Fatal("failed query displayed healthy") + } + store.Lock() + store.fail = false + if store.rows[1].received != before { + t.Fatal("failed write changed heartbeat time") + } + store.Unlock() + // A fresh DAO loses health by design; no health_snapshot SQL column. + items, err := dao.NewDingospeedDao(base).ListNodeHealth(ctx, 0, 100, time.Now()) + if err != nil || items[0].Reported { + t.Fatalf("snapshot unexpectedly persisted through DAO: %v", err) + } + // A legacy heartbeat clears stale health instead of inheriting old success/failure. + req.Health = nil + if _, err = client.Heartbeat(ctx, req); err != nil { + t.Fatal(err) + } + items, err = svc.NodeHealth(ctx, 0, 100) + if err != nil || items[0].Reported || items[2].Communication != "disconnected" { + t.Fatalf("legacy/disconnect handling: %+v %v", items, err) + } +} diff --git a/internal/service/repository_service.go b/internal/service/repository_service.go index 96769c5..fd93a2e 100644 --- a/internal/service/repository_service.go +++ b/internal/service/repository_service.go @@ -19,6 +19,7 @@ import ( "io" "net/http" "net/url" + "strings" "sync" "dingoscheduler/internal/dao" @@ -30,6 +31,7 @@ import ( "dingoscheduler/pkg/config" "dingoscheduler/pkg/consts" myerr "dingoscheduler/pkg/error" + repokey "dingoscheduler/pkg/repository" "dingoscheduler/pkg/util" "github.com/bytedance/sonic" @@ -59,7 +61,7 @@ func NewRepositoryService(dingospeedDao *dao.DingospeedDao, organizationDao: organizationDao, tagDao: tagDao, hfTokenDao: hfTokenDao, - client: &http.Client{}, + client: &http.Client{CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }}, } } @@ -76,13 +78,14 @@ func (s *RepositoryService) RepositoryList(query *query.ModelQuery) ([]*dto.Repo for _, item := range repositories { var repo dto.Repository gocopy.Copy(&repo, &item) - if icon, err := s.organizationDao.GetOrganization(repo.Org); err != nil { + if icon, err := s.organizationDao.GetOrganization(upstreamLogoOrg(repo.Org, repo.Repo)); err != nil { return nil, 0, err } else { if icon != "" { repo.Icon = fmt.Sprintf("%s%s", config.SysConfig.Oss.Path, icon) } } + setRepositoryIdentity(&repo) repos = append(repos, &repo) } return repos, size, nil @@ -102,53 +105,41 @@ func (s *RepositoryService) GetRepositoryById(id int64) (*dto.Repository, error) for _, tag := range tags { repo.Tags = append(repo.Tags, tag.Label) } - if icon, err := s.organizationDao.GetOrganization(repository.Org); err != nil { + if icon, err := s.organizationDao.GetOrganization(upstreamLogoOrg(repository.Org, repository.Repo)); err != nil { return nil, err } else { if icon != "" { repo.Icon = fmt.Sprintf("%s%s", config.SysConfig.Oss.Path, icon) } } + setRepositoryIdentity(&repo) return &repo, nil } func (s *RepositoryService) RepositoryCardById(c echo.Context, instanceId string, id int64) (*common.Response, error) { - cardKey := util.GetCardKey(instanceId, id) - var commResp *common.Response - if v, ok := s.baseData.Cache.Get(cardKey); ok { - commResp = v.(*common.Response) - s.baseData.Cache.Set(cardKey, commResp, config.SysConfig.GetCacheExpiration()) - } else { - targetURL, repository, err := s.getRepository(instanceId, id) - if err != nil { - return nil, err - } - prefix := string(consts.RepoTypeModel) - if repository.Datatype == string(consts.RepoTypeDataset) { - prefix = string(consts.RepoTypeDataset) - } - forwardURL := fmt.Sprintf("%s/%s/%s/resolve/%s/README.md", targetURL.String(), prefix, repository.OrgRepo, repository.Sha) - resp, err := s.requestForward(c, targetURL, forwardURL) - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("读取响应体失败: %v", err) - } - headers := make(map[string]interface{}, 0) - for key, values := range resp.Header { - headers[key] = values - } - commResp = &common.Response{ - StatusCode: resp.StatusCode, - Headers: headers, - Body: body, - } - s.baseData.Cache.Set(cardKey, commResp, config.SysConfig.GetCacheExpiration()) + targetURL, repository, err := s.getRepository(instanceId, id) + if err != nil { + return nil, err + } + uri, err := storageAPIKey(repository.Datatype, repository.Org, repository.Repo).OperationURI("file", repository.Sha, "README.md") + if err != nil { + return nil, err } - return commResp, nil + resp, err := s.requestForward(c, targetURL, targetURL.String()+uri) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + headers := make(map[string]interface{}) + for key, values := range resp.Header { + headers[key] = values + } + // Always pass through DingoSpeed authorization, even for a cached README. + return &common.Response{StatusCode: resp.StatusCode, Headers: headers, Body: body}, nil } func (s *RepositoryService) RepositoryFilesById(c echo.Context, instanceId string, id int64, filePath string) error { @@ -156,14 +147,11 @@ func (s *RepositoryService) RepositoryFilesById(c echo.Context, instanceId strin if err != nil { return err } - prefix := string(consts.RepoTypeModel) - if repository.Datatype == string(consts.RepoTypeDataset) { - prefix = string(consts.RepoTypeDataset) - } - forwardURL := fmt.Sprintf("%s/api/%s/%s/files/%s/", targetURL.String(), prefix, repository.OrgRepo, repository.Sha) - if filePath != "" { - forwardURL += filePath + uri, err := storageAPIKey(repository.Datatype, repository.Org, repository.Repo).OperationURI("files", repository.Sha, filePath) + if err != nil { + return err } + forwardURL := targetURL.String() + uri resp, err := s.requestForward(c, targetURL, forwardURL) if err != nil { return err @@ -270,3 +258,27 @@ func (s *RepositoryService) MountRepository(repoReq *query.RepositoryReq) error } return nil } + +func upstreamLogoOrg(org, repo string) string { + if strings.Contains(org, "/") { + return "" + } + return org +} +func storageAPIKey(repoType, org, repo string) repokey.Key { + k, _ := repokey.FromWire(repoType, org, repo) + return k +} + +func setRepositoryIdentity(r *dto.Repository) { + k, e := repokey.FromWire(r.Datatype, r.Org, r.Repo) + if e != nil { + return + } + r.Namespace = k.Namespace + if strings.HasPrefix(r.Org, "dingo-local/") { + r.Org = k.Namespace + r.Repo = k.Repo + r.OrgRepo = k.ID() + } +} diff --git a/internal/service/scheduler_service.go b/internal/service/scheduler_service.go index af48ee3..4f1374d 100644 --- a/internal/service/scheduler_service.go +++ b/internal/service/scheduler_service.go @@ -17,6 +17,7 @@ package service import ( "context" "fmt" + "net/url" "sync" "time" @@ -28,10 +29,14 @@ import ( "dingoscheduler/pkg/config" "dingoscheduler/pkg/consts" myerr "dingoscheduler/pkg/error" + "dingoscheduler/pkg/nodehealth" pb "dingoscheduler/pkg/proto/manager" + "dingoscheduler/pkg/repository" "dingoscheduler/pkg/util" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" ) @@ -48,6 +53,7 @@ type SchedulerService struct { repositoryDao *dao.RepositoryDao cacheJobDao *dao.CacheJobDao scheudlerLock sync.Mutex + writeMu sync.Mutex } func NewSchedulerService( @@ -69,6 +75,15 @@ func NewSchedulerService( } func (s *SchedulerService) Register(ctx context.Context, req *pb.RegisterRequest) (*pb.RegisterResponse, error) { + for _, raw := range []string{req.ManagementUrl, req.DownloadUrl} { + if raw == "" { + continue + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || len(raw) > 2048 { + return nil, status.Error(codes.InvalidArgument, "invalid advertised URL") + } + } if req.InstanceId == "" || req.Host == "" || req.Port <= 0 { return nil, fmt.Errorf("invalid parameter") } @@ -97,6 +112,11 @@ func (s *SchedulerService) Register(ctx context.Context, req *pb.RegisterRequest } dingospeed.ID = int32(id) } + if req.ManagementUrl != "" || req.DownloadUrl != "" { + if err := s.dingospeedDao.SaveNodeEndpoint(ctx, dingospeed.ID, req.ManagementUrl, req.DownloadUrl); err != nil { + return nil, err + } + } s.updateCache(req.InstanceId, req.Online) zap.S().Infof("register success.instanceId:%s, host:%s, port:%d, online:%v", req.InstanceId, req.Host, req.Port, req.Online) return &pb.RegisterResponse{ @@ -106,8 +126,12 @@ func (s *SchedulerService) Register(ctx context.Context, req *pb.RegisterRequest } func (s *SchedulerService) Heartbeat(ctx context.Context, req *pb.HeartbeatRequest) (*emptypb.Empty, error) { + encoded, err := nodehealth.Encode(req.Health) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } if req.Id > 0 { - err := s.dingospeedDao.HeartbeatUpdate(req.Id) + err := s.dingospeedDao.HeartbeatHealth(ctx, req, encoded, time.Now()) if err != nil { return nil, err } @@ -115,7 +139,19 @@ func (s *SchedulerService) Heartbeat(ctx context.Context, req *pb.HeartbeatReque } else { return nil, myerr.New(fmt.Sprintf("speed id is unlawful.id = %d", req.Id)) } - return nil, nil + return &emptypb.Empty{}, nil +} + +func (s *SchedulerService) NodeHealth(ctx context.Context, after int32, limit int) ([]dao.NodeHealthView, error) { + return s.dingospeedDao.ListNodeHealth(ctx, after, limit, time.Now()) +} + +func (s *SchedulerService) NodeEndpoints(ctx context.Context, ids []int32) (map[int32]model.NodeEndpoint, error) { + return s.dingospeedDao.NodeEndpoints(ctx, ids) +} + +func (s *SchedulerService) NodeRepositories(ctx context.Context, nodeID int32, after int64, limit int) (*dao.NodeRepositoryPage, error) { + return s.dingospeedDao.ListNodeRepositories(ctx, nodeID, after, limit) } func (s *SchedulerService) updateCache(instanceId string, online bool) { @@ -165,7 +201,19 @@ func (s *SchedulerService) getApiLock(apiPath string) *sync.RWMutex { } func (s *SchedulerService) SchedulerFile(ctx context.Context, req *pb.SchedulerFileRequest) (*pb.SchedulerFileResponse, error) { - schedulerFilePath := fmt.Sprintf("scheduler/%s/%s/%s/%s", req.DataType, req.Org, req.Repo, req.Etag) + s.writeMu.Lock() + defer s.writeMu.Unlock() + key, err := repository.FromWire(req.DataType, req.Org, req.Repo) + if err != nil { + return nil, err + } + if req.InstanceId == "" || req.Etag == "" { + return nil, fmt.Errorf("instance and etag are required") + } + if err := repository.ValidatePath(req.Name, true); err != nil { + return nil, err + } + schedulerFilePath := key.LockKey("scheduler", req.Name, req.Etag) lock := s.getApiLock(schedulerFilePath) lock.Lock() defer lock.Unlock() @@ -291,7 +339,25 @@ func (s *SchedulerService) SyncFileProcess(ctx context.Context, req *pb.SyncFile } func (s *SchedulerService) SingleFileProcess(processEntry *pb.FileProcessEntry) (*emptypb.Empty, error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + key, err := repository.FromWire(processEntry.DataType, processEntry.Org, processEntry.Repo) + if err != nil { + return nil, err + } + if processEntry.InstanceId == "" || processEntry.Etag == "" { + return nil, fmt.Errorf("instance and etag are required") + } + if err := repository.ValidatePath(processEntry.Name, true); err != nil { + return nil, err + } + lock := s.getApiLock(key.LockKey("scheduler", processEntry.Name, processEntry.Etag)) + lock.Lock() + defer lock.Unlock() if processEntry.ProcessId != 0 { + if err := s.modelFileProcessDao.ValidateProcessIdentity(processEntry); err != nil { + return nil, err + } if err := s.modelFileProcessDao.ReportFileProcess(&pb.FileProcessRequest{ ProcessId: processEntry.ProcessId, StaPos: processEntry.StartPos, @@ -364,6 +430,14 @@ func (s *SchedulerService) ReportFileProcess(ctx context.Context, req *pb.FilePr } func (s *SchedulerService) DeleteByEtagsAndFields(ctx context.Context, req *pb.DeleteByEtagsAndFieldsRequest) (*emptypb.Empty, error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + if _, err := repository.FromWire(req.Datatype, req.Org, req.Repo); err != nil { + return nil, err + } + if req.InstanceID == "" { + return nil, fmt.Errorf("instance is required") + } recordIds, err := s.modelFileRecordDao.GetIDsByEtagsOrFields(req.Etag, req.Datatype, req.Org, req.Repo, req.Name) if err != nil { return nil, fmt.Errorf("查询recordIds失败: %w", err) @@ -387,6 +461,12 @@ func (s *SchedulerService) DeleteByEtagsAndFields(ctx context.Context, req *pb.D } func (s *SchedulerService) CreateCacheJob(ctx context.Context, req *pb.CreateCacheJobReq) (*pb.CreateCacheJobResp, error) { + if _, err := repository.FromWire(req.Datatype, req.Org, req.Repo); err != nil { + return nil, err + } + if req.InstanceId == "" { + return nil, fmt.Errorf("instance is required") + } cacheJob := &model.CacheJob{ Type: req.Type, InstanceId: req.InstanceId, diff --git a/model_import/model_import.go b/model_import/model_import.go index 5f9e990..adfa814 100644 --- a/model_import/model_import.go +++ b/model_import/model_import.go @@ -264,6 +264,12 @@ func processDirectory(rootPath string) ([]FileInfo, error) { } components := strings.Split(relPath, string(filepath.Separator)) + if len(components) >= 3 && components[2] == "dingo-local" { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } if len(components) < 5 || components[0] != "api" || !(components[1] == "models" || components[1] == "datasets" || components[1] == "spaces") || diff --git a/pkg/nodehealth/health.go b/pkg/nodehealth/health.go new file mode 100644 index 0000000..50e2efa --- /dev/null +++ b/pkg/nodehealth/health.go @@ -0,0 +1,105 @@ +// Package nodehealth validates transported observations, not node readiness. +package nodehealth + +import ( + "encoding/json" + "fmt" + "time" + + pb "dingoscheduler/pkg/proto/manager" + "google.golang.org/protobuf/encoding/protojson" +) + +func Encode(h *pb.NodeHealthSnapshot) (*string, error) { + if h == nil { + return nil, nil + } + if h.Version != 1 || h.ProcessId == "" || len(h.ProcessId) > 128 || h.StartedAt <= 0 || h.CollectedAt < h.StartedAt || h.HeartbeatPeriodSeconds == 0 || h.HeartbeatPeriodSeconds > 86400 { + return nil, fmt.Errorf("invalid health snapshot header") + } + if len(h.Capabilities) != 2 || len(h.Errors) > 16 { + return nil, fmt.Errorf("invalid health observation count") + } + seen := map[string]bool{} + for _, c := range h.Capabilities { + if c == nil || (c.Capability != "metadata_read" && c.Capability != "metadata_write") || seen[c.Capability] || c.State < 0 || c.State > 4 || c.LastObservation < 0 || c.LastFailure < 0 || c.LastObservation > h.CollectedAt || c.LastFailure > h.CollectedAt { + return nil, fmt.Errorf("invalid capability observation") + } + seen[c.Capability] = true + } + seen = map[string]bool{} + for _, e := range h.Errors { + if e == nil { + return nil, fmt.Errorf("invalid error count") + } + if e.Operation != "stat" && e.Operation != "read" && e.Operation != "mkdir" && e.Operation != "write" { + return nil, fmt.Errorf("invalid error operation") + } + if e.Kind != "permission_denied" && e.Kind != "mount_disconnected" && e.Kind != "io_failure" && e.Kind != "path_unavailable" { + return nil, fmt.Errorf("invalid error kind") + } + key := e.Operation + ":" + e.Kind + if seen[key] { + return nil, fmt.Errorf("duplicate error count") + } + seen[key] = true + } + encoded, err := (protojson.MarshalOptions{UseProtoNames: true, EmitUnpopulated: true}).Marshal(h) + if err != nil { + return nil, err + } + value := string(encoded) + return &value, nil +} + +type CapabilityView struct { + Capability string `json:"capability"` + Status string `json:"status"` + Unresolved bool `json:"unresolved"` + LastObservation int64 `json:"lastObservation"` + LastFailure int64 `json:"lastFailure"` +} + +type View struct { + Communication string `json:"communication"` + ReceivedAt time.Time `json:"receivedAt"` + Reported bool `json:"reported"` + ReportError bool `json:"reportError"` + Snapshot json.RawMessage `json:"snapshot,omitempty"` + Capabilities []CapabilityView `json:"capabilities"` +} + +func Present(encoded *string, received, now time.Time) View { + v := View{Communication: "connected", ReceivedAt: received, Capabilities: []CapabilityView{}} + maxAge := 5 * time.Minute + if encoded != nil { + h := &pb.NodeHealthSnapshot{} + err := protojson.Unmarshal([]byte(*encoded), h) + if err == nil { + _, err = Encode(h) + } + if err != nil { + v.ReportError = true + } else { + v.Reported, v.Snapshot = true, json.RawMessage(*encoded) + maxAge = max(30*time.Second, time.Duration(h.HeartbeatPeriodSeconds)*3*time.Second) + for _, c := range h.Capabilities { + status := "unknown" + // Transport age uses scheduler time; evidence age uses speed-relative + // timestamps, avoiding dependence on synchronized node clocks. + evidenceAge := h.CollectedAt - c.LastObservation + fresh := evidenceAge <= 120 && max(time.Duration(0), now.Sub(received)) <= time.Duration(120-evidenceAge)*time.Second + if c.Unresolved { + status = "unresolved" + } else if c.LastObservation > 0 && fresh && c.State == 1 { + status = "observed_healthy" + } + v.Capabilities = append(v.Capabilities, CapabilityView{c.Capability, status, c.Unresolved, c.LastObservation, c.LastFailure}) + } + } + } + if received.IsZero() || now.Sub(received) > maxAge { + v.Communication = "disconnected" + } + return v +} diff --git a/pkg/nodehealth/health_test.go b/pkg/nodehealth/health_test.go new file mode 100644 index 0000000..769d8fd --- /dev/null +++ b/pkg/nodehealth/health_test.go @@ -0,0 +1,84 @@ +package nodehealth + +import ( + pb "dingoscheduler/pkg/proto/manager" + "testing" + "time" +) + +func fixture(now time.Time) *pb.NodeHealthSnapshot { + return &pb.NodeHealthSnapshot{Version: 1, ProcessId: "process-1", StartedAt: now.Unix() - 60, CollectedAt: now.Unix(), HeartbeatPeriodSeconds: 5, + Capabilities: []*pb.CapabilityObservation{ + {Capability: "metadata_read", State: 2, Unresolved: true, LastObservation: now.Unix(), LastFailure: now.Unix()}, + {Capability: "metadata_write", State: 1, LastObservation: now.Unix()}, + }, Errors: []*pb.StorageErrorCount{{Operation: "read", Kind: "mount_disconnected", Count: 12}}, + } +} + +func TestPresentationTimeline(t *testing.T) { + now := time.Now().Truncate(time.Second) + h := fixture(now) + encoded, err := Encode(h) + if err != nil { + t.Fatal(err) + } + v := Present(encoded, now, now) + if v.Communication != "connected" || v.Capabilities[0].Status != "unresolved" || v.Capabilities[1].Status != "observed_healthy" { + t.Fatalf("unexpected view: %+v", v) + } + if v = Present(encoded, now, now.Add(31*time.Second)); v.Communication != "disconnected" || !v.Capabilities[0].Unresolved { + t.Fatalf("lost failure on disconnect: %+v", v) + } + // New heartbeat after communication recovery still carries unresolved storage. + h.CollectedAt = now.Add(time.Minute).Unix() + encoded, _ = Encode(h) + v = Present(encoded, now.Add(time.Minute), now.Add(time.Minute)) + if v.Communication != "connected" || !v.Capabilities[0].Unresolved { + t.Fatal("communication recovery cleared storage failure") + } + // No business traffic: a fresh heartbeat cannot refresh business evidence. + h.CollectedAt = now.Add(3 * time.Minute).Unix() + encoded, _ = Encode(h) + v = Present(encoded, now.Add(3*time.Minute), now.Add(3*time.Minute)) + if v.Capabilities[1].Status != "unknown" || v.Capabilities[0].Status != "unresolved" { + t.Fatalf("stale evidence became healthy: %+v", v) + } + h.ProcessId = "process-2" + h.StartedAt = h.CollectedAt + h.Errors = nil + for _, c := range h.Capabilities { + c.State = 0 + c.Unresolved = false + c.LastObservation = 0 + c.LastFailure = 0 + } + encoded, _ = Encode(h) + v = Present(encoded, now, now) + if v.Capabilities[0].Status != "unknown" { + t.Fatal("restart treated as recovery") + } + if v = Present(nil, now, now); v.Reported || v.Communication != "connected" { + t.Fatal("legacy node not handled") + } + bad := "{broken" + if !Present(&bad, now, now).ReportError { + t.Fatal("corrupt storage silently accepted") + } +} + +func TestInvalidSnapshots(t *testing.T) { + for _, mutate := range []func(*pb.NodeHealthSnapshot){ + func(h *pb.NodeHealthSnapshot) { h.Version = 2 }, + func(h *pb.NodeHealthSnapshot) { h.HeartbeatPeriodSeconds = 0 }, + func(h *pb.NodeHealthSnapshot) { h.Capabilities[0].State = 99 }, + func(h *pb.NodeHealthSnapshot) { h.Capabilities[1].Capability = h.Capabilities[0].Capability }, + func(h *pb.NodeHealthSnapshot) { h.Errors[0].Kind = "raw_secret_error" }, + func(h *pb.NodeHealthSnapshot) { h.Errors = append(h.Errors, h.Errors[0]) }, + } { + h := fixture(time.Now()) + mutate(h) + if _, err := Encode(h); err == nil { + t.Fatal("invalid snapshot accepted") + } + } +} diff --git a/pkg/proto/manager.proto b/pkg/proto/manager.proto index 2813c76..286db6c 100644 --- a/pkg/proto/manager.proto +++ b/pkg/proto/manager.proto @@ -5,8 +5,11 @@ import "google/protobuf/empty.proto"; option go_package = ".;manager"; +// Existing RPC org/repo keep legacy remote coordinates; hosted org is dingo-local/namespace. + // 用户服务定义 service Manager { + rpc IngestRepository (IngestRepositoryRequest) returns (IngestRepositoryResponse); // 启动注册方法 rpc Register (RegisterRequest) returns (RegisterResponse) {}; // 心跳方法 @@ -34,6 +37,8 @@ message RegisterRequest { string host = 2; int32 port = 3; bool online = 4; + string managementUrl = 5; + string downloadUrl = 6; } // 注册响应 @@ -47,6 +52,32 @@ message HeartbeatRequest { int32 id = 1; string instanceId = 2; bool online = 3; + NodeHealthSnapshot health = 4; +} + +// In-memory observations only; absence means the client does not report health. +message NodeHealthSnapshot { + uint32 version = 1; + string process_id = 2; + int64 started_at = 3; + int64 collected_at = 4; + uint32 heartbeat_period_seconds = 5; + repeated CapabilityObservation capabilities = 6; + repeated StorageErrorCount errors = 7; +} + +message CapabilityObservation { + string capability = 1; + int32 state = 2; // 0 unknown, 1 observed healthy, 2 suspect, 3 degraded, 4 recovering + bool unresolved = 3; + int64 last_observation = 4; + int64 last_failure = 5; +} + +message StorageErrorCount { + string operation = 1; + string kind = 2; + uint64 count = 3; // Cumulative within process_id, not an active-failure count. } message SchedulerFileRequest { @@ -135,4 +166,15 @@ message UpdateRepositoryMountStatusReq{ int64 id = 1; int32 status = 2; string errorMsg = 3; -} \ No newline at end of file +} + +message IngestRepositoryRequest { + string instance_id = 1; + string namespace = 2; + string repo_type = 3; + string repo = 4; + string revision = 5; + string commit = 6; + bool online = 7; +} +message IngestRepositoryResponse { int64 repository_id = 1; string commit = 2; int64 file_count = 3; int64 used_storage = 4; } diff --git a/pkg/proto/manager/manager.pb.go b/pkg/proto/manager/manager.pb.go index 2d47159..c094ab3 100644 --- a/pkg/proto/manager/manager.pb.go +++ b/pkg/proto/manager/manager.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.5 -// protoc v5.29.3 +// protoc-gen-go v1.36.6 +// protoc v7.35.1 // source: manager.proto package manager @@ -29,6 +29,8 @@ type RegisterRequest struct { Host string `protobuf:"bytes,2,opt,name=host,proto3" json:"host,omitempty"` Port int32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` Online bool `protobuf:"varint,4,opt,name=online,proto3" json:"online,omitempty"` + ManagementUrl string `protobuf:"bytes,5,opt,name=managementUrl,proto3" json:"managementUrl,omitempty"` + DownloadUrl string `protobuf:"bytes,6,opt,name=downloadUrl,proto3" json:"downloadUrl,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -91,6 +93,20 @@ func (x *RegisterRequest) GetOnline() bool { return false } +func (x *RegisterRequest) GetManagementUrl() string { + if x != nil { + return x.ManagementUrl + } + return "" +} + +func (x *RegisterRequest) GetDownloadUrl() string { + if x != nil { + return x.DownloadUrl + } + return "" +} + // 注册响应 type RegisterResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -150,6 +166,7 @@ type HeartbeatRequest struct { Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` InstanceId string `protobuf:"bytes,2,opt,name=instanceId,proto3" json:"instanceId,omitempty"` Online bool `protobuf:"varint,3,opt,name=online,proto3" json:"online,omitempty"` + Health *NodeHealthSnapshot `protobuf:"bytes,4,opt,name=health,proto3" json:"health,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -205,6 +222,242 @@ func (x *HeartbeatRequest) GetOnline() bool { return false } +func (x *HeartbeatRequest) GetHealth() *NodeHealthSnapshot { + if x != nil { + return x.Health + } + return nil +} + +// In-memory observations only; absence means the client does not report health. +type NodeHealthSnapshot struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + ProcessId string `protobuf:"bytes,2,opt,name=process_id,json=processId,proto3" json:"process_id,omitempty"` + StartedAt int64 `protobuf:"varint,3,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + CollectedAt int64 `protobuf:"varint,4,opt,name=collected_at,json=collectedAt,proto3" json:"collected_at,omitempty"` + HeartbeatPeriodSeconds uint32 `protobuf:"varint,5,opt,name=heartbeat_period_seconds,json=heartbeatPeriodSeconds,proto3" json:"heartbeat_period_seconds,omitempty"` + Capabilities []*CapabilityObservation `protobuf:"bytes,6,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + Errors []*StorageErrorCount `protobuf:"bytes,7,rep,name=errors,proto3" json:"errors,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NodeHealthSnapshot) Reset() { + *x = NodeHealthSnapshot{} + mi := &file_manager_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NodeHealthSnapshot) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeHealthSnapshot) ProtoMessage() {} + +func (x *NodeHealthSnapshot) ProtoReflect() protoreflect.Message { + mi := &file_manager_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeHealthSnapshot.ProtoReflect.Descriptor instead. +func (*NodeHealthSnapshot) Descriptor() ([]byte, []int) { + return file_manager_proto_rawDescGZIP(), []int{3} +} + +func (x *NodeHealthSnapshot) GetVersion() uint32 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *NodeHealthSnapshot) GetProcessId() string { + if x != nil { + return x.ProcessId + } + return "" +} + +func (x *NodeHealthSnapshot) GetStartedAt() int64 { + if x != nil { + return x.StartedAt + } + return 0 +} + +func (x *NodeHealthSnapshot) GetCollectedAt() int64 { + if x != nil { + return x.CollectedAt + } + return 0 +} + +func (x *NodeHealthSnapshot) GetHeartbeatPeriodSeconds() uint32 { + if x != nil { + return x.HeartbeatPeriodSeconds + } + return 0 +} + +func (x *NodeHealthSnapshot) GetCapabilities() []*CapabilityObservation { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *NodeHealthSnapshot) GetErrors() []*StorageErrorCount { + if x != nil { + return x.Errors + } + return nil +} + +type CapabilityObservation struct { + state protoimpl.MessageState `protogen:"open.v1"` + Capability string `protobuf:"bytes,1,opt,name=capability,proto3" json:"capability,omitempty"` + State int32 `protobuf:"varint,2,opt,name=state,proto3" json:"state,omitempty"` // 0 unknown, 1 observed healthy, 2 suspect, 3 degraded, 4 recovering + Unresolved bool `protobuf:"varint,3,opt,name=unresolved,proto3" json:"unresolved,omitempty"` + LastObservation int64 `protobuf:"varint,4,opt,name=last_observation,json=lastObservation,proto3" json:"last_observation,omitempty"` + LastFailure int64 `protobuf:"varint,5,opt,name=last_failure,json=lastFailure,proto3" json:"last_failure,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CapabilityObservation) Reset() { + *x = CapabilityObservation{} + mi := &file_manager_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CapabilityObservation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CapabilityObservation) ProtoMessage() {} + +func (x *CapabilityObservation) ProtoReflect() protoreflect.Message { + mi := &file_manager_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CapabilityObservation.ProtoReflect.Descriptor instead. +func (*CapabilityObservation) Descriptor() ([]byte, []int) { + return file_manager_proto_rawDescGZIP(), []int{4} +} + +func (x *CapabilityObservation) GetCapability() string { + if x != nil { + return x.Capability + } + return "" +} + +func (x *CapabilityObservation) GetState() int32 { + if x != nil { + return x.State + } + return 0 +} + +func (x *CapabilityObservation) GetUnresolved() bool { + if x != nil { + return x.Unresolved + } + return false +} + +func (x *CapabilityObservation) GetLastObservation() int64 { + if x != nil { + return x.LastObservation + } + return 0 +} + +func (x *CapabilityObservation) GetLastFailure() int64 { + if x != nil { + return x.LastFailure + } + return 0 +} + +type StorageErrorCount struct { + state protoimpl.MessageState `protogen:"open.v1"` + Operation string `protobuf:"bytes,1,opt,name=operation,proto3" json:"operation,omitempty"` + Kind string `protobuf:"bytes,2,opt,name=kind,proto3" json:"kind,omitempty"` + Count uint64 `protobuf:"varint,3,opt,name=count,proto3" json:"count,omitempty"` // Cumulative within process_id, not an active-failure count. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StorageErrorCount) Reset() { + *x = StorageErrorCount{} + mi := &file_manager_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StorageErrorCount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StorageErrorCount) ProtoMessage() {} + +func (x *StorageErrorCount) ProtoReflect() protoreflect.Message { + mi := &file_manager_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StorageErrorCount.ProtoReflect.Descriptor instead. +func (*StorageErrorCount) Descriptor() ([]byte, []int) { + return file_manager_proto_rawDescGZIP(), []int{5} +} + +func (x *StorageErrorCount) GetOperation() string { + if x != nil { + return x.Operation + } + return "" +} + +func (x *StorageErrorCount) GetKind() string { + if x != nil { + return x.Kind + } + return "" +} + +func (x *StorageErrorCount) GetCount() uint64 { + if x != nil { + return x.Count + } + return 0 +} + type SchedulerFileRequest struct { state protoimpl.MessageState `protogen:"open.v1"` DataType string `protobuf:"bytes,1,opt,name=dataType,proto3" json:"dataType,omitempty"` @@ -222,7 +475,7 @@ type SchedulerFileRequest struct { func (x *SchedulerFileRequest) Reset() { *x = SchedulerFileRequest{} - mi := &file_manager_proto_msgTypes[3] + mi := &file_manager_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -234,7 +487,7 @@ func (x *SchedulerFileRequest) String() string { func (*SchedulerFileRequest) ProtoMessage() {} func (x *SchedulerFileRequest) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[3] + mi := &file_manager_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -247,7 +500,7 @@ func (x *SchedulerFileRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SchedulerFileRequest.ProtoReflect.Descriptor instead. func (*SchedulerFileRequest) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{3} + return file_manager_proto_rawDescGZIP(), []int{6} } func (x *SchedulerFileRequest) GetDataType() string { @@ -322,7 +575,7 @@ type SyncFileProcessReq struct { func (x *SyncFileProcessReq) Reset() { *x = SyncFileProcessReq{} - mi := &file_manager_proto_msgTypes[4] + mi := &file_manager_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -334,7 +587,7 @@ func (x *SyncFileProcessReq) String() string { func (*SyncFileProcessReq) ProtoMessage() {} func (x *SyncFileProcessReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[4] + mi := &file_manager_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -347,7 +600,7 @@ func (x *SyncFileProcessReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SyncFileProcessReq.ProtoReflect.Descriptor instead. func (*SyncFileProcessReq) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{4} + return file_manager_proto_rawDescGZIP(), []int{7} } func (x *SyncFileProcessReq) GetFileProcessEntries() []*FileProcessEntry { @@ -376,7 +629,7 @@ type FileProcessEntry struct { func (x *FileProcessEntry) Reset() { *x = FileProcessEntry{} - mi := &file_manager_proto_msgTypes[5] + mi := &file_manager_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -388,7 +641,7 @@ func (x *FileProcessEntry) String() string { func (*FileProcessEntry) ProtoMessage() {} func (x *FileProcessEntry) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[5] + mi := &file_manager_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -401,7 +654,7 @@ func (x *FileProcessEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use FileProcessEntry.ProtoReflect.Descriptor instead. func (*FileProcessEntry) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{5} + return file_manager_proto_rawDescGZIP(), []int{8} } func (x *FileProcessEntry) GetDataType() string { @@ -496,7 +749,7 @@ type SchedulerFileResponse struct { func (x *SchedulerFileResponse) Reset() { *x = SchedulerFileResponse{} - mi := &file_manager_proto_msgTypes[6] + mi := &file_manager_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -508,7 +761,7 @@ func (x *SchedulerFileResponse) String() string { func (*SchedulerFileResponse) ProtoMessage() {} func (x *SchedulerFileResponse) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[6] + mi := &file_manager_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -521,7 +774,7 @@ func (x *SchedulerFileResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SchedulerFileResponse.ProtoReflect.Descriptor instead. func (*SchedulerFileResponse) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{6} + return file_manager_proto_rawDescGZIP(), []int{9} } func (x *SchedulerFileResponse) GetSchedulerType() int32 { @@ -578,7 +831,7 @@ type FileProcessRequest struct { func (x *FileProcessRequest) Reset() { *x = FileProcessRequest{} - mi := &file_manager_proto_msgTypes[7] + mi := &file_manager_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -590,7 +843,7 @@ func (x *FileProcessRequest) String() string { func (*FileProcessRequest) ProtoMessage() {} func (x *FileProcessRequest) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[7] + mi := &file_manager_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -603,7 +856,7 @@ func (x *FileProcessRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FileProcessRequest.ProtoReflect.Descriptor instead. func (*FileProcessRequest) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{7} + return file_manager_proto_rawDescGZIP(), []int{10} } func (x *FileProcessRequest) GetProcessId() int64 { @@ -648,7 +901,7 @@ type DeleteByEtagsAndFieldsRequest struct { func (x *DeleteByEtagsAndFieldsRequest) Reset() { *x = DeleteByEtagsAndFieldsRequest{} - mi := &file_manager_proto_msgTypes[8] + mi := &file_manager_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -660,7 +913,7 @@ func (x *DeleteByEtagsAndFieldsRequest) String() string { func (*DeleteByEtagsAndFieldsRequest) ProtoMessage() {} func (x *DeleteByEtagsAndFieldsRequest) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[8] + mi := &file_manager_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -673,7 +926,7 @@ func (x *DeleteByEtagsAndFieldsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteByEtagsAndFieldsRequest.ProtoReflect.Descriptor instead. func (*DeleteByEtagsAndFieldsRequest) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{8} + return file_manager_proto_rawDescGZIP(), []int{11} } func (x *DeleteByEtagsAndFieldsRequest) GetEtag() string { @@ -734,7 +987,7 @@ type CreateCacheJobReq struct { func (x *CreateCacheJobReq) Reset() { *x = CreateCacheJobReq{} - mi := &file_manager_proto_msgTypes[9] + mi := &file_manager_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -746,7 +999,7 @@ func (x *CreateCacheJobReq) String() string { func (*CreateCacheJobReq) ProtoMessage() {} func (x *CreateCacheJobReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[9] + mi := &file_manager_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -759,7 +1012,7 @@ func (x *CreateCacheJobReq) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateCacheJobReq.ProtoReflect.Descriptor instead. func (*CreateCacheJobReq) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{9} + return file_manager_proto_rawDescGZIP(), []int{12} } func (x *CreateCacheJobReq) GetType() int32 { @@ -828,7 +1081,7 @@ type CreateCacheJobResp struct { func (x *CreateCacheJobResp) Reset() { *x = CreateCacheJobResp{} - mi := &file_manager_proto_msgTypes[10] + mi := &file_manager_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -840,7 +1093,7 @@ func (x *CreateCacheJobResp) String() string { func (*CreateCacheJobResp) ProtoMessage() {} func (x *CreateCacheJobResp) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[10] + mi := &file_manager_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -853,7 +1106,7 @@ func (x *CreateCacheJobResp) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateCacheJobResp.ProtoReflect.Descriptor instead. func (*CreateCacheJobResp) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{10} + return file_manager_proto_rawDescGZIP(), []int{13} } func (x *CreateCacheJobResp) GetId() int64 { @@ -878,7 +1131,7 @@ type UpdateCacheJobStatusReq struct { func (x *UpdateCacheJobStatusReq) Reset() { *x = UpdateCacheJobStatusReq{} - mi := &file_manager_proto_msgTypes[11] + mi := &file_manager_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -890,7 +1143,7 @@ func (x *UpdateCacheJobStatusReq) String() string { func (*UpdateCacheJobStatusReq) ProtoMessage() {} func (x *UpdateCacheJobStatusReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[11] + mi := &file_manager_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -903,7 +1156,7 @@ func (x *UpdateCacheJobStatusReq) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateCacheJobStatusReq.ProtoReflect.Descriptor instead. func (*UpdateCacheJobStatusReq) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{11} + return file_manager_proto_rawDescGZIP(), []int{14} } func (x *UpdateCacheJobStatusReq) GetId() int64 { @@ -966,7 +1219,7 @@ type UpdateRepositoryMountStatusReq struct { func (x *UpdateRepositoryMountStatusReq) Reset() { *x = UpdateRepositoryMountStatusReq{} - mi := &file_manager_proto_msgTypes[12] + mi := &file_manager_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -978,7 +1231,7 @@ func (x *UpdateRepositoryMountStatusReq) String() string { func (*UpdateRepositoryMountStatusReq) ProtoMessage() {} func (x *UpdateRepositoryMountStatusReq) ProtoReflect() protoreflect.Message { - mi := &file_manager_proto_msgTypes[12] + mi := &file_manager_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -991,7 +1244,7 @@ func (x *UpdateRepositoryMountStatusReq) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateRepositoryMountStatusReq.ProtoReflect.Descriptor instead. func (*UpdateRepositoryMountStatusReq) Descriptor() ([]byte, []int) { - return file_manager_proto_rawDescGZIP(), []int{12} + return file_manager_proto_rawDescGZIP(), []int{15} } func (x *UpdateRepositoryMountStatusReq) GetId() int64 { @@ -1015,182 +1268,317 @@ func (x *UpdateRepositoryMountStatusReq) GetErrorMsg() string { return "" } +type IngestRepositoryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + RepoType string `protobuf:"bytes,3,opt,name=repo_type,json=repoType,proto3" json:"repo_type,omitempty"` + Repo string `protobuf:"bytes,4,opt,name=repo,proto3" json:"repo,omitempty"` + Revision string `protobuf:"bytes,5,opt,name=revision,proto3" json:"revision,omitempty"` + Commit string `protobuf:"bytes,6,opt,name=commit,proto3" json:"commit,omitempty"` + Online bool `protobuf:"varint,7,opt,name=online,proto3" json:"online,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IngestRepositoryRequest) Reset() { + *x = IngestRepositoryRequest{} + mi := &file_manager_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IngestRepositoryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IngestRepositoryRequest) ProtoMessage() {} + +func (x *IngestRepositoryRequest) ProtoReflect() protoreflect.Message { + mi := &file_manager_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IngestRepositoryRequest.ProtoReflect.Descriptor instead. +func (*IngestRepositoryRequest) Descriptor() ([]byte, []int) { + return file_manager_proto_rawDescGZIP(), []int{16} +} + +func (x *IngestRepositoryRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *IngestRepositoryRequest) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *IngestRepositoryRequest) GetRepoType() string { + if x != nil { + return x.RepoType + } + return "" +} + +func (x *IngestRepositoryRequest) GetRepo() string { + if x != nil { + return x.Repo + } + return "" +} + +func (x *IngestRepositoryRequest) GetRevision() string { + if x != nil { + return x.Revision + } + return "" +} + +func (x *IngestRepositoryRequest) GetCommit() string { + if x != nil { + return x.Commit + } + return "" +} + +func (x *IngestRepositoryRequest) GetOnline() bool { + if x != nil { + return x.Online + } + return false +} + +type IngestRepositoryResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + RepositoryId int64 `protobuf:"varint,1,opt,name=repository_id,json=repositoryId,proto3" json:"repository_id,omitempty"` + Commit string `protobuf:"bytes,2,opt,name=commit,proto3" json:"commit,omitempty"` + FileCount int64 `protobuf:"varint,3,opt,name=file_count,json=fileCount,proto3" json:"file_count,omitempty"` + UsedStorage int64 `protobuf:"varint,4,opt,name=used_storage,json=usedStorage,proto3" json:"used_storage,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IngestRepositoryResponse) Reset() { + *x = IngestRepositoryResponse{} + mi := &file_manager_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IngestRepositoryResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IngestRepositoryResponse) ProtoMessage() {} + +func (x *IngestRepositoryResponse) ProtoReflect() protoreflect.Message { + mi := &file_manager_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IngestRepositoryResponse.ProtoReflect.Descriptor instead. +func (*IngestRepositoryResponse) Descriptor() ([]byte, []int) { + return file_manager_proto_rawDescGZIP(), []int{17} +} + +func (x *IngestRepositoryResponse) GetRepositoryId() int64 { + if x != nil { + return x.RepositoryId + } + return 0 +} + +func (x *IngestRepositoryResponse) GetCommit() string { + if x != nil { + return x.Commit + } + return "" +} + +func (x *IngestRepositoryResponse) GetFileCount() int64 { + if x != nil { + return x.FileCount + } + return 0 +} + +func (x *IngestRepositoryResponse) GetUsedStorage() int64 { + if x != nil { + return x.UsedStorage + } + return 0 +} + var File_manager_proto protoreflect.FileDescriptor -var file_manager_proto_rawDesc = string([]byte{ - 0x0a, 0x0d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x07, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x71, 0x0a, 0x0f, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, - 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x06, 0x6f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x3c, 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, - 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x5a, 0x0a, 0x10, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, - 0x65, 0x61, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6f, 0x6e, 0x6c, 0x69, - 0x6e, 0x65, 0x22, 0xf0, 0x01, 0x0a, 0x14, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, - 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, - 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, - 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x72, 0x74, 0x50, 0x6f, - 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x73, 0x74, 0x61, 0x72, 0x74, 0x50, 0x6f, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x6e, 0x64, 0x50, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x06, 0x65, 0x6e, 0x64, 0x50, 0x6f, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66, 0x69, 0x6c, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x5f, 0x0a, 0x12, 0x53, 0x79, 0x6e, 0x63, 0x46, 0x69, 0x6c, - 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x12, 0x49, 0x0a, 0x12, 0x66, - 0x69, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x12, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0xa2, 0x02, 0x0a, 0x10, 0x46, 0x69, 0x6c, 0x65, 0x50, - 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x64, - 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, - 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, - 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x72, 0x74, 0x50, 0x6f, - 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x73, 0x74, 0x61, 0x72, 0x74, 0x50, 0x6f, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x6e, 0x64, 0x50, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x06, 0x65, 0x6e, 0x64, 0x50, 0x6f, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66, 0x69, 0x6c, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, - 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x0a, - 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x64, 0x22, 0xcd, 0x01, 0x0a, 0x15, - 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x70, - 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, - 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, - 0x74, 0x12, 0x2a, 0x0a, 0x10, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x73, - 0x74, 0x65, 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, - 0x09, 0x6d, 0x61, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x09, 0x6d, 0x61, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x7a, 0x0a, 0x12, 0x46, - 0x69, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x49, 0x64, 0x12, - 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x50, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x50, 0x6f, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x6e, 0x64, 0x50, 0x6f, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x65, 0x6e, 0x64, 0x50, 0x6f, 0x73, 0x12, - 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0xa9, 0x01, 0x0a, 0x1d, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x42, 0x79, 0x45, 0x74, 0x61, 0x67, 0x73, 0x41, 0x6e, 0x64, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, - 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x1a, 0x0a, - 0x08, 0x64, 0x61, 0x74, 0x61, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x64, 0x61, 0x74, 0x61, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x72, - 0x65, 0x70, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, - 0x44, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, - 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, - 0x08, 0x64, 0x61, 0x74, 0x61, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x64, 0x61, 0x74, 0x61, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x72, - 0x65, 0x70, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, - 0x20, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x64, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x64, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x22, 0x24, 0x0a, 0x12, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0xbd, 0x01, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, - 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x18, 0x0a, - 0x07, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x02, 0x52, 0x07, - 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x22, 0x64, 0x0a, 0x1e, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4d, 0x6f, 0x75, 0x6e, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x32, 0xc5, 0x05, - 0x0a, 0x07, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x41, 0x0a, 0x08, 0x52, 0x65, 0x67, - 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x19, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x09, - 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x19, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4e, 0x0a, 0x0d, - 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1d, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, - 0x72, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x11, - 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x12, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x46, 0x69, 0x6c, 0x65, - 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x46, 0x69, - 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x58, - 0x0a, 0x16, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x45, 0x74, 0x61, 0x67, 0x73, 0x41, - 0x6e, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x26, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x45, 0x74, 0x61, 0x67, 0x73, - 0x41, 0x6e, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x1a, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4a, 0x6f, 0x62, 0x52, - 0x65, 0x73, 0x70, 0x12, 0x50, 0x0a, 0x14, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x1b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x27, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4d, - 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x3b, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -}) +const file_manager_proto_rawDesc = "" + + "\n" + + "\rmanager.proto\x12\amanager\x1a\x1bgoogle/protobuf/empty.proto\"\xb9\x01\n" + + "\x0fRegisterRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04host\x18\x02 \x01(\tR\x04host\x12\x12\n" + + "\x04port\x18\x03 \x01(\x05R\x04port\x12\x16\n" + + "\x06online\x18\x04 \x01(\bR\x06online\x12$\n" + + "\rmanagementUrl\x18\x05 \x01(\tR\rmanagementUrl\x12 \n" + + "\vdownloadUrl\x18\x06 \x01(\tR\vdownloadUrl\"<\n" + + "\x10RegisterResponse\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x05R\x02id\x12\x18\n" + + "\asuccess\x18\x02 \x01(\bR\asuccess\"\x8f\x01\n" + + "\x10HeartbeatRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x05R\x02id\x12\x1e\n" + + "\n" + + "instanceId\x18\x02 \x01(\tR\n" + + "instanceId\x12\x16\n" + + "\x06online\x18\x03 \x01(\bR\x06online\x123\n" + + "\x06health\x18\x04 \x01(\v2\x1b.manager.NodeHealthSnapshotR\x06health\"\xc1\x02\n" + + "\x12NodeHealthSnapshot\x12\x18\n" + + "\aversion\x18\x01 \x01(\rR\aversion\x12\x1d\n" + + "\n" + + "process_id\x18\x02 \x01(\tR\tprocessId\x12\x1d\n" + + "\n" + + "started_at\x18\x03 \x01(\x03R\tstartedAt\x12!\n" + + "\fcollected_at\x18\x04 \x01(\x03R\vcollectedAt\x128\n" + + "\x18heartbeat_period_seconds\x18\x05 \x01(\rR\x16heartbeatPeriodSeconds\x12B\n" + + "\fcapabilities\x18\x06 \x03(\v2\x1e.manager.CapabilityObservationR\fcapabilities\x122\n" + + "\x06errors\x18\a \x03(\v2\x1a.manager.StorageErrorCountR\x06errors\"\xbb\x01\n" + + "\x15CapabilityObservation\x12\x1e\n" + + "\n" + + "capability\x18\x01 \x01(\tR\n" + + "capability\x12\x14\n" + + "\x05state\x18\x02 \x01(\x05R\x05state\x12\x1e\n" + + "\n" + + "unresolved\x18\x03 \x01(\bR\n" + + "unresolved\x12)\n" + + "\x10last_observation\x18\x04 \x01(\x03R\x0flastObservation\x12!\n" + + "\flast_failure\x18\x05 \x01(\x03R\vlastFailure\"[\n" + + "\x11StorageErrorCount\x12\x1c\n" + + "\toperation\x18\x01 \x01(\tR\toperation\x12\x12\n" + + "\x04kind\x18\x02 \x01(\tR\x04kind\x12\x14\n" + + "\x05count\x18\x03 \x01(\x04R\x05count\"\xf0\x01\n" + + "\x14SchedulerFileRequest\x12\x1a\n" + + "\bdataType\x18\x01 \x01(\tR\bdataType\x12\x10\n" + + "\x03org\x18\x02 \x01(\tR\x03org\x12\x12\n" + + "\x04repo\x18\x03 \x01(\tR\x04repo\x12\x12\n" + + "\x04name\x18\x04 \x01(\tR\x04name\x12\x12\n" + + "\x04etag\x18\x05 \x01(\tR\x04etag\x12\x1e\n" + + "\n" + + "instanceId\x18\x06 \x01(\tR\n" + + "instanceId\x12\x1a\n" + + "\bstartPos\x18\a \x01(\x03R\bstartPos\x12\x16\n" + + "\x06endPos\x18\b \x01(\x03R\x06endPos\x12\x1a\n" + + "\bfileSize\x18\t \x01(\x03R\bfileSize\"_\n" + + "\x12SyncFileProcessReq\x12I\n" + + "\x12fileProcessEntries\x18\x01 \x03(\v2\x19.manager.FileProcessEntryR\x12fileProcessEntries\"\xa2\x02\n" + + "\x10FileProcessEntry\x12\x1a\n" + + "\bdataType\x18\x01 \x01(\tR\bdataType\x12\x10\n" + + "\x03org\x18\x02 \x01(\tR\x03org\x12\x12\n" + + "\x04repo\x18\x03 \x01(\tR\x04repo\x12\x12\n" + + "\x04name\x18\x04 \x01(\tR\x04name\x12\x12\n" + + "\x04etag\x18\x05 \x01(\tR\x04etag\x12\x1e\n" + + "\n" + + "instanceId\x18\x06 \x01(\tR\n" + + "instanceId\x12\x1a\n" + + "\bstartPos\x18\a \x01(\x03R\bstartPos\x12\x16\n" + + "\x06endPos\x18\b \x01(\x03R\x06endPos\x12\x1a\n" + + "\bfileSize\x18\t \x01(\x03R\bfileSize\x12\x16\n" + + "\x06status\x18\n" + + " \x01(\x05R\x06status\x12\x1c\n" + + "\tprocessId\x18\v \x01(\x03R\tprocessId\"\xcd\x01\n" + + "\x15SchedulerFileResponse\x12$\n" + + "\rschedulerType\x18\x01 \x01(\x05R\rschedulerType\x12\x1c\n" + + "\tprocessId\x18\x02 \x01(\x03R\tprocessId\x12\x12\n" + + "\x04host\x18\x03 \x01(\tR\x04host\x12\x12\n" + + "\x04port\x18\x04 \x01(\x05R\x04port\x12*\n" + + "\x10masterInstanceId\x18\x05 \x01(\tR\x10masterInstanceId\x12\x1c\n" + + "\tmaxOffset\x18\x06 \x01(\x03R\tmaxOffset\"z\n" + + "\x12FileProcessRequest\x12\x1c\n" + + "\tprocessId\x18\x01 \x01(\x03R\tprocessId\x12\x16\n" + + "\x06staPos\x18\x02 \x01(\x03R\x06staPos\x12\x16\n" + + "\x06endPos\x18\x03 \x01(\x03R\x06endPos\x12\x16\n" + + "\x06status\x18\x04 \x01(\x05R\x06status\"\xa9\x01\n" + + "\x1dDeleteByEtagsAndFieldsRequest\x12\x12\n" + + "\x04etag\x18\x01 \x01(\tR\x04etag\x12\x1a\n" + + "\bdatatype\x18\x02 \x01(\tR\bdatatype\x12\x10\n" + + "\x03org\x18\x03 \x01(\tR\x03org\x12\x12\n" + + "\x04repo\x18\x04 \x01(\tR\x04repo\x12\x12\n" + + "\x04name\x18\x05 \x01(\tR\x04name\x12\x1e\n" + + "\n" + + "instanceID\x18\x06 \x01(\tR\n" + + "instanceID\"\xdb\x01\n" + + "\x11CreateCacheJobReq\x12\x12\n" + + "\x04type\x18\x01 \x01(\x05R\x04type\x12\x1e\n" + + "\n" + + "instanceId\x18\x02 \x01(\tR\n" + + "instanceId\x12\x1a\n" + + "\bdatatype\x18\x03 \x01(\tR\bdatatype\x12\x10\n" + + "\x03org\x18\x04 \x01(\tR\x03org\x12\x12\n" + + "\x04repo\x18\x05 \x01(\tR\x04repo\x12 \n" + + "\vusedStorage\x18\x06 \x01(\x03R\vusedStorage\x12\x16\n" + + "\x06commit\x18\a \x01(\tR\x06commit\x12\x16\n" + + "\x06status\x18\b \x01(\x05R\x06status\"$\n" + + "\x12CreateCacheJobResp\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x03R\x02id\"\xbd\x01\n" + + "\x17UpdateCacheJobStatusReq\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x03R\x02id\x12\x16\n" + + "\x06status\x18\x02 \x01(\x05R\x06status\x12\x1a\n" + + "\berrorMsg\x18\x03 \x01(\tR\berrorMsg\x12\x1e\n" + + "\n" + + "instanceId\x18\x04 \x01(\tR\n" + + "instanceId\x12\x10\n" + + "\x03org\x18\x05 \x01(\tR\x03org\x12\x12\n" + + "\x04repo\x18\x06 \x01(\tR\x04repo\x12\x18\n" + + "\aprocess\x18\a \x01(\x02R\aprocess\"d\n" + + "\x1eUpdateRepositoryMountStatusReq\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x03R\x02id\x12\x16\n" + + "\x06status\x18\x02 \x01(\x05R\x06status\x12\x1a\n" + + "\berrorMsg\x18\x03 \x01(\tR\berrorMsg\"\xd5\x01\n" + + "\x17IngestRepositoryRequest\x12\x1f\n" + + "\vinstance_id\x18\x01 \x01(\tR\n" + + "instanceId\x12\x1c\n" + + "\tnamespace\x18\x02 \x01(\tR\tnamespace\x12\x1b\n" + + "\trepo_type\x18\x03 \x01(\tR\brepoType\x12\x12\n" + + "\x04repo\x18\x04 \x01(\tR\x04repo\x12\x1a\n" + + "\brevision\x18\x05 \x01(\tR\brevision\x12\x16\n" + + "\x06commit\x18\x06 \x01(\tR\x06commit\x12\x16\n" + + "\x06online\x18\a \x01(\bR\x06online\"\x99\x01\n" + + "\x18IngestRepositoryResponse\x12#\n" + + "\rrepository_id\x18\x01 \x01(\x03R\frepositoryId\x12\x16\n" + + "\x06commit\x18\x02 \x01(\tR\x06commit\x12\x1d\n" + + "\n" + + "file_count\x18\x03 \x01(\x03R\tfileCount\x12!\n" + + "\fused_storage\x18\x04 \x01(\x03R\vusedStorage2\x9e\x06\n" + + "\aManager\x12W\n" + + "\x10IngestRepository\x12 .manager.IngestRepositoryRequest\x1a!.manager.IngestRepositoryResponse\x12A\n" + + "\bRegister\x12\x18.manager.RegisterRequest\x1a\x19.manager.RegisterResponse\"\x00\x12>\n" + + "\tHeartbeat\x12\x19.manager.HeartbeatRequest\x1a\x16.google.protobuf.Empty\x12N\n" + + "\rSchedulerFile\x12\x1d.manager.SchedulerFileRequest\x1a\x1e.manager.SchedulerFileResponse\x12H\n" + + "\x11ReportFileProcess\x12\x1b.manager.FileProcessRequest\x1a\x16.google.protobuf.Empty\x12F\n" + + "\x0fSyncFileProcess\x12\x1b.manager.SyncFileProcessReq\x1a\x16.google.protobuf.Empty\x12X\n" + + "\x16DeleteByEtagsAndFields\x12&.manager.DeleteByEtagsAndFieldsRequest\x1a\x16.google.protobuf.Empty\x12I\n" + + "\x0eCreateCacheJob\x12\x1a.manager.CreateCacheJobReq\x1a\x1b.manager.CreateCacheJobResp\x12P\n" + + "\x14UpdateCacheJobStatus\x12 .manager.UpdateCacheJobStatusReq\x1a\x16.google.protobuf.Empty\x12^\n" + + "\x1bUpdateRepositoryMountStatus\x12'.manager.UpdateRepositoryMountStatusReq\x1a\x16.google.protobuf.EmptyB\vZ\t.;managerb\x06proto3" var ( file_manager_proto_rawDescOnce sync.Once @@ -1204,48 +1592,58 @@ func file_manager_proto_rawDescGZIP() []byte { return file_manager_proto_rawDescData } -var file_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_manager_proto_goTypes = []any{ (*RegisterRequest)(nil), // 0: manager.RegisterRequest (*RegisterResponse)(nil), // 1: manager.RegisterResponse (*HeartbeatRequest)(nil), // 2: manager.HeartbeatRequest - (*SchedulerFileRequest)(nil), // 3: manager.SchedulerFileRequest - (*SyncFileProcessReq)(nil), // 4: manager.SyncFileProcessReq - (*FileProcessEntry)(nil), // 5: manager.FileProcessEntry - (*SchedulerFileResponse)(nil), // 6: manager.SchedulerFileResponse - (*FileProcessRequest)(nil), // 7: manager.FileProcessRequest - (*DeleteByEtagsAndFieldsRequest)(nil), // 8: manager.DeleteByEtagsAndFieldsRequest - (*CreateCacheJobReq)(nil), // 9: manager.CreateCacheJobReq - (*CreateCacheJobResp)(nil), // 10: manager.CreateCacheJobResp - (*UpdateCacheJobStatusReq)(nil), // 11: manager.UpdateCacheJobStatusReq - (*UpdateRepositoryMountStatusReq)(nil), // 12: manager.UpdateRepositoryMountStatusReq - (*emptypb.Empty)(nil), // 13: google.protobuf.Empty + (*NodeHealthSnapshot)(nil), // 3: manager.NodeHealthSnapshot + (*CapabilityObservation)(nil), // 4: manager.CapabilityObservation + (*StorageErrorCount)(nil), // 5: manager.StorageErrorCount + (*SchedulerFileRequest)(nil), // 6: manager.SchedulerFileRequest + (*SyncFileProcessReq)(nil), // 7: manager.SyncFileProcessReq + (*FileProcessEntry)(nil), // 8: manager.FileProcessEntry + (*SchedulerFileResponse)(nil), // 9: manager.SchedulerFileResponse + (*FileProcessRequest)(nil), // 10: manager.FileProcessRequest + (*DeleteByEtagsAndFieldsRequest)(nil), // 11: manager.DeleteByEtagsAndFieldsRequest + (*CreateCacheJobReq)(nil), // 12: manager.CreateCacheJobReq + (*CreateCacheJobResp)(nil), // 13: manager.CreateCacheJobResp + (*UpdateCacheJobStatusReq)(nil), // 14: manager.UpdateCacheJobStatusReq + (*UpdateRepositoryMountStatusReq)(nil), // 15: manager.UpdateRepositoryMountStatusReq + (*IngestRepositoryRequest)(nil), // 16: manager.IngestRepositoryRequest + (*IngestRepositoryResponse)(nil), // 17: manager.IngestRepositoryResponse + (*emptypb.Empty)(nil), // 18: google.protobuf.Empty } var file_manager_proto_depIdxs = []int32{ - 5, // 0: manager.SyncFileProcessReq.fileProcessEntries:type_name -> manager.FileProcessEntry - 0, // 1: manager.Manager.Register:input_type -> manager.RegisterRequest - 2, // 2: manager.Manager.Heartbeat:input_type -> manager.HeartbeatRequest - 3, // 3: manager.Manager.SchedulerFile:input_type -> manager.SchedulerFileRequest - 7, // 4: manager.Manager.ReportFileProcess:input_type -> manager.FileProcessRequest - 4, // 5: manager.Manager.SyncFileProcess:input_type -> manager.SyncFileProcessReq - 8, // 6: manager.Manager.DeleteByEtagsAndFields:input_type -> manager.DeleteByEtagsAndFieldsRequest - 9, // 7: manager.Manager.CreateCacheJob:input_type -> manager.CreateCacheJobReq - 11, // 8: manager.Manager.UpdateCacheJobStatus:input_type -> manager.UpdateCacheJobStatusReq - 12, // 9: manager.Manager.UpdateRepositoryMountStatus:input_type -> manager.UpdateRepositoryMountStatusReq - 1, // 10: manager.Manager.Register:output_type -> manager.RegisterResponse - 13, // 11: manager.Manager.Heartbeat:output_type -> google.protobuf.Empty - 6, // 12: manager.Manager.SchedulerFile:output_type -> manager.SchedulerFileResponse - 13, // 13: manager.Manager.ReportFileProcess:output_type -> google.protobuf.Empty - 13, // 14: manager.Manager.SyncFileProcess:output_type -> google.protobuf.Empty - 13, // 15: manager.Manager.DeleteByEtagsAndFields:output_type -> google.protobuf.Empty - 10, // 16: manager.Manager.CreateCacheJob:output_type -> manager.CreateCacheJobResp - 13, // 17: manager.Manager.UpdateCacheJobStatus:output_type -> google.protobuf.Empty - 13, // 18: manager.Manager.UpdateRepositoryMountStatus:output_type -> google.protobuf.Empty - 10, // [10:19] is the sub-list for method output_type - 1, // [1:10] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 3, // 0: manager.HeartbeatRequest.health:type_name -> manager.NodeHealthSnapshot + 4, // 1: manager.NodeHealthSnapshot.capabilities:type_name -> manager.CapabilityObservation + 5, // 2: manager.NodeHealthSnapshot.errors:type_name -> manager.StorageErrorCount + 8, // 3: manager.SyncFileProcessReq.fileProcessEntries:type_name -> manager.FileProcessEntry + 16, // 4: manager.Manager.IngestRepository:input_type -> manager.IngestRepositoryRequest + 0, // 5: manager.Manager.Register:input_type -> manager.RegisterRequest + 2, // 6: manager.Manager.Heartbeat:input_type -> manager.HeartbeatRequest + 6, // 7: manager.Manager.SchedulerFile:input_type -> manager.SchedulerFileRequest + 10, // 8: manager.Manager.ReportFileProcess:input_type -> manager.FileProcessRequest + 7, // 9: manager.Manager.SyncFileProcess:input_type -> manager.SyncFileProcessReq + 11, // 10: manager.Manager.DeleteByEtagsAndFields:input_type -> manager.DeleteByEtagsAndFieldsRequest + 12, // 11: manager.Manager.CreateCacheJob:input_type -> manager.CreateCacheJobReq + 14, // 12: manager.Manager.UpdateCacheJobStatus:input_type -> manager.UpdateCacheJobStatusReq + 15, // 13: manager.Manager.UpdateRepositoryMountStatus:input_type -> manager.UpdateRepositoryMountStatusReq + 17, // 14: manager.Manager.IngestRepository:output_type -> manager.IngestRepositoryResponse + 1, // 15: manager.Manager.Register:output_type -> manager.RegisterResponse + 18, // 16: manager.Manager.Heartbeat:output_type -> google.protobuf.Empty + 9, // 17: manager.Manager.SchedulerFile:output_type -> manager.SchedulerFileResponse + 18, // 18: manager.Manager.ReportFileProcess:output_type -> google.protobuf.Empty + 18, // 19: manager.Manager.SyncFileProcess:output_type -> google.protobuf.Empty + 18, // 20: manager.Manager.DeleteByEtagsAndFields:output_type -> google.protobuf.Empty + 13, // 21: manager.Manager.CreateCacheJob:output_type -> manager.CreateCacheJobResp + 18, // 22: manager.Manager.UpdateCacheJobStatus:output_type -> google.protobuf.Empty + 18, // 23: manager.Manager.UpdateRepositoryMountStatus:output_type -> google.protobuf.Empty + 14, // [14:24] is the sub-list for method output_type + 4, // [4:14] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_manager_proto_init() } @@ -1259,7 +1657,7 @@ func file_manager_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_manager_proto_rawDesc), len(file_manager_proto_rawDesc)), NumEnums: 0, - NumMessages: 13, + NumMessages: 18, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/proto/manager/manager_grpc.pb.go b/pkg/proto/manager/manager_grpc.pb.go index f7f5beb..c617498 100644 --- a/pkg/proto/manager/manager_grpc.pb.go +++ b/pkg/proto/manager/manager_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.5.1 -// - protoc v5.29.3 +// - protoc v7.35.1 // source: manager.proto package manager @@ -20,6 +20,7 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( + Manager_IngestRepository_FullMethodName = "/manager.Manager/IngestRepository" Manager_Register_FullMethodName = "/manager.Manager/Register" Manager_Heartbeat_FullMethodName = "/manager.Manager/Heartbeat" Manager_SchedulerFile_FullMethodName = "/manager.Manager/SchedulerFile" @@ -37,6 +38,7 @@ const ( // // 用户服务定义 type ManagerClient interface { + IngestRepository(ctx context.Context, in *IngestRepositoryRequest, opts ...grpc.CallOption) (*IngestRepositoryResponse, error) // 启动注册方法 Register(ctx context.Context, in *RegisterRequest, opts ...grpc.CallOption) (*RegisterResponse, error) // 心跳方法 @@ -63,6 +65,16 @@ func NewManagerClient(cc grpc.ClientConnInterface) ManagerClient { return &managerClient{cc} } +func (c *managerClient) IngestRepository(ctx context.Context, in *IngestRepositoryRequest, opts ...grpc.CallOption) (*IngestRepositoryResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(IngestRepositoryResponse) + err := c.cc.Invoke(ctx, Manager_IngestRepository_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *managerClient) Register(ctx context.Context, in *RegisterRequest, opts ...grpc.CallOption) (*RegisterResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RegisterResponse) @@ -159,6 +171,7 @@ func (c *managerClient) UpdateRepositoryMountStatus(ctx context.Context, in *Upd // // 用户服务定义 type ManagerServer interface { + IngestRepository(context.Context, *IngestRepositoryRequest) (*IngestRepositoryResponse, error) // 启动注册方法 Register(context.Context, *RegisterRequest) (*RegisterResponse, error) // 心跳方法 @@ -185,6 +198,9 @@ type ManagerServer interface { // pointer dereference when methods are called. type UnimplementedManagerServer struct{} +func (UnimplementedManagerServer) IngestRepository(context.Context, *IngestRepositoryRequest) (*IngestRepositoryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method IngestRepository not implemented") +} func (UnimplementedManagerServer) Register(context.Context, *RegisterRequest) (*RegisterResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") } @@ -233,6 +249,24 @@ func RegisterManagerServer(s grpc.ServiceRegistrar, srv ManagerServer) { s.RegisterService(&Manager_ServiceDesc, srv) } +func _Manager_IngestRepository_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IngestRepositoryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ManagerServer).IngestRepository(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Manager_IngestRepository_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ManagerServer).IngestRepository(ctx, req.(*IngestRepositoryRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Manager_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RegisterRequest) if err := dec(in); err != nil { @@ -402,6 +436,10 @@ var Manager_ServiceDesc = grpc.ServiceDesc{ ServiceName: "manager.Manager", HandlerType: (*ManagerServer)(nil), Methods: []grpc.MethodDesc{ + { + MethodName: "IngestRepository", + Handler: _Manager_IngestRepository_Handler, + }, { MethodName: "Register", Handler: _Manager_Register_Handler, diff --git a/pkg/repository/key.go b/pkg/repository/key.go new file mode 100644 index 0000000..654c259 --- /dev/null +++ b/pkg/repository/key.go @@ -0,0 +1,132 @@ +// Package repository defines the namespace identity shared by scheduler boundaries. +package repository + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + "unicode" + "unicode/utf8" +) + +// Key is the HTTP identity; SQL and RPC retain legacy org/repo coordinates. +type Key struct { + Namespace string `json:"namespace"` + RepoType string `json:"repoType"` + Repo string `json:"repo"` +} + +// FromWire decodes unchanged remote identities and explicitly encoded hosted ones. +func FromWire(repoType, org, repo string) (Key, error) { + k := Key{Namespace: "huggingface", RepoType: repoType, Repo: repo} + if strings.HasPrefix(org, "dingo-local/") { + k.Namespace = strings.TrimPrefix(org, "dingo-local/") + } else if strings.HasPrefix(org, "modelscope/") { + k.Namespace = "modelscope" + k.Repo = strings.TrimPrefix(org, "modelscope/") + "/" + repo + } else if org != "" { + k.Repo = org + "/" + repo + } + return k, k.Validate() +} + +// Storage leaves HF owner/repo exactly as old SQL and old peers expect. +func (k Key) Storage() (org, repo string, err error) { + if err = k.Validate(); err != nil { + return + } + repo = k.Repo + switch k.Namespace { + case "huggingface": + parts := strings.SplitN(repo, "/", 2) + if len(parts) == 2 { + org, repo = parts[0], parts[1] + } + case "modelscope": + parts := strings.SplitN(repo, "/", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("ModelScope needs owner/repo") + } + org, repo = "modelscope/"+parts[0], parts[1] + default: + org = "dingo-local/" + k.Namespace + } + if k.Namespace != "huggingface" && (len(org) > 100 || len(repo) > 100 || len(org)+1+len(repo) > 100) { + err = fmt.Errorf("repository identity exceeds unchanged SQL VARCHAR(100) capacity") + } + return +} + +func (k Key) Validate() error { + if k.RepoType != "models" && k.RepoType != "datasets" && k.RepoType != "spaces" { + return fmt.Errorf("invalid repository type") + } + if err := ValidatePath(k.Namespace, false); err != nil { + return fmt.Errorf("invalid namespace: %w", err) + } + if len(k.Namespace) > 255 || len(k.Repo) > 1024 { + return fmt.Errorf("namespace or repository exceeds storage identity limit") + } + return ValidatePath(k.Repo, true) +} + +func ValidatePath(value string, multiple bool) error { + if value == "" || (!multiple && strings.Contains(value, "/")) || (multiple && len(value) > 1024) { + return fmt.Errorf("empty or invalid path") + } + for _, part := range strings.Split(value, "/") { + if part == "" || part == "." || part == ".." || !utf8.ValidString(part) || len(part) > 255 || strings.TrimSpace(part) != part || strings.HasSuffix(part, ".") { + return fmt.Errorf("invalid path segment") + } + for _, ch := range part { + if unicode.IsControl(ch) || strings.ContainsRune(`\<>:"|?*`, ch) { + return fmt.Errorf("unsafe path character") + } + } + base := strings.ToUpper(strings.SplitN(part, ".", 2)[0]) + if base == "CON" || base == "PRN" || base == "AUX" || base == "NUL" || + (len(base) == 4 && (strings.HasPrefix(base, "COM") || strings.HasPrefix(base, "LPT")) && base[3] >= '1' && base[3] <= '9') { + return fmt.Errorf("reserved platform path segment") + } + } + return nil +} + +// LockKey uses a structured encoding: multi-level repositories cannot collide +// with later identity components such as file paths, job types or content IDs. +func (k Key) LockKey(operation string, fields ...string) string { + parts := append([]string{operation, k.Namespace, k.RepoType, k.Repo}, fields...) + encoded, _ := json.Marshal(parts) + return string(encoded) +} + +func (k Key) ID() string { return k.Namespace + "/" + k.Repo } + +// OperationURI addresses DingoSpeed's own API, never the upstream HF protocol. +func (k Key) OperationURI(operation, revision, path string) (string, error) { + if err := k.Validate(); err != nil { + return "", err + } + switch operation { + case "metadata", "snapshot", "file", "files", "tree", "archive": + default: + return "", fmt.Errorf("unsupported repository operation") + } + if revision == "" { + revision = "main" + } + if err := ValidatePath(revision, false); err != nil { + return "", err + } + if path != "" { + if err := ValidatePath(path, true); err != nil { + return "", err + } + } + q := url.Values{"repo": {k.Repo}, "revision": {revision}} + if path != "" { + q.Set("path", path) + } + return "/api/repositories/" + url.PathEscape(k.RepoType) + "/" + url.PathEscape(k.Namespace) + "/" + operation + "?" + q.Encode(), nil +} diff --git a/pkg/repository/storage_test.go b/pkg/repository/storage_test.go new file mode 100644 index 0000000..8e824c4 --- /dev/null +++ b/pkg/repository/storage_test.go @@ -0,0 +1,35 @@ +package repository + +import ( + "strings" + "testing" +) + +func TestLegacyAndHostedStorage(t *testing.T) { + for _, c := range []struct{ ns, api, org, repo string }{ + {"huggingface", "Qwen/demo", "Qwen", "demo"}, {"huggingface", "gpt2", "", "gpt2"}, + {"huggingface", "alice/demo", "alice", "demo"}, {"alice", "demo", "dingo-local/alice", "demo"}, + {"datacanvas", "team/demo", "dingo-local/datacanvas", "team/demo"}, + {"modelscope", "Qwen/demo", "modelscope/Qwen", "demo"}, + } { + k := Key{Namespace: c.ns, RepoType: "models", Repo: c.api} + org, repo, e := k.Storage() + if e != nil || org != c.org || repo != c.repo { + t.Fatalf("%+v => %s/%s %v", c, org, repo, e) + } + back, e := FromWire("models", org, repo) + if e != nil || back != k { + t.Fatalf("round trip %v %v", back, e) + } + } +} +func TestUnchangedSQLCapacity(t *testing.T) { + k := Key{"alice", "models", strings.Repeat("a", 82)} + if _, _, e := k.Storage(); e != nil { + t.Fatal(e) + } + k.Repo += "a" + if _, _, e := k.Storage(); e == nil { + t.Fatal("accepted org_repo >100 bytes") + } +} diff --git a/process_import/process_import.go b/process_import/process_import.go index 9f816ad..17c39cb 100644 --- a/process_import/process_import.go +++ b/process_import/process_import.go @@ -355,6 +355,12 @@ func processDirectory(rootPath string) ([]FileInfo, error) { return nil } components := strings.Split(relPath, string(filepath.Separator)) + if len(components) >= 3 && components[2] == "dingo-local" { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } if len(components) < 5 || components[0] != "api" || !(components[1] == "models" || components[1] == "datasets" || components[1] == "spaces") || components[4] != "paths-info" { diff --git a/repair/data_import.go b/repair/data_import.go index cb22d9d..cb7177b 100644 --- a/repair/data_import.go +++ b/repair/data_import.go @@ -405,6 +405,12 @@ func processDirectory(rootPath string) ([]FileInfo, error) { } components := strings.Split(relPath, string(filepath.Separator)) + if len(components) >= 3 && components[2] == "dingo-local" { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } // 校验路径结构：必须包含 api/[models|datasets|spaces]/org/repo/paths-info 前缀 if len(components) < 5 || components[0] != "api" || From 97deaedcbd1b6ec05c97596d6098b557340053d1 Mon Sep 17 00:00:00 2001 From: dongzhenyangofficial-ctrl Date: Wed, 23 Sep 2026 18:06:46 +0800 Subject: [PATCH 2/2] feat(inventory): accept durable reports and complete ModelScope parity --- config/config.yaml | 4 +- docs/modelscope-parity.md | 62 +++++++ docs/upload-inventory-schema.sql | 12 ++ docs/upload-inventory.md | 59 ++++-- internal/dao/cache_job_dao.go | 21 ++- internal/dao/hf_token_dao.go | 14 ++ internal/dao/ingest_dao.go | 9 + internal/dao/namespace_filter.go | 25 +++ internal/dao/remote_file_identity.go | 101 ++++++++++ internal/dao/remote_parity_test.go | 227 +++++++++++++++++++++++ internal/dao/repository_dao.go | 98 ++++++++-- internal/dao/upload_report.go | 205 ++++++++++++++++++++ internal/dao/upload_report_retry_test.go | 37 ++++ internal/dao/upload_report_test.go | 102 ++++++++++ internal/data/data.go | 2 +- internal/handler/cache_job_handler.go | 2 +- internal/handler/repository_handler.go | 1 + internal/handler/upload_report.go | 63 +++++++ internal/model/dto/dto.go | 5 + internal/model/dto/preheat.go | 6 + internal/model/query/query.go | 16 +- internal/model/upload_report.go | 23 +++ internal/router/http_router.go | 5 + internal/service/cache_job_service.go | 27 +-- internal/service/remote_parity_test.go | 99 ++++++++++ internal/service/repository_service.go | 3 +- internal/service/scheduler_service.go | 4 +- internal/service/upload_report.go | 122 ++++++++++++ pkg/config/config.go | 7 +- pkg/inventory/protocol.go | 53 ++++++ pkg/repository/key.go | 9 +- 31 files changed, 1358 insertions(+), 65 deletions(-) create mode 100644 docs/modelscope-parity.md create mode 100644 internal/dao/namespace_filter.go create mode 100644 internal/dao/remote_file_identity.go create mode 100644 internal/dao/remote_parity_test.go create mode 100644 internal/dao/upload_report.go create mode 100644 internal/dao/upload_report_retry_test.go create mode 100644 internal/dao/upload_report_test.go create mode 100644 internal/handler/upload_report.go create mode 100644 internal/model/upload_report.go create mode 100644 internal/service/remote_parity_test.go create mode 100644 internal/service/upload_report.go create mode 100644 pkg/inventory/protocol.go diff --git a/config/config.yaml b/config/config.yaml index f74b073..3b768e9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -24,6 +24,8 @@ bizDB: maxIdleConn: 10 scheduler: + # ModelScope 私有仓库凭据；不配置时匿名访问，不复用 HF token。 + modelscopeToken: "" port: 19091 persistRepo: enabled: true @@ -75,4 +77,4 @@ aidc: xn-03: xn-03 xn-zl: xn-zl hd-05: hd-05 - xb-01: xb-01 \ No newline at end of file + xb-01: xb-01 diff --git a/docs/modelscope-parity.md b/docs/modelscope-parity.md new file mode 100644 index 0000000..95f281a --- /dev/null +++ b/docs/modelscope-parity.md @@ -0,0 +1,62 @@ +# ModelScope 远端仓库管理补齐 + +2026-09-23。范围为 Scheduler 的远端下载管理；不修改上传库存、仓库范本或节点文件，不迁移数据库。 + +## 行为 + +- 未指定版本的仓库发现按来源取默认分支：HF 为 `main`，ModelScope 为 `master`。 +- 缓存任务完成后使用任务保存的 commit 获取元数据；返回 commit 不一致或清单缺失时拒绝登记。 +- 完整性核验逐一匹配节点、来源、仓库类型、仓库、路径、内容摘要和大小，不再以文件总数代替清单。缺少内容身份的 HF 元数据通过固定 commit 的递归 tree 补齐；Speed 负责上游分页。不会通过文件 GET 下载内容。 +- 下载完成与仓库登记分别表达：登记失败保留任务完成状态，`errorMsg` 标明 `repository registration pending`，RPC 返回错误，进入 Speed 既有持久化通知重试。成功重试清除错误。 +- 同一节点仓库更新原记录并替换标签，保留数据库 ID、挂载状态与节点关系；重复回报不重复插入。旧任务迟到回报不会覆盖编号更新的已完成任务。 +- 不再因为 Scheduler 存在同仓库历史任务而拦截创建；由 Speed 根据当前 commit 判断运行中、已缓存或创建新任务，Scheduler 透传结果。 + +## 凭据 + +```yaml +scheduler: + modelscopeToken: "" # 私有仓库需要配置所属 ModelScope 账号的凭据 +``` + +ModelScope 的创建、恢复、仓库登记及默认挂载使用该配置。空值表示匿名请求，不回退到 HF token。HF 保持原有 `hf_token` 数据库选择方式。挂载请求显式提供的 `token` 仍优先使用。停止和实时进度查询不发送上游凭据。 + +配置修改需在后续部署或服务重启时生效；本次没有修改运行环境或配置真实凭据。 + +## 查询兼容 + +仓库列表 `/api/v1/repositories` 和任务列表 `/api/v1/cacheJob/list` 增加可选参数 `namespace=modelscope` 或 `namespace=huggingface`，也支持具体上传 namespace。过滤发生在数据库分页和计数之前；省略参数保持混合列表行为。 + +普通仓库列表/详情与任务列表新增统一身份字段： + +```json +{ + "namespace": "modelscope", + "fullRepo": "owner/demo", + "repositoryId": "modelscope/owner/demo" +} +``` + +`repositoryId` 是带 namespace 的业务标识，仍需结合 `repoType/datatype` 区分类型；数据库数字 `id` 不变。为兼容已有调用方，不改旧 `org/repo/orgRepo` 含义。任务列表分页改为标准 OFFSET/LIMIT，并使用 ID 作为相同创建时间的排序补充。 + +`POST /api/persistRepo` 可在明确指定仓库时提供 `commit`，用于人工重试固定版本登记。没有指定 commit 的待登记仓库优先使用最新已完成任务的 commit，没有完成任务时才按来源默认分支发现。旧 `offVerify` 显式选项继续保留；任务完成通知不再开启它。 + +## 验证 + +新增 DAO 与 service 测试使用临时 SQLite 数据库及本机临时 HTTP 端口，覆盖： + +- ModelScope 默认 `master`，逐路径校验而非数量比较。 +- 元数据服务失败后补报，任务完成事实及登记错误回读。 +- commit 不匹配、旧内容冒充新内容时拒绝登记。 +- 重试幂等、新版本原位更新、旧通知不回退新版本。 +- HF/ModelScope 凭据隔离及 ModelScope 匿名访问。 +- 两来源同名仓库的筛选、分页、计数及统一身份字段。 +- 已存在任务仍转交 Speed 决定复用，并透传 disposition。 +- HF 固定版本 tree 的 LFS 内容身份解析。 + +使用只读源码挂载、禁用外网的 Linux Go 1.24 容器执行 `go test ./...`。本机默认 Go 1.26 与 sonic 不兼容；Windows Go 1.23.1 还受既有 Linux `syscall.Stat_t` 代码限制，因此未改项目依赖或平台代码来绕过。 + +本次验证为真实 DAO/HTTP 调用及隔离数据库测试，上游和 Speed 响应使用受控 fixture；未连接真实 ModelScope 私有仓库，未进行真实多节点下载或浏览器验收。 + +## 尚未扩展的展示信息 + +当前 Speed 的 ModelScope 元数据适配只提供仓库版本、文件清单和容量，未提供点赞、下载数、标签、任务分类及组织头像。本次不把缺失信息伪造成上游统计，也不根据 HF 标签体系猜测 ModelScope 分类；这些展示字段仍需确认实际上游接口及映射后单独补充。 diff --git a/docs/upload-inventory-schema.sql b/docs/upload-inventory-schema.sql index 87ba0ca..7b65cd7 100644 --- a/docs/upload-inventory-schema.sql +++ b/docs/upload-inventory-schema.sql @@ -32,3 +32,15 @@ CREATE TABLE IF NOT EXISTS upload_inventory_holding ( UNIQUE KEY uk_upload_holding(file_id, instance_id), INDEX idx_upload_holding_instance(instance_id), INDEX idx_upload_holding_file(file_id) ); + +-- v2 fencing and repository watermarks; independent from holdings lifetime. +CREATE TABLE IF NOT EXISTS upload_report_nodes ( + instance_id VARCHAR(191) PRIMARY KEY, + epoch VARCHAR(64), pending_epoch VARCHAR(64), baseline_digest VARCHAR(64), + status VARCHAR(32), error TEXT, updated_at DATETIME(3) +); +CREATE TABLE IF NOT EXISTS upload_report_repos ( + repo_hash VARCHAR(64) PRIMARY KEY, + instance_id VARCHAR(191), epoch VARCHAR(64), sequence BIGINT UNSIGNED, + digest VARCHAR(64), INDEX idx_upload_report_node(instance_id) +); diff --git a/docs/upload-inventory.md b/docs/upload-inventory.md index 4663de7..d3b1aef 100644 --- a/docs/upload-inventory.md +++ b/docs/upload-inventory.md @@ -1,33 +1,54 @@ -# 上传库存（第一阶段） +# 上传库存：持久队列主动推送（协议 v2） -本功能只记录各 DingoSpeed 节点已经生效且可读取的本地上传文件，不参与远端仓库下载，也不提供权威 revision、集群发布事务或自动同步。 +只记录 Speed 已生效、可提供的上传文件。远端下载进度、清理上报、仓库范本与节点间同步保持独立。 -## 持久化边界 +## 正常运行 -Scheduler 使用三个独立表： +Speed 在 `/.upload-inventory/outbox.json` 保存变化序号、按 namespace/repoType/完整仓库名合并的待处理记录、操作恢复日志及固定待确认报告。该目录不在任何仓库目录内，整仓删除不会删除待报告墓碑。 -- `upload_inventory_file`：由 `namespace + repo_type + repo + path + sha256` 唯一确定的文件；`size` 用于描述和一致性校验。 -- `upload_inventory_holding`：文件与 Speed `instance_id` 的多对多持有关系。 -- `upload_inventory_state`：每个节点最后接收的 epoch、单调序号、完整性、确认时间和错误信息。 +本地有效元数据变更前先落盘操作意图，操作完成后清除意图；未完成意图可按确定目标重放。发布、删除和恢复经过同一套记录机制。暂存上传不触发有效库存变更。网络不参与本地提交，日志收尾失败也不会将已生效操作改判为网络失败。 -表结构见 `upload-inventory-schema.sql`。Scheduler 启动时也会以 GORM `AutoMigrate` 创建这些新增表。原有 `repository`、`model_file_record`、`model_file_process` 等远端业务表不会被上传库存的写入、删除、恢复或对账修改。 +工作线程汇总变化仓库所有有效本地 revision 的文件并主动 POST；同路径不同 SHA256 保留，同内容不同路径保留，多个 revision 的相同引用只计一次。扫描绕过下载服务的元数据缓存，读取不完整或内容不可提供时报错，保留队列与最后确认库存。 -## 收敛机制 +报告生成后序号、文件列表、删除标记固定并持久化。请求失败、超时或响应丢失重发同一报告；收到匹配 epoch/sequence/digest 的确认后，仅清除该报告覆盖的记录。发送期间的更新继续保留。 -DingoSpeed 每 30 秒以及本地上传、发布、删除、回收或恢复发生变化后扫描一次有效本地 revision，并将完整快照原子写入 `/.upload-inventory/snapshot.json`。快照 epoch 和 sequence 持久化，因此服务重启后仍能继续单调报告；本地操作已经成功但进程在通知前退出时，下一次启动扫描也会补报。 +连续变化首次延迟 300ms 合并，后续变化不推迟首个截止时间。失败采用有随机抖动的指数退避，最大约 257 秒。注册/重连唤醒待处理队列；空队列不枚举仓库、不发送库存。没有 30 秒周期盘点，心跳不触发盘点。 -Speed 向 Scheduler 的 `IngestRepository` RPC 只发送一次性读取令牌。Scheduler 从已注册节点的内部 HTTP 端点读取对应完整快照。读取或校验失败不会改变现有持有关系；扫描不完整时只记录失败状态，也不会把未出现的文件解释为删除。只有更新的完整快照才能替换该节点的持有集合。重复、旧序号或旧 epoch 快照会被忽略。 +## 事务与序号 -节点失去心跳只影响查询结果中的 `nodeAvailable`，不会删除最后确认的持有关系。只有完整快照确认缺失，或本地明确操作后产生的完整快照，才移除该节点持有关系；某文件不再有任何节点持有后才删除全局文件记录。 +Scheduler 复用 `upload_inventory_file`、`upload_inventory_holding` 和节点查询状态表，新增： -## 查询接口 +- `upload_report_nodes`：每节点当前代次、待对账代次、基线摘要、任务状态。 +- `upload_report_repos`：每节点每仓库接收水位及固定报告摘要。删除持有关系后保留该水位。 -- `GET /api/v1/upload-inventory/repositories` -- `GET /api/v1/upload-inventory/files`，可选 `namespace`、`repoType`、`repo`、`instanceId` -- `GET /api/v1/upload-inventory/nodes/:instanceId/files`（响应的 `state` 即使文件列表为空也会给出节点最后库存确认状态） +一次事务中锁定对应节点状态、比较代次和仓库序号、替换对应节点与仓库的持有关系并保存水位，提交后才确认。相同序号不同内容拒绝；重复报告确认；旧序号不覆盖新状态；旧代次拒绝。节点间没有共享上报互斥锁。无持有关系的文件身份可以保留，但库存目录查询不会显示它，避免跨节点报告与身份清理产生竞争。 -文件结果同时包含仓库身份、相对路径、SHA256、大小、节点、最后确认时间、节点当前可用性，以及该节点最近库存是否完整确认。Speed 的 `/api/upload-inventory` 是 Scheduler 使用的令牌保护内部端点，不是面向用户的查询接口。 +## 初次基线与人为对账 -## 配置和启动 +初次接入 v2 时由 Scheduler 分配基线代次。仅初次接入与用户明确对账才全量盘点；普通重启继续持久队列。由 v1 升级到 v2 会建立一次新基线，此后旧全节点回拉报告不再被接受。 -沿用现有 Scheduler 数据库配置和 Speed 注册/心跳配置，无需新增配置项。先启动 Scheduler，再按原方式启动 Speed；Scheduler 暂时不可用不影响 Speed 的本地上传、发布、删除、回收或恢复。连接恢复后，周期完整快照会自动对账。 +人为对账：Spinfield 节点配置 → **重新对账上传库存** → Scheduler 持久化任务并通知目标 Speed → Speed 固定完整清单并校验内容 SHA256 → POST → Scheduler 原子替换该节点全量上传持有关系 → Speed 收到确认后归零。 + +成功时切换新 epoch，清空被基线覆盖的旧队列，序号重置为 0。盘点固定之后发生的新变化保留，并从新代次的 1 开始重新编号；它们已产生时，观察到的序号可以立即大于 0。其他节点的队列、序号和代次不变。重复基线不会擦除其后已应用的仓库更新。 + +任务失败保持原库存和原代次。已固定报告的发送失败只重发原报告，不重复扫描；本地基线构建最多自动尝试三次，达到上限后持久保存停止状态并上报 `needs_attention` 和错误，不再自动全量扫盘。普通重启、重连、重复投递和新的仓库变更不会重置次数。修复原因后再次显式对账，由 Scheduler 分配新代次重新执行；进行中的非停止任务仍合并重复请求。停止期间普通仓库变化继续持久入队，待完整基线确认后接续，不能用部分基线覆盖库存。任务投递有独立重试；Scheduler 重启恢复未投递任务，Speed 重连也取得待执行任务。页面 GET 和刷新只读取任务状态。 + +Speed 丢失序号但 Scheduler 已有代次时不会擅自从 1 上报，必须显式对账。队列 JSON 损坏时显式请求会先保存 `.damaged-*` 副本再建立重建状态；权限或其他普通 IO 失败不会被当成可清空的损坏文件。直接改磁盘不产生事件，可由显式对账发现。 + +## 接口 + +- `POST /api/v1/upload-inventory/reports`：v2 报告，单请求上限 128 MiB。 +- `POST /api/v1/upload-inventory/nodes/:instanceId/session`：首次基线和重连控制状态，不自行扫描磁盘。 +- `POST /api/v1/upload-inventory/nodes/:instanceId/reconcile`：显式创建/重试投递对账任务。 +- `GET /api/v1/upload-inventory/nodes/:instanceId/reconcile`：只读任务状态。 +- `POST /api/v1/upload-inventory/nodes/:instanceId/reconcile-progress`：扫描/重试/需要处理状态，不修改持有关系。 +- Speed 管理口 `POST /api/upload-inventory/reconcile`：持久化对账请求并唤醒处理。 +- Spinfield `GET/POST /api/v1/node-settings/:nodeId/inventory-reconcile`：经节点管理权限校验后访问 Scheduler。 + +原上传库存 repositories/files/nodes 文件查询接口保持不变。节点离线仅影响可用性，保留最后持有关系。原远端下载接口不变。 + +## 配置与限制 + +Speed 新增 `scheduler.httpUrl`（注册配置为 `httpUrl`），指向 Scheduler HTTP 根地址，不从 gRPC 端口推断。Spinfield 接管注册的节点自动使用全局 Scheduler HTTP 地址；外部管理的节点需在 Speed 配置中提供。先升级 Scheduler，再升级 Speed；失败期间本地操作继续。 + +此版本使用原子文件状态存储，不引入外部消息系统。仓库快照及基线在本地有效元数据变更屏障内生成，网络发送不持有屏障。大仓库/大基线需要相应内存与扫描时间，超过 128 MiB 请求限制会明确失败并保留待处理状态，尚未实现分块清单传输。部署沿用现有内部管理网络和权限边界。 diff --git a/internal/dao/cache_job_dao.go b/internal/dao/cache_job_dao.go index c6bd5d5..de49d70 100644 --- a/internal/dao/cache_job_dao.go +++ b/internal/dao/cache_job_dao.go @@ -124,9 +124,19 @@ func (c *CacheJobDao) UpdateStatusAndRepo(jobStatusReq *query.UpdateJobStatusReq return err } if jobStatusReq.Status == consts.RunningStatusJobComplete { - err = c.repositoryDao.PersistRepo(&query.PersistRepoReq{Datatype: job.Datatype, InstanceIds: []string{jobStatusReq.InstanceId}, - Org: jobStatusReq.Org, Repo: jobStatusReq.Repo, OffVerify: true}) + if job.Commit == "" { + err = fmt.Errorf("completed cache job has no pinned commit") + } else { + err = c.repositoryDao.PersistRepo(&query.PersistRepoReq{Datatype: job.Datatype, InstanceIds: []string{jobStatusReq.InstanceId}, + Org: jobStatusReq.Org, Repo: jobStatusReq.Repo, Commit: job.Commit, CompletedJobID: job.ID}) + } if err != nil { + // Preserve download completion and expose the pending registration. + // Returning the error engages Speed's durable notification retry. + msg, _ := sonic.Marshal(map[string]string{"msg": "repository registration pending: " + err.Error()}) + if saveErr := c.baseData.BizDB.Model(job).Update("error_msg", string(msg)).Error; saveErr != nil { + return fmt.Errorf("%v; storing registration error: %w", err, saveErr) + } return err } } @@ -160,13 +170,18 @@ func (c *CacheJobDao) ListCacheJob(condition *query.CacheJobQuery) ([]*model.Cac if condition.Repo != "" { db = db.Where("repo = ?", condition.Repo) } + var err error + db, err = filterNamespace(db, "org", condition.Namespace) + if err != nil { + return nil, 0, err + } var count int64 if err := db.Count(&count).Error; err != nil { zap.S().Error("统计数量失败", err) return nil, 0, err } offset, pageSize := paginate(condition.Page, condition.PageSize) - db = db.Order(fmt.Sprintf("created_at desc offset %d limit %d", offset, pageSize)) + db = db.Order("created_at DESC, id DESC").Offset(offset).Limit(pageSize) if err := db.Find(&cacheJobs).Error; err != nil { return nil, 0, err } diff --git a/internal/dao/hf_token_dao.go b/internal/dao/hf_token_dao.go index 2852f46..e098fac 100644 --- a/internal/dao/hf_token_dao.go +++ b/internal/dao/hf_token_dao.go @@ -6,6 +6,8 @@ import ( "dingoscheduler/internal/data" "dingoscheduler/internal/model" + "dingoscheduler/pkg/config" + "dingoscheduler/pkg/repository" "go.uber.org/zap" ) @@ -51,3 +53,15 @@ func (d *HfTokenDao) GetHeaders() map[string]string { } return m } + +// ProviderHeaders never sends a credential belonging to another upstream. +func (d *HfTokenDao) ProviderHeaders(key repository.Key) map[string]string { + if key.Namespace == "huggingface" { + return d.GetHeaders() + } + headers := make(map[string]string) + if key.Namespace == "modelscope" && config.SysConfig != nil && config.SysConfig.Scheduler.ModelScopeToken != "" { + headers["Authorization"] = "Bearer " + config.SysConfig.Scheduler.ModelScopeToken + } + return headers +} diff --git a/internal/dao/ingest_dao.go b/internal/dao/ingest_dao.go index 3582c79..5b9f870 100644 --- a/internal/dao/ingest_dao.go +++ b/internal/dao/ingest_dao.go @@ -134,6 +134,15 @@ func uploadIdentity(item *UploadedInventoryItem) string { // exactly one node's upload holdings and cannot touch remote-domain tables. func (r *RepositoryDao) ApplyUploadedInventory(ctx context.Context, snap *UploadedInventorySnapshot) (accepted bool, count, total int64, err error) { err = r.baseData.BizDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if tx.Migrator().HasTable(&model.UploadReportNode{}) { + node, e := lockReportNode(tx, snap.InstanceID) + if e != nil { + return e + } + if node.Epoch != "" { + return fmt.Errorf("legacy full-node reports disabled after inventory protocol upgrade") + } + } var state model.UploadInventoryState findErr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("instance_id = ?", snap.InstanceID).Take(&state).Error if findErr != nil && findErr != gorm.ErrRecordNotFound { diff --git a/internal/dao/namespace_filter.go b/internal/dao/namespace_filter.go new file mode 100644 index 0000000..fe9c8f3 --- /dev/null +++ b/internal/dao/namespace_filter.go @@ -0,0 +1,25 @@ +package dao + +import ( + "dingoscheduler/pkg/repository" + "gorm.io/gorm" +) + +// column is supplied only by DAO code, never by an HTTP request. Prefixes are +// literal; arbitrary hosted namespaces use equality rather than SQL wildcards. +func filterNamespace(db *gorm.DB, column, namespace string) (*gorm.DB, error) { + if namespace == "" { + return db, nil + } + if err := repository.ValidatePath(namespace, false); err != nil { + return nil, err + } + switch namespace { + case "modelscope": + return db.Where(column+" LIKE ?", "modelscope/%"), nil + case "huggingface": + return db.Where(column+" NOT LIKE ? AND "+column+" NOT LIKE ?", "modelscope/%", "dingo-local/%"), nil + default: + return db.Where(column+" = ?", "dingo-local/"+namespace), nil + } +} diff --git a/internal/dao/remote_file_identity.go b/internal/dao/remote_file_identity.go new file mode 100644 index 0000000..25066da --- /dev/null +++ b/internal/dao/remote_file_identity.go @@ -0,0 +1,101 @@ +package dao + +import ( + "encoding/json" + "fmt" + "math" + "net/http" + "time" + + "dingoscheduler/internal/model/dto" + "dingoscheduler/pkg/repository" +) + +type remoteTreeFile struct { + Type string `json:"type"` + Path string `json:"path"` + OID string `json:"oid"` + Size *int64 `json:"size"` + LFS *struct { + OID string `json:"oid"` + Size int64 `json:"size"` + } `json:"lfs"` +} + +// Resolve absent file identities in one recursive tree request at the pinned +// commit. Speed handles provider pagination; all metadata paths must be present. +// This avoids per-file HEAD requests (which can create cache links in Speed). +func (r *RepositoryDao) resolveFileIdentities(meta *dto.CommitHfSha, domain string, key repository.Key, headers map[string]string) error { + needsTree := false + for _, file := range meta.Siblings { + if file.LFS == nil && (file.BlobID == "" || file.Size == nil) { + needsTree = true + } + } + tree := map[string]remoteTreeFile{} + if needsTree { + uri, err := key.OperationURI("tree", meta.Sha, "") + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodGet, domain+uri+"&recursive=true", nil) + if err != nil { + return err + } + for name, value := range headers { + req.Header.Set(name, value) + } + client := &http.Client{Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("snapshot tree returned HTTP %d", resp.StatusCode) + } + var files []remoteTreeFile + if err := json.NewDecoder(resp.Body).Decode(&files); err != nil { + return err + } + for _, file := range files { + if file.Type != "file" { + continue + } + if _, exists := tree[file.Path]; exists { + return fmt.Errorf("duplicate snapshot tree path") + } + tree[file.Path] = file + } + } + seen := make(map[string]bool) + var total int64 + for i := range meta.Siblings { + file := &meta.Siblings[i] + if repository.ValidatePath(file.Rfilename, true) != nil || seen[file.Rfilename] { + return fmt.Errorf("invalid or duplicate snapshot path") + } + seen[file.Rfilename] = true + if file.LFS != nil { + file.BlobID, file.Size = file.LFS.OID, &file.LFS.Size + } + if file.BlobID == "" || file.Size == nil { + item, ok := tree[file.Rfilename] + if !ok { + return fmt.Errorf("snapshot tree is missing %s", file.Rfilename) + } + // HF blobId can identify a Git LFS pointer. The LFS OID identifies + // the downloadable content used by Scheduler's file records. + if item.LFS != nil { + item.OID, item.Size = item.LFS.OID, &item.LFS.Size + } + file.BlobID, file.Size = item.OID, item.Size + } + if repository.ValidatePath(file.BlobID, false) != nil || file.Size == nil || *file.Size < 0 || *file.Size > math.MaxInt64-total { + return fmt.Errorf("invalid snapshot content identity") + } + total += *file.Size + } + meta.UsedStorage = total + return nil +} diff --git a/internal/dao/remote_parity_test.go b/internal/dao/remote_parity_test.go new file mode 100644 index 0000000..a977d68 --- /dev/null +++ b/internal/dao/remote_parity_test.go @@ -0,0 +1,227 @@ +package dao + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "dingoscheduler/internal/data" + "dingoscheduler/internal/model" + "dingoscheduler/internal/model/dto" + "dingoscheduler/internal/model/query" + "dingoscheduler/pkg/config" + "dingoscheduler/pkg/consts" + "dingoscheduler/pkg/repository" + "github.com/glebarez/sqlite" + "github.com/patrickmn/go-cache" + "gorm.io/gorm" +) + +func remoteTestDAO(t *testing.T, handler http.HandlerFunc) (*RepositoryDao, *CacheJobDao, *gorm.DB) { + t.Helper() + previous := config.SysConfig + config.SysConfig = &config.Config{Retry: config.Retry{Attempts: 1}, Scheduler: config.Scheduler{ModelScopeToken: "ms-test"}} + t.Cleanup(func() { config.SysConfig = previous }) + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, _ := db.DB() + t.Cleanup(func() { sqlDB.Close() }) + if err := db.AutoMigrate(&model.Repository{}, &model.RepositoryTag{}, &model.Tag{}, &model.CacheJob{}, &model.Dingospeed{}, &model.ModelFileRecord{}, &model.ModelFileProcess{}, &model.HfToken{}); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + host, port, _ := net.SplitHostPort(strings.TrimPrefix(srv.URL, "http://")) + p, _ := strconv.Atoi(port) + if err := db.Create(&model.Dingospeed{InstanceID: "node", Host: host, Port: int32(p), Online: true}).Error; err != nil { + t.Fatal(err) + } + d := &data.BaseData{BizDB: db, Cache: cache.New(time.Minute, time.Minute)} + r := NewRepositoryDao(d, NewRepositoryTagDao(d), NewTagDao(d), NewDingospeedDao(d), NewOrganizationDao(d), NewHfTokenDao(d)) + return r, NewCacheJobDao(d, r), db +} + +func putRemoteFile(t *testing.T, db *gorm.DB, org, path, etag string, size int64) { + t.Helper() + r := &model.ModelFileRecord{Datatype: "models", Org: org, Repo: "demo", Name: path, Etag: etag, FileSize: size} + if err := db.Create(r).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.ModelFileProcess{RecordID: r.ID, InstanceID: "node", OffsetNum: size, Status: 3}).Error; err != nil { + t.Fatal(err) + } +} + +func TestModelScopeCompletionRetriesPinnedSnapshotAndUpdatesProjection(t *testing.T) { + fail, mismatch := true, false + requests := 0 + _, jobs, db := remoteTestDAO(t, func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Path != "/api/repositories/models/modelscope/metadata" || r.URL.Query().Get("repo") != "owner/demo" || r.Header.Get("Authorization") != "Bearer ms-test" { + t.Errorf("unexpected request %s auth=%q", r.URL, r.Header.Get("Authorization")) + } + commit := r.URL.Query().Get("revision") + if commit != "snapshot-1" && commit != "snapshot-2" { + t.Errorf("queried mutable revision %s", commit) + } + if fail { + http.Error(w, "unavailable", 503) + return + } + if mismatch { + commit = "different-head" + } + fmt.Fprintf(w, `{"sha":%q,"siblings":[{"rfilename":"weights.bin","blobId":%q,"size":8}],"usedStorage":8}`, commit, commit) + }) + putRemoteFile(t, db, "modelscope/owner", "weights.bin", "snapshot-1", 8) + job := &model.CacheJob{Type: consts.CacheTypePreheat, InstanceId: "node", Datatype: "models", Org: "modelscope/owner", Repo: "demo", Commit: "snapshot-1", Status: 1} + if err := jobs.Save(job); err != nil { + t.Fatal(err) + } + complete := func(id int64) error { + return jobs.UpdateStatusAndRepo(&query.UpdateJobStatusReq{Id: id, InstanceId: "node", Status: consts.RunningStatusJobComplete, Process: 100}) + } + if err := complete(job.ID); err == nil { + t.Fatal("registration failure was hidden") + } + stored, _ := jobs.GetCacheJob(&query.CacheJobQuery{Id: job.ID}) + if stored.Status != 3 || !strings.Contains(stored.ErrorMsg, "registration pending") { + t.Fatalf("lost completion or error: %+v", stored) + } + fail = false + if err := complete(job.ID); err != nil { + t.Fatal(err) + } + if err := complete(job.ID); err != nil { + t.Fatal(err) + } + var repos []model.Repository + db.Find(&repos) + if len(repos) != 1 || repos[0].Sha != "snapshot-1" { + t.Fatalf("retry not idempotent: %+v", repos) + } + firstID := repos[0].ID + job2 := &model.CacheJob{Type: consts.CacheTypePreheat, InstanceId: "node", Datatype: "models", Org: "modelscope/owner", Repo: "demo", Commit: "snapshot-2", Status: 1} + if err := jobs.Save(job2); err != nil { + t.Fatal(err) + } + // An old complete file at the same path cannot satisfy the new snapshot. + if err := complete(job2.ID); err == nil { + t.Fatal("wrong content accepted") + } + putRemoteFile(t, db, "modelscope/owner", "weights.bin", "snapshot-2", 8) + mismatch = true + if err := complete(job2.ID); err == nil { + t.Fatal("upstream commit drift accepted") + } + mismatch = false + if err := complete(job2.ID); err != nil { + t.Fatal(err) + } + before := requests + if err := complete(job.ID); err != nil { + t.Fatal(err) + } + if requests != before { + t.Fatal("stale completion fetched an old snapshot") + } + repos = nil + db.Find(&repos) + if len(repos) != 1 || repos[0].ID != firstID || repos[0].Sha != "snapshot-2" { + t.Fatalf("projection replaced or regressed: %+v", repos) + } + stored, _ = jobs.GetCacheJob(&query.CacheJobQuery{Id: job2.ID}) + if stored.ErrorMsg != "" { + t.Fatal("successful retry retained registration error") + } +} + +func TestModelScopeDiscoveryUsesMasterAndExactFiles(t *testing.T) { + repo, _, db := remoteTestDAO(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("revision") != "master" { + t.Errorf("expected master: %s", r.URL) + } + fmt.Fprint(w, `{"sha":"snapshot","siblings":[{"rfilename":"weights.bin","blobId":"content","size":8}]}`) + }) + putRemoteFile(t, db, "modelscope/owner", "other.bin", "content", 8) + if err := repo.PersistRepo(&query.PersistRepoReq{InstanceIds: []string{"node"}}); err == nil { + t.Fatal("same count with a different path accepted") + } + putRemoteFile(t, db, "modelscope/owner", "weights.bin", "content", 8) + if err := repo.PersistRepo(&query.PersistRepoReq{InstanceIds: []string{"node"}}); err != nil { + t.Fatal(err) + } + var result model.Repository + if err := db.First(&result).Error; err != nil { + t.Fatal(err) + } + if result.Sha != "snapshot" || result.UsedStorage != 8 { + t.Fatalf("wrong projection: %+v", result) + } +} + +func TestProviderCredentialsAndNamespacePagination(t *testing.T) { + repos, jobs, db := remoteTestDAO(t, func(http.ResponseWriter, *http.Request) {}) + if err := db.Create(&model.HfToken{Token: "hf-test", Enabled: true}).Error; err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ ns, want string }{{"huggingface", "Bearer hf-test"}, {"modelscope", "Bearer ms-test"}, {"alice", ""}} { + if got := repos.hfTokenDao.ProviderHeaders(repository.Key{Namespace: tc.ns})["Authorization"]; got != tc.want { + t.Fatalf("%s credential = %q", tc.ns, got) + } + } + config.SysConfig.Scheduler.ModelScopeToken = "" + if len(repos.hfTokenDao.ProviderHeaders(repository.Key{Namespace: "modelscope"})) != 0 { + t.Fatal("fell back to HF credential") + } + for _, org := range []string{"owner", "modelscope/owner", "dingo-local/alice"} { + if err := db.Create(&model.Repository{InstanceId: "node", Datatype: "models", Org: org, Repo: "demo"}).Error; err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := jobs.Save(&model.CacheJob{InstanceId: "node", Datatype: "models", Org: org, Repo: "demo"}); err != nil { + t.Fatal(err) + } + } + } + for _, ns := range []string{"huggingface", "modelscope", "alice"} { + r, total, err := repos.ModelList(&query.ModelQuery{Namespace: ns}) + if err != nil || total != 1 || len(r) != 1 || repositoryKey(r[0]).Namespace != ns { + t.Fatalf("repository filter %s: %v %d %v", ns, r, total, err) + } + first, count, err := jobs.ListCacheJob(&query.CacheJobQuery{Namespace: ns, Page: 1, PageSize: 1}) + if err != nil || count != 2 || len(first) != 1 { + t.Fatalf("job filter: %v %d %v", first, count, err) + } + second, _, err := jobs.ListCacheJob(&query.CacheJobQuery{Namespace: ns, Page: 2, PageSize: 1}) + if err != nil || len(second) != 1 || second[0].ID == first[0].ID { + t.Fatal("pagination repeated first result") + } + } +} + +func TestPinnedTreeResolvesHFFileIdentity(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || r.URL.Query().Get("recursive") != "true" || r.URL.Query().Get("revision") != "pinned" || r.Header.Get("Authorization") != "Bearer hf" { + t.Errorf("unexpected tree: %s %s", r.Method, r.URL) + } + fmt.Fprint(w, `[{"type":"file","path":"weights.bin","oid":"git-pointer","size":120,"lfs":{"oid":"content","size":8}}]`) + })) + defer srv.Close() + var meta dto.CommitHfSha + json.Unmarshal([]byte(`{"sha":"pinned","siblings":[{"rfilename":"weights.bin"}]}`), &meta) + if err := (&RepositoryDao{}).resolveFileIdentities(&meta, srv.URL, repository.Key{Namespace: "huggingface", RepoType: "models", Repo: "owner/demo"}, map[string]string{"Authorization": "Bearer hf"}); err != nil { + t.Fatal(err) + } + if meta.Siblings[0].BlobID != "content" || meta.UsedStorage != 8 { + t.Fatalf("unresolved metadata: %+v", meta) + } +} diff --git a/internal/dao/repository_dao.go b/internal/dao/repository_dao.go index e41a629..9e0c364 100644 --- a/internal/dao/repository_dao.go +++ b/internal/dao/repository_dao.go @@ -15,6 +15,7 @@ package dao import ( + "errors" "fmt" "net/http" "strings" @@ -24,6 +25,7 @@ import ( "dingoscheduler/internal/model" "dingoscheduler/internal/model/dto" "dingoscheduler/internal/model/query" + "dingoscheduler/pkg/consts" myerr "dingoscheduler/pkg/error" "dingoscheduler/pkg/repository" "dingoscheduler/pkg/util" @@ -69,6 +71,21 @@ func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error } persistRepoReq.Org, persistRepoReq.Repo = org, repo } + if persistRepoReq.Commit != "" { + if persistRepoReq.Repo == "" { + return fmt.Errorf("commit requires an explicit repository") + } + if err := repository.ValidatePath(persistRepoReq.Commit, false); err != nil { + return err + } + key, err := repository.FromWire(persistRepoReq.Datatype, persistRepoReq.Org, persistRepoReq.Repo) + if err != nil { + return err + } + if key.Namespace != "huggingface" && key.Namespace != "modelscope" { + return fmt.Errorf("remote persistence requires a remote repository") + } + } zap.S().Debugf("PersistRepo start instanceId:%s， org:%s, repo:%s", persistRepoReq.InstanceIds, persistRepoReq.Org, persistRepoReq.Repo) var ( @@ -77,6 +94,7 @@ func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error ) r.persistSync.Lock() defer r.persistSync.Unlock() + var failures []error pipelineMap, err = r.cachePipelineTags() if err != nil { return err @@ -90,6 +108,10 @@ func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error if err != nil { return err } + if persistRepoReq.Commit != "" { + // A completed newer snapshot updates the existing projection in place. + freeRepositories = []*model.Repository{{Datatype: persistRepoReq.Datatype, Org: persistRepoReq.Org, Repo: persistRepoReq.Repo}} + } if len(freeRepositories) == 0 { zap.S().Warnf("instanceId:%s 没有要持久化的仓库。", instanceId) continue @@ -106,19 +128,35 @@ func (r *RepositoryDao) PersistRepo(persistRepoReq *query.PersistRepoReq) error if repositoryKey(repository).Namespace != "huggingface" && repositoryKey(repository).Namespace != "modelscope" { continue } - if err = r.singleRepositoryPersist(repository, instanceId, speedDomain, pipelineMap, persistRepoReq.OffVerify); err != nil { + commit := persistRepoReq.Commit + var jobs []model.CacheJob + if err = r.baseData.BizDB.Where("instance_id = ? AND datatype = ? AND org = ? AND repo = ? AND status = ?", instanceId, repository.Datatype, repository.Org, repository.Repo, consts.RunningStatusJobComplete).Order("id DESC").Limit(1).Find(&jobs).Error; err != nil { + return err + } + if len(jobs) > 0 { + if persistRepoReq.CompletedJobID > 0 && jobs[0].ID > persistRepoReq.CompletedJobID { + continue + } + if commit == "" { + commit = jobs[0].Commit + } + } + if err = r.singleRepositoryPersist(repository, instanceId, speedDomain, pipelineMap, persistRepoReq.OffVerify, commit); err != nil { zap.S().Errorf("singleRepositoryPersist err.%v", err) + failures = append(failures, fmt.Errorf("%s %s: %w", instanceId, repositoryKey(repository).ID(), err)) continue } } } zap.S().Debugf("PersistRepo end instanceId:%s， org:%s, repo:%s", persistRepoReq.InstanceIds, persistRepoReq.Org, persistRepoReq.Repo) - return nil + return errors.Join(failures...) } -func (r *RepositoryDao) singleRepositoryPersist(repository *model.Repository, instanceId, speedDomain string, pipelineMap map[string]string, offVerify bool) error { +func (r *RepositoryDao) singleRepositoryPersist(repository *model.Repository, instanceId, speedDomain string, pipelineMap map[string]string, offVerify bool, commit string) error { orgRepo := util.GetOrgRepo(repository.Org, repository.Repo) - metaResp, err := r.dingospeedDao.RemoteRequestMeta(speedDomain, repositoryKey(repository), "main", r.hfTokenDao.GetHeaders()) + key := repositoryKey(repository) + headers := r.hfTokenDao.ProviderHeaders(key) + metaResp, err := r.dingospeedDao.RemoteRequestMeta(speedDomain, key, commit, headers) if err != nil { return err } @@ -130,15 +168,23 @@ func (r *RepositoryDao) singleRepositoryPersist(repository *model.Repository, in zap.S().Errorf("unmarshal error.orgRepo:%s, %v", orgRepo, err) return err } + if metaData.Sha == "" || metaData.Siblings == nil { + return fmt.Errorf("incomplete repository metadata") + } + if commit != "" && metaData.Sha != commit { + return fmt.Errorf("metadata does not match pinned commit %s", commit) + } if !offVerify { + if err := r.resolveFileIdentities(&metaData, speedDomain, key, headers); err != nil { + return err + } // 根据当前版本的元数据与下载进度、进度比较，只将完整的模型做保存。 isComplete, err := r.verifyRepoComplete(&metaData, instanceId, repository.Datatype, repository.Org, repository.Repo) if err != nil { return err } if !isComplete { - zap.S().Infof("repo file unComplete.%s", orgRepo) - return nil + return fmt.Errorf("repository snapshot is not fully cached: %s", orgRepo) } } // 保存组织图片 @@ -191,15 +237,21 @@ func (r *RepositoryDao) cachePipelineTags() (map[string]string, error) { } func (r *RepositoryDao) verifyRepoComplete(metaData *dto.CommitHfSha, instanceId, datatype, org, repo string) (bool, error) { - size, err := r.VerifyRepoComplete(instanceId, datatype, org, repo) - if err != nil { - return false, err - } - fileCount := len(metaData.Siblings) - if size >= int64(fileCount) { - return true, nil + for _, file := range metaData.Siblings { + if file.BlobID == "" || file.Size == nil { + return false, fmt.Errorf("missing file identity") + } + var count int64 + err := r.baseData.BizDB.Table("model_file_record r").Joins("JOIN model_file_process p ON p.record_id = r.id"). + Where("r.datatype = ? AND r.org = ? AND r.repo = ? AND r.name = ? AND r.etag = ? AND r.file_size = ? AND p.instance_id = ? AND p.offset_num = r.file_size", datatype, org, repo, file.Rfilename, file.BlobID, *file.Size, instanceId).Count(&count).Error + if err != nil { + return false, err + } + if count == 0 { + return false, nil + } } - return false, nil + return true, nil } func (r *RepositoryDao) SaveBySql(tx *gorm.DB, repo *model.Repository) (int64, error) { @@ -225,7 +277,18 @@ func (r *RepositoryDao) Get(id int64) (*model.Repository, error) { func (r *RepositoryDao) RepoAndTagSave(repository *model.Repository, tags []*model.RepositoryTag) error { if err := r.baseData.BizDB.Transaction(func(tx *gorm.DB) error { - lastId, err := r.SaveBySql(tx, repository) + var existing model.Repository + err := tx.Where("instance_id = ? AND datatype = ? AND org = ? AND repo = ?", repository.InstanceId, repository.Datatype, repository.Org, repository.Repo).Take(&existing).Error + var lastId int64 + if err == gorm.ErrRecordNotFound { + lastId, err = r.SaveBySql(tx, repository) + } else if err == nil { + lastId = existing.ID + err = tx.Model(&existing).Select("like_num", "download_num", "pipeline_tag_id", "pipeline_tag", "last_modified", "used_storage", "sha").Updates(repository).Error + if err == nil { + err = tx.Where("repo_id = ?", lastId).Delete(&model.RepositoryTag{}).Error + } + } if err != nil { return err } @@ -309,6 +372,11 @@ func (r *RepositoryDao) ModelList(query *query.ModelQuery) ([]*model.Repository, if len(tags) > 0 { db = db.Where(" t1.id in (select repo_id from repository_tag where tag_id in (?))", tags) } + var filterErr error + db, filterErr = filterNamespace(db, "t1.org", query.Namespace) + if filterErr != nil { + return nil, 0, filterErr + } var count int64 if err := db.Count(&count).Error; err != nil { zap.S().Error("统计数量失败", err) diff --git a/internal/dao/upload_report.go b/internal/dao/upload_report.go new file mode 100644 index 0000000..effa5f5 --- /dev/null +++ b/internal/dao/upload_report.go @@ -0,0 +1,205 @@ +package dao + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "time" + + "dingoscheduler/internal/model" + inv "dingoscheduler/pkg/inventory" + "dingoscheduler/pkg/repository" + "github.com/google/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func lockReportNode(tx *gorm.DB, id string) (model.UploadReportNode, error) { + n := model.UploadReportNode{InstanceID: id, Status: "uninitialized"} + if id == "" || len(id) > 191 { + return n, fmt.Errorf("invalid node identity") + } + if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&n).Error; err != nil { + return n, err + } + err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("instance_id = ?", id).Take(&n).Error + return n, err +} + +func (r *RepositoryDao) UploadReportSession(ctx context.Context, id string, explicit bool) (out model.UploadReportNode, err error) { + err = r.baseData.BizDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var count int64 + if err := tx.Model(&model.Dingospeed{}).Where("instance_id = ?", id).Count(&count).Error; err != nil { + return err + } + if count == 0 { + return fmt.Errorf("node is not registered") + } + var e error + out, e = lockReportNode(tx, id) + if e != nil { + return e + } + if ((explicit || out.Epoch == "") && out.PendingEpoch == "") || (explicit && out.Status == "needs_attention") { + out.PendingEpoch, out.Status, out.Error = uuid.NewString(), "pending", "" + return tx.Save(&out).Error + } + return nil + }) + return +} + +func (r *RepositoryDao) UploadReportStatus(ctx context.Context, id string) (model.UploadReportNode, error) { + var n model.UploadReportNode + err := r.baseData.BizDB.WithContext(ctx).Where("instance_id = ?", id).Take(&n).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return model.UploadReportNode{InstanceID: id, Status: "uninitialized"}, nil + } + return n, err +} + +func validateReport(p *inv.Report) error { + if p.Version != 2 || p.InstanceID == "" || p.Epoch == "" || len(p.Epoch) > 64 { + return fmt.Errorf("invalid report envelope") + } + if p.Baseline { + if p.Sequence != 0 || p.Deleted || p.Key != (inv.Key{}) { + return fmt.Errorf("invalid baseline") + } + } else { + if p.Sequence == 0 { + return fmt.Errorf("repository sequence must be positive") + } + if err := validateReportKey(p.Key); err != nil { + return err + } + if p.Deleted && len(p.Files) > 0 { + return fmt.Errorf("deleted repository has files") + } + } + seen := map[string]bool{} + for _, f := range p.Files { + if err := validateReportKey(f.Key); err != nil { + return err + } + if !p.Baseline && f.Key != p.Key { + return fmt.Errorf("report contains another repository") + } + if err := repository.ValidatePath(f.Path, true); err != nil || len(f.Path) > 1000 { + return fmt.Errorf("invalid file path") + } + h, e := hex.DecodeString(f.SHA256) + if e != nil || len(h) != 32 || hex.EncodeToString(h) != f.SHA256 || f.Size < 0 { + return fmt.Errorf("invalid file content") + } + id := f.Key.ID() + "\x00" + f.Path + "\x00" + f.SHA256 + if seen[id] { + return fmt.Errorf("duplicate file") + } + seen[id] = true + } + return nil +} +func validateReportKey(k inv.Key) error { + if k.Namespace == "huggingface" || k.Namespace == "modelscope" { + return fmt.Errorf("remote namespace is not uploaded inventory") + } + return (repository.Key{Namespace: k.Namespace, RepoType: k.RepoType, Repo: k.Repo}).Validate() +} + +// ApplyUploadReport serializes on this node's row, never on a global service mutex. +// Epoch, watermark and holdings commit together; a lost response can be retried verbatim. +func (r *RepositoryDao) ApplyUploadReport(ctx context.Context, p *inv.Report) (ack inv.Ack, err error) { + if err = validateReport(p); err != nil { + return + } + ack = inv.Ack{Epoch: p.Epoch, Sequence: p.Sequence, Digest: p.Digest(), Status: "accepted"} + err = r.baseData.BizDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + n, e := lockReportNode(tx, p.InstanceID) + if e != nil { + return e + } + var state model.UploadReportRepo + if p.Baseline { + if n.Epoch == p.Epoch { + if n.BaselineDigest != ack.Digest { + return fmt.Errorf("baseline content conflict") + } + ack.Status = "confirmed" + return nil + } + if n.PendingEpoch != p.Epoch { + return fmt.Errorf("baseline epoch is not authorized") + } + } else { + if n.Epoch != p.Epoch { + return fmt.Errorf("report epoch is obsolete; explicit reconciliation required") + } + h := sha256.Sum256([]byte(p.InstanceID + "\x00" + p.Key.ID())) + state.RepoHash = hex.EncodeToString(h[:]) + e = tx.Where("repo_hash = ?", state.RepoHash).Take(&state).Error + if e != nil && !errors.Is(e, gorm.ErrRecordNotFound) { + return e + } + if e == nil && state.Sequence >= p.Sequence { + if state.Sequence == p.Sequence && state.Digest != ack.Digest { + return fmt.Errorf("sequence content conflict") + } + ack.Status = "confirmed" + if state.Sequence > p.Sequence { + ack.Status = "obsolete" + } + return nil + } + } + // Delete only the selected scope, then insert the replacement in the same transaction. + remove := tx.Where("instance_id = ?", p.InstanceID) + if !p.Baseline { + ids := tx.Model(&model.UploadInventoryFile{}).Select("id").Where("namespace = ? AND repo_type = ? AND repo = ?", p.Namespace, p.RepoType, p.Repo) + remove = remove.Where("file_id IN (?)", ids) + } + if e = remove.Delete(&model.UploadInventoryHolding{}).Error; e != nil { + return e + } + for _, f := range p.Files { + item := UploadedInventoryItem{Namespace: f.Namespace, RepoType: f.RepoType, Repo: f.Repo, Path: f.Path, SHA256: f.SHA256, Size: f.Size} + file := model.UploadInventoryFile{IdentityHash: uploadIdentity(&item), Namespace: f.Namespace, RepoType: f.RepoType, Repo: f.Repo, Path: f.Path, SHA256: f.SHA256, Size: f.Size} + if e = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&file).Error; e != nil { + return e + } + if e = tx.Where("identity_hash = ?", file.IdentityHash).Take(&file).Error; e != nil { + return e + } + if file.Size != f.Size || file.Namespace != f.Namespace || file.RepoType != f.RepoType || file.Repo != f.Repo || file.Path != f.Path || file.SHA256 != f.SHA256 { + return fmt.Errorf("file identity conflict") + } + if e = tx.Create(&model.UploadInventoryHolding{FileID: file.ID, InstanceID: p.InstanceID, Sequence: p.Sequence, ConfirmedAt: time.Now().UTC()}).Error; e != nil { + return e + } + } + if p.Baseline { + if e = tx.Where("instance_id = ?", p.InstanceID).Delete(&model.UploadReportRepo{}).Error; e != nil { + return e + } + n.Epoch, n.PendingEpoch, n.BaselineDigest, n.Status, n.Error = p.Epoch, "", ack.Digest, "completed", "" + if e = tx.Save(&n).Error; e != nil { + return e + } + } else { + state.InstanceID, state.Epoch, state.Sequence, state.Digest = p.InstanceID, p.Epoch, p.Sequence, ack.Digest + if e = tx.Save(&state).Error; e != nil { + return e + } + } + now := time.Now().UTC() + view := model.UploadInventoryState{InstanceID: p.InstanceID, Epoch: p.Epoch, EpochStartedAt: now, LastSequence: p.Sequence, InventoryComplete: true, LastAttemptAt: now, LastConfirmedAt: &now} + if e = tx.Clauses(clause.OnConflict{UpdateAll: true}).Create(&view).Error; e != nil { + return e + } + // Orphan identities are retained: concurrent reports from another node may be inserting holdings. + return nil + }) + return +} diff --git a/internal/dao/upload_report_retry_test.go b/internal/dao/upload_report_retry_test.go new file mode 100644 index 0000000..fc9cc1f --- /dev/null +++ b/internal/dao/upload_report_retry_test.go @@ -0,0 +1,37 @@ +package dao + +import ( + "context" + "dingoscheduler/internal/model" + "testing" +) + +func TestExplicitReconcileRestartsOnlyAttentionGeneration(t *testing.T) { + d, db := inventoryTestDAO(t) + if err := db.AutoMigrate(&model.UploadReportNode{}, &model.UploadReportRepo{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&model.Dingospeed{InstanceID: "retry-node"}).Error; err != nil { + t.Fatal(err) + } + ctx := context.Background() + first, err := d.UploadReportSession(ctx, "retry-node", true) + if err != nil { + t.Fatal(err) + } + same, err := d.UploadReportSession(ctx, "retry-node", true) + if err != nil || same.PendingEpoch != first.PendingEpoch { + t.Fatal("duplicate active request changed epoch", err) + } + if err := db.Model(&model.UploadReportNode{}).Where("instance_id = ?", "retry-node").Update("status", "needs_attention").Error; err != nil { + t.Fatal(err) + } + same, err = d.UploadReportSession(ctx, "retry-node", false) + if err != nil || same.PendingEpoch != first.PendingEpoch { + t.Fatal("reconnect reset stopped task", err) + } + next, err := d.UploadReportSession(ctx, "retry-node", true) + if err != nil || next.PendingEpoch == first.PendingEpoch || next.Status != "pending" { + t.Fatal("explicit retry did not reset", err) + } +} diff --git a/internal/dao/upload_report_test.go b/internal/dao/upload_report_test.go new file mode 100644 index 0000000..9b0bf8b --- /dev/null +++ b/internal/dao/upload_report_test.go @@ -0,0 +1,102 @@ +package dao + +import ( + "context" + "dingoscheduler/internal/model" + inv "dingoscheduler/pkg/inventory" + "strings" + "testing" +) + +func TestRepositoryReportsAndManualResetAreFencedPerNode(t *testing.T) { + dbDAO, db := inventoryTestDAO(t) + if err := db.AutoMigrate(&model.UploadReportNode{}, &model.UploadReportRepo{}); err != nil { + t.Fatal(err) + } + for _, id := range []string{"A", "B"} { + if err := db.Create(&model.Dingospeed{InstanceID: id}).Error; err != nil { + t.Fatal(err) + } + } + ctx := context.Background() + key := inv.Key{Namespace: "team", RepoType: "models", Repo: "full/name"} + other := inv.Key{Namespace: "team", RepoType: "models", Repo: "other"} + file := inv.File{Key: key, Path: "weights.bin", SHA256: strings.Repeat("a", 64), Size: 5} + baseline := func(id string) *inv.Report { + t.Helper() + n, e := dbDAO.UploadReportSession(ctx, id, true) + if e != nil { + t.Fatal(e) + } + p := &inv.Report{Version: 2, InstanceID: id, Epoch: n.PendingEpoch, Baseline: true, Files: []inv.File{file}} + if _, e = dbDAO.ApplyUploadReport(ctx, p); e != nil { + t.Fatal(e) + } + return p + } + a := baseline("A") + b := baseline("B") + apply := func(p *inv.Report) { + t.Helper() + if _, e := dbDAO.ApplyUploadReport(ctx, p); e != nil { + t.Fatal(e) + } + } + one := inv.Report{Version: 2, InstanceID: "A", Epoch: a.Epoch, Sequence: 10, Key: other, Files: []inv.File{{Key: other, Path: "x", SHA256: strings.Repeat("b", 64), Size: 1}}} + apply(&one) + del := inv.Report{Version: 2, InstanceID: "A", Epoch: a.Epoch, Sequence: 11, Key: key, Deleted: true} + apply(&del) + apply(&del) + old := del + old.Sequence = 9 + old.Deleted = false + old.Files = []inv.File{file} + apply(&old) + var count int64 + db.Model(&model.UploadInventoryHolding{}).Count(&count) + if count != 2 { + t.Fatalf("delete affected another node or repository: %d", count) + } + conflict := del + conflict.Deleted = false + conflict.Files = []inv.File{file} + if _, e := dbDAO.ApplyUploadReport(ctx, &conflict); e == nil { + t.Fatal("same sequence different bytes accepted") + } + // One failed identity validation must roll back deletion and inserted files. + bad := one + bad.Sequence = 12 + bad.Files = []inv.File{{Key: other, Path: "x", SHA256: strings.Repeat("b", 64), Size: 999}} + if _, e := dbDAO.ApplyUploadReport(ctx, &bad); e == nil { + t.Fatal("conflicting size accepted") + } + db.Model(&model.UploadInventoryHolding{}).Count(&count) + if count != 2 { + t.Fatal("failed transaction changed holdings") + } + newA := baseline("A") + if newA.Epoch == a.Epoch { + t.Fatal("manual reset reused epoch") + } + if _, e := dbDAO.ApplyUploadReport(ctx, &one); e == nil { + t.Fatal("old epoch revived inventory") + } + // Lost baseline response replay cannot erase later repository reports. + one.Epoch = newA.Epoch + one.Sequence = 1 + apply(&one) + apply(newA) + db.Model(&model.UploadInventoryHolding{}).Count(&count) + if count != 3 { + t.Fatalf("baseline replay changed inventory: %d", count) + } + n, e := dbDAO.UploadReportStatus(ctx, "B") + if e != nil || n.Epoch != b.Epoch { + t.Fatal("reset changed another node") + } + var waters []model.UploadReportRepo + db.Where("instance_id = ?", "A").Find(&waters) + if len(waters) != 1 || waters[0].Sequence != 1 { + t.Fatalf("reset watermark: %+v", waters) + } +} diff --git a/internal/data/data.go b/internal/data/data.go index d138cb5..10f403f 100644 --- a/internal/data/data.go +++ b/internal/data/data.go @@ -66,7 +66,7 @@ func NewBaseData(conf *config.Config) (*BaseData, func(), error) { // First-phase uploaded inventory is an additive schema. AutoMigrate only // creates/extends these dedicated tables and never migrates legacy remote // repository or download records. - if err = bizClient.AutoMigrate(&model.NodeEndpoint{}, &model.UploadInventoryState{}, &model.UploadInventoryFile{}, &model.UploadInventoryHolding{}, &authority.Definition{}, &authority.Receipt{}); err != nil { + if err = bizClient.AutoMigrate(&model.NodeEndpoint{}, &model.UploadInventoryState{}, &model.UploadInventoryFile{}, &model.UploadInventoryHolding{}, &model.UploadReportNode{}, &model.UploadReportRepo{}, &authority.Definition{}, &authority.Receipt{}); err != nil { cleanup() return nil, nil, fmt.Errorf("migrate uploaded inventory schema: %w", err) } diff --git a/internal/handler/cache_job_handler.go b/internal/handler/cache_job_handler.go index 78452ba..fb0db64 100644 --- a/internal/handler/cache_job_handler.go +++ b/internal/handler/cache_job_handler.go @@ -86,7 +86,7 @@ func (handler *CacheJobHandler) ListCacheJobHandler(c echo.Context) error { return util.ErrorRequestParamCN(c) } datatype := c.QueryParam("datatype") - cacheJobResps, total, err := handler.cacheJobService.ListCacheJob(instanceId, datatype, page, pageSize) + cacheJobResps, total, err := handler.cacheJobService.ListCacheJob(instanceId, datatype, page, pageSize, c.QueryParam("namespace")) if err != nil { return util.ResponseError(c, err) } diff --git a/internal/handler/repository_handler.go b/internal/handler/repository_handler.go index 8b0f2e5..20bb0f5 100644 --- a/internal/handler/repository_handler.go +++ b/internal/handler/repository_handler.go @@ -61,6 +61,7 @@ func (handler *RepositoryHandler) RepositoriesHandler(c echo.Context) error { datatype := c.QueryParam("datatype") status := c.QueryParam("status") models, total, err := handler.repositoryService.RepositoryList(&query.ModelQuery{ + Namespace: c.QueryParam("namespace"), InstanceId: instanceId, Name: name, Page: page, diff --git a/internal/handler/upload_report.go b/internal/handler/upload_report.go new file mode 100644 index 0000000..41a0a25 --- /dev/null +++ b/internal/handler/upload_report.go @@ -0,0 +1,63 @@ +package handler + +import ( + "dingoscheduler/pkg/inventory" + "encoding/json" + "github.com/labstack/echo/v4" + "io" + "net/http" +) + +func (h *ManagerHandler) UploadReport(c echo.Context) error { + var p inventory.Report + d := json.NewDecoder(http.MaxBytesReader(c.Response(), c.Request().Body, 128<<20)) + d.DisallowUnknownFields() + if err := d.Decode(&p); err != nil { + return echo.NewHTTPError(400, err.Error()) + } + if d.Decode(&struct{}{}) != io.EOF { + return echo.NewHTTPError(400, "expected one report") + } + a, err := h.schedulerService.UploadReport(c.Request().Context(), &p) + if err != nil { + return c.JSON(409, map[string]string{"error": err.Error()}) + } + return c.JSON(200, a) +} +func (h *ManagerHandler) UploadReportSession(c echo.Context) error { + n, err := h.schedulerService.UploadReportSession(c.Request().Context(), c.Param("instanceId")) + if err != nil { + return c.JSON(409, map[string]string{"error": err.Error()}) + } + return c.JSON(200, n) +} + +func (h *ManagerHandler) UploadReconcileProgress(c echo.Context) error { + var p struct { + Epoch string `json:"epoch"` + Status string `json:"status"` + Error string `json:"error"` + } + d := json.NewDecoder(http.MaxBytesReader(c.Response(), c.Request().Body, 8192)) + if err := d.Decode(&p); err != nil { + return echo.NewHTTPError(400, err.Error()) + } + if err := h.schedulerService.UploadReconcileProgress(c.Request().Context(), c.Param("instanceId"), p.Epoch, p.Status, p.Error); err != nil { + return echo.NewHTTPError(400, err.Error()) + } + return c.JSON(200, map[string]bool{"ok": true}) +} +func (h *ManagerHandler) UploadReconcile(c echo.Context) error { + if c.Request().Method == http.MethodGet { + n, err := h.schedulerService.UploadReportStatus(c.Request().Context(), c.Param("instanceId")) + if err != nil { + return c.JSON(500, map[string]string{"error": err.Error()}) + } + return c.JSON(200, n) + } + n, err := h.schedulerService.ReconcileUploadInventory(c.Request().Context(), c.Param("instanceId")) + if err != nil { + return c.JSON(409, map[string]string{"error": err.Error()}) + } + return c.JSON(202, n) +} diff --git a/internal/model/dto/dto.go b/internal/model/dto/dto.go index 2723a61..d4359db 100644 --- a/internal/model/dto/dto.go +++ b/internal/model/dto/dto.go @@ -1,6 +1,8 @@ package dto type Repository struct { + FullRepo string `json:"fullRepo"` + RepositoryID string `json:"repositoryId"` Namespace string `json:"namespace,omitempty"` Repo string `json:"repo,omitempty"` Datatype string `json:"repoType,omitempty"` @@ -19,6 +21,9 @@ type Repository struct { } type CacheJobResp struct { + Namespace string `json:"namespace"` + FullRepo string `json:"fullRepo"` + RepositoryID string `json:"repositoryId"` ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"id"` Type int32 `gorm:"column:type;not null" json:"type"` InstanceId string `gorm:"column:instance_id;not null" json:"instanceId"` diff --git a/internal/model/dto/preheat.go b/internal/model/dto/preheat.go index 084de35..dbde4e7 100644 --- a/internal/model/dto/preheat.go +++ b/internal/model/dto/preheat.go @@ -9,6 +9,12 @@ type CommitHfSha struct { LastModified string `json:"lastModified"` Siblings []struct { Rfilename string `json:"rfilename"` + BlobID string `json:"blobId"` + Size *int64 `json:"size"` + LFS *struct { + OID string `json:"oid"` + Size int64 `json:"size"` + } `json:"lfs"` } `json:"siblings"` UsedStorage int64 `json:"usedStorage"` } diff --git a/internal/model/query/query.go b/internal/model/query/query.go index 76c1c2b..891dac4 100644 --- a/internal/model/query/query.go +++ b/internal/model/query/query.go @@ -23,6 +23,7 @@ type CreateCacheJobReq struct { } type CacheJobQuery struct { + Namespace string `json:"namespace"` Id int64 `json:"id"` Type int32 `json:"type"` InstanceId string `json:"instanceId"` @@ -90,15 +91,18 @@ type PathInfoQuery struct { } type PersistRepoReq struct { - Datatype string `json:"datatype"` - Namespace string `json:"namespace"` - InstanceIds []string `json:"instanceIds"` - Org string `json:"org"` - Repo string `json:"repo"` - OffVerify bool `json:"offVerify"` + Commit string `json:"commit,omitempty"` + CompletedJobID int64 `json:"-"` + Datatype string `json:"datatype"` + Namespace string `json:"namespace"` + InstanceIds []string `json:"instanceIds"` + Org string `json:"org"` + Repo string `json:"repo"` + OffVerify bool `json:"offVerify"` } type ModelQuery struct { + Namespace string `json:"namespace"` InstanceId string `json:"instanceId"` Name string Page int diff --git a/internal/model/upload_report.go b/internal/model/upload_report.go new file mode 100644 index 0000000..dac8d7b --- /dev/null +++ b/internal/model/upload_report.go @@ -0,0 +1,23 @@ +package model + +import "time" + +// UploadReportNode fences all reports from older baselines. It survives empty inventories. +type UploadReportNode struct { + InstanceID string `gorm:"primaryKey;size:191" json:"instanceId"` + Epoch string `gorm:"size:64" json:"epoch"` + PendingEpoch string `gorm:"size:64" json:"pendingEpoch,omitempty"` + BaselineDigest string `gorm:"size:64" json:"-"` + Status string `gorm:"size:32" json:"status"` + Error string `gorm:"type:text" json:"error,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// RepoHash includes node and the full repository identity; deletion keeps this row. +type UploadReportRepo struct { + RepoHash string `gorm:"primaryKey;size:64"` + InstanceID string `gorm:"size:191;index"` + Epoch string `gorm:"size:64"` + Sequence uint64 + Digest string `gorm:"size:64"` +} diff --git a/internal/router/http_router.go b/internal/router/http_router.go index e2847a9..62590fc 100644 --- a/internal/router/http_router.go +++ b/internal/router/http_router.go @@ -50,6 +50,11 @@ func (r *HttpRouter) GetHandler() *echo.Echo { } func (r *HttpRouter) initRouter() { + r.echo.POST("/api/v1/upload-inventory/reports", r.managerHandler.UploadReport) + r.echo.POST("/api/v1/upload-inventory/nodes/:instanceId/session", r.managerHandler.UploadReportSession) + r.echo.POST("/api/v1/upload-inventory/nodes/:instanceId/reconcile", r.managerHandler.UploadReconcile) + r.echo.GET("/api/v1/upload-inventory/nodes/:instanceId/reconcile", r.managerHandler.UploadReconcile) + r.echo.POST("/api/v1/upload-inventory/nodes/:instanceId/reconcile-progress", r.managerHandler.UploadReconcileProgress) r.echo.POST("/internal/v1/official-revisions", r.managerHandler.OfficialRevision) r.echo.GET("/node-health", handler.NodeHealthPage) r.echo.GET("/api/v1/nodes/health", r.managerHandler.NodeHealth) diff --git a/internal/service/cache_job_service.go b/internal/service/cache_job_service.go index e4764db..d79db0d 100644 --- a/internal/service/cache_job_service.go +++ b/internal/service/cache_job_service.go @@ -50,8 +50,13 @@ func NewCacheJobService(dingospeedDao *dao.DingospeedDao, modelFileProcessDao *d } } -func (c *CacheJobService) ListCacheJob(instanceId, datatype string, page, pageSize int) ([]*dto.CacheJobResp, int64, error) { +func (c *CacheJobService) ListCacheJob(instanceId, datatype string, page, pageSize int, namespaces ...string) ([]*dto.CacheJobResp, int64, error) { + namespace := "" + if len(namespaces) > 0 { + namespace = namespaces[0] + } cacheJobs, size, err := c.cacheJobDao.ListCacheJob(&query.CacheJobQuery{ + Namespace: namespace, Type: consts.CacheTypePreheat, InstanceId: instanceId, Datatype: datatype, @@ -75,6 +80,9 @@ func (c *CacheJobService) ListCacheJob(instanceId, datatype string, page, pageSi for _, job := range cacheJobs { cacheJobResp := &dto.CacheJobResp{} gocopy.Copy(cacheJobResp, job) + if key, err := repository.FromWire(job.Datatype, job.Org, job.Repo); err == nil { + cacheJobResp.Namespace, cacheJobResp.FullRepo, cacheJobResp.RepositoryID = key.Namespace, key.Repo, key.ID() + } if status, ok := statusMap[job.ID]; ok { cacheJobResp.StockSpeed = status.StockSpeed cacheJobResp.StockProcess = status.StockProcess @@ -102,7 +110,7 @@ func (c *CacheJobService) getJobRealtimeStatus(jobIds []int64, instanceId string if err != nil { return nil, err } - resp, err := util.PostForDomain(speedDomain, "/api/cacheJob/realtime", "application/json", b, c.hfTokenDao.GetHeaders()) + resp, err := util.PostForDomain(speedDomain, "/api/cacheJob/realtime", "application/json", b, nil) if err != nil { return nil, err } @@ -127,14 +135,7 @@ func (c *CacheJobService) CreateCacheJob(createCacheJobReq *query.CreateCacheJob lock := c.lockDao.GetCacheJobReqLock(key.LockKey("job", createCacheJobReq.InstanceId, fmt.Sprint(createCacheJobReq.Type))) lock.Lock() defer lock.Unlock() - cacheJob, err := c.cacheJobDao.GetCacheJob(&query.CacheJobQuery{InstanceId: createCacheJobReq.InstanceId, Type: createCacheJobReq.Type, - Org: createCacheJobReq.Org, Repo: createCacheJobReq.Repo, Datatype: createCacheJobReq.Datatype}) - if err != nil { - return nil, err - } - if cacheJob != nil { - return nil, myerr.New("已存在该任务，不能再创建。") - } + // Speed resolves the current commit and owns durable task reuse/admission. entity, err := c.dingospeedDao.GetEntity(createCacheJobReq.InstanceId, true) if err != nil { return nil, err @@ -147,7 +148,7 @@ func (c *CacheJobService) CreateCacheJob(createCacheJobReq *query.CreateCacheJob if err != nil { return nil, err } - return util.PostForDomain(speedDomain, "/api/cacheJob/create", "application/json", b, c.hfTokenDao.GetHeaders()) + return util.PostForDomain(speedDomain, "/api/cacheJob/create", "application/json", b, c.hfTokenDao.ProviderHeaders(key)) } func (c *CacheJobService) StopCacheJob(jobStatusReq *query.JobStatusReq) error { @@ -183,7 +184,7 @@ func (c *CacheJobService) StopCacheJob(jobStatusReq *query.JobStatusReq) error { if err != nil { return err } - _, err = util.PostForDomain(speedDomain, "/api/cacheJob/stop", "application/json", b, c.hfTokenDao.GetHeaders()) + _, err = util.PostForDomain(speedDomain, "/api/cacheJob/stop", "application/json", b, nil) if err != nil { err = c.cacheJobDao.UpdateCacheStatus(&query.UpdateJobStatusReq{Id: jobStatusReq.Id, Status: consts.RunningStatusJobStop}) if err != nil { @@ -233,7 +234,7 @@ func (c *CacheJobService) ResumeCacheJob(resumeCacheJobReq *query.ResumeCacheJob if err != nil { return err } - _, err = util.PostForDomain(speedDomain, "/api/cacheJob/resume", "application/json", b, c.hfTokenDao.GetHeaders()) + _, err = util.PostForDomain(speedDomain, "/api/cacheJob/resume", "application/json", b, c.hfTokenDao.ProviderHeaders(storageAPIKey(cacheJob.Datatype, cacheJob.Org, cacheJob.Repo))) if err != nil { return err } diff --git a/internal/service/remote_parity_test.go b/internal/service/remote_parity_test.go new file mode 100644 index 0000000..1fa7871 --- /dev/null +++ b/internal/service/remote_parity_test.go @@ -0,0 +1,99 @@ +package service + +import ( + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "dingoscheduler/internal/dao" + "dingoscheduler/internal/data" + "dingoscheduler/internal/model" + "dingoscheduler/internal/model/dto" + "dingoscheduler/internal/model/query" + "dingoscheduler/pkg/config" + "dingoscheduler/pkg/consts" + "github.com/glebarez/sqlite" + "github.com/patrickmn/go-cache" + "gorm.io/gorm" +) + +func TestRemoteCreationDelegatesCommitReuseAndSelectsProviderCredentials(t *testing.T) { + previous := config.SysConfig + config.SysConfig = &config.Config{Scheduler: config.Scheduler{ModelScopeToken: "ms-test"}} + t.Cleanup(func() { config.SysConfig = previous }) + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, _ := db.DB() + t.Cleanup(func() { sqlDB.Close() }) + if err := db.AutoMigrate(&model.CacheJob{}, &model.Dingospeed{}, &model.HfToken{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&model.HfToken{Token: "hf-test", Enabled: true}).Error; err != nil { + t.Fatal(err) + } + count := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count++ + var req query.CreateCacheJobReq + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + want := "Bearer hf-test" + if req.Org == "modelscope/owner" { + want = "Bearer ms-test" + } + if r.URL.Path != "/api/cacheJob/create" || r.Header.Get("Authorization") != want { + t.Errorf("wrong provider request: %s %s", r.URL, r.Header.Get("Authorization")) + } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"id":42,"disposition":"cached","commit":"pinned"}`)) + })) + defer srv.Close() + host, port, _ := net.SplitHostPort(strings.TrimPrefix(srv.URL, "http://")) + p, _ := strconv.Atoi(port) + if err := db.Create(&model.Dingospeed{InstanceID: "node", Host: host, Port: int32(p), Online: true}).Error; err != nil { + t.Fatal(err) + } + d := &data.BaseData{BizDB: db, Cache: cache.New(time.Minute, time.Minute)} + jobs := dao.NewCacheJobDao(d, nil) + s := NewCacheJobService(dao.NewDingospeedDao(d), nil, jobs, dao.NewHfTokenDao(d), dao.NewLockDao(d)) + for _, org := range []string{"owner", "modelscope/owner"} { + if err := jobs.Save(&model.CacheJob{Type: consts.CacheTypePreheat, InstanceId: "node", Datatype: "models", Org: org, Repo: "demo", Commit: "previous", Status: consts.RunningStatusJobComplete}); err != nil { + t.Fatal(err) + } + resp, err := s.CreateCacheJob(&query.CreateCacheJobReq{Type: consts.CacheTypePreheat, InstanceId: "node", Datatype: "models", Org: org, Repo: "demo"}) + if err != nil || resp.StatusCode != 200 || !strings.Contains(string(resp.Body), `"disposition":"cached"`) { + t.Fatalf("prior job blocked reuse: %+v %v", resp, err) + } + } + if count != 2 { + t.Fatalf("Speed admission called %d times", count) + } + items, total, err := s.ListCacheJob("node", "models", 1, 10, "modelscope") + if err != nil || total != 1 || len(items) != 1 { + t.Fatalf("list: %v %d %v", items, total, err) + } + if items[0].Namespace != "modelscope" || items[0].FullRepo != "owner/demo" || items[0].RepositoryID != "modelscope/owner/demo" || items[0].Org != "modelscope/owner" { + t.Fatalf("identity contract: %+v", items[0]) + } +} + +func TestRepositoryIdentityAddsCanonicalFieldsWithoutChangingLegacyHF(t *testing.T) { + for _, tc := range []struct{ org, namespace, full string }{{"owner", "huggingface", "owner/demo"}, {"modelscope/owner", "modelscope", "owner/demo"}, {"dingo-local/alice", "alice", "demo"}} { + r := &dto.Repository{Datatype: "models", Org: tc.org, Repo: "demo"} + setRepositoryIdentity(r) + if r.Namespace != tc.namespace || r.FullRepo != tc.full || r.RepositoryID != tc.namespace+"/"+tc.full { + t.Fatalf("identity: %+v", r) + } + if tc.namespace == "huggingface" && (r.Org != "owner" || r.Repo != "demo") { + t.Fatal("changed legacy HF fields") + } + } +} diff --git a/internal/service/repository_service.go b/internal/service/repository_service.go index fd93a2e..5ba9113 100644 --- a/internal/service/repository_service.go +++ b/internal/service/repository_service.go @@ -242,7 +242,7 @@ func (s *RepositoryService) MountRepository(repoReq *query.RepositoryReq) error if repoReq.Token != "" { authHeaders["Authorization"] = fmt.Sprintf("Bearer %s", repoReq.Token) } else { - authHeaders = s.hfTokenDao.GetHeaders() + authHeaders = s.hfTokenDao.ProviderHeaders(storageAPIKey(repository.Datatype, repository.Org, repository.Repo)) } var status int32 = consts.RunningStatusJobIng _, err = util.PostForDomain(speedDomain, "/api/cacheJob/create", "application/json", b, authHeaders) @@ -276,6 +276,7 @@ func setRepositoryIdentity(r *dto.Repository) { return } r.Namespace = k.Namespace + r.FullRepo, r.RepositoryID = k.Repo, k.ID() if strings.HasPrefix(r.Org, "dingo-local/") { r.Org = k.Namespace r.Repo = k.Repo diff --git a/internal/service/scheduler_service.go b/internal/service/scheduler_service.go index 4f1374d..fa94662 100644 --- a/internal/service/scheduler_service.go +++ b/internal/service/scheduler_service.go @@ -64,7 +64,7 @@ func NewSchedulerService( repositoryDao *dao.RepositoryDao, cacheJobDao *dao.CacheJobDao, ) *SchedulerService { - return &SchedulerService{ + s := &SchedulerService{ baseData: baseData, dingospeedDao: dingospeedDao, modelFileRecordDao: modelFileRecordDao, @@ -72,6 +72,8 @@ func NewSchedulerService( repositoryDao: repositoryDao, cacheJobDao: cacheJobDao, } + go s.resumeInventoryReconciles() + return s } func (s *SchedulerService) Register(ctx context.Context, req *pb.RegisterRequest) (*pb.RegisterResponse, error) { diff --git a/internal/service/upload_report.go b/internal/service/upload_report.go new file mode 100644 index 0000000..5b4f672 --- /dev/null +++ b/internal/service/upload_report.go @@ -0,0 +1,122 @@ +package service + +import ( + "bytes" + "context" + "dingoscheduler/internal/model" + "dingoscheduler/pkg/inventory" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" +) + +func (s *SchedulerService) UploadReport(ctx context.Context, p *inventory.Report) (inventory.Ack, error) { + return s.repositoryDao.ApplyUploadReport(ctx, p) +} +func (s *SchedulerService) UploadReportSession(ctx context.Context, id string) (model.UploadReportNode, error) { + return s.repositoryDao.UploadReportSession(ctx, id, false) +} +func (s *SchedulerService) UploadReportStatus(ctx context.Context, id string) (model.UploadReportNode, error) { + return s.repositoryDao.UploadReportStatus(ctx, id) +} + +func (s *SchedulerService) UploadReconcileProgress(ctx context.Context, id, epoch, status, message string) error { + if status != "scanning" && status != "retrying" && status != "needs_attention" { + return fmt.Errorf("invalid reconciliation status") + } + if len(message) > 4096 { + message = message[:4096] + } + return s.baseData.BizDB.WithContext(ctx).Model(&model.UploadReportNode{}).Where("instance_id = ? AND pending_epoch = ? AND pending_epoch <> ''", id, epoch).Updates(map[string]any{"status": status, "error": message}).Error +} +func (s *SchedulerService) ReconcileUploadInventory(ctx context.Context, id string) (model.UploadReportNode, error) { + n, err := s.repositoryDao.UploadReportSession(ctx, id, true) + if err != nil { + return n, err + } + // Delivery is only a wake-up. The durable pending epoch is also returned on reconnection. + err = s.deliverInventoryReconcile(ctx, n) + if err != nil { + n.Error = err.Error() + s.retryInventoryDelivery(n) + } + return n, nil +} + +var inventoryDeliveries sync.Map + +func (s *SchedulerService) resumeInventoryReconciles() { + if s.baseData == nil || s.baseData.BizDB == nil { + return + } + for { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + var pending []model.UploadReportNode + err := s.baseData.BizDB.WithContext(ctx).Where("pending_epoch <> ''").Find(&pending).Error + cancel() + if err == nil { + for _, n := range pending { + s.retryInventoryDelivery(n) + } + return + } + time.Sleep(10 * time.Second) + } +} +func (s *SchedulerService) retryInventoryDelivery(n model.UploadReportNode) { + key := n.InstanceID + "/" + n.PendingEpoch + if _, loaded := inventoryDeliveries.LoadOrStore(key, true); loaded { + return + } + go func() { + defer inventoryDeliveries.Delete(key) + for delay := time.Second; ; { + time.Sleep(delay) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + current, err := s.repositoryDao.UploadReportStatus(ctx, n.InstanceID) + if err == nil && current.PendingEpoch != n.PendingEpoch { + cancel() + return + } + if err == nil { + err = s.deliverInventoryReconcile(ctx, n) + } + cancel() + if err == nil { + return + } + if delay < time.Minute { + delay *= 2 + } + } + }() +} +func (s *SchedulerService) deliverInventoryReconcile(ctx context.Context, n model.UploadReportNode) error { + var endpoint model.NodeEndpoint + err := s.baseData.BizDB.WithContext(ctx).Where("node_id = (?)", s.baseData.BizDB.Model(&model.Dingospeed{}).Select("MAX(id)").Where("instance_id = ?", n.InstanceID)).Take(&endpoint).Error + if err != nil { + return err + } + if endpoint.ManagementURL == "" { + return fmt.Errorf("node management URL unavailable; task retained until reconnect") + } + b, _ := json.Marshal(map[string]string{"epoch": n.PendingEpoch}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(endpoint.ManagementURL, "/")+"/api/upload-inventory/reconcile", bytes.NewReader(b)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + client := http.Client{Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + return fmt.Errorf("node reconciliation returned HTTP %d", resp.StatusCode) + } + return nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index c9b64ea..7197790 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -91,9 +91,10 @@ type Proxy struct { } type Scheduler struct { - Port int32 `json:"port" yaml:"port"` - PersistRepo PersistRepo `json:"persistRepo" yaml:"persistRepo"` - GlobalHfToken string `json:"globalHfToken" yaml:"globalHfToken"` + Port int32 `json:"port" yaml:"port"` + PersistRepo PersistRepo `json:"persistRepo" yaml:"persistRepo"` + GlobalHfToken string `json:"globalHfToken" yaml:"globalHfToken"` + ModelScopeToken string `json:"-" yaml:"modelscopeToken"` } type PersistRepo struct { diff --git a/pkg/inventory/protocol.go b/pkg/inventory/protocol.go new file mode 100644 index 0000000..fb550ab --- /dev/null +++ b/pkg/inventory/protocol.go @@ -0,0 +1,53 @@ +// Package inventory owns only uploaded inventory, never remote cache progress. +package inventory + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +type Key struct { + Namespace string `json:"namespace"` + RepoType string `json:"repoType"` + Repo string `json:"repo"` +} + +func (k Key) ID() string { return k.Namespace + "\x00" + k.RepoType + "\x00" + k.Repo } + +type File struct { + Key + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} +type Report struct { + Version int `json:"version"` + InstanceID string `json:"instanceId"` + Epoch string `json:"epoch"` + Sequence uint64 `json:"sequence"` + Baseline bool `json:"baseline"` + Key + Deleted bool `json:"deleted"` + Files []File `json:"files"` +} + +func (r Report) Digest() string { + b, _ := json.Marshal(r) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +type Ack struct { + Epoch string `json:"epoch"` + Sequence uint64 `json:"sequence"` + Digest string `json:"digest"` + Status string `json:"status"` +} +type Session struct { + InstanceID string `json:"instanceId"` + Epoch string `json:"epoch"` + PendingEpoch string `json:"pendingEpoch"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} diff --git a/pkg/repository/key.go b/pkg/repository/key.go index 654c259..827418a 100644 --- a/pkg/repository/key.go +++ b/pkg/repository/key.go @@ -103,6 +103,13 @@ func (k Key) LockKey(operation string, fields ...string) string { func (k Key) ID() string { return k.Namespace + "/" + k.Repo } +func (k Key) DefaultRevision() string { + if k.Namespace == "modelscope" { + return "master" + } + return "main" +} + // OperationURI addresses DingoSpeed's own API, never the upstream HF protocol. func (k Key) OperationURI(operation, revision, path string) (string, error) { if err := k.Validate(); err != nil { @@ -114,7 +121,7 @@ func (k Key) OperationURI(operation, revision, path string) (string, error) { return "", fmt.Errorf("unsupported repository operation") } if revision == "" { - revision = "main" + revision = k.DefaultRevision() } if err := ValidatePath(revision, false); err != nil { return "", err