diff --git a/src/tinker-sft/index.ts b/src/tinker-sft/index.ts index 8e514d24..2d1ed66c 100644 --- a/src/tinker-sft/index.ts +++ b/src/tinker-sft/index.ts @@ -45,6 +45,8 @@ export const TINKER_LORA_SCOPE: Readonly = Object.freeze({ }); const RUN_SCHEMA = "understudy.tinker_sft.run.v1"; +export const TINKER_FINAL_STATE_TTL_SECONDS = null; +export const TINKER_SAMPLER_TTL_SECONDS = 24 * 60 * 60; const MAX_STDIO_BYTES = 1024 * 1024; const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; @@ -102,8 +104,10 @@ const RuntimeResultSchema = z.object({ backend: z.literal("tinker"), model: z.string(), renderer: z.string(), + training_state_path: z.string().min(1), + training_state_ttl_seconds: z.null(), sampler_state_path: z.string().min(1), - checkpoint_ttl_seconds: z.number().int().positive().max(3600), + checkpoint_ttl_seconds: z.literal(TINKER_SAMPLER_TTL_SECONDS), training: z.object({ steps: z.number().int().positive(), tokens: z.number().int().positive(), diff --git a/src/tinker-sft/runtime-source.ts b/src/tinker-sft/runtime-source.ts index ed60b161..94944028 100644 --- a/src/tinker-sft/runtime-source.ts +++ b/src/tinker-sft/runtime-source.ts @@ -211,9 +211,13 @@ async def main(request: dict) -> dict: "total": math.ceil(len(train_datums) / batch_size) * request["epochs"], }) - emit({"type": "phase", "phase": "evaluating", "message": "Saving one-hour sampler weights and re-running the same holdout."}) + emit({"type": "phase", "phase": "checkpointing", "message": "Saving a non-expiring resumable state before creating bounded sampler weights."}) + training_state = training_client.save_state( + name=f"understudy-{request['run_id'][:32]}-state", ttl_seconds=None + ).result() + emit({"type": "phase", "phase": "evaluating", "message": "Saving 24-hour sampler weights and re-running the same holdout."}) saved = training_client.save_weights_for_sampler( - name=f"understudy-{request['run_id'][:32]}", ttl_seconds=3600 + name=f"understudy-{request['run_id'][:32]}-sampler", ttl_seconds=86400 ).result() tuned_client = await service.create_sampling_client_async(model_path=saved.path) heldout = await evaluate(tuned_client, renderer, heldout_rows, request["max_generation_tokens"], 44) @@ -242,8 +246,10 @@ async def main(request: dict) -> dict: "backend": "tinker", "model": model, "renderer": renderer_name, + "training_state_path": training_state.path, + "training_state_ttl_seconds": None, "sampler_state_path": saved.path, - "checkpoint_ttl_seconds": 3600, + "checkpoint_ttl_seconds": 86400, "training": { "steps": steps, "tokens": train_tokens, diff --git a/tests/fixtures/tinker-sft-deterministic-runner.mjs b/tests/fixtures/tinker-sft-deterministic-runner.mjs index 31cb696a..be5c8cba 100644 --- a/tests/fixtures/tinker-sft-deterministic-runner.mjs +++ b/tests/fixtures/tinker-sft-deterministic-runner.mjs @@ -61,8 +61,10 @@ process.stdout.write(`${JSON.stringify({ backend: "tinker", model, renderer: "deterministic_renderer", + training_state_path: `tinker://checkpoint/${request.run_id}-state`, + training_state_ttl_seconds: null, sampler_state_path: `tinker://checkpoint/${request.run_id}`, - checkpoint_ttl_seconds: 3600, + checkpoint_ttl_seconds: 86400, training: { steps: 2, tokens: 256, loss_mask: "last_assistant_message", lora_scope: loraScope }, baseline, heldout: trained, diff --git a/tests/tinker-sft.test.mjs b/tests/tinker-sft.test.mjs index 0b480707..ba9f81aa 100644 --- a/tests/tinker-sft.test.mjs +++ b/tests/tinker-sft.test.mjs @@ -109,7 +109,7 @@ describe("portable Tinker SFT backend", () => { assert.equal(existsSync(outputRoot), false); }); - it("executes the same immutable evaluator contract with bounded spend and one-hour cleanup", async () => { + it("executes the same immutable evaluator contract with durable recovery state and bounded sampler lifetime", async () => { const fixture = portablePlan(); const result = await startTinkerSftTraining({ planPath: fixture.planPath, @@ -134,7 +134,9 @@ describe("portable Tinker SFT backend", () => { assert.equal(result.cost.approved_max_usd, 0.5); assert.ok(result.cost.actual_estimated_usd <= result.cost.worst_case_usd); assert.ok(result.cost.worst_case_usd <= result.cost.approved_max_usd); - assert.equal(result.checkpoint_ttl_seconds, 3600); + assert.match(result.training_state_path, /^tinker:\/\//); + assert.equal(result.training_state_ttl_seconds, null); + assert.equal(result.checkpoint_ttl_seconds, 86400); assert.equal(result.privacy.provider_training_data_sent, true); assert.equal(result.privacy.raw_artifact_uploaded, false); assert.equal(result.runtime.maximum_seconds, 900);