Preserve durable Tinker training state

src/tinker-sft/index.ts

@@ -45,6 +45,8 @@ export const TINKER_LORA_SCOPE: Readonly<TinkerLoraScope> = Object.freeze({
 });
 
 const RUN_SCHEMA = "understudy.tinker_sft.run.v1";
+export const TINKER_FINAL_STATE_TTL_SECONDS = null;
+export const TINKER_SAMPLER_TTL_SECONDS = 24 * 60 * 60;
 const MAX_STDIO_BYTES = 1024 * 1024;
 const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
 
@@ -102,8 +104,10 @@ const RuntimeResultSchema = z.object({
   backend: z.literal("tinker"),
   model: z.string(),
   renderer: z.string(),
+  training_state_path: z.string().min(1),
+  training_state_ttl_seconds: z.null(),
   sampler_state_path: z.string().min(1),
-  checkpoint_ttl_seconds: z.number().int().positive().max(3600),
+  checkpoint_ttl_seconds: z.literal(TINKER_SAMPLER_TTL_SECONDS),
   training: z.object({
     steps: z.number().int().positive(),
     tokens: z.number().int().positive(),

src/tinker-sft/runtime-source.ts

@@ -211,9 +211,13 @@ async def main(request: dict) -> dict:
                 "total": math.ceil(len(train_datums) / batch_size) * request["epochs"],
             })
 
-    emit({"type": "phase", "phase": "evaluating", "message": "Saving one-hour sampler weights and re-running the same holdout."})
+    emit({"type": "phase", "phase": "checkpointing", "message": "Saving a non-expiring resumable state before creating bounded sampler weights."})
+    training_state = training_client.save_state(
+        name=f"understudy-{request['run_id'][:32]}-state", ttl_seconds=None
+    ).result()
+    emit({"type": "phase", "phase": "evaluating", "message": "Saving 24-hour sampler weights and re-running the same holdout."})
     saved = training_client.save_weights_for_sampler(
-        name=f"understudy-{request['run_id'][:32]}", ttl_seconds=3600
+        name=f"understudy-{request['run_id'][:32]}-sampler", ttl_seconds=86400
     ).result()
     tuned_client = await service.create_sampling_client_async(model_path=saved.path)
     heldout = await evaluate(tuned_client, renderer, heldout_rows, request["max_generation_tokens"], 44)
@@ -242,8 +246,10 @@ async def main(request: dict) -> dict:
         "backend": "tinker",
         "model": model,
         "renderer": renderer_name,
+        "training_state_path": training_state.path,
+        "training_state_ttl_seconds": None,
         "sampler_state_path": saved.path,
-        "checkpoint_ttl_seconds": 3600,
+        "checkpoint_ttl_seconds": 86400,
         "training": {
             "steps": steps,
             "tokens": train_tokens,

tests/fixtures/tinker-sft-deterministic-runner.mjs

@@ -61,8 +61,10 @@ process.stdout.write(`${JSON.stringify({
     backend: "tinker",
     model,
     renderer: "deterministic_renderer",
+    training_state_path: `tinker://checkpoint/${request.run_id}-state`,
+    training_state_ttl_seconds: null,
     sampler_state_path: `tinker://checkpoint/${request.run_id}`,
-    checkpoint_ttl_seconds: 3600,
+    checkpoint_ttl_seconds: 86400,
     training: { steps: 2, tokens: 256, loss_mask: "last_assistant_message", lora_scope: loraScope },
     baseline,
     heldout: trained,

tests/tinker-sft.test.mjs

@@ -109,7 +109,7 @@ describe("portable Tinker SFT backend", () => {
     assert.equal(existsSync(outputRoot), false);
   });
 
-  it("executes the same immutable evaluator contract with bounded spend and one-hour cleanup", async () => {
+  it("executes the same immutable evaluator contract with durable recovery state and bounded sampler lifetime", async () => {
     const fixture = portablePlan();
     const result = await startTinkerSftTraining({
       planPath: fixture.planPath,
@@ -134,7 +134,9 @@ describe("portable Tinker SFT backend", () => {
     assert.equal(result.cost.approved_max_usd, 0.5);
     assert.ok(result.cost.actual_estimated_usd <= result.cost.worst_case_usd);
     assert.ok(result.cost.worst_case_usd <= result.cost.approved_max_usd);
-    assert.equal(result.checkpoint_ttl_seconds, 3600);
+    assert.match(result.training_state_path, /^tinker:\/\//);
+    assert.equal(result.training_state_ttl_seconds, null);
+    assert.equal(result.checkpoint_ttl_seconds, 86400);
     assert.equal(result.privacy.provider_training_data_sent, true);
     assert.equal(result.privacy.raw_artifact_uploaded, false);
     assert.equal(result.runtime.maximum_seconds, 900);