From 9fd9310ed310f153b72e96d3e40abd8a32d510ef Mon Sep 17 00:00:00 2001 From: Midhat Nashar <35053211+midhatn@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:16:17 +0300 Subject: [PATCH 1/3] WIP: automatic desktop recovery of pending tool operations after worker loss Adds automatic worker-loss detection in DesktopComposition, host terminal receipt export, bounded reconcile and replacement worker that keeps reopen context and the durable execution budget. Includes source-lineage records and focused tests. Not yet validated for publication: the local full core run (1595 tests) reported 2 errors in test_compiler_composition and test_video_composition. compiler_launch_args now calls compiler_host.lifecycle(), which the existing test doubles do not provide. No public evidence records or doc updates are included. --- provenance/automatic-tool-recovery/README.md | 25 + .../adaptive_inference/desktop/composition.py | 989 ++++++++++++++++++ .../desktop/desktop_controls.py | 507 +++++++++ .../desktop/task_handover.py | 773 ++++++++++++++ .../predecessor/tests/pending_tool_lineage.py | 89 ++ .../tests/test_desktop_composition.py | 377 +++++++ .../tests/test_pending_tool_lineage.py | 107 ++ .../source-lineage.json | 156 +++ src/adaptive_inference/desktop/composition.py | 181 +++- .../desktop/desktop_controls.py | 43 + .../desktop/task_handover.py | 11 + tests/automatic_tool_recovery_lineage.py | 89 ++ tests/pending_tool_lineage.py | 12 +- tests/test_automatic_tool_recovery_lineage.py | 107 ++ tests/test_desktop_automatic_recovery.py | 676 ++++++++++++ tests/test_desktop_composition.py | 194 ++++ tests/test_pending_tool_lineage.py | 7 +- 17 files changed, 4338 insertions(+), 5 deletions(-) create mode 100644 provenance/automatic-tool-recovery/README.md create mode 100644 provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/composition.py create mode 100644 provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/desktop_controls.py create mode 100644 provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/task_handover.py create mode 100644 provenance/automatic-tool-recovery/predecessor/tests/pending_tool_lineage.py create mode 100644 provenance/automatic-tool-recovery/predecessor/tests/test_desktop_composition.py create mode 100644 provenance/automatic-tool-recovery/predecessor/tests/test_pending_tool_lineage.py create mode 100644 provenance/automatic-tool-recovery/source-lineage.json create mode 100644 tests/automatic_tool_recovery_lineage.py create mode 100644 tests/test_automatic_tool_recovery_lineage.py create mode 100644 tests/test_desktop_automatic_recovery.py diff --git a/provenance/automatic-tool-recovery/README.md b/provenance/automatic-tool-recovery/README.md new file mode 100644 index 0000000..1cccd52 --- /dev/null +++ b/provenance/automatic-tool-recovery/README.md @@ -0,0 +1,25 @@ +# Automatic tool recovery source preservation + +The source-lineage record binds automatic desktop tool recovery to the exact +published pending-effect increment at `9242434991f2c2c21461061d61d898649299b847`. +Six complete predecessor Git blobs preserve the three changed runtime modules +(`composition.py`, `desktop_controls.py`, `task_handover.py`), one modified +desktop composition test module, and two source-lineage files. They remain data +under this repository's Apache-2.0 license; tests do not import or execute the +retained copies. + +The test-only view reuses [pending tool recovery's source-preservation mechanism](../../tests/pending_tool_lineage.py) +and the [new layer](../../tests/automatic_tool_recovery_lineage.py). Complete-file +SHA-256 and a portable Python syntax-tree fingerprint must both identify a +declared successor before projection to its exact predecessor. Unknown complete +bytes, including an added comment, do not earn that projection. + +The complete 77-module runtime is checked before the preceding pending-effect, +recipe, video and compiler views run. Existing projected views remain idempotent. +The 24 original pending-effect, recipe, video and compiler JSON records keep +their original bytes, source identities and qualification limits. + +This preserves the meaning of historical evidence. It does not qualify the new +automatic recovery behavior: component checks, the complete local suite, +distribution, installed-agent recovery and physical IQ3_S tests remain separate +gates. diff --git a/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/composition.py b/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/composition.py new file mode 100644 index 0000000..0033acd --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/composition.py @@ -0,0 +1,989 @@ +# SPDX-License-Identifier: Apache-2.0 +"""One installed Windows host/worker/GUI composition of the accepted lifecycle. + +Dedicated process only. No work occurs on import. An explicit bounded file task +(or the retained orders example) starts paused and requires genuine GUI Resume. +Unknown child/pipe obligations remain retained; exit124 is an emergency failure, +never a clean Stop or completed task. +""" +from __future__ import annotations +import asyncio +from copy import deepcopy +from dataclasses import asdict +import json +import os +from pathlib import Path +import secrets +import subprocess +import sys +import threading +import time +import traceback +import uuid + +from adaptive_inference.demand import DemandIdentity, DemandRegistry, Registration, NAMESPACE +from adaptive_inference.task_store import TaskStore +from adaptive_inference.control_mailbox import Mailbox, execution_outcome +from adaptive_inference.execution_clock import CLOCK_ID, execution_now, verify_execution_clock +from adaptive_inference.handoff import ToolRequirements +from adaptive_inference.desktop.installation import package_identity, sha, verify_interpreter, verify_sdk +from adaptive_inference.desktop.profile import WindowsProfile, COOPERATIVE_POLICY +from adaptive_inference.desktop.control_transport import PreparedTask, ControlTransport, credential +from adaptive_inference.desktop.mailbox_runtime import MailboxRuntime, ExpiredObservation +from adaptive_inference.desktop.ui_intent import UIIntentGate +from adaptive_inference.desktop.serial_listener import SerialListener +from adaptive_inference.desktop.hermes import recovery as rec +from adaptive_inference.desktop.hermes.run_recovery import MODEL, QUALITY, PROMPT +from adaptive_inference.desktop.hermes.service_stop import WorkerStopBinding, STOP_EXIT + +HERE = Path(__file__).resolve().parent +INITIAL_PAUSE = 'initial-pause-before-bootstrap' +TITLE = 'Adaptive inference' + +def read(path): + return json.loads(Path(path).read_text(encoding='utf-8')) + +def diagnostic(exc): + chain=[];seen=set();cause=exc + while cause is not None and id(cause) not in seen and len(chain)<16: + seen.add(id(cause)) + row={'type':type(cause).__name__,'message':str(cause)} + for key in ('sqlite_errorcode','sqlite_errorname','winerror','errno'): + value=getattr(cause,key,None) + if type(value) in (int,str):row[key]=value + chain.append(row);cause=cause.__cause__ or cause.__context__ + return {'type':type(exc).__name__,'message':str(exc),'chain':chain, + 'traceback':''.join(traceback.format_exception(exc))} + +def hard_deadline(seconds=1800, *, wait=time.sleep, terminate=os._exit): + # No receipt/log/file IO: blocked or failed diagnostics cannot prevent exit. + # This daemon remains armed through asyncio and interpreter teardown. + wait(seconds) + terminate(124) + +def complete_ledger(events,prefix): + rows=deepcopy(list(events));maximum=getattr(events,'maxlen',None) + # A full deque cannot prove whether append discarded a prefix. Conservatively + # refuse even the exactly-full case; no shared controller change is needed. + if (not prefix or type(maximum) is not int or len(rows)>=maximum + or rows[:len(prefix)]!=prefix): + raise RuntimeError('Controller ledger prefix missing or bounded buffer full') + return rows + +def no_work(cp,original,deadline,host,work, *, cold=True): + """Read-only actual checkpoint/runtime gate, used before GUI and at terminal.""" + if (cp.prompt!=original.prompt or cp.metadata!=original.metadata or cp.max_attempts!=original.max_attempts + or cp.state['messages']!=original.state['messages'] + or cp.state.get('deadline_monotonic')!=deadline or cp.state['status']!='pending' + or cp.state['phase']!='ready' or cp.operation is not None or cp.requires_reconciliation + or cp.state['requests'] or cp.state['attempts'] or cp.state['admission_attempts'] + or cp.state['tools'] or cp.state['native_history'] or cp.state['pending_request'] is not None + or cp.state['admission_deferrals'] or cp.state[NAMESPACE]['state']!='paused'): + raise RuntimeError('Paused original task contains new/unknown work or changed accounting') + gateway=host.gateway;scheduler=gateway.scheduler + if cold and (host.adapter.launch_intent is not None or host.adapter.started_owner + or gateway._runtime_context is not None or gateway.history or scheduler.active is not None + or scheduler.pending or scheduler.uncertain or host.controller.fault is not None): + raise RuntimeError('Cold zero-work runtime evidence differs') + from adaptive_inference.task_spec import TaskSpec + spec = TaskSpec.from_metadata(cp.metadata) + outputs = ['report.json'] if spec is None else [row['name'] for row in spec.data['outputs']] + for filename in ('effect-receipts.jsonl','wire-observations.json', *(f'work/{name}' for name in outputs)): + path=work/filename + if path.exists() and path.read_bytes().strip() not in (b'',b'[]'): + raise RuntimeError('Unexpected provider/effect artifact in cold trial') + +async def join_retained(tasks): + """Join every known obligation even when a sibling failed; never cancel it.""" + unique={id(task):task for task in tasks if task is not None} + if not unique:return [] + return await asyncio.gather(*(asyncio.shield(task) for task in unique.values()),return_exceptions=True) + + +class DesktopComposition: + """Shared lifecycle; qualification scenarios may extend its existing seams.""" + def __init__(self, profile_path, run, *, qualification_hold_after_report=False, + task_spec=None, task_inputs=None, saved_task=None, enable_reopen=False): + self.profile_path = Path(profile_path).resolve(strict=True) + self.profile_sha256 = sha(self.profile_path) + self.profile = WindowsProfile.load(self.profile_path) + if type(qualification_hold_after_report) is not bool: + raise TypeError('Qualification hold must be an explicit boolean') + self.qualification_hold_after_report = qualification_hold_after_report + if type(enable_reopen) is not bool: + raise TypeError('Saved-task support must be explicit') + if saved_task is not None and (task_spec is not None or task_inputs is not None or qualification_hold_after_report): + raise ValueError('Open one saved task without new task or qualification inputs') + self.saved_task = None if saved_task is None else Path(saved_task).resolve(strict=True) + self.enable_reopen = enable_reopen or self.saved_task is not None + self.reopen_context = self.saved = self.saved_diagnostics = None + self.reopen_locks = [] + self.reopen_pause_id = 'reopen-pause-' + uuid.uuid4().hex + if (task_spec is None) != (task_inputs is None): + raise ValueError('Task specification and input directory must be supplied together') + self.task_spec_path = self.task_inputs = self.task_spec_sha256 = None + if task_spec is not None: + from adaptive_inference.task_spec import TaskSpec, MAX_SPEC_BYTES + self.task_spec_path = Path(task_spec).resolve(strict=True) + self.task_inputs = Path(task_inputs).resolve(strict=True) + if not self.task_spec_path.is_file() or not self.task_inputs.is_dir(): + raise ValueError('Explicit task specification file and input directory required') + with self.task_spec_path.open('rb') as stream: + raw = stream.read(MAX_SPEC_BYTES + 1) + # Decode now, before retaining any process. Preparation independently + # verifies these same bytes and stages the declared input files. + initial_spec = TaskSpec.from_bytes(raw) + if initial_spec.compiler is not None and self.profile.compiler is None: + raise ValueError('Compiler task requires explicit compiler deployment settings') + if initial_spec.video is not None and self.profile.video is None: + raise ValueError('Video task requires explicit video deployment settings') + self.task_spec_sha256 = rec.digest(raw) + if qualification_hold_after_report: + raise ValueError('The orders report qualification hold cannot select another task') + self.root = Path(run).absolute() + if self.root.exists() or not self.root.parent.is_dir() or self.root.parent.resolve() != self.root.parent: + raise ValueError('Use a fresh run directory under an existing non-redirected parent') + self.handles=[]; self.logs=[]; self.waits={}; self.host=self.job=self.worker=None + self.gui=self.facade=self.runtime=self.transport=self.listener=self.stop_binding=None + self.deadline=self.original=self.initial_events=None + self.progress=None; self.stop_id='service-stop-'+uuid.uuid4().hex + self.result={'schema':'adaptive.desktop.windows-orders.result.v1', 'clean_stop':False, + 'task_completed':False, 'execution_expired':False, + 'model':MODEL, 'quality':deepcopy(QUALITY)} + if self.task_spec_path is not None: + self.result['schema'] = 'adaptive.desktop.windows-file-task.result.v1' + self.worker_exit_verified=False + self.continuation = None + self.handover = None + self.compiler_host = None + self.compiler_kind = 'video' if self.profile.video is not None else 'compiler' + self.compiler_bindings = {} + self.compiler_native_reader = None + + def compiler_launch_args(self, work, plan, *, execution_id, task_id, deadline, diagnostics): + from adaptive_inference.desktop.hermes.run_recovery import prepared_contract + from adaptive_inference.desktop.recipe_dispatch import recipe_kind + spec = prepared_contract(plan) + kind = recipe_kind(spec) + if kind is None: + return [] + if self.compiler_host is None: + raise RuntimeError('Selected task has no configured recipe host') + if kind != getattr(self, 'compiler_kind', 'compiler'): + raise RuntimeError('Selected task differs from the configured useful-tool deployment') + from adaptive_inference.desktop.hermes.compiler_worker import binding_schema + key = (task_id, execution_id) + if key in self.compiler_bindings: + raise RuntimeError('Compiler worker launch binding cannot be reused') + binding = {'schema': binding_schema(kind), 'host_epoch': self.host_epoch, 'task_id': task_id, + 'execution_id': execution_id, 'task_deadline': deadline, + 'gateway_instance': self.host.gateway.instance, 'capability': secrets.token_hex(32)} + path = Path(diagnostics)/('compiler-binding-'+uuid.uuid4().hex+'.private.json') + raw = json.dumps(binding, sort_keys=True).encode('utf-8') + with path.open('xb') as stream: + stream.write(raw); stream.flush(); os.fsync(stream.fileno()) + self.compiler_bindings[key] = binding + return ['--compiler-binding', str(path), '--compiler-binding-sha256', rec.digest(raw)] + + def compiler_worker_joined(self, child): + task = self.waits.get(child) + return (task is not None and task.done() and not task.cancelled() + and task.exception() is None and type(task.result()) is int) + + async def bind_compiler_worker(self, runtime, child, work, deadline): + if not getattr(self, 'compiler_bindings', None): + return + key = (runtime.prepared.task_id, runtime.selected_execution()['execution_id']) + binding = getattr(self, 'compiler_bindings', {}).get(key) + if binding is None: + return + await self.compiler_host.bind(runtime=runtime, worker=child, + is_current_worker=lambda value: value is self.worker and value.poll() is None, + is_worker_joined=self.compiler_worker_joined, + read_native_history=lambda session: self.compiler_native_reader(work, session), + task_root=work, capability=binding['capability'], task_deadline=deadline) + + def launch(self, command, label): + stream=(self.root/(label+'.log')).open('wb'); self.logs.append(stream) + before=len(self.handles) + try: + child=subprocess.Popen(command, stdout=stream, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, cwd=self.root, + creationflags=subprocess.CREATE_NO_WINDOW if os.name=='nt' else 0) + self.handles.append(child) + self.event('worker_launched',label=label,pid=child.pid) + return child + finally: + if label=='worker' and len(self.handles)==before+1: + self.worker=self.handles[-1] + + def event(self,kind,**fields): + value={'kind':kind,'at_monotonic':time.monotonic(),**fields} + with (self.root/'events.private.jsonl').open('a',encoding='utf-8') as stream: + stream.write(json.dumps(value,allow_nan=False)+'\n');stream.flush() + + async def until(self,predicate,child=None,timeout=240): + limit=min(execution_now()+timeout,self.deadline or float('inf')) + while True: + value=predicate() + if value:return value + if child is not None and child.poll() is not None: + raise RuntimeError('Owned worker exited before required boundary; see retained log') + if execution_now()>=limit:raise TimeoutError('Qualification boundary deadline') + if self.host and self.host.controller.fault:raise RuntimeError('Controller fault: '+str(self.host.controller.fault)) + await asyncio.sleep(.05) + + def remaining(self): + remaining=self.deadline-execution_now() + if remaining<=0:raise TimeoutError('Original900second task deadline exhausted') + return remaining + + def status(self,request_id):return self.box.status(request_id) + + def submit(self,request_id,command): + receipt=self.box.submit(self.capability,request_id,command) + self.event('control_recorded',**receipt);return receipt + + async def ack(self,request_id,child): + receipt=await self.until(lambda:self.status(request_id) if self.status(request_id)['owner_acknowledged'] else None,child) + if receipt['outcome']!='applied':raise RuntimeError('Required user control was rejected') + self.event('control_acknowledged',**receipt) + return receipt + + def checkpoint(self):return self.store.load(self.identity.task_id) + + def assert_original(self,cp): + if (cp.prompt!=self.original.prompt or cp.metadata!=self.original.metadata + or cp.state['messages'][0]!=self.original.state['messages'][0] + or cp.state.get('deadline_monotonic')!=self.deadline + or cp.state.get('deadline_clock')!=CLOCK_ID): + raise RuntimeError('Original identity, objective or deadline changed') + + def worker_diagnostics(self): + return self.saved_diagnostics if self.work == self.saved_task else self.work + + def initial_pause_id(self): + return self.reopen_pause_id if self.work == self.saved_task else INITIAL_PAUSE + + def assert_paused(self, cp): + if self.work == self.saved_task: + from adaptive_inference.desktop.saved_tasks import assert_paused_saved + assert_paused_saved(cp, self.original, self.deadline, self.host) + else: + no_work(cp,self.original,self.deadline,self.host,self.work) + + async def ready(self,child,path): + await self.until(lambda:(self.worker_diagnostics()/f'bootstrap-{child.pid}.json').exists(),child) + boot=read(self.worker_diagnostics()/f'bootstrap-{child.pid}.json');cp=self.checkpoint() + self.assert_original(cp) + if (boot['pid']!=child.pid or boot['task_id']!=self.identity.task_id + or boot['registration_id']!=self.identity.registration_id + or boot['deadline_monotonic']!=self.deadline or boot.get('deadline_clock')!=CLOCK_ID + or boot['owner_epoch']!=cp.owner_epoch): + raise RuntimeError('Bootstrap receipt is not the retained owner') + if not self.registry.observe(self.identity.registration_id).eligible: + raise RuntimeError('Readiness waiting lacks actual trusted durable demand') + self.event('demand_before_readiness',pid=child.pid,revision=cp.revision) + await self.host.wait_ready(timeout_s=min(240,self.remaining())) + rec.save(path,{'ready':True,'model':MODEL,'quality':QUALITY,'gateway':self.plan['gateway'], + 'gateway_instance':self.host.gateway.instance,'observed_at':time.time()}) + self.event('controller_ready',pid=child.pid,owner_id=self.host.controller.identity['owner_id']) + + def bind_worker_stop(self): + if self.stop_binding is None: + self.worker_cap=object() + self.stop_binding=WorkerStopBinding(self.worker,self.work,self.box,requester_capability=self.capability, + capability=self.worker_cap,stop_request_id=self.stop_id,original=self.original, + deadline_monotonic=self.deadline,model=MODEL,quality=QUALITY,fence=self.fence, + deadline_clock=CLOCK_ID,diagnostics=self.worker_diagnostics()) + + async def join_child(self,child): + if child not in self.handles:raise RuntimeError('Cannot adopt a child for cleanup') + task=self.waits.get(child) + if task is None: + # The retained object owns the OS handle; a reused PID must never + # borrow the completed wait of an earlier process in this host. + task=self.waits[child]=asyncio.create_task(asyncio.to_thread(child.wait)) + code=await asyncio.shield(task) + self.event('retained_process_exit',pid=child.pid,exit_code=code,terminated=False) + return code + + def fence(self): + if getattr(self, 'compiler_host', None) is not None: + self.compiler_host.interrupt() + if self.facade is not None:self.facade.fence_admission() + elif self.host is not None: + self.host.closing=True;self.host.gateway.scheduler.closing=True + + def paused_ack(self): + snapshot=self.box.snapshot();ack=snapshot.status(self.initial_pause_id());cp=snapshot.target + if ack['outcome']!='applied' or ack['owner_acknowledged'] is not True:return None + if type(cp.owner_epoch) is not str or len(cp.owner_epoch)!=32:return None + if self.worker.poll() is not None:raise RuntimeError('Worker exited before paused ownership proof') + self.assert_paused(cp) + if snapshot.demand is None or snapshot.demand['state']!='paused':raise RuntimeError('No durable paused demand') + return {'ack':ack,'checkpoint':asdict(cp),'worker_pid':self.worker.pid} + + def bind_gui(self,binding,verifier,bootstrap): + self.facade.bind_window(binding,verifier,bootstrap) + self.transport.bind_session(self.transport_cap,binding.session_id,bootstrap.token) + return True + + def obligations(self): + tasks=[] + if getattr(self, 'compiler_host', None) is not None: + for row in self.compiler_host.obligations(): + tasks.append(row['close_task']) + for operation in row['operations']: + tasks.extend((operation['task'], operation['runner'])) + if self.facade is not None:tasks += [self.facade._wake,self.facade._stop,*self.facade._joins] + if self.runtime is not None:tasks += [self.runtime._stop_task,*tuple(self.runtime._pending), + *tuple(self.runtime._continuation_tasks)] + if self.stop_binding is not None:tasks += [self.stop_binding._work] + if self.gui is not None:tasks += [self.gui._start_task,*self.gui._io] + if self.listener is not None: + tasks += [self.listener._driver,self.listener._close_task] + if self.listener._record is not None: + tasks += [self.listener._record.get(key) for key in ('setup','handler','cleanup')] + if self.transport is not None: + tasks += [self.transport._stop_task,self.transport._close_task,*tuple(self.transport._handlers)] + tasks += [row.get('task') for row in self.transport._records.values()] + tasks += [row.get('cleanup') for row in self.transport._connections.values()] + return tasks + + def control_io_released(self): + listener_ok=(self.listener is None or self.listener.lifecycle()['resources_closed'] is True) + transport_ok=(self.transport is None or not self.transport._handlers and not self.transport._connections + and self.transport._close_task is not None and self.transport._close_task.done() + and not self.transport._close_task.cancelled()) + return listener_ok and transport_ok + + async def join_gui_resources(self): + if self.gui is None:return + if self.gui._start_task is not None and not self.gui._bound:self.gui.abort_startup() + while True: + receipt=await self.gui.join(timeout_s=2) + if (receipt['pending_pipe_operations']==0 and receipt['pipe_handles_closed'] + and receipt['process_handle_closed'] and receipt['verifier_handle_closed'] + and (not receipt['child_retained'] or receipt['child_exit_observed'])): + self.result['gui_cleanup']=receipt + return + await asyncio.sleep(.05) + + async def dispose_control_io(self): + if self.listener is not None: + while True: + value=await self.listener.aclose(timeout=2) + if value['resources_closed']:break + await asyncio.sleep(.05) + self.result['listener_cleanup']=value + if self.transport is not None: + while True: + value=await self.transport.aclose(timeout=2) + if (value['handlers']==0 and value['pending_connections']==0 + and self.transport._close_task.done()):break + await asyncio.sleep(.05) + self.result['transport_cleanup']=value + + async def dispose(self,failed): + """Cooperative cleanup only. Live/unknown obligations stay retained.""" + if getattr(self,'handover',None) is not None: + return await self.handover.dispose(failed) + self.fence() + if getattr(self, 'compiler_host', None) is not None: + await self.compiler_host.close() + if self.worker is not None and self.stop_binding is None:self.bind_worker_stop() + if self.transport is not None: + self.transport._request_stop() # Trusted failure cleanup; no network command or auto Resume. + await asyncio.shield(self.transport._stop_task) + elif self.stop_binding is not None: + self.stop_binding.request_stop(self.worker_cap) + while self.worker.poll() is None:await self.stop_binding.join(timeout=2) + # Failed startup closes only its owned input when safe. A started GUI + # sees stop_requested through the still-live listener and exits. + await self.join_gui_resources() + for child in self.handles:await self.join_child(child) + # The listener must remain usable for the GUI's final close reply, but + # its live accept driver cannot be joined until acceptance is fenced. + if self.listener is not None:self.listener.close() + errors=await join_retained(self.obligations()) + self.result['retained_obligations_joined']=True + if any(isinstance(value,BaseException) for value in errors): + self.result['joined_obligation_errors']=[diagnostic(value) for value in errors if isinstance(value,BaseException)] + if self.host is not None: + # Even a failed trial may safely clean native resources once every + # actual retained child and pipe/handle obligation is positively joined. + self.result['cleanup']=await self.host.close(timeout_s=180) + await self.dispose_control_io() + self.result['all_retained_process_exits']=[{'pid':child.pid,'exit_code':child.poll()} for child in self.handles] + + async def prepare(self): + # Verify installation and explicit deployment inputs before owning any + # child. Windows APIs/native modules are imported only on this path. + if os.name != 'nt' or sys.version_info[:3] != (3, 11, 9): + raise RuntimeError('This first desktop profile requires full Windows CPython 3.11.9') + verify_execution_clock() + profile = self.profile + verify_interpreter(profile.host_python, profile.host_python_sha256, current=True) + self.installed = package_identity() + if sha(self.profile_path) != self.profile_sha256: + raise RuntimeError('Deployment profile changed') + rec.save(self.root/'installed-package.private.json', self.installed) + self.result['package_version'] = self.installed['version'] + if self.enable_reopen: + from adaptive_inference.desktop import saved_tasks + if self.saved_task is not None: + self.saved = saved_tasks.read_saved(self.saved_task) + if self.saved['completed']: + self.result.update(saved_result=saved_tasks.completed_result(self.saved), + task_completed=True, worker_outcome='saved_result', clean_stop=True) + return + boot = saved_tasks.observe_boot() + self.reopen_context = saved_tasks.new_context(self.profile_sha256, self.installed, boot) + rec.save(self.root/'reopen-context.private.json', self.reopen_context) + if self.saved_task is not None: + self.saved = saved_tasks.inspect_saved(self.saved_task, + profile_sha256=self.profile_sha256, installed=self.installed, boot=boot, now=execution_now()) + self.reopen_locks.append(saved_tasks.claim_host(self.saved_task)) + # Selection and owner acquisition are separate; re-read after + # locking before any Pause, host start or worker launch. + self.saved = saved_tasks.inspect_saved(self.saved_task, + profile_sha256=self.profile_sha256, installed=self.installed, boot=boot, now=execution_now()) + verify_interpreter(profile.worker_python, profile.worker_python_sha256) + verify_sdk(profile.hermes_source) + from adaptive_inference.desktop import owned_job + from adaptive_inference.desktop.activity import WindowsActivityReader + from adaptive_inference.desktop.cold_start import ColdStartAdapter + from adaptive_inference.desktop.desktop_controls import ControlledDesktopHost, DesktopControlLifecycle + from adaptive_inference.desktop.foreground_policy import ForegroundPolicy + from adaptive_inference.desktop.gui_launch import OwnedUILaunch, WindowsLaunchAPI, LaunchSpec, CHILD_SOURCES + from adaptive_inference.desktop.control_client import Bootstrap + + self.job = owned_job + self.job.contain_current_process() + self.work = self.root/'hermes-task' if self.saved is None else self.saved['work'] + prepare_command = [str(profile.worker_python), '-I', '-B', '-m', + 'adaptive_inference.desktop.hermes.run_recovery', 'prepare', '--run', str(self.work), + '--profile', str(self.profile_path), '--profile-sha256', self.profile_sha256, + '--package-identity', str(self.root/'installed-package.private.json'), + '--control-case', 'complete'] + if self.task_spec_path is None: + prepare_command.append('--validate-report-before-write') + else: + prepare_command.extend(['--task-spec', str(self.task_spec_path), + '--task-spec-sha256', self.task_spec_sha256, '--task-inputs', str(self.task_inputs), + '--task-id', 'task-'+uuid.uuid4().hex, '--session-id', 'adaptive-'+uuid.uuid4().hex]) + if self.reopen_context is not None: + prepare_command.extend(['--reopen-context', str(self.root/'reopen-context.private.json')]) + if self.saved is None: + child = self.launch(prepare_command, 'prepare') + await self.until(lambda:child.poll() is not None, timeout=60) + if await self.join_child(child) != 0: + raise RuntimeError('Fresh paused-task preparation failed; see prepare.log') + if self.enable_reopen: + self.reopen_locks.append(saved_tasks.claim_host(self.work)) + else: + self.saved_diagnostics = self.work/'episodes'/self.root.name + self.saved_diagnostics.mkdir(parents=True, exist_ok=False) + if self.saved['spec'] is not None: + self.task_spec_path = self.work/'plan.json' + self.task_spec_sha256 = self.saved['spec'].sha256 + self.plan = read(self.work/'plan.json') + self.store = TaskStore(self.work/'task.sqlite3') + if self.saved is not None: + self.saved = saved_tasks.activate_saved_budget(self.saved, self.store, boot=boot) + self.identity = DemandIdentity(**self.plan['demand_identity']) + self.registry = DemandRegistry((Registration(self.store,self.identity),)) + self.capability = object() + self.box = Mailbox(self.registry, self.identity.registration_id, requester_capability=self.capability) + self.original = self.checkpoint() + if (self.plan.get('deadline_clock')!=CLOCK_ID or self.original.metadata.get('deadline_clock')!=CLOCK_ID + or self.original.state.get('deadline_clock')!=CLOCK_ID): + raise RuntimeError('Fresh installed task lacks the explicit execution clock') + rec.save(self.root/'original-checkpoint.private.json', asdict(self.original)) + self.submit(self.initial_pause_id(), 'pause') + self.host_epoch = 'desktop-'+uuid.uuid4().hex + self.intent_cap = object() + self.gate = UIIntentGate(host_epoch=self.host_epoch, capability=self.intent_cap, + verify_current=lambda binding:None if self.facade is None else self.facade.verify_current(binding)) + if profile.foreground_policy == COOPERATIVE_POLICY: + from adaptive_inference.desktop.ui_progress import CooperativeProgress + self.progress = CooperativeProgress(self.gate, self.intent_cap) + self.result['foreground_policy_id'] = profile.foreground_policy + owner = dict(profile.owner) + self.reader = WindowsActivityReader() + adapter = ColdStartAdapter(**owner, job=self.job, max_runtime_seconds=None) + self.host = ControlledDesktopHost(owner, profile.gateway, self.root/'host', + ForegroundPolicy(ui_intent=self.gate, cooperative_progress=self.progress), + demand=self.registry, restore_requirements=ToolRequirements(3,3,250), adapter=adapter, + reader=self.reader, job=self.job, max_runtime_seconds=None, idle_grace_s=30, runtime_timing=True) + settings = profile.video if profile.video is not None else profile.compiler + if settings is not None: + if self.compiler_kind == 'video': + from adaptive_inference.desktop.video_profile import verify_settings + else: + from adaptive_inference.desktop.compiler_profile import verify_settings + from adaptive_inference.desktop.compiler_host import CompilerHost + from adaptive_inference.desktop.hermes.native_reader import native_reader_factory + verify_settings(settings) + self.compiler_native_reader = native_reader_factory(profile.hermes_source, + profile.hermes_site_packages, home=self.root/'compiler-native-reader-home') + self.compiler_host = CompilerHost(self.host.controller, settings, + host_epoch=self.host_epoch, kind=self.compiler_kind) + self.compiler_host.add_route(self.host.app) + self.host.compiler_operations = self.compiler_host + await self.host.start(timeout_s=30) + self.initial_events = deepcopy(list(self.host.controller.events)) + complete_ledger(self.host.controller.events, self.initial_events) + if adapter.launch_intent is not None or adapter.started_owner or self.registry.observe(self.identity.registration_id).eligible: + raise RuntimeError('Cold host unexpectedly loaded before paused owner bootstrap') + self.deadline = execution_now()+900 if self.saved is None else self.saved['deadline'] + self.result['deadline_monotonic'] = self.deadline + self.result['deadline_clock'] = CLOCK_ID + self.first_ready = self.worker_diagnostics()/'controller-ready-1.json' + command = [self.plan['base_python'], '-I', '-B', '-m', + 'adaptive_inference.desktop.hermes.run_recovery', 'worker', '--run', str(self.work), + '--controller-ready', str(self.first_ready), '--gateway-instance', self.host.gateway.instance, + '--deadline-monotonic', str(self.deadline), '--stop-request-id', self.stop_id] + command.extend(self.compiler_launch_args(self.work, self.plan, execution_id=self.identity.task_id, + task_id=self.identity.task_id, deadline=self.deadline, diagnostics=self.worker_diagnostics())) + if self.reopen_context is not None: + command.extend(['--execution-boot-id', rec.digest(self.reopen_context['boot'])]) + if self.saved is not None: + command.extend(['--diagnostics', str(self.saved_diagnostics)]) + if self.qualification_hold_after_report: + command.append('--qualification-hold-after-report') + self.worker = self.launch(command, 'worker') + self.bind_worker_stop() + self.launch_api = WindowsLaunchAPI() + handle = self.launch_api.handle(self.worker) + self.worker_identity = self.launch_api.identity(handle) + if (self.launch_api.in_job(handle,self.job._HANDLE) is not True + or self.worker_identity[0] != self.worker.pid or self.worker.poll() is not None): + raise RuntimeError('Retained installed worker is not live in this exact containing job') + self.result['retained_worker_identity'] = {'pid':self.worker_identity[0], + 'creation_time_100ns':self.worker_identity[1], 'containing_job_verified':True} + paused = await self.until(self.paused_ack, self.worker, timeout=60) + rec.save(self.root/'paused-before-gui.private.json', paused) + self.gui = OwnedUILaunch(api=self.launch_api) + prepared = PreparedTask(self.host_epoch,self.identity.task_id,self.original.prompt) + self.facade = DesktopControlLifecycle(self.host,prepared,worker_stop=self.stop_binding, + worker_capability=self.worker_cap,gui=self.gui,shutdown_timeout_s=120) + self.runtime = MailboxRuntime(prepared,self.box,requester_capability=self.capability, + gate=self.gate,intent_capability=self.intent_cap,lifecycle=self.facade) + await self.runtime.refresh() + await self.bind_compiler_worker(self.runtime, self.worker, self.work, self.deadline) + self.host.attach_control_runtime(self.runtime) + from adaptive_inference.desktop.continuation_host import HostContinuation + self.continuation = HostContinuation(self,self.runtime) + from adaptive_inference.desktop.task_handover import HostTaskHandover + self.handover = HostTaskHandover(self) + self.transport_cap = object() + self.transport = ControlTransport(prepared,self.runtime,capability=self.transport_cap, + runner_credential='local-no-key',cooperative_progress=self.progress, + handover=self.handover.adapter(self.runtime)) + self.listener = SerialListener(self.transport) + endpoint = self.listener.start() + self.handover.capture_initial() + bootstrap = Bootstrap(self.host_epoch,'session-'+uuid.uuid4().hex,self.identity.task_id,self.original.prompt, + credential(),endpoint[0],endpoint[1],'AdaptiveInference.'+uuid.uuid4().hex,secrets.randbelow(2**31-1)+1) + spec = LaunchSpec(profile.host_python,profile.host_python_sha256,HERE/'run_gui.py', + tuple((name,sha(HERE.parent/name)) for name in sorted(CHILD_SOURCES))) + await self.gui.start(spec,bootstrap,containing_job=self.job._HANDLE, + launch_ready=lambda:self.host.started_at is not None and not self.host.closing + and self.host.adapter.launch_intent is None and self.paused_ack() is not None, + bind_session=self.bind_gui,revoke_session=self.facade.window_closed,timeout_s=10) + await self.until(lambda:self.transport._ready,self.worker,timeout=30) + self.assert_paused(self.checkpoint()) + self.result['retained_gui_binding'] = asdict(self.gui.binding) + rec.save(self.root/'ready.private.json',{'ready':True,'gui_pid':self.gui.process.pid, + 'worker_pid':self.worker.pid,'task_id':self.identity.task_id,'paused_owner_acknowledged':True, + 'model_started':False,'instructions':'Resume to run the task. Pause pauses at a safe boundary. Stop closes this session.'}) + + async def _paused_expiry_candidate(self): + # Reuse the runtime's retained off-loop coherent read; no new database + # observer or blocking SQLite work is introduced on the host loop. + await self.runtime.refresh() + snapshot = self.runtime._cache.snapshot + cp = snapshot.target + marker = execution_outcome(cp.state) + return (cp.state['status'] == 'pending' and cp.operation is None + and not cp.requires_reconciliation and cp.state.get('deadline_monotonic') == self.deadline + and cp.state.get('deadline_clock') == CLOCK_ID + and cp.state.get('phase') in ('ready','waiting_admission') + and cp.state.get('pending_request') is None + and snapshot.demand is not None and snapshot.demand['state'] == 'paused' + and bool(snapshot.requests) and len(snapshot.requests) == len(snapshot.receipts) + and snapshot.requests[-1]['command'] == 'pause' and snapshot.receipts[-1]['outcome'] == 'applied' + and (marker is None or marker['deadline_monotonic'] == self.deadline)) + + async def _admission_wait_expiry_candidate(self): + # Like the paused path this retains finalization, never authorizes work. + from adaptive_inference.control_mailbox import ControlBlocked + await self.runtime.refresh() + snapshot = self.runtime._cache.snapshot + cp = snapshot.target + if (cp.task_id != self.stop_binding.original.task_id + or cp.state.get('deadline_monotonic') != self.deadline + or cp.state.get('deadline_clock') != CLOCK_ID): + return False + # Pure immutable binding checks; durable selected lifetime verification + # still belongs to the exact exit verifier after the retained join. + self.stop_binding._admission_wait_identity(cp) + try: + self.stop_binding.mailbox.admission_wait_candidate(snapshot, allow_sealed=True) + except ControlBlocked: + return False + return True + + def assert_continuation_parked(self, release): + controller,scheduler = self.host.controller,self.host.gateway.scheduler + owner = None if controller.identity is None else controller.identity['owner_id'] + if (controller.state != 'parked' or controller.fault is not None or controller.tool is not None + or scheduler.active is not None or scheduler.pending or scheduler.uncertain + or owner != release['native']['owner_id'] + or self.host.adapter._barrier is None + or self.host.adapter._cold_start_error is not None or self.host.adapter._startup_failure is not None): + raise RuntimeError('Continuation requires the same verified parked native owner') + if owner is not None and self.host.adapter._released != release['native']['release']: + raise RuntimeError('Parked release identity changed before selection') + + async def continuation_release(self, request, intent_valid, *, task_handover=False): + from adaptive_inference.desktop.control_transport import Refused + from adaptive_inference.desktop.hermes.service_stop import PAUSED_EXPIRY_EXIT, WAIT_EXPIRY_EXIT + child,binding = self.worker,self.stop_binding + if request['execution_id'] != binding.original.task_id: + raise Refused('stale_execution') + limit=execution_now()+120 + while child.poll() is None: + if self.runtime._stopping or not intent_valid():raise Refused('continuation_blocked') + if execution_now()>=limit:raise TimeoutError('Previous worker has not released') + await asyncio.sleep(.05) + code=await self.join_child(child) + if not task_handover and code not in (PAUSED_EXPIRY_EXIT, WAIT_EXPIRY_EXIT): + raise Refused('continuation_unavailable') + outcome=await asyncio.to_thread(binding._verify_exit,code) + expired = {PAUSED_EXPIRY_EXIT: 'paused_budget_expired', + WAIT_EXPIRY_EXIT: 'admission_wait_budget_expired'} + allowed = ('task_completed','task_cancelled') if task_handover else (expired[code],) + if outcome not in allowed:raise Refused('continuation_unavailable') + while self.host.controller.state!='parked': + if self.runtime._stopping or not intent_valid():raise Refused('continuation_blocked') + if self.host.controller.fault or execution_now()>=limit: + raise RuntimeError('Previous native owner did not reach verified parking') + await asyncio.sleep(.05) + async with self.host.controller.lock: + adapter,controller=self.host.adapter,self.host.controller + def parked(): + if (adapter._barrier is None or adapter._cold_start_error is not None + or adapter._startup_failure is not None): + raise RuntimeError('Native startup/release ownership is unresolved') + state=adapter.driver.status() + adapter._stopped_identity(state) + if adapter.driver.owner_idle() is not True or adapter.driver.port_free(18098) is not True: + raise RuntimeError('Native owner or backend port is still occupied') + owner=None if controller.identity is None else controller.identity['owner_id'] + proof=deepcopy(adapter._released) + if owner is None: + if adapter.started_owner or not adapter._cold_prepared or adapter.parked_observation is None: + raise RuntimeError('Cold native parking is not proven') + elif (type(proof) is not dict or proof.get('owner_id')!=owner + or any(proof.get(key) is not True for key in ('released','cleanup_confirmed','port_free')) + or type(proof.get('retained_processes')) is not int or proof['retained_processes']<1 + or proof.get('confirmed_process_exits')!=proof['retained_processes']): + raise RuntimeError('Exact retained native release is missing') + return {'owner_id':owner,'release':proof,'status_sha256':rec.digest(state), + 'owner_lock_idle':True,'backend_port_free':True,'startup_barrier_held':True} + native=await adapter._call(parked) + release={'worker_joined':True,'worker_pid':child.pid,'exit_code':code,'outcome':outcome, + 'native':native,'execution_id':request['execution_id']} + self.assert_continuation_parked(release) + if getattr(self, 'compiler_host', None) is not None and self.compiler_host.lifecycle()['bound']: + await self.compiler_host.retire() + return release + + async def launch_continuation(self, record, coordinator, intent_valid): + from adaptive_inference.desktop.continuation_host import SelectedWorkerStopBinding,selection + grant=record['grant'];suffix=grant['payload_sha256'][:24] + record.update(pause_id='continue-pause-'+suffix,resume_id='continue-resume-'+suffix, + stop_id='continue-stop-'+suffix) + diagnostics=self.work/'executions'/grant['payload']['successor_id'] + record['diagnostics']=diagnostics + def files(): + diagnostics.mkdir(parents=True,exist_ok=False) + path=diagnostics/'grant.json';rec.save(path,grant) + return path + path=await asyncio.to_thread(files) + if not coordinator.live(intent_valid):return + await self.runtime.submit_control(record['pause_id'],'pause',execution=selection(grant),_transition=True) + if not coordinator.live(intent_valid):return + self.first_ready=diagnostics/'controller-ready.json' + command=[self.plan['base_python'],'-I','-B','-m','adaptive_inference.desktop.hermes.run_recovery', + 'worker','--run',str(self.work),'--continuation-grant',str(path), + '--host-generation',self.runtime.prepared.host_epoch,'--controller-ready',str(self.first_ready), + '--gateway-instance',self.host.gateway.instance,'--deadline-monotonic',str(grant['deadline_monotonic']), + '--stop-request-id',record['stop_id']] + command.extend(self.compiler_launch_args(self.work, self.plan, + execution_id=grant['payload']['successor_id'], task_id=self.runtime.prepared.task_id, + deadline=grant['deadline_monotonic'], diagnostics=diagnostics)) + # The original trial's report hold is not inherited: this click supplies + # one fresh Resume, and existing artifact/receipt verification still wins. + record['launch_attempted']=True + before=len(self.handles) + try: + self.launch(command,'worker-continue-'+str(grant['execution_generation'])) + finally: + if len(self.handles)==before+1: + child=self.handles[-1] + record['child']=child;self.worker=child + capability=object() + binding=SelectedWorkerStopBinding(child,self.work,record['mailbox'], + requester_capability=record['requester'],capability=capability, + stop_request_id=record['stop_id'],original=record['original'], + deadline_monotonic=grant['deadline_monotonic'],model=MODEL,quality=QUALITY, + fence=self.fence,deadline_clock=CLOCK_ID,grant=grant,authority=record['authority']) + self.facade.bind_selected_worker(record,binding,capability) + self.stop_binding,self.worker_cap,self.stop_id=binding,capability,record['stop_id'] + self.worker_exit_verified=False + child=record['child'] + handle=self.launch_api.handle(child) + observed=self.launch_api.identity(handle) + if self.launch_api.in_job(handle,self.job._HANDLE) is not True or observed[0]!=child.pid or child.poll() is not None: + raise RuntimeError('Selected child is not live in the retained containing job') + record['retained_child_identity']={'pid':observed[0],'creation_time_100ns':observed[1], + 'containing_job_verified':True} + await self.wait_continuation_ack(record,record['pause_id'],intent_valid) + if coordinator.live(intent_valid): + await self.bind_compiler_worker(self.runtime, child, self.work, grant['deadline_monotonic']) + + async def wait_continuation_ack(self,record,request_id,intent_valid): + child,mailbox=record['child'],record['mailbox'] + def install(observation): + try: + self.runtime._install(observation) + except ExpiredObservation: + # Read only: retain prior cache/intent, use no ACK from this read. + # A fresh read must still pass every binding/history check. + return False + return True + while True: + if self.runtime._stopping or not intent_valid():return None + if child.poll() is not None or execution_now()>=record['grant']['deadline_monotonic']: + raise RuntimeError('Selected worker did not acknowledge before its deadline') + fresh=await self.runtime._database(lambda:self.runtime._observe(mailbox),install) + if self.runtime._stopping or not intent_valid():return None + if child.poll() is not None or execution_now()>=record['grant']['deadline_monotonic']: + raise RuntimeError('Selected worker did not acknowledge before its deadline') + if fresh: + snapshot=self.runtime._cache.snapshot + ack=snapshot.status(request_id) + if ack['owner_acknowledged']: + if (ack['outcome']!='applied' or snapshot.target.owner_epoch is None + or snapshot.target.task_id!=record['grant']['payload']['successor_id'] + or child.poll() is not None): + raise RuntimeError('Selected control lacks exact live owner acknowledgement') + return ack + await asyncio.sleep(.05) + + async def ready_continuation(self,record,intent_valid): + from adaptive_inference.desktop.continuation_host import selection + child=record['child'];boot_path=record['diagnostics']/f'bootstrap-{child.pid}.json' + # Keep the existing readiness bound, independent of a fresh task budget. + limit=min(execution_now()+240,record['grant']['deadline_monotonic']) + record['readiness_deadline_monotonic']=limit + while True: + if self.runtime._stopping or not intent_valid():return + if child.poll() is not None: + raise RuntimeError('Selected readiness exceeded the retained worker lifetime') + if execution_now()>=limit:raise TimeoutError('Selected readiness boundary deadline') + if self.host.controller.fault:raise RuntimeError('Controller fault during selected readiness') + if boot_path.exists() and self.host.controller():break + await asyncio.sleep(.05) + mailbox=record['mailbox'] + await self.runtime._database(lambda:self.runtime._observe(mailbox),self.runtime._install) + if self.runtime._stopping or not intent_valid():return + boot=await asyncio.to_thread(read,boot_path) + cp=self.runtime._cache.snapshot.target + if (boot['pid']!=child.pid or boot['task_id']!=cp.task_id + or boot['registration_id']!=record['identity'].registration_id + or boot['owner_epoch']!=cp.owner_epoch or boot['deadline_clock']!=CLOCK_ID + or boot['deadline_monotonic']!=record['grant']['deadline_monotonic']): + raise RuntimeError('Selected bootstrap identity differs from its retained owner') + if self.runtime._stopping or not intent_valid() or not self.host.controller():return + value={'ready':True,'model':MODEL,'quality':QUALITY,'gateway':self.plan['gateway'], + 'gateway_instance':self.host.gateway.instance,'observed_at':time.time(), + 'selected_execution':selection(record['grant'])} + # The verified selected worker may act as soon as this file appears. + # Finish the observation-only transition before publishing readiness. + self.runtime._transitioning = False + await asyncio.to_thread(rec.save,self.first_ready,value) + + async def interact(self): + # No invented admission or automatic Resume: the worker's saved demand, + # host capacity and authenticated GUI intent must agree first. + while not self.transport.lifecycle()['stop_requested']: + if getattr(self,'handover',None) is not None and self.handover.transitioning: + await asyncio.sleep(.05) + continue + selected = None if self.continuation is None else self.continuation.current + if self.runtime._transitioning or selected is not None and selected['child'] is None: + if self.gui.process.poll() is not None: + raise RuntimeError('GUI exited without confirmed Stop') + await asyncio.sleep(.05) + continue + code = self.worker.poll() + if code is None: + if execution_now() >= self.deadline: + if (not await self._paused_expiry_candidate() + and not await self._admission_wait_expiry_candidate()): + self.remaining() + else: + self.remaining() + if code is not None and not self.worker_exit_verified: + await self.join_child(self.worker) + outcome = self.stop_binding._verify_exit(code) + self.worker_exit_verified = True + self.result['worker_outcome'] = outcome + if outcome == 'task_completed' and self.task_spec_path is not None: + cp = self.stop_binding.mailbox.snapshot().target + spec = rec.task_contract(cp) + if spec is None: + raise RuntimeError('Completed user task lost its declared contract') + from adaptive_inference.desktop.recipe_dispatch import completion_kwargs + self.result.update(spec.completion(self.stop_binding.run/'work',cp.state['tools'],cp.state['output'], + **completion_kwargs(spec, rec.verified_recipe_receipts(cp, self.stop_binding.run)))) + self.result['task_spec_sha256'] = spec.sha256 + self.result['task_completed'] = outcome == 'task_completed' + self.result['execution_expired'] = outcome in ('paused_budget_expired', 'admission_wait_budget_expired') + await self.runtime.refresh() + if (selected is None and code is None and not self.first_ready.exists() + and (self.worker_diagnostics()/f'bootstrap-{self.worker.pid}.json').exists() + and self.registry.observe(self.identity.registration_id).eligible + and self.host.controller()): + await self.ready(self.worker,self.first_ready) + if self.gui.process.poll() is not None: + raise RuntimeError('GUI exited without confirmed Stop') + if self.host.controller.fault: + raise RuntimeError('Controller fault: '+str(self.host.controller.fault)) + await asyncio.sleep(.1) + await asyncio.shield(self.transport._stop_task) + if not self.transport.lifecycle()['stop_confirmed']: + raise RuntimeError('Stop is not confirmed') + self.result['lifecycle'] = await self.facade.cooperative_stop() + if self.result['lifecycle'] != {'worker_joined':True,'host_closed':True,'uncertain':False,'fault':None}: + raise RuntimeError('Cooperative Stop lacks exact verified closure') + + def validate_cleanup(self): + if getattr(self,'handover',None) is not None: + self.handover.validate_cleanup() + selected = None if self.continuation is None else self.continuation.current + if selected is not None and selected['child'] is None: + self.continuation.verify_unlaunched(selected) + else: + self.stop_binding._verify_exit(self.worker.poll()) + if (self.result.get('joined_obligation_errors') + or self.result.get('retained_obligations_joined') is not True + or self.result.get('gui_cleanup',{}).get('closed') is not True + or getattr(self,'handover',None) is None and self.result.get('gui_cleanup',{}).get('faults') != [] + or not self.result.get('listener_cleanup',{}).get('clean_close') + or not self.result.get('transport_cleanup',{}).get('closed')): + raise RuntimeError('Child, pipe or control ownership is not cleanly joined') + if self.job.active_process_ids() != {os.getpid()}: + raise RuntimeError('Actual containing job still has children') + cleanup = self.result['cleanup'] + if (not self.host.closed or self.host.controller.fault + or cleanup.get('closed') is not True + or any(cleanup.get(k) is not None for k in ('primary_error','cleanup_error','controller_fault'))): + raise RuntimeError('Native cleanup has a fault') + package_identity(self.installed) + + async def run(self): + if self.root.is_relative_to(HERE.parent): + raise ValueError('Run state must be outside the installed package') + self.root.mkdir(exist_ok=False) + try: + await self.prepare() + if 'saved_result' in self.result: + rec.save(self.root/'result.private.json', self.result) + return self.result + await self.interact() + except BaseException as error: + self.result['primary_error'] = diagnostic(error) + try: + await self.dispose('primary_error' in self.result) + if 'primary_error' not in self.result: + self.validate_cleanup() + except BaseException as error: + self.result['cleanup_error'] = diagnostic(error) + # Preserve the accepted failure path: uncertain retained operations + # are never abandoned just because the caller's wait expired. + try: + rec.save(self.root/'unresolved.private.json',self.result) + except BaseException as failure: + self.result['diagnostic_write_error'] = diagnostic(failure) + try: + if getattr(self,'handover',None) is not None: + await self.handover.failure_cleanup() + await self.join_gui_resources() + if self.listener is not None: + self.listener.close() + await join_retained(self.obligations()) + for child in self.handles: + await self.join_child(child) + try: + if self.host is not None: + self.result['failure_cleanup'] = await self.host.close(timeout_s=180) + await self.dispose_control_io() + except BaseException as failure: + self.result['failure_cleanup_error'] = diagnostic(failure) + if not self.control_io_released(): + raise RuntimeError('Control listener/stream ownership remains unresolved') from failure + if self.job is not None: + while True: + try: + if self.job.active_process_ids() == {os.getpid()}: + break + except BaseException as failure: + self.result.setdefault('job_observation_error',diagnostic(failure)) + await asyncio.sleep(1) + except BaseException as failure: + self.result['retention_error'] = diagnostic(failure) + try: + rec.save(self.root/'retention-failure.private.json',self.result) + except BaseException: + pass + while True: + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + continue + if self.progress is not None: + try: + self.result['cooperative_foreground_final'] = self.progress.snapshot() + except Exception as error: + self.result['cooperative_diagnostic_error'] = diagnostic(error) + self.result['clean_stop'] = not any(k in self.result for k in ('primary_error','cleanup_error')) + for stream in self.logs: + stream.close() + for lock in self.reopen_locks: + lock.close() + rec.save(self.root/'result.private.json',self.result) + return self.result + + +def execute(profile_path, run, *, qualification_timeout_s=None, task_spec=None, task_inputs=None, + saved_task=None, enable_reopen=False): + # Ordinary host lifetime is persistent, as in PersistentDesktopHost. Only a + # caller explicitly running a qualification may arm its emergency bound. + # Unknown ordinary cleanup remains retained; it is never reported clean. + if qualification_timeout_s is not None: + if type(qualification_timeout_s) is not int or qualification_timeout_s != 1800: + raise ValueError('The accepted qualification guard is exactly 1800 seconds') + threading.Thread(target=hard_deadline,args=(qualification_timeout_s,),daemon=True).start() + options = {} if task_spec is None and task_inputs is None else {'task_spec':task_spec,'task_inputs':task_inputs} + if saved_task is not None: options['saved_task'] = saved_task + if enable_reopen: options['enable_reopen'] = True + result = asyncio.run(DesktopComposition(profile_path,run,**options).run()) + result['asyncio_teardown_completed'] = True + rec.save(Path(run)/'result.private.json',result) + return result diff --git a/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/desktop_controls.py b/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/desktop_controls.py new file mode 100644 index 0000000..37770b0 --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/desktop_controls.py @@ -0,0 +1,507 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Private composition of existing host, mailbox, worker and GUI lifecycles. + +No process launch, listener creation, credentials, task adoption or new policy. +The trusted caller supplies the exact already-retained worker and GUI records. +""" +import asyncio +from copy import deepcopy +import math +import threading +import time + +from adaptive_inference.desktop.control_transport import PreparedTask +from adaptive_inference.desktop.mailbox_runtime import MailboxRuntime +from adaptive_inference.desktop.persistent_host import PersistentDesktopHost +from adaptive_inference.desktop.ui_intent import UIBinding +from adaptive_inference.control_mailbox import execution_outcome + + +def bounded(value, maximum): + if type(value) not in (int, float) or not math.isfinite(value) or not 0 < value <= maximum: + raise ValueError('A finite positive lifecycle deadline is required') + return value + + +class ControlledDesktopHost(PersistentDesktopHost): + """Reuse the existing sequential observer; each UI poll remains retained.""" + def __init__(self, *args, **kwargs): + if kwargs.get('max_runtime_seconds') is not None: + raise ValueError('Controlled service lifetime is persistent; qualification watchdog is external') + super().__init__(*args, **kwargs) + self.control_runtime = None + self._control_thread = threading.current_thread() + self._control_loop = asyncio.get_running_loop() + self._control_rebinding = False + self.retired_control_runtimes = [] + + def attach_control_runtime(self, runtime): + if (threading.current_thread() is not self._control_thread + or asyncio.get_running_loop() is not self._control_loop + or type(runtime) is not MailboxRuntime or self.control_runtime is not None + or self.closing or runtime.lifecycle.host is not self): + raise RuntimeError('Bind one actual control runtime on the original host loop') + self.control_runtime = runtime + + def snapshot(self): + value = super().snapshot() + progress = getattr(self.controller.policy, 'cooperative_progress', None) + if progress is not None: + value['cooperative_foreground'] = progress.snapshot() + return value + + async def _observe_demand(self): + try: + while not self.closing: + if self._control_rebinding: + await asyncio.sleep(.05) + continue + await self.controller.refresh_demand() + if self.closing: + break + if self.control_runtime is not None and not self._control_rebinding: + await self.control_runtime.refresh() + for runtime in tuple(self.retired_control_runtimes): + if not self.closing and not runtime._stopping: + await runtime.refresh() + if not self.closing: + await asyncio.sleep(.1) + except BaseException as exc: + self.demand_error = type(exc).__name__ + self.controller._demand_valid = False + raise + + def commit_task_runtime(self, previous, runtime, record, coordinator): + """Select a new immutable logical task under the existing two locks.""" + if (previous is not self.control_runtime or type(runtime) is not MailboxRuntime + or runtime is previous or runtime.lifecycle.host is not self + or previous._pending or not previous._task_retired + or runtime.prepared.host_epoch != previous.prepared.host_epoch + or runtime.prepared.task_id == previous.prepared.task_id + or threading.current_thread() is not self._control_thread + or asyncio.get_running_loop() is not self._control_loop + or not self._control_rebinding or not self.controller.lock.locked() + or not self.controller._demand_lock.locked() + or not record['prepared'] or self.closing or self.closed): + raise RuntimeError('Task rebind lacks exact quiescent host authority') + coordinator.composition.assert_continuation_parked(record['release']) + observation = record['observation'] + snapshot = runtime._check_observation(observation) + if (snapshot.target.task_id != record['reservation']['payload']['task_id'] + or runtime.mailbox.registry is not record['episode']['registry'] + or runtime.selected_execution() != {'execution_id': snapshot.target.task_id, + 'execution_generation': 0}): + raise RuntimeError('Fresh task projection differs from its actual store') + controller = self.controller + controller.demand = runtime.mailbox.registry + controller._demand_rows = () + controller._demand_stamp = None + controller._demand_valid = False + controller._demand_revisions = {} + controller._idle_since = None + runtime._cache = observation + self.retired_control_runtimes.append(previous) + self.control_runtime = runtime + coordinator.commit_episode(record) + + def commit_selected_runtime(self, runtime, record, staged, continuation): + """Synchronous complete rebind while both existing controller locks hold.""" + if (runtime is not self.control_runtime or runtime._continuation is not continuation + or continuation.composition.host is not self or runtime._pending + or threading.current_thread() is not self._control_thread + or asyncio.get_running_loop() is not self._control_loop + or not self._control_rebinding or not self.controller.lock.locked() + or not self.controller._demand_lock.locked() or not record['committed']): + raise RuntimeError('Selected rebind lacks exact quiescent host authority') + from adaptive_inference.desktop.continuation_host import selection + registry,mailbox,capability,observation = staged + selected = selection(record['grant']) + if (mailbox.registry is not registry or mailbox._capability is not capability + or observation.snapshot.target.task_id != selected['execution_id'] + or observation.snapshot.target.prompt != runtime.prepared.objective): + raise RuntimeError('Selected binding does not describe its actual successor') + continuation.composition.assert_continuation_parked(record['release']) + runtime._check_observation(observation,execution=selected) + controller = self.controller + controller.demand = registry + controller._demand_rows = () + controller._demand_stamp = None + controller._demand_valid = False + controller._demand_revisions = {} + controller._idle_since = None + runtime.mailbox,runtime._requester = mailbox,capability + runtime._selection = selected + runtime._cache = observation + composition = continuation.composition + composition.registry,composition.box,composition.capability = registry,mailbox,capability + composition.identity = record['identity'] + composition.original = observation.snapshot.target + composition.deadline = record['grant']['deadline_monotonic'] + composition.worker_exit_verified = False + record['registry'],record['mailbox'],record['requester'] = registry,mailbox,capability + record['original'] = observation.snapshot.target + + +class DesktopControlLifecycle: + """One prepared-task session, no automatic reopening or task replacement. + + The launch callback binds the current GUI before its private ready ACK. + Product Stop waits the supplied safe worker result and actual GUI cleanup, + then invokes the existing native host close. Transport teardown is separate. + """ + def __init__(self, host, prepared, *, worker_stop, worker_capability, gui, + shutdown_timeout_s=120, clock=time.monotonic): + if type(prepared) is not PreparedTask or type(worker_capability) is not object: + raise TypeError('Exact prepared task and private worker capability required') + for target, names in ((worker_stop, ('request_stop', 'join')), + (gui, ('session_live', 'window_live', 'revoke_session', 'join'))): + if any(not callable(getattr(target, name, None)) for name in names): + raise TypeError('Explicit retained lifecycle methods required') + if not callable(clock): + raise TypeError('Monotonic clock required') + self.host, self.prepared, self.worker_stop, self.gui = host, prepared, worker_stop, gui + self._worker_capability = worker_capability + self.timeout = bounded(shutdown_timeout_s, 600) + self.clock = clock + self._loop, self._thread = asyncio.get_running_loop(), threading.current_thread() + self.binding = None + self._session_closed = self._fenced = False + self._wake = self._stop = None + self._joins = [] + self._faults = set() + self.worker_receipt = self.gui_receipt = self.native_receipt = None + self._worker_bindings = [(worker_stop, worker_capability)] + self._replacement_pending = False + self.worker_replacements = [] + self.selected_transitions = [] + self._selected = self._selection_coordinator = None + self.service = None + self._gui_optional_unstarted = False + self._gui_stop_aborted = False + + def _owner(self): + if threading.current_thread() is not self._thread or asyncio.get_running_loop() is not self._loop: + raise RuntimeError('Desktop control belongs to the original host loop/thread') + + def replace_worker(self, expected, create_replacement_binding): + """Trusted synchronous replacement after the harness's safe restart join. + + No await may separate precheck, retained launch, construction and commit. + The callback must retain Popen before diagnostics, including on failure, + and return (WorkerStopBinding, private capability) for that exact child. + This method never launches by PID, signals a process or retries a failure. + The harness owns failed-launch cleanup through its existing retained list. + """ + self._owner() + from adaptive_inference.desktop.hermes.service_stop import WorkerStopBinding + if (expected is not self.worker_stop or type(expected) is not WorkerStopBinding + or self._worker_capability is not expected._capability + or self._fenced or self._stop is not None or self._replacement_pending + or self._faults or self.host.closing or self.host.closed + or expected._work is not None or expected._fault is not None + or not callable(create_replacement_binding)): + raise RuntimeError('Replacement requires the exact current unfenced worker binding') + expected._owner() + code = expected.worker.poll() + if type(code) is not int or expected.worker.wait(timeout=0) != code: + raise RuntimeError('The exact previous worker must already have exited and joined') + snapshot = expected.mailbox.snapshot() + expected._original(snapshot.target) + if (snapshot.target.task_id != self.prepared.task_id + or execution_outcome(snapshot.target.state) is not None + or snapshot.target.state['status'] != 'pending' + or snapshot.target.operation is not None or snapshot.target.requires_reconciliation + or snapshot.demand is None or snapshot.demand['state'] != 'paused'): + raise RuntimeError('Replacement requires the same known paused task') + # Failure after child creation must never let the old binding authorize + # native close. Retain this obligation before calling fallible launch. + record = {'previous_binding': expected, 'previous_exit_code': code, + 'worker': None, 'binding': None, 'capability': None, 'committed': False} + self.worker_replacements.append(record) + self._replacement_pending = True + try: + replacement, capability = create_replacement_binding() + record['binding'], record['capability'] = replacement, capability + if type(replacement) is WorkerStopBinding: + record['worker'] = replacement.worker + if (type(replacement) is not WorkerStopBinding or type(capability) is not object + or replacement._capability is not capability + or replacement._loop is not self._loop or replacement._thread is not self._thread + or replacement._work is not None or replacement._fault is not None + or replacement.mailbox is not expected.mailbox + or replacement._requester is not expected._requester + or replacement.original is not expected.original or replacement.run != expected.run + or replacement.deadline != expected.deadline or replacement.model != expected.model + or replacement.quality != expected.quality + or any(replacement.worker is old.worker or replacement.request_id == old.request_id + or capability is old_cap for old, old_cap in self._worker_bindings) + or any(row['request_id'] == replacement.request_id for row in snapshot.requests)): + raise RuntimeError('Replacement changed task accounting or reused a worker/Stop identity') + # Owner-loop callbacks cannot interleave here, but reject reentrant + # Stop/fencing initiated by a trusted callback before committing. + if self._fenced or self._stop is not None or self.host.closing or self.host.closed: + raise RuntimeError('Stop won before replacement commit') + self._worker_bindings.append((replacement, capability)) + self.worker_stop, self._worker_capability = replacement, capability + self.worker_receipt = None + record['committed'] = True + self._replacement_pending = False + return replacement + except BaseException as primary: + self._faults.add('worker_replacement_failed') + try: + self.fence_admission() + except BaseException as cleanup: + raise BaseExceptionGroup('Worker replacement and admission fence failed', [primary, cleanup]) + raise + + def bind_window(self, binding, verifier, bootstrap): + """Trusted launch callback only; network requests cannot supply these.""" + self._owner() + if (self._fenced or self.binding is not None or self._session_closed + or type(binding) is not UIBinding or binding.host_epoch != self.prepared.host_epoch + or binding.task_id != self.prepared.task_id or bootstrap.host_epoch != binding.host_epoch + or bootstrap.task_id != binding.task_id or bootstrap.session_id != binding.session_id + or bootstrap.objective != self.prepared.objective or self.gui.binding != binding + or self.gui.verifier is not verifier): + raise RuntimeError('Retained launcher/session binding mismatch') + self.binding = binding + + def select_execution(self, record, coordinator): + """Retain committed/no-child obligation before any fallible staging.""" + self._owner() + if not record['committed'] or record['grant'] is None or record in self.selected_transitions: + raise RuntimeError('Exact new committed selection required') + self.selected_transitions.append(record) + self._selected,self._selection_coordinator = record,coordinator + self.worker_receipt = None + + def bind_selected_worker(self, record, binding, capability): + self._owner() + from adaptive_inference.desktop.continuation_host import SelectedWorkerStopBinding + if (record is not self._selected or type(binding) is not SelectedWorkerStopBinding + or binding.worker is not record['child'] or binding.mailbox is not record['mailbox'] + or binding.grant != record['grant'] or binding._capability is not capability + or binding._loop is not self._loop or binding._thread is not self._thread + or any(binding.worker is previous.worker for previous,_ in self._worker_bindings)): + raise RuntimeError('Selected binding does not own its retained child') + record['binding'],record['capability'] = binding,capability + self._worker_bindings.append((binding,capability)) + self.worker_stop,self._worker_capability = binding,capability + self.worker_receipt = None + + def session_is_live(self, session_id): + self._owner() + return (not self._session_closed and self.binding is not None + and session_id == self.binding.session_id + and self.gui.session_live(session_id) is True) + + def window_ready(self, session_id): + self._owner() + if (self._fenced or not self.session_is_live(session_id) + or self.gui.window_live(session_id) is not True): + return None + return self.binding + + def window_closed(self, session_id): + """Never wait here: the child needs this reply before its thread can exit.""" + self._owner() + if self.binding is None or self.binding.session_id != session_id: + raise RuntimeError('Cannot revoke a different session') + if not self._session_closed: + self._session_closed = True + self.gui.revoke_session(session_id) + + def verify_current(self, binding): + self._owner() + if type(binding) is not UIBinding or binding != self.binding or not self.session_is_live(binding.session_id): + return None + return self.gui.verifier.verify_current(binding) + + def status(self): + self._owner() + host, controller = self.host, self.host.controller + if (self._faults or controller.fault is not None or host.primary_error is not None + or host.cleanup_error is not None or host.demand_error is not None): + return {'host_phase': 'fault', 'waiting_reason': 'unknown'} + if host.closed: + return {'host_phase': 'closed', 'waiting_reason': None} + if self._fenced or host.closing: + return {'host_phase': 'stopping', 'waiting_reason': 'stop_requested'} + state = controller.state + if state == 'uninitialized': + return {'host_phase': 'starting', 'waiting_reason': 'unknown'} + if state == 'restoring': + return {'host_phase': 'restoring', 'waiting_reason': 'resources'} + if controller(): + return {'host_phase': 'ready', 'waiting_reason': None} + reason = controller.policy.reason + if reason in ('host_memory_pressure', 'host_cpu_pressure', 'cooperative_host_cpu_unavailable'): + waiting = 'resources' + elif reason in ('foreground_activity', 'foreground_recovery_margin', 'foreground_recovery_dwell'): + waiting = 'foreground' + elif state == 'parked' and reason == 'foreground_clear': + waiting = 'resources' + elif reason == 'cooperative_clear': + waiting = 'resources' + elif reason.startswith('cooperative_'): + waiting = 'foreground' + else: + waiting = 'unknown' + return {'host_phase': 'parked', 'waiting_reason': waiting} + + def fence_admission(self): + self._owner() + if self.service is not None: + return self.service.fence() + self._fence_local() + + def _fence_local(self): + if self._fenced: + return + self._fenced = True + compiler = getattr(self.host, 'compiler_operations', None) + if compiler is not None: + compiler.interrupt() + # These existing owner-loop latches are checked by actual acquire, + # capacity and observer/drive paths. No worker command is appended yet. + self.host.closing = True + self.host.phase = 'closing' + self.host.gateway.scheduler.closing = True + self._wake = self._loop.create_task(self.host.gateway.scheduler.close()) + self._wake.add_done_callback(lambda task: None if task.cancelled() else task.exception()) + + async def cooperative_stop(self): + self._owner() + if self.service is not None: + return await self.service.cooperative_stop() + if self._stop is None: + self.fence_admission() + self._stop = self._loop.create_task(self._finish_stop()) + return deepcopy(await asyncio.shield(self._stop)) + + async def _wait_worker(self, deadline): + while True: + remaining = deadline - self.clock() + if remaining <= 0: + return + value = await self.worker_stop.join(timeout=min(30, remaining)) + self.worker_receipt = deepcopy(value) + if value.get('worker_joined') is True or value.get('fault') is not None: + return + await asyncio.sleep(0) + + async def _wait_gui(self, deadline): + while True: + remaining = deadline - self.clock() + if remaining <= 0: + return + value = await self.gui.join(timeout_s=min(30, remaining)) + self.gui_receipt = deepcopy(value) + if value.get('closed') is True or value.get('faults'): + return + await asyncio.sleep(0) + + def _worker_safe(self): + value = self.worker_receipt + return (not self._replacement_pending and type(value) is dict and value.get('worker_joined') is True + and value.get('safe_checkpoint') is True and value.get('uncertain') is False + and value.get('fault') is None + and value.get('outcome') in ('service_stopped', 'task_completed', 'task_cancelled', 'paused_budget_expired', + 'admission_wait_budget_expired', 'selected_not_launched')) + + def _gui_safe(self): + value = self.gui_receipt + if (self._gui_optional_unstarted and self.gui._start_task is None + and self.gui.process is None and type(value) is dict): + return (value.get('closed') is True and value.get('faults') == [] + and value.get('child_retained') is False + and value.get('pending_pipe_operations') == 0) + if self._gui_stop_aborted and type(value) is dict: + from adaptive_inference.desktop.gui_launch import StartupError + error = self.gui.primary_error + known = (type(error) is StartupError and str(error) in ( + 'Cold host/containing-job launch prerequisite failed', + 'Launch completed after startup was fenced', 'Startup was fenced before readiness', + 'Startup was fenced before session binding', 'Startup was fenced during session binding', + 'Readiness completed after startup was fenced')) + if (known and self.gui.cleanup_error is None and value.get('closed') is True + and value.get('startup_complete') is True and value.get('startup_aborted') is True + and set(value.get('faults',())) <= {'startup_failed'} + and value.get('pending_pipe_operations') == 0 + and all(value.get(key) is True for key in ('pipe_threads_joined','pipe_handles_closed', + 'verifier_handle_closed','process_handle_closed')) + and (value.get('child_retained') is False or value.get('child_exit_observed') is True)): + return True + return (type(value) is dict and value.get('closed') is True and value.get('faults') == [] + and value.get('startup_complete') is True and value.get('startup_aborted') is False + and value.get('child_retained') is True and value.get('child_exit_observed') is True + and type(value.get('child_exit_code')) is int and value['child_exit_code'] == 0 + and type(value.get('pending_pipe_operations')) is int and value['pending_pipe_operations'] == 0 + and all(value.get(key) is True for key in ('pipe_threads_joined', 'pipe_handles_closed', + 'verifier_handle_closed', 'process_handle_closed', 'session_revoked', 'authentication_revoked'))) + + def _receipt(self): + native = self.native_receipt + closed = (type(native) is dict and native.get('closed') is True) + return {'worker_joined': bool(not self._replacement_pending and self.worker_receipt + and self.worker_receipt.get('worker_joined') is True), + 'host_closed': closed, + 'uncertain': not (self._worker_safe() and self._gui_safe() and closed) or bool(self._faults), + 'fault': sorted(self._faults)[0] if self._faults else None} + + async def _finish_stop(self, *, close_host=True): + deadline = self.clock() + self.timeout + try: + await asyncio.shield(self._wake) + compiler = getattr(self.host, 'compiler_operations', None) + if compiler is not None: + while True: + remaining = deadline - self.clock() + if remaining <= 0: + raise TimeoutError('Compiler cleanup did not finish before Stop deadline') + state = await compiler.close(timeout=min(2, remaining)) + if state['bridge'] is None or state['bridge']['closed']: + break + if state['bridge']['unknown']: + raise RuntimeError('Compiler cleanup is unresolved') + # close retains its actual tasks even after a bounded wait. + await asyncio.sleep(.05) + if self._replacement_pending: + # A replacement may exist without a constructed Stop binding. + # Its retained exceptional cleanup must run; the old worker's + # receipt cannot authorize native cleanup or clean product Stop. + self._faults.add('worker_replacement_unbound') + return self._receipt() + # MailboxRuntime invokes us only after already accepted DB work; + # appending this final Pause earlier would race a pending Resume. + if self._selected is not None and self._selected['binding'] is None: + self.worker_receipt = await asyncio.to_thread(self._selection_coordinator.verify_unlaunched,self._selected) + self._joins = [self._loop.create_task(self._wait_gui(deadline))] + else: + self.worker_stop.request_stop(self._worker_capability) + self._joins = [self._loop.create_task(self._wait_worker(deadline)), + self._loop.create_task(self._wait_gui(deadline))] + await asyncio.gather(*(asyncio.shield(task) for task in self._joins), return_exceptions=False) + if not self._worker_safe(): + self._faults.add('worker_stop_unconfirmed') + if not self._gui_safe(): + self._faults.add('gui_close_unconfirmed') + if self._faults: + return self._receipt() + if not close_host: + return self._receipt() + remaining = deadline - self.clock() + if remaining < 1: + self._faults.add('shutdown_deadline') + return self._receipt() + self.native_receipt = await self.host.close(timeout_s=min(600, remaining)) + if (type(self.native_receipt) is not dict or self.native_receipt.get('closed') is not True + or not {'primary_error', 'cleanup_error', 'controller_fault'} <= self.native_receipt.keys() + or any(self.native_receipt.get(key) is not None for key in + ('primary_error', 'cleanup_error', 'controller_fault'))): + self._faults.add('native_close_unconfirmed') + except BaseException: + self._faults.add('desktop_stop_failed') + raise + return self._receipt() diff --git a/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/task_handover.py b/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/task_handover.py new file mode 100644 index 0000000..93532c2 --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/task_handover.py @@ -0,0 +1,773 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit same-host task handover using TaskAdmission and retained lifecycles. + +Task/session/GUI identities are never rewritten. The existing prepare command, +paused worker bootstrap, TaskStore claim and cooperative Stop own every effect. +No task is queued or resumed automatically. See docs/task-spec.md and the Python +3.11 asyncio shielding guidance: retaining a task is separate from joining it. +""" +from __future__ import annotations + +import asyncio +from copy import deepcopy +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path +import secrets +import uuid + +from adaptive_inference.control_mailbox import Mailbox +from adaptive_inference.demand import DemandIdentity, DemandRegistry, Registration, NAMESPACE +from adaptive_inference.execution_clock import CLOCK_ID, execution_now +from adaptive_inference.task_spec import TaskSpec +from adaptive_inference.task_store import TaskStore +from adaptive_inference.desktop.control_transport import PreparedTask, ControlTransport, Refused, credential +from adaptive_inference.desktop.desktop_controls import DesktopControlLifecycle +from adaptive_inference.desktop.mailbox_runtime import MailboxRuntime +from adaptive_inference.desktop.serial_listener import SerialListener +from adaptive_inference.desktop.task_admission import TaskAdmission, MAX_ADMISSIONS +from adaptive_inference.desktop.hermes import recovery as rec +from adaptive_inference.desktop.hermes.run_recovery import MODEL, QUALITY +from adaptive_inference.desktop.hermes.service_stop import WorkerStopBinding + + +EPISODE_FIELDS = ('work','plan','store','identity','registry','capability','box','original', + 'deadline','first_ready','worker','worker_cap','stop_binding','stop_id','gui','facade', + 'runtime','continuation','transport','transport_cap','listener','task_spec_path', + 'task_spec_sha256','task_inputs','worker_exit_verified') + + +class TaskHandoverAdapter: + """One immutable endpoint's view of the shared host coordinator.""" + def __init__(self, coordinator, runtime): + self.coordinator, self.runtime = coordinator, runtime + + def projection(self): + return self.coordinator.projection(self.runtime) + + async def select_task(self, payload, *, intent_valid): + return await self.coordinator.retain( + lambda: self.coordinator.select(self.runtime, payload, intent_valid)) + + async def open_selected_task(self, grant, *, intent_valid): + return await self.coordinator.retain( + lambda: self.coordinator.open(self.runtime, grant, intent_valid)) + + async def lookup_task(self, payload): + return await self.coordinator.lookup(self.runtime, payload) + + +class HostTaskHandover: + def __init__(self, composition): + self.composition = composition + self.records, self.episodes, self.tasks = {}, [], set() + self.current = self.active_record = None + self.authority = self.owner = self.claim = None + self.selection_revision = 0 + self.transitioning = self.stopping = False + self._stop = self._receipt = None + self._fencing = False + self._serial = asyncio.Lock() + self._database_serial = asyncio.Lock() + self._database_tasks = set() + self._failure_drains = [] + + def adapter(self, runtime): + return TaskHandoverAdapter(self, runtime) + + def capture_initial(self): + if self.episodes: + raise RuntimeError('Initial episode already retained') + c = self.composition + self.current = {name: getattr(c, name) for name in EPISODE_FIELDS} + self.current['facade'].service = self + self.episodes.append(self.current) + + def episode(self, runtime): + for episode in self.episodes: + if episode['runtime'] is runtime: + return episode + raise Refused('task_blocked') + + def live(self, runtime, intent_valid): + c = self.composition + return (not self.stopping and not c.host.closing and not c.host.closed + and not runtime._stopping and intent_valid() is True) + + def projection(self, runtime): + phase, request_id = 'unavailable', None + rows = [r for r in self.records.values() if r['runtime'] is runtime] + if rows: + record = rows[-1] + phase, request_id = record['phase'], record['request']['request_id'] + elif (self.current is not None and runtime is self.current['runtime'] + and not self.stopping and not self.transitioning + and self.selection_revision < MAX_ADMISSIONS): + try: + value = runtime.status() + if value['task_status'] == 'accepted' or value['demand'] == 'cancelled': + phase = 'available' + except Exception: + pass # No fresh status means no new admission. + return {'phase': phase, 'request_id': request_id, + 'selection_revision': self.selection_revision} + + def reply(self, record): + grant = record['reservation'] + if grant is None: + return None + return {'request_id': record['request']['request_id'], + 'predecessor_task_id': record['runtime'].prepared.task_id, + 'task_id': grant['payload']['task_id'], + 'selection_revision': grant['selection_revision'], 'phase': record['phase']} + + async def retain(self, operation): + task = asyncio.create_task(operation()) + self.tasks.add(task) + def done(value): + if not value.cancelled(): + value.exception() + self.tasks.discard(value) + task.add_done_callback(done) + return await asyncio.shield(task) + + async def database(self, function): + async def run(): + async with self._database_serial: + return await asyncio.to_thread(function) + task = asyncio.create_task(run()) + self._database_tasks.add(task) + task.add_done_callback(self._database_tasks.discard) + return await asyncio.shield(task) + + async def drain(self, runtime): + while runtime._pending or runtime._continuation_tasks: + values = tuple(runtime._pending | runtime._continuation_tasks) + await asyncio.gather(*(asyncio.shield(task) for task in values)) + await asyncio.sleep(0) + + def _authority(self): + if self.authority is not None: + return + c = self.composition + directory = c.root/'tasks' + directory.mkdir(exist_ok=False) + authority = TaskAdmission(TaskStore(c.root/'admission.sqlite3'), + host_generation=c.host_epoch, tasks_root=directory, + initial_task_id=self.current['runtime'].prepared.task_id, + initial_run=self.current['work']) + self.authority = authority + checkpoint = authority.store.load(authority.task_id) + self.claim = authority.store.claim(authority.task_id, + expected_revision=checkpoint.revision, owner_id=c.host_epoch) + self.owner = self.claim.__enter__() + + async def lookup(self, runtime, request): + record = self.records.get(request['request_id']) + if record is None: + return None + if record['runtime'] is not runtime or record['request'] != request: + raise Refused('conflicting_request_id') + if record['payload'] is not None: + grant = await self.database(lambda: self.authority.lookup(record['payload'])) + if record['reservation'] is not None and grant != record['reservation']: + raise RuntimeError('Historical task reservation changed') + if grant is not None: + record['reservation'] = grant + record['resolved'] = True + return self.reply(record) + + async def select(self, runtime, request, intent_valid): + async with self._serial: + if request['request_id'] in self.records: + return await self.lookup(runtime, request) + if (self.current is None or runtime is not self.current['runtime'] + or self.transitioning or not self.live(runtime, intent_valid) + or len(self.records) >= MAX_ADMISSIONS): + raise Refused('task_blocked') + runtime._require_execution({key: request[key] for key in ('execution_id','execution_generation')}) + await runtime.refresh() + status = runtime.status() + if (status['task_revision'] != request['expected_task_revision'] + or request['expected_selection_revision'] != self.selection_revision + or not (status['task_status'] == 'accepted' or status['demand'] == 'cancelled')): + raise Refused('task_unavailable') + c = self.composition + self.current.update({name: getattr(c, name) for name in EPISODE_FIELDS}) + record = {'runtime': runtime, 'request': deepcopy(request), 'payload': None, + 'reservation': None, 'resolved': True, 'phase': 'reserved', + 'prepared': False, 'started': False, 'launch_attempted': False, + 'prepare_child': None, 'child': None, 'episode': None, + 'prepare_operation': None, 'launch_operation': None, 'release': None} + self.records[request['request_id']] = record + self.transitioning = True + runtime.retire_task() + try: + await self.drain(runtime) + if not self.live(runtime, intent_valid): + raise Refused('task_blocked') + # Leave the normal demand observer active until real parking. + record['release'] = await c.continuation_release(request, intent_valid, task_handover=True) + if not self.live(runtime, intent_valid): + raise Refused('task_blocked') + def reserve(): + self._authority() + from adaptive_inference.desktop.continuation_host import SelectedWorkerStopBinding + if type(c.stop_binding) is SelectedWorkerStopBinding: + self.authority.retain_execution_binding(c.stop_binding) + if self.active_record is not None: + old = self.active_record + try: + self.authority.finish(self.owner, old['reservation'], + execution_id=request['execution_id'], task_revision=request['expected_task_revision'], + cleanup_sha256=rec.digest(record['release']), operation_id=old['launch_operation']) + except BaseException: + # A committed handback can lose its return. Observe + # that exact row only; never repeat finish or reserve. + status = self.authority.status() + rows = [row for row in status['admissions'] if row['reservation']==old['reservation']] + if (len(rows)==1 and rows[0]['phase']=='released' + and rows[0]['release']['execution_id']==request['execution_id'] + and rows[0]['release']['revision']==request['expected_task_revision'] + and rows[0]['release']['cleanup_sha256']==rec.digest(record['release']) + and status['operation'] is None and not status['requires_reconciliation']): + old['released'] = True + self.active_record = None + raise + old['released'] = True + self.active_record = None + source = {key: request[key] for key in ('task_spec_path','task_spec_sha256','task_inputs')} + spec = TaskSpec.read(source['task_spec_path'], source['task_spec_sha256']) + spec.verify_inputs(Path(source['task_inputs'])) + if spec.compiler is not None and getattr(c.profile, 'compiler', None) is None: + raise ValueError('Compiler task requires explicit compiler deployment settings') + if spec.video is not None and getattr(c.profile, 'video', None) is None: + raise ValueError('Video task requires explicit video deployment settings') + payload = self.authority.request(command_id=request['request_id'], + expected_selection_revision=request['expected_selection_revision'], + predecessor_execution_id=request['execution_id'], + predecessor_revision=request['expected_task_revision'], spec=spec, + arrived_at_utc=datetime.now(timezone.utc).isoformat().replace('+00:00','Z'), source=source) + record['payload'], record['resolved'] = payload, False + return self.authority.reserve(self.owner, payload, + predecessor_cleanup_sha256=rec.digest(record['release'])) + grant = await self.database(reserve) + record['reservation'], record['resolved'] = grant, True + return self.reply(record) + except BaseException: + record['phase'] = 'blocked' + if record['payload'] is not None and not record['resolved']: + try: + await self.lookup(runtime, request) + except Exception: + pass + raise + finally: + # The durable reservation remains a Stop obligation between + # select and open. No child is inferred from a missing record. + if record['reservation'] is None: + self.transitioning = False + + async def open(self, runtime, reply, intent_valid): + record = self.records.get(reply['request_id']) + if record is None or record['runtime'] is not runtime or self.reply(record) != reply: + raise Refused('task_blocked') + if record['started']: + return self.reply(record) + record['started'] = True + try: + if not self.live(runtime, intent_valid): + record['phase'] = 'blocked'; return self.reply(record) + record['phase'] = 'preparing' + await self.prepare_episode(record) + if not self.live(runtime, intent_valid): + record['phase'] = 'blocked'; return self.reply(record) + await self.launch_episode(record) + if not self.live(runtime, intent_valid): + record['phase'] = 'blocked'; return self.reply(record) + c, episode = self.composition, record['episode'] + # Retire the old endpoint before B can bind the shared intent gate. + self.current['transport'].retire_task(self.current['transport_cap'], reply['request_id']) + c.host._control_rebinding = True + try: + await self.drain(runtime) + controller = c.host.controller + async with controller.lock: + async with controller._demand_lock: + record['observation'] = await episode['runtime']._database( + episode['runtime']._observe, lambda value: value) + if not self.live(runtime, intent_valid): + record['phase'] = 'blocked'; return self.reply(record) + c.host.commit_task_runtime(runtime, episode['runtime'], record, self) + finally: + c.host._control_rebinding = False + await c.bind_compiler_worker(episode['runtime'], episode['worker'], episode['work'], episode['deadline']) + await self.open_view(record) + if not self.live(runtime, intent_valid): + record['phase'] = 'blocked'; return self.reply(record) + record['phase'] = 'opened' + return self.reply(record) + except BaseException as error: + record['phase'] = 'blocked' + episode = record['episode'] + if (self.stopping and episode is not None and episode['facade'] is not None + and episode['facade']._gui_stop_aborted + and error is episode['gui'].primary_error): + return self.reply(record) # Cleanup still checks the exact abort reason and joins. + raise + finally: + self.transitioning = False + + async def _until(self, predicate, child, seconds, *, deadline=None): + limit = min(execution_now()+seconds, float('inf') if deadline is None else deadline) + while True: + value = predicate() + if value: + return value + if child is not None and child.poll() is not None: + raise RuntimeError('Retained task child exited before its required boundary') + if execution_now() >= limit: + raise TimeoutError('Task handover phase deadline') + await asyncio.sleep(.05) + + async def prepare_episode(self, record): + c, grant = self.composition, record['reservation'] + payload = grant['payload']; source = payload['source'] + record['prepare_operation'] = await self.database( + lambda: self.authority.begin(self.owner, grant, action='prepare')) + identity_path = c.root/('task-demand-'+grant['payload_sha256'][:24]+'.json') + await self.database(lambda: rec.save(identity_path, {key:payload['demand_identity'][key] + for key in ('registration_id','task_id','owner_id','arrived_at_utc')})) + command = [str(c.profile.worker_python), '-I', '-B', '-m', + 'adaptive_inference.desktop.hermes.run_recovery', 'prepare', '--run', payload['run'], + '--profile', str(c.profile_path), '--profile-sha256', c.profile_sha256, + '--package-identity', str(c.root/'installed-package.private.json'), '--control-case', 'complete', + '--task-spec', source['task_spec_path'], '--task-spec-sha256', source['task_spec_sha256'], + '--task-inputs', source['task_inputs'], '--task-id', payload['task_id'], + '--session-id', payload['session_id'], '--demand-identity', str(identity_path), + '--demand-identity-sha256', rec.digest(identity_path.read_bytes())] + if c.reopen_context is not None: + command.extend(['--reopen-context', str(c.root/'reopen-context.private.json')]) + before = len(c.handles) + try: + c.launch(command, 'prepare-task-'+str(grant['selection_revision'])) + finally: + if len(c.handles) == before+1: + record['prepare_child'] = c.handles[-1] + child = record['prepare_child'] + await self._until(lambda: child.poll() is not None, None, 60) + if await c.join_child(child) != 0: + raise RuntimeError('Fresh task preparation failed') + if c.reopen_context is not None: + from adaptive_inference.desktop.saved_tasks import claim_host + # Retain through every episode until the composition's ordinary + # final cleanup; a prepared task must not acquire a second host. + c.reopen_locks.append(claim_host(Path(payload['run']))) + await self.database(lambda: self.authority.prepared(self.owner, grant, + operation_id=record['prepare_operation'])) + record['prepared'] = True + self.selection_revision = grant['selection_revision'] + self.active_record = record + + async def launch_episode(self, record): + c, grant = self.composition, record['reservation'] + payload = grant['payload']; work = Path(payload['run']) + from adaptive_inference.desktop.composition import read, INITIAL_PAUSE, no_work + from adaptive_inference.desktop.gui_launch import OwnedUILaunch + from adaptive_inference.desktop.continuation_host import HostContinuation + plan = await self.database(lambda: read(work/'plan.json')) + store = TaskStore(work/'task.sqlite3') + identity = DemandIdentity(**plan['demand_identity']) + registry = DemandRegistry((Registration(store, identity),)) + capability = object(); mailbox = Mailbox(registry, identity.registration_id, requester_capability=capability) + original = await self.database(lambda: store.load(identity.task_id)) + deadline = execution_now()+900 + episode = {'work':work,'plan':plan,'store':store,'identity':identity,'registry':registry, + 'capability':capability,'box':mailbox,'original':original,'deadline':deadline, + 'first_ready':work/'controller-ready-1.json','worker':None,'worker_cap':object(), + 'stop_binding':None,'stop_id':'task-stop-'+grant['payload_sha256'][:24], + 'gui':None,'facade':None,'runtime':None,'continuation':None,'transport':None, + 'transport_cap':object(),'listener':None,'worker_exit_verified':False, + 'task_spec_path':Path(payload['source']['task_spec_path']), + 'task_spec_sha256':payload['source']['task_spec_sha256'], + 'task_inputs':Path(payload['source']['task_inputs'])} + record['episode'] = episode + # Retain before any launch, diagnostics or binding may fail. + self.episodes.append(episode) + record['launch_operation'] = await self.database(lambda: self.authority.begin(self.owner, grant, action='launch')) + await self.database(lambda: mailbox.submit(capability, INITIAL_PAUSE, 'pause')) + command = [plan['base_python'],'-I','-B','-m','adaptive_inference.desktop.hermes.run_recovery', + 'worker','--run',str(work),'--controller-ready',str(episode['first_ready']), + '--gateway-instance',c.host.gateway.instance,'--deadline-monotonic',str(deadline), + '--stop-request-id',episode['stop_id']] + command.extend(c.compiler_launch_args(work, plan, execution_id=identity.task_id, + task_id=identity.task_id, deadline=deadline, diagnostics=work)) + if c.reopen_context is not None: + command.extend(['--execution-boot-id', rec.digest(c.reopen_context['boot'])]) + record['launch_attempted'] = True + before = len(c.handles) + try: + c.launch(command, 'worker-task-'+str(grant['selection_revision'])) + finally: + if len(c.handles) == before+1: + child = record['child'] = episode['worker'] = c.handles[-1] + episode['stop_binding'] = WorkerStopBinding(child,work,mailbox,requester_capability=capability, + capability=episode['worker_cap'],stop_request_id=episode['stop_id'],original=original, + deadline_monotonic=deadline,model=MODEL,quality=QUALITY,fence=self.fence,deadline_clock=CLOCK_ID) + episode['gui'] = OwnedUILaunch(api=c.launch_api) + prepared = PreparedTask(c.host_epoch,identity.task_id,original.prompt) + episode['facade'] = DesktopControlLifecycle(c.host,prepared,worker_stop=episode['stop_binding'], + worker_capability=episode['worker_cap'],gui=episode['gui'],shutdown_timeout_s=120) + episode['facade'].service = self + episode['facade']._gui_optional_unstarted = True + runtime = episode['runtime'] = MailboxRuntime(prepared,mailbox,requester_capability=capability, + gate=c.gate,intent_capability=c.intent_cap,lifecycle=episode['facade']) + episode['continuation'] = HostContinuation(c,runtime) + if self.stopping: + self.fence() + return + child = episode['worker']; handle = c.launch_api.handle(child) + observed = c.launch_api.identity(handle) + if (c.launch_api.in_job(handle,c.job._HANDLE) is not True + or observed[0] != child.pid or child.poll() is not None): + raise RuntimeError('New task child is not live in the retained containing job') + record['retained_child_identity'] = {'pid':observed[0], 'creation_time_100ns':observed[1], + 'containing_job_verified':True} + def paused(): + snapshot = mailbox.snapshot(); ack = snapshot.status(INITIAL_PAUSE) + if not ack['owner_acknowledged']: + return None + if ack['outcome'] != 'applied' or snapshot.target.owner_epoch is None or child.poll() is not None: + raise RuntimeError('New task lacks exact live paused owner ACK') + no_work(snapshot.target,original,deadline,c.host,work,cold=False) + c.assert_continuation_parked(record['release']) + return {'checkpoint':asdict(snapshot.target),'ack':ack} + # SQLite reads remain off the host event loop and bounded by this phase. + limit = min(execution_now()+60,deadline) + while True: + if self.stopping: + return + value = await self.database(paused) + if self.stopping: + return + if value: + record['paused'] = value; break + if child.poll() is not None or execution_now() >= limit: + raise RuntimeError('New task paused bootstrap did not complete') + await asyncio.sleep(.05) + await runtime.refresh() + + def commit_episode(self, record): + episode = record['episode']; c = self.composition + if episode not in self.episodes or episode['runtime'] is not c.host.control_runtime: + raise RuntimeError('New task episode is not the selected host runtime') + self.current = episode + for name in EPISODE_FIELDS: + setattr(c,name,episode[name]) + record['rebound'] = True + c.result['task_completed'] = c.result['execution_expired'] = False + c.result.pop('worker_outcome',None) + for name in ('semantic_validation','validated_artifacts','artifacts','task_spec_sha256'): + c.result.pop(name,None) + c.result['task_selection_revision'] = self.selection_revision + + async def open_view(self, record): + from adaptive_inference.desktop.composition import HERE, no_work + from adaptive_inference.desktop.gui_launch import LaunchSpec, CHILD_SOURCES + from adaptive_inference.desktop.control_client import Bootstrap + from adaptive_inference.desktop.installation import sha + c, episode = self.composition, record['episode'] + runtime = episode['runtime']; prepared = runtime.prepared + transport = episode['transport'] = ControlTransport(prepared,runtime, + capability=episode['transport_cap'],runner_credential='local-no-key', + cooperative_progress=c.progress,handover=self.adapter(runtime)) + listener = episode['listener'] = SerialListener(transport) + # Reflect retained endpoint objects before the first fallible start. + c.transport,c.listener,c.transport_cap = transport,listener,episode['transport_cap'] + endpoint = listener.start() + bootstrap = Bootstrap(c.host_epoch,'session-'+uuid.uuid4().hex,prepared.task_id,prepared.objective, + credential(),endpoint[0],endpoint[1],'AdaptiveInference.'+uuid.uuid4().hex,secrets.randbelow(2**31-1)+1) + spec = LaunchSpec(c.profile.host_python,c.profile.host_python_sha256,HERE/'run_gui.py', + tuple((name,sha(HERE.parent/name)) for name in sorted(CHILD_SOURCES))) + def bind(binding, verifier, supplied): + episode['facade'].bind_window(binding,verifier,supplied) + transport.bind_session(episode['transport_cap'],binding.session_id,supplied.token) + return True + def launch_ready(): + c.assert_continuation_parked(record['release']) + return not self.stopping and not c.host.closing and episode['worker'].poll() is None + await episode['gui'].start(spec,bootstrap,containing_job=c.job._HANDLE,launch_ready=launch_ready, + bind_session=bind,revoke_session=runtime.window_closed,timeout_s=10) + await self._until(lambda: transport._ready or self.stopping,episode['worker'],30,deadline=episode['deadline']) + if self.stopping: + return + snapshot = await runtime._database(runtime._observe,runtime._install) + no_work(snapshot.target,episode['original'],episode['deadline'],c.host,episode['work'],cold=False) + record['ready'] = {'paused_owner_acknowledged':True,'task_id':prepared.task_id, + 'selection_revision':self.selection_revision,'gui_binding':asdict(episode['gui'].binding)} + + def fence(self): + self.stopping = True + if self._fencing: + return + self._fencing = True + c = self.composition + try: + c.gate.deactivate(c.intent_cap) + for episode in self.episodes: + runtime,facade = episode['runtime'],episode['facade'] + if runtime is not None: + runtime._stopping = True + if facade is not None: + facade._fence_local() + transport = episode['transport'] + if transport is not None and not transport.lifecycle()['stop_requested']: + transport._request_stop() + gui = episode['gui'] + if gui is not None and gui._start_task is not None and not gui._bound: + if facade is not None and getattr(gui,'primary_error',None) is None: + facade._gui_stop_aborted = True + gui.abort_startup() + if c.host is not None: + c.host.closing = True + c.host.gateway.scheduler.closing = True + finally: + self._fencing = False + + async def cooperative_stop(self): + if self._stop is None: + self.fence() + self._stop = asyncio.create_task(self._finish_stop()) + return deepcopy(await asyncio.shield(self._stop)) + + async def _finish_stop(self): + c = self.composition + errors = [] + while self.tasks or self._database_tasks: + await asyncio.gather(*(asyncio.shield(task) for task in tuple(self.tasks | self._database_tasks)), + return_exceptions=True) + if any(not row['resolved'] for row in self.records.values()): + errors.append('Task reservation outcome remains unresolved') + for episode in self.episodes: + runtime = episode['runtime'] + if runtime is not None: + await self.drain(runtime) + transport = episode['transport'] + if transport is not None: + await asyncio.gather(*(asyncio.shield(task) for task in + getattr(transport,'_mutation_tasks',lambda:())()),return_exceptions=True) + # Staging/launch failures never borrow A's safe worker receipt. + for record in self.records.values(): + if record['reservation'] is None: + continue + if record['prepare_operation'] is None and record['episode'] is None: + await self.database(lambda r=record: self.authority.abandon(self.owner,r['reservation'], + cleanup_sha256=rec.digest(r['release']))) + record['phase'] = 'abandoned' + elif not record['prepared']: + errors.append('Task preparation outcome remains unresolved') + for episode in self.episodes: + facade = episode['facade'] + if facade is None: + binding = episode['stop_binding'] + if binding is not None: + binding.request_stop(episode['worker_cap']) + while not (await binding.join(timeout=2))['worker_joined']: + await asyncio.sleep(.05) + errors.append('Task launch lacks a complete retained lifecycle') + continue + try: + await facade._finish_stop(close_host=False) + if not facade._worker_safe() or not facade._gui_safe() or facade._faults: + errors.append('A retained task/window did not confirm cooperative cleanup') + except BaseException as error: + errors.append(type(error).__name__) + for child in c.handles: + await c.join_child(child) + errors.extend(self.episode_faults()) + if errors: + raise RuntimeError('; '.join(errors)) + cleanup = await c.host.close(timeout_s=180) + if (cleanup.get('closed') is not True or any(cleanup.get(k) is not None + for k in ('primary_error','cleanup_error','controller_fault'))): + raise RuntimeError('Shared native host cleanup is unconfirmed') + c.result['cleanup'] = cleanup + if self.active_record is not None: + record = self.active_record + episode = record['episode'] + binding = None if episode is None else (c.stop_binding if episode is self.current else episode['stop_binding']) + continuation = None + if episode is not None and episode['facade']._selected is not None and episode['facade']._selected['binding'] is None: + binding,continuation = None,episode['continuation'] + arguments = {'cleanup_sha256':rec.digest(cleanup),'operation_id':record['launch_operation'],'binding':binding} + if continuation is not None: + arguments['continuation'] = continuation + await self.database(lambda: self.authority.stop(self.owner,record['reservation'],**arguments)) + if self.claim is not None: + await self.database(lambda: self.claim.__exit__(None,None,None)) + self.claim = self.owner = None + self._receipt = {'worker_joined':True,'host_closed':True,'uncertain':False,'fault':None} + for episode in self.episodes: + episode['facade'].native_receipt = cleanup + return self._receipt + + def episode_faults(self): + errors = [] + for episode in self.episodes: + runtime,transport = episode['runtime'],episode['transport'] + if runtime is not None: + if runtime._faults or runtime._pending or runtime._continuation_tasks: + errors.append('Retained runtime operation failed or remains pending') + continuation = runtime._continuation + if continuation is not None and any(not row['authorization_resolved'] for row in continuation.records.values()): + errors.append('Retained continuation authorization is unresolved') + if transport is not None: + if getattr(transport,'_faults',None) or any(row['outcome']=='unknown' + for row in getattr(transport,'_records',{}).values()): + errors.append('Retained transport operation is unresolved') + mutations = getattr(transport,'_mutation_tasks',lambda:())() + if any(not task.done() for task in mutations): + errors.append('Retained transport mutation is pending') + return errors + + async def dispose(self, failed): + c = self.composition + await self.cooperative_stop() + # Per-view cleanup uses existing methods without changing current task + # aliases. Old transports already share the same completed Stop owner. + errors = [] + for episode in self.episodes: + gui,listener,transport = episode['gui'],episode['listener'],episode['transport'] + if gui is not None: + receipt = await gui.join(timeout_s=2) + episode['gui_cleanup'] = receipt + if listener is not None: + value = await listener.aclose(timeout=2) + while not value['resources_closed']: + value = await listener.aclose(timeout=2) + episode['listener_cleanup'] = value + if transport is not None: + value = await transport.aclose(timeout=2) + while not value['closed']: + value = await transport.aclose(timeout=2) + episode['transport_cleanup'] = value + errors.extend(await asyncio.gather(*(asyncio.shield(t) for t in + (transport._stop_task,transport._close_task) if t is not None),return_exceptions=True)) + for child in c.handles: + await c.join_child(child) + c.result['retained_obligations_joined'] = True + if any(isinstance(value,BaseException) for value in errors): + raise RuntimeError('Retained endpoint shutdown failed') + if self.episode_faults() or any(not episode['transport'].lifecycle().get('stop_confirmed') + for episode in self.episodes if episode['transport'] is not None): + raise RuntimeError('A retained endpoint did not confirm shared Stop') + for name in ('gui_cleanup','listener_cleanup','transport_cleanup'): + c.result[name] = deepcopy(self.current[name]) + c.result['all_retained_process_exits'] = [{'pid':child.pid,'exit_code':child.poll()} for child in c.handles] + c.result['task_episodes'] = [{'task_id':episode['runtime'].prepared.task_id, + 'gui_cleanup':episode['gui_cleanup'],'listener_cleanup':episode['listener_cleanup'], + 'transport_cleanup':episode['transport_cleanup']} for episode in self.episodes] + + async def failure_cleanup(self): + """Join all retained episodes even when a sibling's proof failed. + + This is physical containment cleanup only. It cannot return a successful + product Stop or reconcile an unknown admission operation. + """ + self.fence() + c = self.composition + failures = [] + while self.tasks or self._database_tasks: + await asyncio.gather(*(asyncio.shield(task) for task in tuple(self.tasks | self._database_tasks)), + return_exceptions=True) + async def drain_worker(binding): + while binding.worker.poll() is None: + await binding.join(timeout=2) + work = getattr(binding,'_work',None) + if work is not None: + await asyncio.shield(work) + + async def drain_view(episode): + gui = episode['gui'] + if gui is not None: + if gui._start_task is not None and not gui._bound: + gui.abort_startup() + while True: + receipt = await gui.join(timeout_s=2) + if (receipt['pending_pipe_operations']==0 and receipt['pipe_handles_closed'] + and receipt['process_handle_closed'] and receipt['verifier_handle_closed'] + and (not receipt['child_retained'] or receipt['child_exit_observed'])): + episode['gui_cleanup'] = receipt + break + await asyncio.sleep(.05) + listener,transport = episode['listener'],episode['transport'] + if listener is not None: + listener.close() + if transport is not None: + while True: + receipt = await transport.aclose(timeout=2) + if receipt['handlers']==0 and receipt['pending_connections']==0 and transport._close_task.done(): + break + if listener is not None: + while not (await listener.aclose(timeout=2))['resources_closed']: + await asyncio.sleep(.05) + runtime = episode['runtime'] + if runtime is not None: + await asyncio.gather(*(asyncio.shield(task) for task in + (*runtime._pending,*runtime._continuation_tasks,runtime._stop_task) if task is not None), + return_exceptions=True) + # Start every retained obligation before awaiting an uncertain sibling. + # A's unresolved GUI handle must not delay B's Stop, worker join, GUI + # close or endpoint drain. These tasks remain owned even if this waiter + # is cancelled; an unfinished drain never permits native close below. + for episode in self.episodes: + binding = c.stop_binding if episode is self.current else episode['stop_binding'] + if binding is not None: + try: + binding.request_stop(binding._capability) + self._failure_drains.append(asyncio.create_task(drain_worker(binding))) + except BaseException as error: + failures.append(type(error).__name__) + self._failure_drains.append(asyncio.create_task(drain_view(episode))) + self._failure_drains.extend(asyncio.create_task(c.join_child(child)) for child in c.handles) + compiler = getattr(c, 'compiler_host', None) + if compiler is not None: + compiler.interrupt() + async def drain_compiler(): + await compiler.close() + tasks = [] + for row in compiler.obligations(): + tasks.append(row['close_task']) + for operation in row['operations']: + tasks.extend((operation['task'], operation['runner'])) + # Cancelled recipe runners are an expected interruption path; + # the bridge, not cancellation alone, proves cleanup completion. + await asyncio.gather(*(asyncio.shield(task) for task in tasks if task is not None), + return_exceptions=True) + state = compiler.lifecycle() + if state['bridge'] is not None and not state['bridge']['closed']: + raise RuntimeError('Retained compiler cleanup is unresolved') + self._failure_drains.append(asyncio.create_task(drain_compiler())) + drained = await asyncio.gather(*(asyncio.shield(task) for task in self._failure_drains), + return_exceptions=True) + errors = [value for value in drained if isinstance(value,BaseException)] + if errors: + raise BaseExceptionGroup('Retained episode cleanup failed',errors) + if c.host is not None: + c.result['failure_cleanup'] = await c.host.close(timeout_s=180) + if self.claim is not None: + await self.database(lambda:self.claim.__exit__(None,None,None)) + self.claim = self.owner = None # Existing unfinished-effect fence survives release. + c.result['task_handover_failure_cleanup'] = {'episodes_joined':len(self.episodes), + 'failures':failures,'product_stop_confirmed':False} + + def validate_cleanup(self): + if self._receipt != {'worker_joined':True,'host_closed':True,'uncertain':False,'fault':None}: + raise RuntimeError('Shared host Stop is unconfirmed') + if self.episode_faults(): + raise RuntimeError('Retained episode has unresolved operations') + for episode in self.episodes: + if (not episode['facade']._worker_safe() or not episode['facade']._gui_safe() + or not episode.get('listener_cleanup',{}).get('clean_close') + or not episode.get('transport_cleanup',{}).get('closed')): + raise RuntimeError('A retained task episode is not cleanly closed') diff --git a/provenance/automatic-tool-recovery/predecessor/tests/pending_tool_lineage.py b/provenance/automatic-tool-recovery/predecessor/tests/pending_tool_lineage.py new file mode 100644 index 0000000..03a7546 --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/tests/pending_tool_lineage.py @@ -0,0 +1,89 @@ +"""Exact pending-tool-recovery source view before historical recipe evidence. + +Reuse recipe_reconciliation_lineage's complete-file SHA and portable-AST projection. Retained +93eba3c sources are data, never imported or executed. Current recovery behavior +has separate tests; this view does not transfer historical qualification. +""" +import ast +import hashlib +import json +from pathlib import Path + +from paused_expiry_lineage import fingerprint + +ROOT = Path(__file__).resolve().parents[1] +ARCHIVE = 'provenance/pending-tool-recovery' + + +def record(root=None): + root = ROOT if root is None else root + path = root / ARCHIVE / 'source-lineage.json' + return json.loads(path.read_bytes()) if path.exists() else None + + +def inputs(root=None): + root = ROOT if root is None else root + rows = record(root) + if rows is None: + return [] + return [root / ARCHIVE / 'source-lineage.json', + *(root / row['predecessor_source'] for row in rows['files'].values())] + + +def retained(relative, row, root): + expected = ARCHIVE + '/predecessor/' + relative + if (not relative.startswith(('src/adaptive_inference/', 'tests/')) + or '\\' in relative or any(part in ('', '.', '..') for part in relative.split('/')) + or row['predecessor_source'] != expected): + raise AssertionError('Changed pending-tool-recovery predecessor path: ' + relative) + before = (root / expected).read_bytes() + if (hashlib.sha256(before).hexdigest() != row['predecessor_sha256'] + or fingerprint(ast.parse(before)) != row['predecessor_ast_sha256']): + raise AssertionError('Changed pending-tool-recovery predecessor identity: ' + relative) + return before + + +def predecessor(relative, data, root=None): + """Project known complete successor bytes; unknown bytes remain unknown.""" + root = ROOT if root is None else root + rows = record(root) + row = None if rows is None else rows['files'].get(relative) + if row is None: + return data + before = retained(relative, row, root) + if hashlib.sha256(data).hexdigest() == row['successor_sha256']: + if fingerprint(ast.parse(data)) != row['successor_ast_sha256']: + raise AssertionError('Changed pending-tool-recovery successor AST: ' + relative) + return before + return data + + +def runtime_view(current, root=None): + """Require the complete 77-module closure before restoring recipe sources.""" + root = ROOT if root is None else root + rows = record(root) + result = dict(current) + if rows is None: + return result + baseline = rows['baseline_runtime'] + if set(result) != set(baseline): + raise AssertionError('Changed pending-tool-recovery runtime closure') + for name, pin in baseline.items(): + result[name] = predecessor('src/adaptive_inference/' + name, result[name], root) + if hashlib.sha256(result[name]).hexdigest() != pin: + raise AssertionError('Changed pending-tool-recovery runtime source: ' + name) + return result + + +def restore(tree, relative, root=None): + """AST restoration also requires exact complete on-disk successor bytes.""" + root = ROOT if root is None else root + rows = record(root) + source = 'src/adaptive_inference/' + relative + row = None if rows is None else rows['files'].get(source) + if row is None or fingerprint(tree) != row['successor_ast_sha256']: + return tree + data = (root / source).read_bytes() + if hashlib.sha256(data).hexdigest() != row['successor_sha256']: + raise AssertionError('Changed complete reboot-budget source (pending-tool-recovery source): ' + relative) + return ast.parse(predecessor(source, data, root)) diff --git a/provenance/automatic-tool-recovery/predecessor/tests/test_desktop_composition.py b/provenance/automatic-tool-recovery/predecessor/tests/test_desktop_composition.py new file mode 100644 index 0000000..08a14ec --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/tests/test_desktop_composition.py @@ -0,0 +1,377 @@ +"""Shared installed composition ownership/ordering fixtures, no GUI or model.""" +import asyncio +from collections import deque +from copy import deepcopy +from pathlib import Path +import json +import os +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest import mock + +from adaptive_inference.desktop import composition as q +from adaptive_inference.desktop.hermes.control_bridge import prepare_task +from test_desktop_installation import profile_value +from adaptive_inference.task_spec import TaskSpec, BUDGET + + +def custom_spec(): + return TaskSpec({'schema':'adaptive.task-spec.v1', + 'prompt':'Write a short welcome message to greeting.txt using write_file.', + 'inputs':[], 'outputs':[{'name':'greeting.txt','required':True,'format':'text','max_chars':500}], + 'budget':dict(BUDGET),'validation':{'validator':None}}) + + +def make_case(root): + profile=root/'profile.json';profile.write_text(json.dumps(profile_value(root)),encoding='utf-8') + return q.DesktopComposition(profile,root/'run') + + +class Child: + def __init__(self,pid=41,code=None): + self.pid=pid;self.code=code;self.done=threading.Event();self.waits=0 + if code is not None:self.done.set() + def poll(self):return self.code + def wait(self): + self.waits+=1 + if not self.done.wait(3):raise TimeoutError('Fixture child not released') + return self.code + def finish(self,code):self.code=code;self.done.set() + def kill(self):raise AssertionError('No product kill') + def terminate(self):raise AssertionError('No product terminate') + + +class CompositionTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + temp=tempfile.TemporaryDirectory();self.addCleanup(temp.cleanup) + self.root=Path(temp.name).resolve();self.case=make_case(self.root);self.case.root.mkdir() + + async def test_noncanonical_parent_is_rejected_until_explicitly_resolved(self): + nested=self.root/'nested';nested.mkdir() + parent=nested/'..' + with self.assertRaisesRegex(ValueError,'non-redirected parent'): + q.DesktopComposition(self.case.profile_path,parent/'fresh-run') + case=q.DesktopComposition(self.case.profile_path,parent.resolve()/'fresh-run') + self.assertEqual(case.root,self.root/'fresh-run') + self.assertFalse(case.root.exists()) + + async def test_diagnostic_failure_still_retains_worker_and_log(self): + c=self.case;child=Child() + with mock.patch.object(q.subprocess,'Popen',return_value=child), \ + mock.patch.object(c,'event',side_effect=OSError('diagnostic failed')): + with self.assertRaises(OSError):c.launch(['fixture'],'worker') + self.assertIs(c.worker,child);self.assertEqual(c.handles,[child]) + self.assertEqual(len(c.logs),1) + child.finish(2) + self.assertEqual(await c.join_child(child),2) + for stream in c.logs:stream.close() + + async def test_cancelled_wait_retains_exact_child_without_signal(self): + c=self.case;child=Child();c.handles.append(child) + task=asyncio.create_task(c.join_child(child));await asyncio.sleep(.02) + task.cancel() + with self.assertRaises(asyncio.CancelledError):await task + self.assertFalse(c.waits[child].done()) + child.finish(0) + self.assertEqual(await c.join_child(child),0) + self.assertEqual(child.waits,1) + + async def test_reused_pid_cannot_borrow_another_retained_child_wait(self): + c=self.case;old=Child(41,5);new=Child(41) + c.handles.extend((old,new)) + self.assertEqual(await c.join_child(old),5) + task=asyncio.create_task(c.join_child(new)) + try: + await asyncio.sleep(.02);self.assertFalse(task.done()) + self.assertIsNot(c.waits[old],c.waits[new]) + finally:new.finish(0) + self.assertEqual(await task,0);self.assertEqual(old.waits,1);self.assertEqual(new.waits,1) + + async def test_join_retained_collects_sibling_error_and_waits_other(self): + release=asyncio.Event() + async def fail():raise ValueError('fixture') + async def pending():await release.wait();return 'joined' + one=asyncio.create_task(fail());two=asyncio.create_task(pending()) + join=asyncio.create_task(q.join_retained([one,two,two])) + await asyncio.sleep(.01);self.assertFalse(join.done()) + release.set();result=await join + self.assertIsInstance(result[0],ValueError);self.assertEqual(result[1],'joined') + + async def test_initial_pause_checkpoint_is_unchanged_before_gui(self): + c=self.case;c.work=c.root/'hermes-task';(c.work/'work').mkdir(parents=True) + c.store=q.TaskStore(c.work/'task.sqlite3') + c.identity=q.DemandIdentity('registration','orders-report','worker','2026-10-25T10:00:00Z') + prepare_task(c.store,c.identity,q.PROMPT,'session',metadata={'model':q.MODEL,'quality':q.QUALITY, + 'deadline_clock':q.CLOCK_ID}) + c.registry=q.DemandRegistry((q.Registration(c.store,c.identity),));c.capability=object() + c.box=q.Mailbox(c.registry,c.identity.registration_id,requester_capability=c.capability) + c.original=c.checkpoint();c.deadline=q.execution_now()+900;c.worker=Child() + c.host=SimpleNamespace(adapter=SimpleNamespace(launch_intent=None,started_owner=False), + gateway=SimpleNamespace(_runtime_context=None,history=[],scheduler=SimpleNamespace( + active=None,pending=0,uncertain=False)),controller=SimpleNamespace(fault=None)) + with c.store.claim(c.identity.task_id,expected_revision=c.original.revision) as owned: + state=deepcopy(owned.checkpoint.state);state['deadline_monotonic']=c.deadline + owned.save_checkpoint(state,expected_revision=owned.checkpoint.revision) + c.submit(q.INITIAL_PAUSE,'pause');c.box.consume(owned,updated_at_utc='2026-10-25T10:00:00Z') + before=c.checkpoint();value=c.paused_ack() + self.assertTrue(value['ack']['owner_acknowledged']);self.assertEqual(before,c.checkpoint()) + c.host.adapter.launch_intent={'unexpected':True} + with self.assertRaises(RuntimeError):c.paused_ack() + + async def test_failed_prepare_without_children_records_failure_and_closes(self): + c=self.case;c.root.rmdir() + with mock.patch.object(c,'prepare',side_effect=RuntimeError('invalid deployment')): + result=await c.run() + self.assertFalse(result['clean_stop']);self.assertFalse(result['task_completed']) + self.assertEqual(result['all_retained_process_exits'],[]) + self.assertEqual(result['primary_error']['type'],'RuntimeError') + + async def test_completed_host_outlives_task_deadline_until_real_stop(self): + await self._completed_host() + + async def test_completed_custom_task_exposes_format_only_validation(self): + await self._completed_host(custom=True) + self.assertEqual(self.case.result['semantic_validation'],'not_checked') + self.assertEqual(self.case.result['validated_artifacts'],[]) + self.assertEqual(self.case.result['artifacts'],{'greeting.txt':q.rec.digest(b'Welcome!')}) + + async def _completed_host(self,custom=False): + c=self.case;c.deadline=q.execution_now()-1;c.worker=Child(code=0) + c.handles=[c.worker];c.worker_exit_verified=False + c.stop_binding=SimpleNamespace(_verify_exit=lambda code:'task_completed') + if custom: + spec=custom_spec();c.task_spec_path=self.root/'task.json' + work=self.root/'work';work.mkdir();(work/'greeting.txt').write_bytes(b'Welcome!') + checkpoint=SimpleNamespace(prompt=spec.prompt,max_attempts=8, + metadata={**spec.metadata(),'pre_effect_validator':spec.validator_identity()}, + state={'output':'Created greeting.txt.','tools':[{'name':'write_file', + 'args':{'path':'greeting.txt'},'artifact_sha256':q.rec.digest(b'Welcome!')}]}) + c.stop_binding.mailbox=SimpleNamespace(snapshot=lambda:SimpleNamespace(target=checkpoint)) + c.stop_binding.run=self.root + async def refresh():pass + async def stopped():return dict(worker_joined=True,host_closed=True,uncertain=False,fault=None) + c.runtime=SimpleNamespace(refresh=refresh,_transitioning=False) + c.facade=SimpleNamespace(cooperative_stop=stopped) + c.host=SimpleNamespace(controller=SimpleNamespace(fault=None)) + c.gui=SimpleNamespace(process=Child(42)) + state={'stop_requested':False,'stop_confirmed':False} + c.transport=SimpleNamespace(lifecycle=lambda:dict(state),_stop_task=None) + task=asyncio.create_task(c.interact()) + for _ in range(100): + if c.result['task_completed']: + break + await asyncio.sleep(.01) + self.assertFalse(task.done());self.assertTrue(c.result['task_completed']) + state.update(stop_requested=True,stop_confirmed=True) + c.transport._stop_task=asyncio.create_task(asyncio.sleep(0)) + await asyncio.wait_for(task,1) + self.assertTrue(c.result['lifecycle']['host_closed']) + + async def test_expired_active_task_still_refuses_more_work(self): + c=self.case;c.deadline=q.execution_now()-1;c.worker=Child() + # The successor reads the existing runtime before distinguishing an + # observed Pause from active expiry. Supply that actual active shape. + cp=SimpleNamespace(state={'status':'pending','phase':'ready', + 'deadline_monotonic':c.deadline,'pending_request':None},operation=None, + requires_reconciliation=False) + c.runtime=SimpleNamespace(refresh=mock.AsyncMock(),_transitioning=False,_cache=SimpleNamespace(snapshot= + SimpleNamespace(target=cp,demand={'state':'active'},requests=(),receipts=()))) + c.transport=SimpleNamespace(lifecycle=lambda:{'stop_requested':False}) + # A ready active checkpoint is not a supported admission wait. Detailed + # wait identity/prefix checks use real bindings in the safe-wait tests. + with mock.patch.object(c,'_admission_wait_expiry_candidate', + new=mock.AsyncMock(return_value=False)) as wait_candidate: + with self.assertRaises(TimeoutError):await c.interact() + wait_candidate.assert_awaited_once() + + async def test_prepare_order_uses_one_job_and_real_initial_pause(self): + await self._prepare_order() + + async def test_custom_prompt_reaches_task_and_gui_with_same_cold_start_order(self): + spec=custom_spec();path=self.root/'task.json';path.write_bytes(json.dumps(spec.data).encode()) + inputs=self.root/'inputs';inputs.mkdir() + self.case=q.DesktopComposition(self.case.profile_path,self.root/'custom-run',task_spec=path,task_inputs=inputs) + self.case.root.mkdir() + await self._prepare_order(spec) + + async def test_saved_task_reuses_progress_deadline_and_fresh_paused_owner(self): + from adaptive_inference.desktop import saved_tasks + spec = custom_spec(); work = self.root/'saved-task' + (work/'work').mkdir(parents=True) + identity=q.DemandIdentity('registration','saved-task','worker','2026-10-25T10:00:00Z') + boot={'source':saved_tasks.BOOT_SOURCE,'boot_utc':'2026-11-01T08:12:13.1234567Z','computer_name':'fixture'} + context=saved_tasks.bind_context(saved_tasks.new_context(q.sha(self.case.profile_path), + {'version':'fixture','files':{}},boot),work) + store=q.TaskStore(work/'task.sqlite3') + cp=prepare_task(store,identity,spec.prompt,'saved-native-session',metadata={ + 'model':q.MODEL,'quality':q.QUALITY,'deadline_clock':q.CLOCK_ID,**spec.metadata(), + 'pre_effect_validator':spec.validator_identity(),saved_tasks.CONTEXT:context}) + deadline=q.execution_now()+600 + with store.claim(cp.task_id,expected_revision=cp.revision) as owned: + state=deepcopy(owned.checkpoint.state) + state.update(deadline_monotonic=deadline,requests=2,attempts=1,admission_attempts=3, + native_history=[{'role':'user','content':spec.prompt}],output='Saved progress.') + owned.save_checkpoint(state,expected_revision=owned.checkpoint.revision) + q.rec.save(work/'plan.json',{'task_id':cp.task_id,'demand_identity':q.asdict(identity), + 'deadline_clock':q.CLOCK_ID,'base_python':str(self.case.profile.worker_python), + 'gateway':'http://127.0.0.1:18100/v1','pins':{},'task_spec':spec.data,'task_spec_sha256':spec.sha256, + saved_tasks.CONTEXT:context}) + old=(work/'controller-ready-1.json');old.write_text('{"old":true}') + self.case=q.DesktopComposition(self.case.profile_path,self.root/'reopened',saved_task=work) + self.case.root.mkdir() + with mock.patch.object(saved_tasks,'observe_boot',return_value=boot): + await self._prepare_order(spec,reopened=True) + self.assertEqual(self.case.deadline,deadline) + self.assertEqual(self.case.original.state['requests'],2) + self.assertEqual(self.case.original.state['output'],'Saved progress.') + self.assertNotEqual(self.case.first_ready,old) + self.assertEqual(old.read_text(),'{"old":true}') + self.assertEqual(self.case.stop_binding.diagnostics,self.case.saved_diagnostics) + + async def _prepare_order(self,spec=None,reopened=False): + from adaptive_inference.desktop import owned_job, activity, cold_start, desktop_controls, gui_launch + c=self.case;events=[];children=set();owned_context=[] + job=SimpleNamespace(_HANDLE=object(),contain_current_process=lambda:events.append('own-job'), + active_process_ids=lambda:{os.getpid(),*children}) + prepare_child=Child(101,0);worker=Child(102) + original_join=c.join_child + async def joined(child): + value=await original_join(child);children.discard(child.pid);events.append('join-prepare');return value + c.join_child=joined + def launch(command,label): + if label=='prepare': + self.assertFalse(reopened,'Saved task must not invoke fresh prepare') + self.assertEqual(events,['own-job']) + c.work.mkdir();(c.work/'work').mkdir() + task_id='orders-report' if spec is None else command[command.index('--task-id')+1] + if spec is None:self.assertIn('--validate-report-before-write',command) + else: + self.assertNotIn('--validate-report-before-write',command) + self.assertEqual(command[command.index('--task-spec-sha256')+1],q.sha(c.task_spec_path)) + self.assertEqual(command[command.index('--task-inputs')+1],str(c.task_inputs)) + identity=q.DemandIdentity('registration',task_id,'worker','2026-10-25T10:00:00Z') + store=q.TaskStore(c.work/'task.sqlite3') + prepare_task(store,identity,q.PROMPT if spec is None else spec.prompt,'session', + metadata={'model':q.MODEL,'quality':q.QUALITY,'deadline_clock':q.CLOCK_ID, + **({} if spec is None else spec.metadata())}) + q.rec.save(c.work/'plan.json',{'demand_identity':q.asdict(identity),'task_id':task_id, + **({} if spec is None else {'task_spec':spec.data,'task_spec_sha256':spec.sha256}), + 'deadline_clock':q.CLOCK_ID, + 'base_python':str(c.profile.worker_python),'gateway':'http://127.0.0.1:18100/v1'}) + events.append('prepare');child=prepare_child + else: + self.assertEqual(events[-1],'parked-host') + if reopened: + self.assertEqual(command[command.index('--deadline-monotonic')+1],str(c.original.state['deadline_monotonic'])) + self.assertEqual(command[command.index('--diagnostics')+1],str(c.saved_diagnostics)) + self.assertIn('-m',command);self.assertNotIn('--canonical',command) + cp=c.store.load(c.identity.task_id) + context=c.store.claim(cp.task_id,expected_revision=cp.revision) + owned=context.__enter__();owned_context.append(context) + state=deepcopy(owned.checkpoint.state);state['deadline_monotonic']=c.deadline + owned.save_checkpoint(state,expected_revision=owned.checkpoint.revision) + c.box.consume(owned,updated_at_utc='2026-10-25T10:00:00Z') + events.append('paused-worker');child=worker + c.handles.append(child);children.add(child.pid);return child + c.launch=launch + class Adapter: + launch_intent=None;started_owner=False + def __init__(self,**kw): + self.job=kw['job'];self.max_runtime_seconds=kw['max_runtime_seconds'] + class Host: + closing=False;started_at=None + def __init__(self,*a,**kw): + self.adapter=kw['adapter'];self.job=kw['job'] + self.gateway=SimpleNamespace(instance='gateway',_runtime_context=None,history=[], + scheduler=SimpleNamespace(active=None,pending=0,uncertain=False)) + self.controller=SimpleNamespace(events=deque([{'parked':True}],maxlen=256),fault=None) + async def start(self,**kw): + self_outer.assertEqual(self.job.active_process_ids(),{os.getpid()}) + self_outer.assertIs(self.job,self.adapter.job) + self.started_at=time.monotonic();events.append('parked-host') + def attach_control_runtime(self,value):events.append('attach-runtime') + class Runtime: + def __init__(self,*a,**kw):pass + async def refresh(self):pass + def configure_continuation(self,value):pass + class Transport: + _ready=True + def __init__(self,*a,**kw):pass + def bind_session(self,*a):events.append('bind-session') + class Listener: + def __init__(self,*a):pass + def start(self):events.append('listener');return ('127.0.0.1',12345) + class GUI: + def __init__(self,**kw):self.binding=None;self.verifier=object();self.process=Child(103) + async def start(self,spec,bootstrap,**kw): + self_outer.assertIs(kw['containing_job'],job._HANDLE) + self_outer.assertEqual(spec.interpreter,c.profile.host_python) + self_outer.assertEqual(bootstrap.objective,c.original.prompt) + self_outer.assertEqual(bootstrap.task_id,c.original.task_id) + self_outer.assertTrue(kw['launch_ready']()) + from adaptive_inference.desktop.ui_intent import UIBinding + self.binding=UIBinding(bootstrap.host_epoch,bootstrap.session_id,bootstrap.task_id,1,103,2) + kw['bind_session'](self.binding,self.verifier,bootstrap);events.append('gui') + self_outer=self + api=SimpleNamespace(handle=lambda child:child,identity=lambda child:(child.pid,5),in_job=lambda *a:True) + facade=SimpleNamespace(bind_window=lambda *a:None,window_closed=lambda *a:None,verify_current=lambda *a:None) + try: + with mock.patch.object(q,'os',SimpleNamespace(name='nt',getpid=os.getpid)), \ + mock.patch.object(q,'sys',SimpleNamespace(version_info=(3,11,9))), \ + mock.patch.object(q,'verify_interpreter'), mock.patch.object(q,'verify_sdk'), \ + mock.patch.object(q,'verify_execution_clock',return_value=q.CLOCK_ID), \ + mock.patch.object(q,'package_identity',return_value={'version':'fixture','files':{}}), \ + mock.patch('adaptive_inference.desktop.owned_job',job), \ + mock.patch.object(activity,'WindowsActivityReader',return_value=object()), \ + mock.patch.object(cold_start,'ColdStartAdapter',Adapter), \ + mock.patch.object(desktop_controls,'ControlledDesktopHost',Host), \ + mock.patch.object(desktop_controls,'DesktopControlLifecycle',return_value=facade), \ + mock.patch.object(gui_launch,'WindowsLaunchAPI',return_value=api), \ + mock.patch.object(gui_launch,'OwnedUILaunch',GUI), \ + mock.patch.object(q,'MailboxRuntime',Runtime), \ + mock.patch.object(q,'ControlTransport',Transport), \ + mock.patch.object(q,'SerialListener',Listener): + await c.prepare() + self.assertEqual(events,['own-job',*([] if reopened else ['prepare','join-prepare']),'parked-host', + 'paused-worker','attach-runtime','listener','bind-session','gui']) + self.assertIs(c.job,job) + self.assertTrue(c.paused_ack()['ack']['owner_acknowledged']) + finally: + for context in owned_context:context.__exit__(None,None,None) + for lock in c.reopen_locks:lock.close() + + +class LifetimeTests(unittest.TestCase): + def test_custom_input_options_require_a_pair_and_reject_bad_spec_before_launch(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory).resolve();case=make_case(root) + path=root/'task.json';path.write_text('{"schema":1,"schema":2}',encoding='utf-8') + with mock.patch.object(q.subprocess,'Popen') as launch: + for options in ({'task_spec':path},{'task_inputs':root}, + {'task_spec':path,'task_inputs':root}): + with self.subTest(options=options),self.assertRaises(ValueError): + q.DesktopComposition(case.profile_path,root/'run',**options) + launch.assert_not_called() + + def test_ordinary_execute_has_no_lifetime_watchdog(self): + async def run():return {'clean_stop':True} + case=SimpleNamespace(run=run) + with mock.patch.object(q,'DesktopComposition',return_value=case), \ + mock.patch.object(q.threading,'Thread') as thread, mock.patch.object(q.rec,'save'): + self.assertTrue(q.execute('profile','state')['clean_stop']) + thread.assert_not_called() + + def test_only_explicit_qualification_arms_original_no_io_guard(self): + async def run():return {'clean_stop':True} + with mock.patch.object(q,'DesktopComposition',return_value=SimpleNamespace(run=run)), \ + mock.patch.object(q.threading,'Thread') as thread, mock.patch.object(q.rec,'save'): + q.execute('profile','state',qualification_timeout_s=1800) + thread.assert_called_once_with(target=q.hard_deadline,args=(1800,),daemon=True) + thread.return_value.start.assert_called_once() + with self.assertRaises(ValueError):q.execute('profile','state',qualification_timeout_s=1801) + + +if __name__=='__main__':unittest.main() diff --git a/provenance/automatic-tool-recovery/predecessor/tests/test_pending_tool_lineage.py b/provenance/automatic-tool-recovery/predecessor/tests/test_pending_tool_lineage.py new file mode 100644 index 0000000..6cc3612 --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/tests/test_pending_tool_lineage.py @@ -0,0 +1,107 @@ +"""Pending tool recovery preserves exact published recipe source identities.""" +import ast +from copy import deepcopy +import hashlib +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import pending_tool_lineage as lineage +import recipe_reconciliation_lineage +from paused_expiry_lineage import fingerprint + + +class PendingToolLineageTests(unittest.TestCase): + def current_runtime(self): + runtime = lineage.ROOT / 'src/adaptive_inference' + return {path.relative_to(runtime).as_posix(): path.read_bytes() + for path in runtime.rglob('*.py')} + + def test_exact_recipe_predecessors_and_evidence_remain_unchanged(self): + rows = lineage.record() + self.assertEqual(rows['predecessor_revision'], '93eba3ce4eee9609283f360fb7ca51b6c136e8bb') + self.assertTrue(rows['files']) + for relative, row in rows['files'].items(): + with self.subTest(relative=relative): + data = (lineage.ROOT / relative).read_bytes() + before = lineage.retained(relative, row, lineage.ROOT) + self.assertEqual(hashlib.sha256(data).hexdigest(), row['successor_sha256']) + self.assertEqual(fingerprint(ast.parse(data)), row['successor_ast_sha256']) + self.assertEqual(lineage.predecessor(relative, data), before) + self.assertEqual(lineage.predecessor(relative, before), before) + for relative, pin in rows['preserved_records'].items(): + self.assertEqual(hashlib.sha256((lineage.ROOT / relative).read_bytes()).hexdigest(), pin) + self.assertIn('provenance/recipe-reconciliation/source-lineage.json', rows['preserved_records']) + self.assertIn('provenance/recipe-reconciliation/installed-worker-loss-2026-11-02.json', + rows['preserved_records']) + + def test_unknown_complete_bytes_and_semantics_are_not_projected(self): + for relative in lineage.record()['files']: + data = (lineage.ROOT / relative).read_bytes() + for extra in (b'\n# unreviewed reconciliation source\n', b'\nunreviewed = True\n'): + changed = data + extra + with self.subTest(relative=relative, extra=extra): + self.assertEqual(lineage.predecessor(relative, changed), changed) + + def test_runtime_restores_all_77_recipe_modules_and_composes_idempotently(self): + rows = lineage.record() + self.assertEqual(len(rows['baseline_runtime']), 77) + current = self.current_runtime(); original = dict(current) + result = lineage.runtime_view(current) + self.assertEqual(current, original) + self.assertEqual({name: hashlib.sha256(data).hexdigest() for name, data in result.items()}, + rows['baseline_runtime']) + self.assertEqual(lineage.runtime_view(result), result) + prior = recipe_reconciliation_lineage.runtime_view(current) + self.assertEqual(len(prior), 77) + self.assertEqual(recipe_reconciliation_lineage.runtime_view(result), prior) + self.assertEqual(recipe_reconciliation_lineage.runtime_view(prior), prior) + for name in current: + changed = dict(current); changed[name] += b'\n# unreviewed\n' + with self.subTest(name=name), self.assertRaisesRegex(AssertionError, 'runtime source'): + lineage.runtime_view(changed) + changed = dict(current); del changed['task_spec.py'] + with self.assertRaisesRegex(AssertionError, 'runtime closure'): + lineage.runtime_view(changed) + with self.assertRaisesRegex(AssertionError, 'runtime closure'): + lineage.runtime_view({**current, 'unreviewed.py': b'pass\n'}) + + def test_ast_projection_requires_complete_current_bytes(self): + relative = 'desktop/hermes/compiler_recovery.py' + source = 'src/adaptive_inference/' + relative + target = lineage.ROOT / source; data = target.read_bytes(); tree = ast.parse(data) + row = lineage.record()['files'][source] + self.assertEqual(fingerprint(lineage.restore(tree, relative)), row['predecessor_ast_sha256']) + self.assertEqual(fingerprint(tree), row['successor_ast_sha256']) + changed = ast.parse(data + b'\nunreviewed = True\n') + self.assertEqual(fingerprint(lineage.restore(changed, relative)), fingerprint(changed)) + original_read = Path.read_bytes + def read(path): + raw = original_read(path) + return raw + b'\n# same AST\n' if path == target else raw + with mock.patch.object(Path, 'read_bytes', read): + with self.assertRaisesRegex(AssertionError, 'pending-tool-recovery source'): + lineage.restore(tree, relative) + + def test_predecessor_and_successor_identity_tampering_is_refused(self): + rows = lineage.record(); relative = 'src/adaptive_inference/desktop/hermes/compiler_recovery.py' + data = (lineage.ROOT / relative).read_bytes() + for field in ('predecessor_source', 'predecessor_sha256', 'predecessor_ast_sha256', + 'successor_ast_sha256'): + changed = deepcopy(rows); changed['files'][relative][field] = '0' * 64 + with self.subTest(field=field), mock.patch.object(lineage, 'record', return_value=changed): + with self.assertRaisesRegex(AssertionError, 'pending-tool-recovery'): + lineage.predecessor(relative, data) + + def test_missing_archive_never_authorizes_an_unbound_projection(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); data = b'pass\n'; tree = ast.parse(data) + self.assertEqual(lineage.inputs(root), []) + self.assertEqual(lineage.predecessor('src/adaptive_inference/task_spec.py', data, root), data) + self.assertEqual(lineage.runtime_view({'unknown.py': data}, root), {'unknown.py': data}) + self.assertIs(lineage.restore(tree, 'task_spec.py', root), tree) + + +if __name__ == '__main__': + unittest.main() diff --git a/provenance/automatic-tool-recovery/source-lineage.json b/provenance/automatic-tool-recovery/source-lineage.json new file mode 100644 index 0000000..8b7b52d --- /dev/null +++ b/provenance/automatic-tool-recovery/source-lineage.json @@ -0,0 +1,156 @@ +{ + "baseline_runtime": { + "__init__.py": "1d906cca6d5581290d73292cce949dcaf74001de67bda63b903dfb2e70edacba", + "__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "adapters.py": "378ee41c1bdbb16873e31752d704e656a70009862d41b36898d30f57a36dfc58", + "cli.py": "a13d6c2c52445884f2ee9897eb7fadb75188339be4472db98bd760c18bce5ecf", + "client_routing.py": "d57178cfbd5666e75aef311f1362cf8486f48085c6f4995861b4fbd0cb7e41d1", + "config.py": "a6800a081a1e14a46ee18235dd21c1d1a714e6ef4e879840cd2768c590fde669", + "context_budget.py": "cd16b57bc105a179e94dbc9e429c02b901bcd8602cfaf04c9cf8776e9d799ccf", + "control_mailbox.py": "12e6ceaef36b41099422040fb23db0a9bccf96a6afb799f0e301a64bccb95888", + "costs.py": "9e8fcd06d39af57392f191b5d5701aa828f134e5141c7e298aa62e0c232224e1", + "demand.py": "76a9fbb26128c1e458348a37f0d15c36f6a9a5ae2d0f89b15d372bd2d1e394ad", + "demo.py": "5a3b1ff743bee6c45f85df8417bedbc18657f2c59f3e894f1f3298a480b22b93", + "desktop/__init__.py": "6169073159f3c0e506752f54aa51b40b058f71ed058d5272e160aa4a61ee7daa", + "desktop/activity.py": "3282dc37b493ba3fd347e279b074fa3dfd5d2c1ea5eaba8131dc568d97ec0750", + "desktop/cold_start.py": "1264f34910a0d619c6502c7565e66abbfe7ec642391a461390271b8e0799bf68", + "desktop/compiler_bridge.py": "a33613e13995c098488144557a87ee4b34b1ed3f4342610b3a43fa8923945299", + "desktop/compiler_host.py": "95a7d000bea96f527996e360eadb7324aaafd583506d4587dae89288d47ab3d2", + "desktop/compiler_profile.py": "872b8babf3fe26828ce50422485489fdde35bd59f010ff063be66ef253e323fc", + "desktop/compiler_recipe.py": "12b31514327a7c168c915b3a321ee1c17bf8446b49181eed6bed2bced55b01d2", + "desktop/compiler_template.py": "bf7a3123eb147caded7700c7f67e0c072cfe915239fa3a5cb004e7e1772f7f85", + "desktop/composition.py": "066aa1454c88cc08ed9c58452023fb47f1a223981d08428a32b20581ecc5e321", + "desktop/continuation_host.py": "9bd258aa8b1a2516e80cd94641f2a9fce3584c5b8aa484badf9854dfc121d7c7", + "desktop/control_client.py": "ba950c73328e084329fdb29ab78fdfe86cfed1715a792cdbf9872a74e8c75919", + "desktop/control_transport.py": "280d8a55793c241b3ec30e253b758de42386f3329bb6477ba47537f9ca264c6b", + "desktop/desktop_controls.py": "081cfc11a5335299ad119007dfc7a428ed011f55386af50d4d1289ec0f62e786", + "desktop/execution_budget.py": "77ad74ddbf0eaa89379fbecb305574d5b70cc32f9e63934a14f439ed2c744728", + "desktop/foreground_policy.py": "35781fb869c03220c6f48c3e138083b15bbcf7e40187edbfe546ad4da09b7faa", + "desktop/gui_launch.py": "c93e837ae15092de1a7814f766731ef38dc118f3e8a50280ee7658b91966355b", + "desktop/gui_session.py": "f232609a01f11977d7f011a8b1c7f2b7c2a0c3ca697766b63ec0873ec9f4cc64", + "desktop/hermes/__init__.py": "43830f7aa3deb6001eb8aa8646fc515f5d8d74a26f50590984c8278e4db67532", + "desktop/hermes/admission.py": "6f12fd6ad21fa194cb337590cf0bbd71b1d394e6a2be0cd79dfdd634af5e0f07", + "desktop/hermes/compiler_client.py": "fb0372c6b16738cdb1b0c8d527765e8b7f60a038513950988c84d40b741bf8a1", + "desktop/hermes/compiler_evidence.py": "8b2144e5b1634d8e2bc132711d948308ebc62320e98f9129c44817848a806d56", + "desktop/hermes/compiler_recovery.py": "03d19056b0c2f73679754aa3eab215d3fb2728b22c71a5ab5e1c327ad6a8c114", + "desktop/hermes/compiler_worker.py": "8d18ef60025318804e81f848546e1c3e0e84f61cf4a09fe7bfd1c759b243d91e", + "desktop/hermes/continuation.py": "d3b396bba2ed794b7304e3bebb888e2d8d51568de846fbe6549363b42d6de5c6", + "desktop/hermes/continuation_worker.py": "6871ecbfaec346ccd658d957b5f593f0dc242db17c1b13b93ab603f693967b67", + "desktop/hermes/control_bridge.py": "7a8e0aa672efeb1c6f676446e0f3b6886fd778f5d0f6c185a87d005a0256a3db", + "desktop/hermes/native_reader.py": "1f51ea1180ef0d7ad52359430831084de7cde45e21e758068fee5b9a93079464", + "desktop/hermes/pause_activity.py": "3cffa8eae39b956de3d873e82593a052c2682aad5f06b36a8ff845080baf0e5d", + "desktop/hermes/recovery.py": "c6f07435ae29f4e2542af585ace107f7952583ed3c0735cf6df2f3a0df438f37", + "desktop/hermes/report_contract.py": "d86aa6ee0f18912400aa80d96c343128af08ae80e7d8c367ee9c8733ea5d69de", + "desktop/hermes/report_validator.py": "378dd445a088a075d300fc85b86160f3b96790ee046adecf926f9e59e8e7d9af", + "desktop/hermes/run_recovery.py": "3d78a3f1f78e024d04b0020e3702c784dfc3723a802bdb87c9cbca25ec827fd3", + "desktop/hermes/sdk_pins.py": "538a5ea8651e1106c7c535ccd73b33b2f56de6c736d926ce6b92c5fb6eb075ce", + "desktop/hermes/service_stop.py": "235b61ad248083f8f0536767ad79473955d96ad1b40189b73c9cfe71439a51c9", + "desktop/installation.py": "13e13d532bedb68f150d1096767c98796de83f74dce54021d286c6c18c521e85", + "desktop/load_feasibility.py": "23e94f3ee770b065fb323f30bb104d8d5732bf710bd3cc9b2a27daa746014dda", + "desktop/mailbox_runtime.py": "aad1c3b383f6954be89ebe6c9f2e680cd286c672d7bb26bd7027402e0a7d6b72", + "desktop/native_owner_adapter.py": "5a79c48911c798310104be07ebb978f9ad9fe4f7ab000c328afd992f75415977", + "desktop/owned_job.py": "90b59dbdae226419fc9680c961c21dfa9e061df08512a7a8089d6f6d11b2ccc4", + "desktop/persistent_host.py": "acff567d5eb6f2c569588bfa684363907897bdb66be8008bc20c10d3ca7c0011", + "desktop/profile.py": "caac6f07fc87ccbf51d97003a0d8cb209d21e04bfc00acda6a5c9e05b2d3fbbc", + "desktop/recipe_dispatch.py": "3a4813e156448305e7880c3cc3ac3b82e992b34a0e05b7f6b74f09bc9e2c6f4b", + "desktop/retained_ui.py": "749b947510377518c452ae7a70fa21c32f27c3f10225f6eb90d87ef03406f5cb", + "desktop/run_gui.py": "eb67c9bac75b72bcc545c234b28c54b8c9c5fbfb3c2183f99e63a47abb3a6ef0", + "desktop/saved_tasks.py": "75703584b3c4cda42aed6de0e4645ea75b2a3b1e2e81c702e919f626cc8ec1b7", + "desktop/serial_listener.py": "048e8ab061e175a978f755801b9a27e3daa292c1dc7dabe3c8a8763be81199b2", + "desktop/startup_pipe.py": "0617c527461b4c437474a947a497300e4c28c06d11f8bb43f1e67fc51a159fae", + "desktop/task_admission.py": "2ca007580bca43678c529599b6296ff4b97c15e210b9daeb651a8f2c4f12b25d", + "desktop/task_handover.py": "6905f542ac26e5b1ad51fe94cfe3bf07accd7ec057b4d8521a4446a807bdf809", + "desktop/ui_intent.py": "ff67208a3890656bc76717ac18c8d0c02a19dc2e09b11b81d88fc1f63c0a29d1", + "desktop/ui_progress.py": "466deda7ba15d73ea779abf3b1b2e690e92217950b15afdbdf8acf9cda3d2de9", + "desktop/video_profile.py": "2ac53c40937e60d984e2903d643d2ee83372760756d1afedf5406b71644b41e4", + "desktop/video_recipe.py": "b255d3f939aa3a1e29449bc397e39b50859fcc0cf8a10201859b2ec2dc50bcff", + "desktop/video_template.py": "1a74d771d83eed358196467857e11c5986cfc6c196b6d172c614224db311a835", + "desktop/window_marker.py": "e2613ef662993be88c332657e4e53c598fdc006634bf8fabf9e0d0a9765c0d36", + "desktop/windows_host_base.py": "ad08c31435bc4da77b43c68321a7c08130ab1c4f7bc688c71191c6497abc97bd", + "execution_clock.py": "c9d04e8e029d4d88830560482e2437cab3d06ef140a942d16543c3be95642cff", + "feedback.py": "c76eba0110659ae67f4a0efd2ca21aff0987af7e9095998e4db9382de20826c3", + "gateway.py": "b2009a8a776bcea4dfc3bedac5296c930ae2211bb8e29d7ccdf76a227fefdef8", + "handoff.py": "6cd7f593698f6d72fc3ac16d9a541436f65b85103bed2420372364262fcb4db1", + "resource_policy.py": "53768d4b95b589bb76d1788a9ccceefe60c05b332b1b1f43dece6b04bfec212a", + "resources.py": "a72e376e4dfc6021e61756812f93275af20c28ec1438faf5eb381a491d3db4d9", + "scheduler.py": "c7f962bd9ef8e44587cadcc02a41fae8a00de7fc15b105848dcdfc4862bd5b4f", + "task_prepare.py": "9cddd6e09b3f5435da08316f8d9ccdfbcff67a336f6f74b420108c4c856223b2", + "task_spec.py": "2975c0c4489ac863502189777699d13d7b2b8aca6d19191247fd860ed4a0ce76", + "task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6" + }, + "files": { + "src/adaptive_inference/desktop/composition.py": { + "predecessor_ast_sha256": "4966b22db00b7aec7b7554488966d656415601d832fbc3e2561d0a63b53a9606", + "predecessor_sha256": "066aa1454c88cc08ed9c58452023fb47f1a223981d08428a32b20581ecc5e321", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/composition.py", + "successor_ast_sha256": "3e46746633683ebf3c7b72c1dfb9c8dc408c6848fa74d8add64c193c856528b2", + "successor_sha256": "a8688f7dadceac26747e139a8d2e53e3a7a1dd5bdb71751049ab72698650165f" + }, + "src/adaptive_inference/desktop/desktop_controls.py": { + "predecessor_ast_sha256": "a5e6a6bb28b947cad24d21218c87bdfb6fa39d0a7ac204253ca098b3d19b5f24", + "predecessor_sha256": "081cfc11a5335299ad119007dfc7a428ed011f55386af50d4d1289ec0f62e786", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/desktop_controls.py", + "successor_ast_sha256": "d8de8c2594e2d9db01e9cc51b76d5efe2d99c42418dc4e10e45ff37ff72517a1", + "successor_sha256": "89eeb900ce8bcdb079d5400f09408a56e1df15383d58ec74d405a027795469c9" + }, + "src/adaptive_inference/desktop/task_handover.py": { + "predecessor_ast_sha256": "36ec79b0a6957333cb2d5c6c915b55b8656b1e7bd6a5ac880121d1b542038f2a", + "predecessor_sha256": "6905f542ac26e5b1ad51fe94cfe3bf07accd7ec057b4d8521a4446a807bdf809", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/src/adaptive_inference/desktop/task_handover.py", + "successor_ast_sha256": "2bbda7233be2cc2b2aa6074c0118778258831d208f1720c5bfa8ebcd321c28c9", + "successor_sha256": "8fd90d86635a3b5968a39d32b20e54e67b17c8dc9bb4a90b9fcdcabff97f01b4" + }, + "tests/pending_tool_lineage.py": { + "predecessor_ast_sha256": "8c7c88b348e80da3f44aed33ab854e581b95b17de63b91a124a64d8cc6c93522", + "predecessor_sha256": "519bd7115aca134617ca22751dee7f35f79a410abdf15e8c5464787025cce0d8", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/tests/pending_tool_lineage.py", + "successor_ast_sha256": "c6fc69456720937c5dbac32f04b99d1b11d6cce8559faf9d60e258d075f14011", + "successor_sha256": "2727e96a5280c0e24b1b2c5428bbff4a8d18cec7d628dd951bf1d99645b80cb4" + }, + "tests/test_desktop_composition.py": { + "predecessor_ast_sha256": "040422da37507cfb592cab3fa3f24bf10e5307f129e28850921f022416b6874f", + "predecessor_sha256": "b84ea2ce1b5e8484387dbe1fc5d0557dfb62589452442ea3e48b08ab2f50f1c0", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/tests/test_desktop_composition.py", + "successor_ast_sha256": "76f9c2f7f3f43eea819956480347c4263b99bfc70cf55508c8715f8050b7ae5a", + "successor_sha256": "29efb54d499c69c00d44ef5b5660c9aa7c747ccdb0ef109c9c431d0787b35df1" + }, + "tests/test_pending_tool_lineage.py": { + "predecessor_ast_sha256": "e1789a3af4289c067a8233c636c60473b4d95012f512421d33814276cffe1e4c", + "predecessor_sha256": "dfa87eece5a7db895e574a43a989f76a4cca1ec58a98e3d494c26f269168f479", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/tests/test_pending_tool_lineage.py", + "successor_ast_sha256": "d279f1055c39474b0a08f7d6a314542c1a3cb4ed6775e745020a7fd1261250fa", + "successor_sha256": "b63114b5896666122d4bebd30d346356c3b50489e0c7a7ae674cf9d9ecdcfa89" + } + }, + "license": "Apache-2.0", + "mechanism": "Reuse tests/pending_tool_lineage.py complete-file SHA-256 and portable-AST identity before projection.", + "predecessor_revision": "9242434991f2c2c21461061d61d898649299b847", + "preserved_records": { + "provenance/compiler/component-validation-2026-11-02.json": "0369b9620d235350c4124510e62511ed1fd04c9e4ef6aca72e320029cf0c8275", + "provenance/compiler/installed-sdk-evidence-2026-11-02.json": "e5aa49d90535bc0f87267a3d11b0977c16162c3123dbbfd4d69c47bdbe929e2b", + "provenance/compiler/local-validation-2026-11-02.json": "b5564e1af03b854a008dde93d514a3cc3697ddfc4ee07b59851bec17fb0f2485", + "provenance/compiler/physical-compiler-evidence-2026-11-02.json": "bafc4718848f7b685f5b5243cd7dcb0410a1cfef19e539b87f1b0f8433c8300f", + "provenance/compiler/physical-foreground-compiler-2026-11-02.json": "b539716c48889ce95e70bdfec53838f1dc92c7136a97268227efa3a3479a1125", + "provenance/compiler/setup-validation-2026-11-02.json": "01f4d13842a6d64f598cf1887651fb7d5d5505d92bd2b578788bb4c6fa565976", + "provenance/compiler/source-lineage.json": "21606221cf65fba78fab867228d7aee4a82a1d52593c3aa13402266a5ef81f8d", + "provenance/compiler/template-native-validation-2026-11-02.json": "4bd4ad52c583b83b5af3f3d636e0ac5814f9f123fe27a1ead22fb3934fd02ea2", + "provenance/pending-tool-recovery/installed-worker-loss-2026-11-02.json": "8fa982de35baf497bb5a8f2dc136a108eaae9db45c755259e463355e65ed3389", + "provenance/pending-tool-recovery/local-validation-2026-11-02.json": "5819446522d2537dd5e29892931a1f600cea20d6584d9c060b5ab595a5a3b854", + "provenance/pending-tool-recovery/source-lineage.json": "8aba39ffa3869903e60ab460b718e014fc1cbc7d6a10e5f9a8717f43162be212", + "provenance/recipe-reconciliation/installed-worker-loss-2026-11-02.json": "853482533179691dd0aa939c9e12bfcc89e8b80920b48ca8c68b071add3d0965", + "provenance/recipe-reconciliation/local-validation-2026-11-02.json": "899983e2377295852fa1d4a09f5765a89fa5a89ce10c36937bcc928b7e568dc2", + "provenance/recipe-reconciliation/source-lineage.json": "53bb4b13f8fce84395a6599015e4db14e0d39167815c0963bb7c760a419ecb43", + "provenance/video-tool/hosted-ci-fixture-2026-11-02.json": "e8b8155b4910d7cf9175ce9662c98bdbca8e0ea68e8efed06330e1eed481da8a", + "provenance/video-tool/hosted-ci-timeout-2026-11-02.json": "b2df23d6b3286226e4cf00f35cefb7d4b1a79d00fedcc04f4a7ab2f154645b47", + "provenance/video-tool/hosted-observation-clock-2026-11-02.json": "91a8e07af741871cf46ae1ea2a2d4432ce4143bac13b16b3a97c81030142a253", + "provenance/video-tool/installed-sdk-evidence-2026-11-02.json": "2dc88ca761ec7d6093d0ca92d7d295df078f8d08c3fcdec01f23fbb51affa579", + "provenance/video-tool/installed-worker-restart-2026-11-02.json": "e89428645c061240109228a21f21e3989d4a2dc9e596185c8baf28691cdde0cc", + "provenance/video-tool/local-validation-2026-11-02.json": "d7046c3dbe0b6ece2f525886e675c929be3db70cb24b8c9c3fab0c7f9203afaa", + "provenance/video-tool/native-component-2026-11-02.json": "2e52df681d22598e882ee97706921e1ebe5b2b489effca5bcbce1c0461087780", + "provenance/video-tool/physical-foreground-video-2026-11-02.json": "8c1cc857aea0b0f4599f84cce54e53b9ec2ae25b6e3fd10d9eecf6cd56b640f3", + "provenance/video-tool/physical-video-evidence-2026-11-02.json": "6e8d7555a11b22886ddf2624ae48a079f580f78d4469d1c14ae0cbe851a8fb7a", + "provenance/video-tool/source-lineage.json": "2f6fe653edd7865aacc2302444b865f541f44e8419fa1046a0a6dee23a79556b" + }, + "schema": "adaptive.automatic-tool-recovery.source-lineage.v1", + "scope": "Exact source-only projection; no historical qualification is transferred to automatic desktop tool recovery." +} diff --git a/src/adaptive_inference/desktop/composition.py b/src/adaptive_inference/desktop/composition.py index 0033acd..33e7c79 100644 --- a/src/adaptive_inference/desktop/composition.py +++ b/src/adaptive_inference/desktop/composition.py @@ -158,6 +158,10 @@ def __init__(self, profile_path, run, *, qualification_hold_after_report=False, self.compiler_host = None self.compiler_kind = 'video' if self.profile.video is not None else 'compiler' self.compiler_bindings = {} + self.compiler_bindings_history = [] + self.compiler_generations = {} + self.recovery_records = {} + self.worker_replacements_count = 0 self.compiler_native_reader = None def compiler_launch_args(self, work, plan, *, execution_id, task_id, deadline, diagnostics): @@ -174,7 +178,18 @@ def compiler_launch_args(self, work, plan, *, execution_id, task_id, deadline, d from adaptive_inference.desktop.hermes.compiler_worker import binding_schema key = (task_id, execution_id) if key in self.compiler_bindings: - raise RuntimeError('Compiler worker launch binding cannot be reused') + if (self.compiler_host is not None + and not self.compiler_host.lifecycle()['bound'] + and getattr(self, 'worker', None) is not None + and self.compiler_worker_joined(self.worker)): + if not hasattr(self, 'compiler_bindings_history'): + self.compiler_bindings_history = [] + self.compiler_bindings_history.append((key, self.compiler_bindings.pop(key))) + if not hasattr(self, 'compiler_generations'): + self.compiler_generations = {} + self.compiler_generations[key] = self.compiler_generations.get(key, 0) + 1 + else: + raise RuntimeError('Compiler worker launch binding cannot be reused') binding = {'schema': binding_schema(kind), 'host_epoch': self.host_epoch, 'task_id': task_id, 'execution_id': execution_id, 'task_deadline': deadline, 'gateway_instance': self.host.gateway.instance, 'capability': secrets.token_hex(32)} @@ -203,6 +218,47 @@ async def bind_compiler_worker(self, runtime, child, work, deadline): read_native_history=lambda session: self.compiler_native_reader(work, session), task_root=work, capability=binding['capability'], task_deadline=deadline) + def create_replacement_binding(self): + """Trusted synchronous replacement worker launch callback.""" + generation = getattr(self, 'worker_replacements_count', 0) + 1 + self.worker_replacements_count = generation + self.stop_id = 'service-stop-' + uuid.uuid4().hex + self.worker_cap = object() + ready = self.worker_diagnostics() / f'controller-ready-{generation + 1}.json' + if ready.exists(): + raise RuntimeError('Replacement readiness must be fresh') + command = [self.plan['base_python'], '-I', '-B', '-m', + 'adaptive_inference.desktop.hermes.run_recovery', 'worker', '--run', str(self.work), + '--controller-ready', str(ready), '--gateway-instance', self.host.gateway.instance, + '--deadline-monotonic', str(self.deadline), '--stop-request-id', self.stop_id] + command.extend(self.compiler_launch_args(self.work, self.plan, execution_id=self.identity.task_id, + task_id=self.identity.task_id, deadline=self.deadline, diagnostics=self.worker_diagnostics())) + if self.reopen_context is not None: + command.extend(['--execution-boot-id', rec.digest(self.reopen_context['boot'])]) + if self.saved is not None: + command.extend(['--diagnostics', str(self.saved_diagnostics)]) + if self.qualification_hold_after_report: + command.append('--qualification-hold-after-report') + before = len(self.handles) + try: + child = self.launch(command, f'worker-{generation + 1}') + finally: + if len(self.handles) == before + 1: + self.worker = self.handles[-1] + self.stop_binding = None + self.bind_worker_stop() + if hasattr(self, 'launch_api') and self.launch_api is not None and getattr(self, 'job', None) is not None: + handle = self.launch_api.handle(child) + identity = self.launch_api.identity(handle) + if (identity[0] != child.pid or self.launch_api.in_job(handle, self.job._HANDLE) is not True + or child.poll() is not None): + raise RuntimeError('Replacement is not retained and live in the containing job') + self.result['replacement_retained_identity'] = { + 'pid': identity[0], 'creation_time_100ns': identity[1], 'containing_job_verified': True + } + self.first_ready = ready + return self.stop_binding, self.worker_cap + def launch(self, command, label): stream=(self.root/(label+'.log')).open('wb'); self.logs.append(stream) before=len(self.handles) @@ -818,6 +874,127 @@ async def ready_continuation(self,record,intent_valid): self.runtime._transitioning = False await asyncio.to_thread(rec.save,self.first_ready,value) + async def _try_recover_pending_tool_worker(self, code): + """Host-owned automatic recovery of a known terminal tool result. + + If a worker exits with non-clean exit code while the compiler host retains + the terminal recipe result for that worker's pending effect operation, + exports the receipt, retires the old binding, reconciles TaskStore via + run_recovery CLI, launches a replacement worker under the unchanged deadline, + awaits its bootstrap, re-binds compiler host, and publishes readiness. + """ + if code == 0: + return False + if getattr(self, 'transport', None) is None: + return False + if self.transport.lifecycle()['stop_requested']: + return False + if getattr(self, 'host', None) is not None: + if getattr(self.host, 'closing', False) or getattr(self.host, 'closed', False): + return False + controller = getattr(self.host, 'controller', None) + if controller is not None and getattr(controller, 'fault', None): + return False + if getattr(self, 'stop_binding', None) is None: + return False + mailbox = getattr(self.stop_binding, 'mailbox', None) + if mailbox is None: + return False + snapshot = mailbox.snapshot() + cp = getattr(snapshot, 'target', None) + if cp is None or cp.operation is None or cp.operation.get('kind') != 'effect': + return False + op_id = cp.operation.get('id') + if not op_id: + return False + if self.compiler_host is None or not self.compiler_host.lifecycle()['bound']: + return False + if execution_now() >= self.deadline or (self.deadline - execution_now()) < 5.0: + return False + if cp.state.get('attempts', 0) >= getattr(cp, 'max_attempts', 8): + return False + if not hasattr(self, 'recovery_records'): + self.recovery_records = {} + if (self.worker.pid, op_id) in self.recovery_records: + return False + + # Export terminal receipt from compiler host + try: + exported = await self.compiler_host.terminal_receipt(op_id) + except Exception: + return False + + self.recovery_records[(self.worker.pid, op_id)] = { + 'worker_pid': self.worker.pid, + 'operation_id': op_id, + 'receipt': exported, + } + + # 1. Durably save receipt to disk + reconcile_diagnostics = self.work / 'episodes' / f'reconcile-{self.worker.pid}' + reconcile_diagnostics.mkdir(parents=True, exist_ok=True) + explicit_receipt = reconcile_diagnostics / 'retained-terminal.json' + rec.save(explicit_receipt, exported) + + # 2. Retire old host binding + await self.compiler_host.retire(timeout=30) + + # 3. Run reconcile subcommand via child process + base_py = getattr(self, 'plan', {}).get('base_python', sys.executable) + reconcile_cmd = [ + base_py, '-I', '-B', '-m', + 'adaptive_inference.desktop.hermes.run_recovery', 'reconcile', + '--run', str(self.work), + '--diagnostics', str(reconcile_diagnostics), + '--receipt', str(explicit_receipt), + '--operation-id', op_id, + ] + reconciler = self.launch(reconcile_cmd, f'reconcile-{self.worker.pid}') + reconcile_code = await self.join_child(reconciler) + if reconcile_code != 0: + raise RuntimeError(f'Reconcile subcommand failed with exit code {reconcile_code}') + + # 4. Refresh runtime & verify checkpoint + await self.runtime.refresh() + snapshot = self.stop_binding.mailbox.snapshot() + repaired_cp = snapshot.target + if (repaired_cp.operation is not None or repaired_cp.requires_reconciliation + or repaired_cp.state.get('status') != 'pending' + or repaired_cp.state.get('phase') != 'ready'): + raise RuntimeError('Checkpoint not cleanly repaired after reconciliation') + + # 5. Replace worker via narrow recovered-state lifecycle entry + prev_binding = self.stop_binding + replacement_binding = self.facade.replace_recovered_worker( + prev_binding, self.create_replacement_binding + ) + + # 6. Synchronously update composition & handover references + self.stop_binding = replacement_binding + self.worker = replacement_binding.worker + self.worker_cap = replacement_binding._capability + self.stop_id = replacement_binding.request_id + self.worker_exit_verified = False + if getattr(self, 'handover', None) is not None: + self.handover.replace_worker_episode( + self.worker, self.stop_binding, self.worker_cap, self.stop_id, first_ready=self.first_ready + ) + + # 7. Await replacement bootstrap + await self.until( + lambda: (self.worker_diagnostics() / f'bootstrap-{self.worker.pid}.json').exists(), + self.worker + ) + + # 8. Re-bind compiler worker + await self.runtime.refresh() + await self.bind_compiler_worker(self.runtime, self.worker, self.work, self.deadline) + + # 9. Publish readiness + await self.ready(self.worker, self.first_ready) + + return True + async def interact(self): # No invented admission or automatic Resume: the worker's saved demand, # host capacity and authenticated GUI intent must agree first. @@ -841,6 +1018,8 @@ async def interact(self): self.remaining() if code is not None and not self.worker_exit_verified: await self.join_child(self.worker) + if await self._try_recover_pending_tool_worker(code): + continue outcome = self.stop_binding._verify_exit(code) self.worker_exit_verified = True self.result['worker_outcome'] = outcome diff --git a/src/adaptive_inference/desktop/desktop_controls.py b/src/adaptive_inference/desktop/desktop_controls.py index 37770b0..6bfcaad 100644 --- a/src/adaptive_inference/desktop/desktop_controls.py +++ b/src/adaptive_inference/desktop/desktop_controls.py @@ -213,6 +213,41 @@ def replace_worker(self, expected, create_replacement_binding): or snapshot.target.operation is not None or snapshot.target.requires_reconciliation or snapshot.demand is None or snapshot.demand['state'] != 'paused'): raise RuntimeError('Replacement requires the same known paused task') + return self._commit_replacement(expected, code, snapshot, create_replacement_binding) + + def replace_recovered_worker(self, expected, create_replacement_binding): + """Trusted synchronous replacement after automatic tool recovery. + + Shares synchronous launch, binding, and commit mechanics with replace_worker, + with narrow recovered-state preconditions requiring a reconciled pending task + with active/waiting/paused demand and ready phase. + """ + self._owner() + from adaptive_inference.desktop.hermes.service_stop import WorkerStopBinding + if (expected is not self.worker_stop or type(expected) is not WorkerStopBinding + or self._worker_capability is not expected._capability + or self._fenced or self._stop is not None or self._replacement_pending + or self._faults or self.host.closing or self.host.closed + or expected._work is not None or expected._fault is not None + or not callable(create_replacement_binding)): + raise RuntimeError('Replacement requires the exact current unfenced worker binding') + expected._owner() + code = expected.worker.poll() + if type(code) is not int or expected.worker.wait(timeout=0) != code: + raise RuntimeError('The exact previous worker must already have exited and joined') + snapshot = expected.mailbox.snapshot() + expected._original(snapshot.target) + if (snapshot.target.task_id != self.prepared.task_id + or execution_outcome(snapshot.target.state) is not None + or snapshot.target.state['status'] != 'pending' + or snapshot.target.operation is not None or snapshot.target.requires_reconciliation + or snapshot.target.state.get('phase') != 'ready' + or snapshot.demand is None or snapshot.demand['state'] not in ('active', 'waiting', 'paused')): + raise RuntimeError('Recovered replacement requires reconciled pending task with active/waiting/paused demand') + return self._commit_replacement(expected, code, snapshot, create_replacement_binding) + + def _commit_replacement(self, expected, code, snapshot, create_replacement_binding): + from adaptive_inference.desktop.hermes.service_stop import WorkerStopBinding # Failure after child creation must never let the old binding authorize # native close. Retain this obligation before calling fallible launch. record = {'previous_binding': expected, 'previous_exit_code': code, @@ -244,6 +279,14 @@ def replace_worker(self, expected, create_replacement_binding): self._worker_bindings.append((replacement, capability)) self.worker_stop, self._worker_capability = replacement, capability self.worker_receipt = None + if self.service is not None and getattr(self.service, 'current', None) is not None: + if hasattr(self.service, 'replace_worker_episode'): + self.service.replace_worker_episode(replacement.worker, replacement, capability, replacement.request_id) + else: + self.service.current['worker'] = replacement.worker + self.service.current['stop_binding'] = replacement + self.service.current['worker_cap'] = capability + self.service.current['stop_id'] = replacement.request_id record['committed'] = True self._replacement_pending = False return replacement diff --git a/src/adaptive_inference/desktop/task_handover.py b/src/adaptive_inference/desktop/task_handover.py index 93532c2..e27cc52 100644 --- a/src/adaptive_inference/desktop/task_handover.py +++ b/src/adaptive_inference/desktop/task_handover.py @@ -83,6 +83,17 @@ def capture_initial(self): self.current['facade'].service = self self.episodes.append(self.current) + def replace_worker_episode(self, worker, stop_binding, worker_cap, stop_id, *, first_ready=None): + """Update current episode references upon worker replacement.""" + if self.current is not None: + self.current['worker'] = worker + self.current['stop_binding'] = stop_binding + self.current['worker_cap'] = worker_cap + self.current['stop_id'] = stop_id + if first_ready is not None: + self.current['first_ready'] = first_ready + self.current['worker_exit_verified'] = False + def episode(self, runtime): for episode in self.episodes: if episode['runtime'] is runtime: diff --git a/tests/automatic_tool_recovery_lineage.py b/tests/automatic_tool_recovery_lineage.py new file mode 100644 index 0000000..31474dd --- /dev/null +++ b/tests/automatic_tool_recovery_lineage.py @@ -0,0 +1,89 @@ +"""Exact automatic-tool-recovery source view before historical pending-tool evidence. + +Reuse pending_tool_lineage's complete-file SHA and portable-AST projection. Retained +9242434 sources are data, never imported or executed. Current automatic recovery behavior +has separate tests; this view does not transfer historical qualification. +""" +import ast +import hashlib +import json +from pathlib import Path + +from paused_expiry_lineage import fingerprint + +ROOT = Path(__file__).resolve().parents[1] +ARCHIVE = 'provenance/automatic-tool-recovery' + + +def record(root=None): + root = ROOT if root is None else root + path = root / ARCHIVE / 'source-lineage.json' + return json.loads(path.read_bytes()) if path.exists() else None + + +def inputs(root=None): + root = ROOT if root is None else root + rows = record(root) + if rows is None: + return [] + return [root / ARCHIVE / 'source-lineage.json', + *(root / row['predecessor_source'] for row in rows['files'].values())] + + +def retained(relative, row, root): + expected = ARCHIVE + '/predecessor/' + relative + if (not relative.startswith(('src/adaptive_inference/', 'tests/')) + or '\\' in relative or any(part in ('', '.', '..') for part in relative.split('/')) + or row['predecessor_source'] != expected): + raise AssertionError('Changed automatic-tool-recovery predecessor path: ' + relative) + before = (root / expected).read_bytes() + if (hashlib.sha256(before).hexdigest() != row['predecessor_sha256'] + or fingerprint(ast.parse(before)) != row['predecessor_ast_sha256']): + raise AssertionError('Changed automatic-tool-recovery predecessor identity: ' + relative) + return before + + +def predecessor(relative, data, root=None): + """Project known complete successor bytes; unknown bytes remain unknown.""" + root = ROOT if root is None else root + rows = record(root) + row = None if rows is None else rows['files'].get(relative) + if row is None: + return data + before = retained(relative, row, root) + if hashlib.sha256(data).hexdigest() == row['successor_sha256']: + if fingerprint(ast.parse(data)) != row['successor_ast_sha256']: + raise AssertionError('Changed automatic-tool-recovery successor AST: ' + relative) + return before + return data + + +def runtime_view(current, root=None): + """Require the complete 77-module closure before restoring pending-tool sources.""" + root = ROOT if root is None else root + rows = record(root) + result = dict(current) + if rows is None: + return result + baseline = rows['baseline_runtime'] + if set(result) != set(baseline): + raise AssertionError('Changed automatic-tool-recovery runtime closure') + for name, pin in baseline.items(): + result[name] = predecessor('src/adaptive_inference/' + name, result[name], root) + if hashlib.sha256(result[name]).hexdigest() != pin: + raise AssertionError('Changed automatic-tool-recovery runtime source: ' + name) + return result + + +def restore(tree, relative, root=None): + """AST restoration also requires exact complete on-disk successor bytes.""" + root = ROOT if root is None else root + rows = record(root) + source = 'src/adaptive_inference/' + relative + row = None if rows is None else rows['files'].get(source) + if row is None or fingerprint(tree) != row['successor_ast_sha256']: + return tree + data = (root / source).read_bytes() + if hashlib.sha256(data).hexdigest() != row['successor_sha256']: + raise AssertionError('Changed complete reboot-budget source (automatic-tool-recovery source): ' + relative) + return ast.parse(predecessor(source, data, root)) diff --git a/tests/pending_tool_lineage.py b/tests/pending_tool_lineage.py index 03a7546..d5058d1 100644 --- a/tests/pending_tool_lineage.py +++ b/tests/pending_tool_lineage.py @@ -10,6 +10,7 @@ from pathlib import Path from paused_expiry_lineage import fingerprint +import automatic_tool_recovery_lineage ROOT = Path(__file__).resolve().parents[1] ARCHIVE = 'provenance/pending-tool-recovery' @@ -27,7 +28,8 @@ def inputs(root=None): if rows is None: return [] return [root / ARCHIVE / 'source-lineage.json', - *(root / row['predecessor_source'] for row in rows['files'].values())] + *(root / row['predecessor_source'] for row in rows['files'].values()), + *automatic_tool_recovery_lineage.inputs(root)] def retained(relative, row, root): @@ -46,6 +48,7 @@ def retained(relative, row, root): def predecessor(relative, data, root=None): """Project known complete successor bytes; unknown bytes remain unknown.""" root = ROOT if root is None else root + data = automatic_tool_recovery_lineage.predecessor(relative, data, root) rows = record(root) row = None if rows is None else rows['files'].get(relative) if row is None: @@ -66,6 +69,10 @@ def runtime_view(current, root=None): if rows is None: return result baseline = rows['baseline_runtime'] + if set(result) == set(baseline): + if {name: hashlib.sha256(data).hexdigest() for name, data in result.items()} == baseline: + return result + result = automatic_tool_recovery_lineage.runtime_view(result, root) if set(result) != set(baseline): raise AssertionError('Changed pending-tool-recovery runtime closure') for name, pin in baseline.items(): @@ -78,12 +85,13 @@ def runtime_view(current, root=None): def restore(tree, relative, root=None): """AST restoration also requires exact complete on-disk successor bytes.""" root = ROOT if root is None else root + tree = automatic_tool_recovery_lineage.restore(tree, relative, root) rows = record(root) source = 'src/adaptive_inference/' + relative row = None if rows is None else rows['files'].get(source) if row is None or fingerprint(tree) != row['successor_ast_sha256']: return tree - data = (root / source).read_bytes() + data = automatic_tool_recovery_lineage.predecessor(source, (root / source).read_bytes(), root) if hashlib.sha256(data).hexdigest() != row['successor_sha256']: raise AssertionError('Changed complete reboot-budget source (pending-tool-recovery source): ' + relative) return ast.parse(predecessor(source, data, root)) diff --git a/tests/test_automatic_tool_recovery_lineage.py b/tests/test_automatic_tool_recovery_lineage.py new file mode 100644 index 0000000..fd806f3 --- /dev/null +++ b/tests/test_automatic_tool_recovery_lineage.py @@ -0,0 +1,107 @@ +"""Automatic tool recovery preserves exact published pending-tool source identities.""" +import ast +from copy import deepcopy +import hashlib +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import automatic_tool_recovery_lineage as lineage +import pending_tool_lineage +from paused_expiry_lineage import fingerprint + + +class AutomaticToolRecoveryLineageTests(unittest.TestCase): + def current_runtime(self): + runtime = lineage.ROOT / 'src/adaptive_inference' + return {path.relative_to(runtime).as_posix(): path.read_bytes() + for path in runtime.rglob('*.py')} + + def test_exact_pending_tool_predecessors_and_evidence_remain_unchanged(self): + rows = lineage.record() + self.assertEqual(rows['predecessor_revision'], '9242434991f2c2c21461061d61d898649299b847') + self.assertTrue(rows['files']) + for relative, row in rows['files'].items(): + with self.subTest(relative=relative): + data = (lineage.ROOT / relative).read_bytes() + before = lineage.retained(relative, row, lineage.ROOT) + self.assertEqual(hashlib.sha256(data).hexdigest(), row['successor_sha256']) + self.assertEqual(fingerprint(ast.parse(data)), row['successor_ast_sha256']) + self.assertEqual(lineage.predecessor(relative, data), before) + self.assertEqual(lineage.predecessor(relative, before), before) + for relative, pin in rows['preserved_records'].items(): + self.assertEqual(hashlib.sha256((lineage.ROOT / relative).read_bytes()).hexdigest(), pin) + self.assertIn('provenance/pending-tool-recovery/source-lineage.json', rows['preserved_records']) + self.assertIn('provenance/pending-tool-recovery/installed-worker-loss-2026-11-02.json', + rows['preserved_records']) + + def test_unknown_complete_bytes_and_semantics_are_not_projected(self): + for relative in lineage.record()['files']: + data = (lineage.ROOT / relative).read_bytes() + for extra in (b'\n# unreviewed automatic source\n', b'\nunreviewed = True\n'): + changed = data + extra + with self.subTest(relative=relative, extra=extra): + self.assertEqual(lineage.predecessor(relative, changed), changed) + + def test_runtime_restores_all_77_pending_tool_modules_and_composes_idempotently(self): + rows = lineage.record() + self.assertEqual(len(rows['baseline_runtime']), 77) + current = self.current_runtime(); original = dict(current) + result = lineage.runtime_view(current) + self.assertEqual(current, original) + self.assertEqual({name: hashlib.sha256(data).hexdigest() for name, data in result.items()}, + rows['baseline_runtime']) + self.assertEqual(lineage.runtime_view(result), result) + prior = pending_tool_lineage.runtime_view(current) + self.assertEqual(len(prior), 77) + self.assertEqual(pending_tool_lineage.runtime_view(result), prior) + self.assertEqual(pending_tool_lineage.runtime_view(prior), prior) + for name in current: + changed = dict(current); changed[name] += b'\n# unreviewed\n' + with self.subTest(name=name), self.assertRaisesRegex(AssertionError, 'runtime source'): + lineage.runtime_view(changed) + changed = dict(current); del changed['task_spec.py'] + with self.assertRaisesRegex(AssertionError, 'runtime closure'): + lineage.runtime_view(changed) + with self.assertRaisesRegex(AssertionError, 'runtime closure'): + lineage.runtime_view({**current, 'unreviewed.py': b'pass\n'}) + + def test_ast_projection_requires_complete_current_bytes(self): + relative = 'desktop/composition.py' + source = 'src/adaptive_inference/' + relative + target = lineage.ROOT / source; data = target.read_bytes(); tree = ast.parse(data) + row = lineage.record()['files'][source] + self.assertEqual(fingerprint(lineage.restore(tree, relative)), row['predecessor_ast_sha256']) + self.assertEqual(fingerprint(tree), row['successor_ast_sha256']) + changed = ast.parse(data + b'\nunreviewed = True\n') + self.assertEqual(fingerprint(lineage.restore(changed, relative)), fingerprint(changed)) + original_read = Path.read_bytes + def read(path): + raw = original_read(path) + return raw + b'\n# same AST\n' if path == target else raw + with mock.patch.object(Path, 'read_bytes', read): + with self.assertRaisesRegex(AssertionError, 'automatic-tool-recovery source'): + lineage.restore(tree, relative) + + def test_predecessor_and_successor_identity_tampering_is_refused(self): + rows = lineage.record(); relative = 'src/adaptive_inference/desktop/composition.py' + data = (lineage.ROOT / relative).read_bytes() + for field in ('predecessor_source', 'predecessor_sha256', 'predecessor_ast_sha256', + 'successor_ast_sha256'): + changed = deepcopy(rows); changed['files'][relative][field] = '0' * 64 + with self.subTest(field=field), mock.patch.object(lineage, 'record', return_value=changed): + with self.assertRaisesRegex(AssertionError, 'automatic-tool-recovery'): + lineage.predecessor(relative, data) + + def test_missing_archive_never_authorizes_an_unbound_projection(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); data = b'pass\n'; tree = ast.parse(data) + self.assertEqual(lineage.inputs(root), []) + self.assertEqual(lineage.predecessor('src/adaptive_inference/task_spec.py', data, root), data) + self.assertEqual(lineage.runtime_view({'unknown.py': data}, root), {'unknown.py': data}) + self.assertIs(lineage.restore(tree, 'task_spec.py', root), tree) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_desktop_automatic_recovery.py b/tests/test_desktop_automatic_recovery.py new file mode 100644 index 0000000..4b682ff --- /dev/null +++ b/tests/test_desktop_automatic_recovery.py @@ -0,0 +1,676 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for Milestone 2: Automatic Desktop Tool Recovery (FEAT-RECOV-04). + +Covers UT-M2-01 through UT-M2-09: +- UT-M2-01: Compiler recipe automatic recovery on worker exit 86. +- UT-M2-02: Video recipe automatic recovery on worker exit 86. +- UT-M2-03: Clean worker exit (code 0) skips recovery hook. +- UT-M2-04: Worker exit during stop_requested skips recovery hook. +- UT-M2-05: In-flight or failed host tool raises ValueError, falling through to RecoveryStop. +- UT-M2-06: DesktopControlLifecycle.replace_recovered_worker accepts active demand and commits. +- UT-M2-07: DesktopComposition.compiler_launch_args generation & retirement bookkeeping. +- UT-M2-08: Monotonic deadline exhaustion (< 5.0s remaining) aborts recovery. +- UT-M2-09: Max attempts exhaustion aborts recovery. +""" +import asyncio +from copy import deepcopy +import json +import os +import sys +from pathlib import Path +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest import mock +import uuid + +ROOT = Path(__file__).resolve().parents[1] +for candidate in (ROOT / 'src', ROOT / 'tests'): + if candidate.is_dir() and str(candidate) not in sys.path: + sys.path.insert(0, str(candidate)) + +from adaptive_inference.desktop import composition as comp +from adaptive_inference.desktop.desktop_controls import DesktopControlLifecycle, PreparedTask +from adaptive_inference.desktop.hermes import recovery as rec +from adaptive_inference.desktop.hermes.service_stop import WorkerStopBinding, RecoveryStop +from adaptive_inference.execution_clock import execution_now, CLOCK_ID +from adaptive_inference.desktop.installation import sha +from adaptive_inference.task_store import TaskStore + + +class Child: + def __init__(self, pid=41, code=None): + self.pid = pid + self.code = code + self.done = threading.Event() + self.waits = 0 + if code is not None: + self.done.set() + + def poll(self): + return self.code + + def wait(self, timeout=None): + self.waits += 1 + if timeout == 0: + if not self.done.is_set(): + raise TimeoutError('Child still running') + return self.code + if not self.done.wait(timeout if timeout is not None else 3): + raise TimeoutError('Fixture child not released') + return self.code + + def finish(self, code): + self.code = code + self.done.set() + + def kill(self): + raise AssertionError('No product kill') + + def terminate(self): + raise AssertionError('No product terminate') + + +def make_case(root, *, video=False): + c = object.__new__(comp.DesktopComposition) + c.root = root / 'run' + c.root.mkdir(parents=True, exist_ok=True) + c.work = c.root / 'work' + c.work.mkdir(parents=True, exist_ok=True) + c.handles = [] + c.logs = [] + c.waits = {} + c.result = {'schema': 'adaptive.desktop.windows-orders.result.v1', 'clean_stop': False, + 'task_completed': False, 'execution_expired': False, + 'model': comp.MODEL, 'quality': deepcopy(comp.QUALITY)} + c.worker_exit_verified = False + c.continuation = None + c.handover = None + c.compiler_kind = 'video' if video else 'compiler' + c.compiler_bindings = {} + c.compiler_bindings_history = [] + c.compiler_generations = {} + c.recovery_records = {} + c.worker_replacements_count = 0 + c.compiler_native_reader = None + c.profile = SimpleNamespace(compiler=object() if not video else None, video=object() if video else None) + c.task_spec_path = None + c.first_ready = c.work / 'controller-ready-1.json' + c.host_epoch = 'epoch-test-1' + c.stop_id = 'stop-init-1' + c.saved_task = None + c.saved_diagnostics = None + c.saved = None + c.reopen_context = None + c.qualification_hold_after_report = False + c.box = SimpleNamespace(snapshot=mock.Mock()) + c.capability = object() + c.worker_cap = object() + c.fence = mock.Mock() + return c + + +class AutomaticDesktopRecoveryTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.case = make_case(self.root) + from test_compiler_task_spec import compiler_data + from adaptive_inference.task_spec import TaskSpec + spec = TaskSpec(compiler_data()) + self.case.plan = { + 'base_python': os.path.abspath(comp.sys.executable), + 'gateway': 'http://127.0.0.1:8000/v1', + 'task_spec': spec.data, + 'task_spec_sha256': spec.sha256 + } + self.case.identity = SimpleNamespace(task_id='task-auto-recov-01', registration_id='reg-01') + self.case.original = SimpleNamespace( + prompt='Build harness', + metadata={'deadline_clock': CLOCK_ID}, + max_attempts=4, + state={'messages': [{'role': 'user', 'content': 'Build harness'}], 'deadline_monotonic': 900.0, 'deadline_clock': CLOCK_ID} + ) + + def _setup_mock_host(self, c, *, bound=True, receipt=None): + host_mock = SimpleNamespace( + closing=False, closed=False, + controller=SimpleNamespace(fault=None, identity={'owner_id': 'ctrl-1'}), + gateway=SimpleNamespace(instance='inst-1'), + wait_ready=mock.AsyncMock() + ) + c.host = host_mock + c.transport = SimpleNamespace( + lifecycle=lambda: {'stop_requested': False, 'stop_confirmed': False}, + _stop_task=None + ) + c.registry = SimpleNamespace(observe=lambda rid: SimpleNamespace(eligible=True)) + + bound_state = {'bound': bound} + compiler_host = SimpleNamespace( + lifecycle=lambda: dict(bound_state), + terminal_receipt=mock.AsyncMock(return_value=receipt or { + 'operation_id': 'op-comp-1', + 'tool': 'clang-cl', + 'units': 23, + 'owned_processes_exited': True, + 'returncode': 0 + }), + retire=mock.AsyncMock(), + bind=mock.AsyncMock(), + interrupt=mock.Mock() + ) + async def fake_retire(timeout=30): + bound_state['bound'] = False + compiler_host.retire.side_effect = fake_retire + c.compiler_host = compiler_host + + async def test_ut_m2_01_worker_exit_86_compiler_recovery(self): + """UT-M2-01: Worker exits with code 86 while host holds compiler terminal result.""" + c = self.case + c.deadline = execution_now() + 600.0 + worker_1 = Child(pid=101, code=86) + c.handles = [worker_1] + c.worker = worker_1 + c.worker_exit_verified = False + + receipt_data = { + 'operation_id': 'op-comp-1', + 'tool': 'clang-cl', + 'units': 23, + 'owned_processes_exited': True, + 'returncode': 0 + } + self._setup_mock_host(c, bound=True, receipt=receipt_data) + + # Mock TaskStore checkpoint target + cp_target = SimpleNamespace( + task_id=c.identity.task_id, + revision=1, + max_attempts=4, + operation={'id': 'op-comp-1', 'kind': 'effect', 'tool': 'clang-cl'}, + requires_reconciliation=True, + state={'status': 'pending', 'phase': 'tool_running', 'attempts': 1, 'tools': []} + ) + mailbox_mock = SimpleNamespace( + snapshot=lambda: SimpleNamespace(target=cp_target, requests=[], demand={'state': 'active'}) + ) + stop_binding = SimpleNamespace( + worker=worker_1, + mailbox=mailbox_mock, + _verify_exit=mock.Mock(return_value='task_completed'), + run=c.root + ) + c.stop_binding = stop_binding + + # Mock reconcile step to transition cp_target to repaired + worker_2 = Child(pid=102, code=None) + replacement_binding = SimpleNamespace( + worker=worker_2, + mailbox=mailbox_mock, + _capability=object(), + request_id='stop-req-2', + _verify_exit=mock.Mock(return_value='task_completed'), + run=c.root + ) + + def mock_launch(cmd, label): + if 'reconcile' in label: + reconciler = Child(pid=201, code=0) + c.handles.append(reconciler) + # Repair checkpoint + cp_target.operation = None + cp_target.requires_reconciliation = False + cp_target.state['phase'] = 'ready' + cp_target.state['status'] = 'pending' + return reconciler + elif 'worker' in label: + # Write bootstrap file for worker 2 + boot_path = c.worker_diagnostics() / f'bootstrap-{worker_2.pid}.json' + boot_path.parent.mkdir(parents=True, exist_ok=True) + boot_path.write_text(json.dumps({ + 'pid': worker_2.pid, 'task_id': c.identity.task_id, + 'registration_id': c.identity.registration_id, + 'deadline_monotonic': c.deadline, + 'deadline_clock': CLOCK_ID, + 'owner_epoch': 'epoch-1' + })) + c.handles.append(worker_2) + return worker_2 + child = Child(pid=301, code=0) + c.handles.append(child) + return child + + c.launch = mock.Mock(side_effect=mock_launch) + def fake_bind_worker_stop(): + c.stop_binding = replacement_binding + c.worker_cap = replacement_binding._capability + c.bind_worker_stop = fake_bind_worker_stop + def fake_replace(expected, cb): + cb() + return replacement_binding + c.facade = SimpleNamespace( + replace_recovered_worker=mock.Mock(side_effect=fake_replace), + cooperative_stop=mock.AsyncMock(return_value={'worker_joined': True, 'host_closed': True, 'uncertain': False, 'fault': None}) + ) + c.runtime = SimpleNamespace( + refresh=mock.AsyncMock(), _transitioning=False, + prepared=SimpleNamespace(task_id=c.identity.task_id), + selected_execution=lambda: {'execution_id': c.identity.task_id} + ) + c.gui = SimpleNamespace(process=Child(pid=42)) + c.original.state['deadline_monotonic'] = c.deadline + c.checkpoint = mock.Mock(return_value=SimpleNamespace( + revision=2, + owner_epoch='epoch-1', + prompt=c.original.prompt, + metadata=c.original.metadata, + state=deepcopy(c.original.state) + )) + + # Run recovery helper directly + recovered = await c._try_recover_pending_tool_worker(86) + self.assertTrue(recovered) + self.assertIn((101, 'op-comp-1'), c.recovery_records) + c.compiler_host.terminal_receipt.assert_called_once_with('op-comp-1') + c.compiler_host.retire.assert_called_once() + c.facade.replace_recovered_worker.assert_called_once() + self.assertIs(c.worker, worker_2) + self.assertIs(c.stop_binding, replacement_binding) + + async def test_ut_m2_02_worker_exit_86_video_recovery(self): + """UT-M2-02: Worker exits with code 86 while host holds video terminal result.""" + c = self.case + c.deadline = execution_now() + 600.0 + worker_1 = Child(pid=111, code=86) + c.handles = [worker_1] + c.worker = worker_1 + c.worker_exit_verified = False + + video_receipt = { + 'operation_id': 'op-vid-99', + 'tool': 'ffmpeg', + 'frames': 90, + 'owned_processes_exited': True, + 'returncode': 0 + } + self._setup_mock_host(c, bound=True, receipt=video_receipt) + + cp_target = SimpleNamespace( + task_id=c.identity.task_id, + revision=1, + max_attempts=4, + operation={'id': 'op-vid-99', 'kind': 'effect', 'tool': 'ffmpeg'}, + requires_reconciliation=True, + state={'status': 'pending', 'phase': 'tool_running', 'attempts': 1, 'tools': []} + ) + mailbox_mock = SimpleNamespace( + snapshot=lambda: SimpleNamespace(target=cp_target, requests=[], demand={'state': 'waiting'}) + ) + stop_binding = SimpleNamespace( + worker=worker_1, + mailbox=mailbox_mock, + _verify_exit=mock.Mock(return_value='task_completed'), + run=c.root + ) + c.stop_binding = stop_binding + + worker_2 = Child(pid=112, code=None) + replacement_binding = SimpleNamespace( + worker=worker_2, + mailbox=mailbox_mock, + _capability=object(), + request_id='stop-req-video-2', + _verify_exit=mock.Mock(return_value='task_completed'), + run=c.root + ) + + def mock_launch(cmd, label): + if 'reconcile' in label: + reconciler = Child(pid=202, code=0) + c.handles.append(reconciler) + cp_target.operation = None + cp_target.requires_reconciliation = False + cp_target.state['phase'] = 'ready' + cp_target.state['status'] = 'pending' + return reconciler + elif 'worker' in label: + boot_path = c.worker_diagnostics() / f'bootstrap-{worker_2.pid}.json' + boot_path.parent.mkdir(parents=True, exist_ok=True) + boot_path.write_text(json.dumps({ + 'pid': worker_2.pid, 'task_id': c.identity.task_id, + 'registration_id': c.identity.registration_id, + 'deadline_monotonic': c.deadline, + 'deadline_clock': CLOCK_ID, + 'owner_epoch': 'epoch-video' + })) + c.handles.append(worker_2) + return worker_2 + child = Child(pid=302, code=0) + c.handles.append(child) + return child + + from test_video_task_spec import video_data + from adaptive_inference.task_spec import TaskSpec + video_spec = TaskSpec(video_data()) + c.plan['task_spec'] = video_spec.data + c.plan['task_spec_sha256'] = video_spec.sha256 + c.compiler_kind = 'video' + c.profile.video = object() + c.profile.compiler = None + + c.launch = mock.Mock(side_effect=mock_launch) + def fake_bind_worker_stop(): + c.stop_binding = replacement_binding + c.worker_cap = replacement_binding._capability + c.bind_worker_stop = fake_bind_worker_stop + def fake_replace(expected, cb): + cb() + return replacement_binding + c.facade = SimpleNamespace( + replace_recovered_worker=mock.Mock(side_effect=fake_replace), + cooperative_stop=mock.AsyncMock() + ) + c.runtime = SimpleNamespace( + refresh=mock.AsyncMock(), _transitioning=False, + prepared=SimpleNamespace(task_id=c.identity.task_id), + selected_execution=lambda: {'execution_id': c.identity.task_id} + ) + c.gui = SimpleNamespace(process=Child(pid=43)) + c.original.state['deadline_monotonic'] = c.deadline + c.checkpoint = mock.Mock(return_value=SimpleNamespace( + revision=2, + owner_epoch='epoch-video', + prompt=c.original.prompt, + metadata=c.original.metadata, + state=deepcopy(c.original.state) + )) + + recovered = await c._try_recover_pending_tool_worker(86) + self.assertTrue(recovered) + self.assertIn((111, 'op-vid-99'), c.recovery_records) + self.assertEqual(c.recovery_records[(111, 'op-vid-99')]['receipt']['frames'], 90) + + async def test_ut_m2_03_clean_worker_exit_0_skips_recovery(self): + """UT-M2-03: Clean worker exit (code 0) skips recovery hook.""" + c = self.case + c.deadline = execution_now() + 600.0 + worker = Child(pid=103, code=0) + c.worker = worker + self._setup_mock_host(c, bound=True) + + recovered = await c._try_recover_pending_tool_worker(0) + self.assertFalse(recovered) + c.compiler_host.terminal_receipt.assert_not_called() + + async def test_ut_m2_04_worker_exit_during_stop_skips_recovery(self): + """UT-M2-04: Worker exit while stop was requested skips recovery hook.""" + c = self.case + c.deadline = execution_now() + 600.0 + worker = Child(pid=104, code=3) + c.worker = worker + self._setup_mock_host(c, bound=True) + # Stop requested + c.transport.lifecycle = lambda: {'stop_requested': True, 'stop_confirmed': False} + + recovered = await c._try_recover_pending_tool_worker(3) + self.assertFalse(recovered) + c.compiler_host.terminal_receipt.assert_not_called() + + async def test_ut_m2_05_host_tool_in_flight_or_failed_aborts_recovery(self): + """UT-M2-05: Host tool still in-flight or failed causes terminal_receipt to fail, aborting recovery.""" + c = self.case + c.deadline = execution_now() + 600.0 + worker = Child(pid=105, code=86) + c.worker = worker + self._setup_mock_host(c, bound=True) + # terminal_receipt raises ValueError + c.compiler_host.terminal_receipt.side_effect = ValueError('Exact joined terminal recipe result is unavailable') + + cp_target = SimpleNamespace( + task_id=c.identity.task_id, + revision=1, + max_attempts=4, + operation={'id': 'op-comp-inflight', 'kind': 'effect', 'tool': 'clang-cl'}, + requires_reconciliation=True, + state={'status': 'pending', 'phase': 'tool_running', 'attempts': 1, 'tools': []} + ) + mailbox_mock = SimpleNamespace( + snapshot=lambda: SimpleNamespace(target=cp_target, requests=[], demand={'state': 'active'}) + ) + c.stop_binding = SimpleNamespace(worker=worker, mailbox=mailbox_mock) + + recovered = await c._try_recover_pending_tool_worker(86) + self.assertFalse(recovered) + # Binding was NOT retired since recovery was aborted + c.compiler_host.retire.assert_not_called() + + async def test_ut_m2_06_lifecycle_replace_recovered_worker_accepts_active_demand(self): + """UT-M2-06: DesktopControlLifecycle.replace_recovered_worker accepts active/waiting demand and commits.""" + old_worker = Child(pid=106, code=86) + old_cap = object() + prepared = PreparedTask('epoch-host', 'task-lifecycle', 'Objective') + + cp = SimpleNamespace( + task_id='task-lifecycle', + revision=2, + operation=None, + requires_reconciliation=False, + owner_epoch=None, + state={'status': 'pending', 'phase': 'ready', 'messages': [{'role': 'user', 'content': 'Objective'}]} + ) + snapshot = SimpleNamespace(target=cp, requests=[], demand={'state': 'active'}) + mailbox_mock = SimpleNamespace(snapshot=lambda: snapshot, store=SimpleNamespace(path=':memory:')) + + old_binding = object.__new__(WorkerStopBinding) + old_binding.worker = old_worker + old_binding._capability = old_cap + old_binding._work = None + old_binding._fault = None + old_binding.mailbox = mailbox_mock + old_binding._owner = lambda: None + old_binding._original = lambda cp: None + old_binding.deadline = 900.0 + old_binding.run = self.root + old_binding.model = 'model-1' + old_binding.quality = {} + old_binding.request_id = 'stop-1' + old_binding._requester = object() + old_binding.original = object() + + gui_mock = SimpleNamespace(session_live=mock.Mock(), window_live=mock.Mock(), + revoke_session=mock.Mock(), join=mock.AsyncMock()) + host_mock = SimpleNamespace(closing=False, closed=False) + + lifecycle = DesktopControlLifecycle(host_mock, prepared, worker_stop=old_binding, + worker_capability=old_cap, gui=gui_mock) + + # Confirm old replace_worker rejects active demand + with self.assertRaisesRegex(RuntimeError, 'known paused task'): + lifecycle.replace_worker(old_binding, lambda: None) + + # Confirm replace_recovered_worker succeeds with active demand + new_worker = Child(pid=107, code=None) + new_cap = object() + new_binding = object.__new__(WorkerStopBinding) + new_binding.worker = new_worker + new_binding._capability = new_cap + new_binding._work = None + new_binding._fault = None + new_binding.mailbox = mailbox_mock + new_binding._requester = old_binding._requester + new_binding.original = old_binding.original + new_binding.run = old_binding.run + new_binding.deadline = 900.0 + new_binding.model = old_binding.model + new_binding.quality = old_binding.quality + new_binding.request_id = 'stop-2' + new_binding._loop = lifecycle._loop + new_binding._thread = lifecycle._thread + + # Attach mock service (HostTaskHandover) + service_mock = SimpleNamespace(current={}, replace_worker_episode=mock.Mock()) + lifecycle.service = service_mock + + res = lifecycle.replace_recovered_worker(old_binding, lambda: (new_binding, new_cap)) + self.assertIs(res, new_binding) + self.assertIs(lifecycle.worker_stop, new_binding) + self.assertIs(lifecycle._worker_capability, new_cap) + service_mock.replace_worker_episode.assert_called_once_with(new_worker, new_binding, new_cap, 'stop-2') + + async def test_ut_m2_07_compiler_launch_args_retirement_and_generation(self): + """UT-M2-07: DesktopComposition.compiler_launch_args generation & retirement bookkeeping.""" + c = self.case + worker = Child(pid=108, code=86) + c.worker = worker + c.handles = [worker] + c.waits[worker] = asyncio.create_task(asyncio.sleep(0, result=86)) + await c.waits[worker] + + from test_compiler_task_spec import compiler_data + from adaptive_inference.task_spec import TaskSpec + spec = TaskSpec(compiler_data()) + c.plan = {'base_python': os.path.abspath(comp.sys.executable), 'gateway': 'http://127.0.0.1:8000/v1', + 'task_spec': spec.data, 'task_spec_sha256': spec.sha256} + + self._setup_mock_host(c, bound=True) + c.host_epoch = 'epoch-host-7' + c.compiler_kind = 'compiler' + + # First launch args call + args_1 = c.compiler_launch_args(c.work, c.plan, execution_id='exec-1', task_id='task-1', + deadline=900.0, diagnostics=c.work) + self.assertTrue(any('--compiler-binding' in arg for arg in args_1)) + + # Second launch args while host is still bound raises RuntimeError + with self.assertRaisesRegex(RuntimeError, 'cannot be reused'): + c.compiler_launch_args(c.work, c.plan, execution_id='exec-1', task_id='task-1', + deadline=900.0, diagnostics=c.work) + + # Retire host + await c.compiler_host.retire() + self.assertFalse(c.compiler_host.lifecycle()['bound']) + + # Now second launch args succeeds with incremented generation and fresh capability + args_2 = c.compiler_launch_args(c.work, c.plan, execution_id='exec-1', task_id='task-1', + deadline=900.0, diagnostics=c.work) + self.assertTrue(any('--compiler-binding' in arg for arg in args_2)) + self.assertEqual(len(c.compiler_bindings_history), 1) + self.assertEqual(c.compiler_generations[('task-1', 'exec-1')], 1) + + async def test_ut_m2_08_monotonic_deadline_exhaustion_aborts_recovery(self): + """UT-M2-08: Monotonic deadline exhaustion (< 5.0s remaining) aborts recovery.""" + c = self.case + # Deadline expired or remaining < 5.0 + c.deadline = execution_now() + 3.0 + worker = Child(pid=109, code=86) + c.worker = worker + self._setup_mock_host(c, bound=True) + + cp_target = SimpleNamespace( + task_id=c.identity.task_id, + revision=1, + max_attempts=4, + operation={'id': 'op-comp-8', 'kind': 'effect', 'tool': 'clang-cl'}, + requires_reconciliation=True, + state={'status': 'pending', 'phase': 'tool_running', 'attempts': 1, 'tools': []} + ) + mailbox_mock = SimpleNamespace( + snapshot=lambda: SimpleNamespace(target=cp_target, requests=[], demand={'state': 'active'}) + ) + c.stop_binding = SimpleNamespace(worker=worker, mailbox=mailbox_mock) + + recovered = await c._try_recover_pending_tool_worker(86) + self.assertFalse(recovered) + c.compiler_host.terminal_receipt.assert_not_called() + + async def test_ut_m2_09_max_attempts_exceeded_aborts_recovery(self): + """UT-M2-09: Repeated worker loss reaching max_attempts aborts recovery.""" + c = self.case + c.deadline = execution_now() + 600.0 + worker = Child(pid=110, code=86) + c.worker = worker + self._setup_mock_host(c, bound=True) + + cp_target = SimpleNamespace( + task_id=c.identity.task_id, + revision=1, + max_attempts=4, + operation={'id': 'op-comp-9', 'kind': 'effect', 'tool': 'clang-cl'}, + requires_reconciliation=True, + state={'status': 'pending', 'phase': 'tool_running', 'attempts': 4, 'tools': []} + ) + mailbox_mock = SimpleNamespace( + snapshot=lambda: SimpleNamespace(target=cp_target, requests=[], demand={'state': 'active'}) + ) + c.stop_binding = SimpleNamespace(worker=worker, mailbox=mailbox_mock) + + recovered = await c._try_recover_pending_tool_worker(86) + self.assertFalse(recovered) + c.compiler_host.terminal_receipt.assert_not_called() + + async def test_ut_m2_10_reopen_context_and_durable_budget_preserved_across_automatic_recovery(self): + """UT-M2-10: Reopen context and durable execution_budget.v1 are preserved across replacement launch.""" + from adaptive_inference.desktop.execution_budget import ( + BUDGET, create_budget, reopen_budget, tighten_budget, validate_budget, + ) + c = self.case + c.deadline = 608.0 + boot_a = 'a' * 64 + boot_b = {'boot_utc': '2026-11-08T12:00:00Z', 'computer_name': 'TEST-HOST'} + boot_b_digest = rec.digest(boot_b) + obs0 = {'utc_ns': 1_793_568_000_000_000_000, 'qpc_before': 100.0, 'qpc_after': 102.0} + obs1 = {'utc_ns': 1_793_568_200_000_000_000, 'qpc_before': 300.0, 'qpc_after': 302.0} + obs2 = {'utc_ns': 1_793_568_300_000_000_000, 'qpc_before': 10.0, 'qpc_after': 12.0} + budget = reopen_budget( + tighten_budget(create_budget(boot_id=boot_a, deadline=1000.0, observation=obs0), + boot_id=boot_a, observation=obs1), + boot_id=boot_b_digest, observation=obs2, + ) + c.reopen_context = {'boot': boot_b} + c.saved = {'task_id': c.identity.task_id} + c.saved_task = c.work + c.saved_diagnostics = self.root / 'saved-episodes' + c.saved_diagnostics.mkdir(parents=True, exist_ok=True) + c.original.state['deadline_monotonic'] = c.deadline + c.original.state[BUDGET] = deepcopy(budget) + + worker_1 = Child(pid=121, code=86) + c.handles = [worker_1] + c.worker = worker_1 + c.waits[worker_1] = asyncio.create_task(asyncio.sleep(0, result=86)) + await c.waits[worker_1] + self._setup_mock_host(c, bound=True) + await c.compiler_host.retire() + + launched_commands = [] + def mock_launch(cmd, label): + launched_commands.append((label, list(cmd))) + child = Child(pid=122, code=None) + c.handles.append(child) + return child + + c.launch = mock.Mock(side_effect=mock_launch) + def fake_bind_worker_stop(): + c.stop_binding = SimpleNamespace(worker=c.worker, request_id=c.stop_id) + c.bind_worker_stop = fake_bind_worker_stop + binding, cap = c.create_replacement_binding() + self.assertIsNotNone(binding) + self.assertIsNotNone(cap) + self.assertEqual(len(launched_commands), 1) + label, cmd = launched_commands[0] + self.assertEqual(label, 'worker-2') + self.assertIn('--execution-boot-id', cmd) + self.assertEqual(cmd[cmd.index('--execution-boot-id') + 1], boot_b_digest) + self.assertIn('--diagnostics', cmd) + self.assertEqual(cmd[cmd.index('--diagnostics') + 1], str(c.saved_diagnostics)) + self.assertIn('--deadline-monotonic', cmd) + self.assertEqual(float(cmd[cmd.index('--deadline-monotonic') + 1]), c.deadline) + self.assertEqual(validate_budget(c.original.state[BUDGET]), budget) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_desktop_composition.py b/tests/test_desktop_composition.py index 08a14ec..e617503 100644 --- a/tests/test_desktop_composition.py +++ b/tests/test_desktop_composition.py @@ -5,6 +5,7 @@ from pathlib import Path import json import os +import sys import tempfile import threading import time @@ -12,6 +13,13 @@ import unittest from unittest import mock +WORKTREE_SRC = r"D:\AI\Milestone-20261024-Handoff\pending-tool-recovery-work\src" +WORKTREE_TESTS = r"D:\AI\Milestone-20261024-Handoff\pending-tool-recovery-work\tests" +if os.path.isdir(WORKTREE_SRC) and WORKTREE_SRC not in sys.path: + sys.path.append(WORKTREE_SRC) +if os.path.isdir(WORKTREE_TESTS) and WORKTREE_TESTS not in sys.path: + sys.path.append(WORKTREE_TESTS) + from adaptive_inference.desktop import composition as q from adaptive_inference.desktop.hermes.control_bridge import prepare_task from test_desktop_installation import profile_value @@ -231,6 +239,192 @@ async def test_saved_task_reuses_progress_deadline_and_fresh_paused_owner(self): self.assertEqual(old.read_text(),'{"old":true}') self.assertEqual(self.case.stop_binding.diagnostics,self.case.saved_diagnostics) + async def test_it_m2_01_automatic_recovery_compiler_harness(self): + """IT-M2-01: End-to-end DesktopComposition.interact automatic recovery with real TaskStore and Mailbox.""" + from test_compiler_task_spec import compiler_data + c = self.case + c.work = self.root / 'run' + c.work.mkdir(parents=True, exist_ok=True) + (c.work / 'work').mkdir(parents=True, exist_ok=True) + spec = TaskSpec(compiler_data()) + c.plan = { + 'task_id': 'task-it-1', + 'base_python': os.path.abspath(q.sys.executable), + 'gateway': 'http://127.0.0.1:8000/v1', + 'task_spec': spec.data, + 'task_spec_sha256': spec.sha256, + } + c.task_spec_path = None + c.compiler_kind = 'compiler' + c.compiler_bindings = {} + c.compiler_bindings_history = [] + c.compiler_generations = {} + c.recovery_records = {} + c.worker_replacements_count = 0 + c.deadline = q.execution_now() + 600.0 + c.capability = object() + c.host_epoch = 'epoch-it' + c.first_ready = c.work / 'controller-ready-1.json' + c.reopen_context = None + c.saved = None + c.qualification_hold_after_report = False + + # Set up real TaskStore SQLite & DemandRegistry & Mailbox + db_path = c.work / 'task.sqlite3' + store = q.TaskStore(db_path) + identity = q.DemandIdentity('reg-it-1', 'task-it-1', 'worker', '2026-11-08T00:00:00Z') + c.identity = SimpleNamespace(task_id='task-it-1', registration_id='reg-it-1') + cp = store.create('task-it-1', spec.prompt, max_attempts=8, metadata={ + q.NAMESPACE: identity.metadata(), + 'model': q.MODEL, 'quality': q.QUALITY, 'deadline_clock': q.CLOCK_ID, **spec.metadata() + }) + registry = q.DemandRegistry((q.Registration(store, identity),)) + c.registry = registry + mailbox = q.Mailbox(registry, 'reg-it-1', requester_capability=c.capability) + + # Advance checkpoint to simulate in-flight effect operation + with store.claim('task-it-1', expected_revision=cp.revision) as owned: + registry.transition('reg-it-1', owned, 'active', expected_revision=owned.checkpoint.revision, updated_at_utc='2026-11-08T00:00:00Z') + st = deepcopy(owned.checkpoint.state) + st.update(status='pending', attempts=1) + effect = owned.begin_operation('effect', details={'tool': 'clang-cl'}, state=st, expected_revision=owned.checkpoint.revision) + op_id = effect.operation['id'] + + worker_1 = Child(pid=501, code=86) + c.handles = [worker_1] + c.worker = worker_1 + c.worker_exit_verified = False + + c.stop_binding = SimpleNamespace( + worker=worker_1, + mailbox=mailbox, + run=c.work, + request_id='stop-it-1', + _verify_exit=lambda code: 'task_completed' if code == 0 else 'unexpected_exit' + ) + c.stop_id = 'stop-it-1' + + receipt_data = { + 'operation_id': op_id, + 'tool': 'clang-cl', + 'units': 5, + 'owned_processes_exited': True, + 'returncode': 0 + } + bound_state = {'bound': True} + compiler_host = SimpleNamespace( + lifecycle=lambda: dict(bound_state), + terminal_receipt=mock.AsyncMock(return_value=receipt_data), + retire=mock.AsyncMock(), + bind=mock.AsyncMock(), + interrupt=mock.Mock() + ) + async def fake_retire(timeout=30): + bound_state['bound'] = False + compiler_host.retire.side_effect = fake_retire + c.compiler_host = compiler_host + + worker_2 = Child(pid=502, code=None) + replacement_binding = SimpleNamespace( + worker=worker_2, + mailbox=mailbox, + run=c.work, + _capability=object(), + request_id='stop-it-2', + _verify_exit=lambda code: 'task_completed' if code == 0 else 'unexpected_exit' + ) + + def mock_launch(cmd, label): + if 'reconcile' in label: + reconciler = Child(pid=601, code=0) + c.handles.append(reconciler) + # Repair real SQLite checkpoint via reconcile_operation + latest = store.load('task-it-1') + with store.claim('task-it-1', expected_revision=latest.revision) as owned: + st = deepcopy(owned.checkpoint.state) + st.update(phase='ready', status='pending') + owned.reconcile_operation(op_id, resolution='completed', result=receipt_data, state=st, expected_revision=owned.checkpoint.revision) + return reconciler + elif 'worker' in label: + boot_path = c.worker_diagnostics() / f'bootstrap-{worker_2.pid}.json' + boot_path.parent.mkdir(parents=True, exist_ok=True) + boot_path.write_text(json.dumps({ + 'pid': worker_2.pid, + 'task_id': c.identity.task_id, + 'registration_id': c.identity.registration_id, + 'deadline_monotonic': c.deadline, + 'deadline_clock': q.CLOCK_ID, + 'owner_epoch': 'epoch-it' + })) + c.handles.append(worker_2) + return worker_2 + ch = Child(pid=701, code=0) + c.handles.append(ch) + return ch + + c.launch = mock.Mock(side_effect=mock_launch) + def fake_bind_worker_stop(): + c.stop_binding = replacement_binding + c.worker_cap = replacement_binding._capability + c.bind_worker_stop = fake_bind_worker_stop + def fake_replace(expected, cb): + cb() + return replacement_binding + c.facade = SimpleNamespace( + replace_recovered_worker=mock.Mock(side_effect=fake_replace), + cooperative_stop=mock.AsyncMock(return_value={'worker_joined': True, 'host_closed': True, 'uncertain': False, 'fault': None}) + ) + c.checkpoint = mock.Mock(return_value=SimpleNamespace( + revision=5, + owner_epoch='epoch-it', + prompt=spec.prompt, + metadata=spec.metadata(), + state=deepcopy(cp.state) + )) + c.assert_original = mock.Mock() + c.host = SimpleNamespace( + closing=False, closed=False, + controller=SimpleNamespace(fault=None, identity={'owner_id': 'ctrl-it'}), + gateway=SimpleNamespace(instance='inst-it'), + wait_ready=mock.AsyncMock() + ) + c.gui = SimpleNamespace(process=Child(pid=42)) + transport_state = {'stop_requested': False, 'stop_confirmed': False} + c.transport = SimpleNamespace(lifecycle=lambda: dict(transport_state), _stop_task=None) + c.runtime = SimpleNamespace( + refresh=mock.AsyncMock(), _transitioning=False, + prepared=SimpleNamespace(task_id=c.identity.task_id), + selected_execution=lambda: {'execution_id': c.identity.task_id} + ) + c.bind_compiler_worker = mock.AsyncMock() + + interact_task = asyncio.create_task(c.interact()) + # Let interact() run through worker_1 joining and recovery + await asyncio.sleep(0.05) + self.assertIn((501, op_id), c.recovery_records) + self.assertIs(c.worker, worker_2) + + # Now simulate worker_2 completing cleanly + worker_2.finish(0) + # Wait for interact to observe worker_2 completion + for _ in range(50): + if c.result['task_completed']: + break + await asyncio.sleep(0.01) + self.assertTrue(c.result['task_completed']) + + # End interact via stop + transport_state.update(stop_requested=True, stop_confirmed=True) + c.transport._stop_task = asyncio.create_task(asyncio.sleep(0)) + await asyncio.wait_for(interact_task, timeout=2.0) + + # Verify real SQLite store was repaired + final_cp = store.load('task-it-1') + self.assertIsNone(final_cp.operation) + self.assertFalse(final_cp.requires_reconciliation) + self.assertEqual(final_cp.state['status'], 'pending') + self.assertEqual(final_cp.state['phase'], 'ready') + async def _prepare_order(self,spec=None,reopened=False): from adaptive_inference.desktop import owned_job, activity, cold_start, desktop_controls, gui_launch c=self.case;events=[];children=set();owned_context=[] diff --git a/tests/test_pending_tool_lineage.py b/tests/test_pending_tool_lineage.py index 6cc3612..bc1b8fb 100644 --- a/tests/test_pending_tool_lineage.py +++ b/tests/test_pending_tool_lineage.py @@ -8,6 +8,7 @@ from unittest import mock import pending_tool_lineage as lineage +import automatic_tool_recovery_lineage import recipe_reconciliation_lineage from paused_expiry_lineage import fingerprint @@ -24,7 +25,8 @@ def test_exact_recipe_predecessors_and_evidence_remain_unchanged(self): self.assertTrue(rows['files']) for relative, row in rows['files'].items(): with self.subTest(relative=relative): - data = (lineage.ROOT / relative).read_bytes() + data = automatic_tool_recovery_lineage.predecessor( + relative, (lineage.ROOT / relative).read_bytes()) before = lineage.retained(relative, row, lineage.ROOT) self.assertEqual(hashlib.sha256(data).hexdigest(), row['successor_sha256']) self.assertEqual(fingerprint(ast.parse(data)), row['successor_ast_sha256']) @@ -70,7 +72,8 @@ def test_runtime_restores_all_77_recipe_modules_and_composes_idempotently(self): def test_ast_projection_requires_complete_current_bytes(self): relative = 'desktop/hermes/compiler_recovery.py' source = 'src/adaptive_inference/' + relative - target = lineage.ROOT / source; data = target.read_bytes(); tree = ast.parse(data) + target = lineage.ROOT / source + data = automatic_tool_recovery_lineage.predecessor(source, target.read_bytes()); tree = ast.parse(data) row = lineage.record()['files'][source] self.assertEqual(fingerprint(lineage.restore(tree, relative)), row['predecessor_ast_sha256']) self.assertEqual(fingerprint(tree), row['successor_ast_sha256']) From f4ac2853030e6f0e33cdb3e02f052e796ae1d9e9 Mon Sep 17 00:00:00 2001 From: Midhat Nashar <35053211+midhatn@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:11:58 +0300 Subject: [PATCH 2/3] Qualify and document automatic desktop tool recovery Fix two launch tests whose inert tool-host doubles lacked lifecycle(); tests only, all 77 runtime modules unchanged. Both modules are added to the automatic-tool-recovery source lineage. Add sanitized evidence: local validation (1,595 core tests, 16 workflow commands with 90 provenance tests, package and isolated install; the failed first core run is retained), the installed SDK automatic recovery trial (21 closed-review checks) and the physical IQ3_S foreground recovery trial (49 closed-audit checks; two earlier failed attempts and the observer shutdown failure retained). Update HANDOFF, PLAN, evidence and video recipe docs, and cite Erlang/OTP supervision as a design reference. --- HANDOFF.md | 18 +- PLAN.md | 13 + REFERENCES.bib | 7 + docs/evidence.md | 50 ++ docs/video-tool-recipe.md | 29 + provenance/automatic-tool-recovery/README.md | 24 +- .../installed-worker-loss-2026-11-08.json | 285 +++++++++ .../local-validation-2026-11-08.json | 597 ++++++++++++++++++ ...ysical-foreground-recovery-2026-11-08.json | 344 ++++++++++ .../tests/test_compiler_composition.py | 124 ++++ .../tests/test_video_composition.py | 50 ++ .../source-lineage.json | 14 + tests/test_compiler_composition.py | 2 +- tests/test_video_composition.py | 2 +- 14 files changed, 1553 insertions(+), 6 deletions(-) create mode 100644 provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json create mode 100644 provenance/automatic-tool-recovery/local-validation-2026-11-08.json create mode 100644 provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json create mode 100644 provenance/automatic-tool-recovery/predecessor/tests/test_compiler_composition.py create mode 100644 provenance/automatic-tool-recovery/predecessor/tests/test_video_composition.py diff --git a/HANDOFF.md b/HANDOFF.md index 4730bc5..3c5a9f2 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -40,7 +40,23 @@ citations and unsuccessful evidence. The full milestone sequence is in ## Current continuation state -**Active isolated M4 increment:** recover a pending compiler/video operation when +**Active isolated M4 increment:** automatic desktop recovery of a pending +compiler/video operation after worker loss. `DesktopComposition` now detects +a non-zero worker exit with one pending effect whose exact terminal result is +held by the live tool host, exports `CompilerHost.terminal_receipt`, runs the +ordinary `reconcile` command and launches one replacement worker that keeps the +task, session, original deadline, counters, reopen context and durable budget. +It passed the complete [1,595-test local core suite](provenance/automatic-tool-recovery/local-validation-2026-11-08.json) +and all 16 remaining workflow commands after a tests-only correction (the +failed first run is retained). The [installed SDK trial](provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json) +passed 21 closed-review checks, and the [physical IQ3_S foreground trial](provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json) +passed 49 closed-audit checks: Explorer F5 parked inference, NVENC ran, the +first worker exited with code 86, and the host recovered automatically. Draft +[PR #39](https://github.com/midhatn/adaptive-inference/pull/39) is stacked on +[PR #38](https://github.com/midhatn/adaptive-inference/pull/38). Next: active-tool +interruption, host loss, observer shutdown diagnosis and M5 restart/reboot. + +**Preceding increment (PR #38):** recover a pending compiler/video operation when the tool host has retained its exact terminal result but the worker disappeared before recording completion. Reuse the existing explicit reconciliation path, native lease and task journal; unknown outcomes remain unresolved. This new diff --git a/PLAN.md b/PLAN.md index 1617cc3..a5f67f5 100644 --- a/PLAN.md +++ b/PLAN.md @@ -998,6 +998,19 @@ verified host export and reconciliation path, then qualify automatic SDK recover and unchanged-IQ3_S recovery. Interruption during active tool execution remains a subsequent gate. +**Automatic desktop recovery result:** `DesktopComposition` now runs that verified +export, reconcile and replacement sequence itself, through a narrow recovered-state +lifecycle entry that keeps the task, session, original deadline, counters, reopen +context and durable budget. The first complete local suite found two test-double +errors (bare `object()` hosts without `lifecycle()`); tests were corrected, runtime +unchanged, and the failed run retained. The [local validation record](provenance/automatic-tool-recovery/local-validation-2026-11-08.json) +then passed 1,595 core tests and all 16 remaining workflow commands. The +[installed SDK trial](provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json) +(21 closed-review checks) and the [physical IQ3_S foreground trial](provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json) +(49 closed-audit checks, two earlier failed attempts retained) both completed with +automatic recovery after worker exit 86. Next: interruption during an active tool, +host loss, observer shutdown diagnosis and the M5 restart/reboot gates. + **Validation adjustment:** the [original video CI](provenance/video-tool/hosted-ci-timeout-2026-11-02.json) closed with all 12 core-test steps passing, but two final distribution checks were cancelled at the whole-job 15-minute limit. Preserve those failures. Give diff --git a/REFERENCES.bib b/REFERENCES.bib index 90bfbf8..08ee883 100644 --- a/REFERENCES.bib +++ b/REFERENCES.bib @@ -1124,3 +1124,10 @@ @misc{ffprobeFixedVideoOracle url = {https://www.ffmpeg.org/ffprobe.html}, note = {Inspected 2026-11-02 with https://www.ffmpeg.org/ffmpeg-all.html and https://github.com/FFmpeg/FFmpeg/blob/master/doc/ffprobe.texi. Reuses count_frames, select_streams, show_entries and JSON output for the existing exact H264 1280x720 90-frame oracle, followed by full decode. FFmpeg and FFprobe remain external pinned executables. Build-dependent licensing is recorded at https://ffmpeg.org/legal.html; no FFmpeg code or binary is bundled} } + +@misc{erlangOtpSupervision, + author = {{Ericsson AB and Erlang/OTP contributors}}, + title = {Supervisor Behaviour, Erlang/OTP System Documentation}, + url = {https://www.erlang.org/doc/system/sup_princ.html}, + note = {Inspected 2026-11-08 for automatic desktop tool recovery. Design reference only for a supervisor-owned, bounded replacement of a failed worker (restart intensity). Implemented through this repository's existing retained-child lifecycle, task journal and reconcile command; no Erlang/OTP code copied} +} diff --git a/docs/evidence.md b/docs/evidence.md index 012eeb4..0d1fcf6 100644 --- a/docs/evidence.md +++ b/docs/evidence.md @@ -179,6 +179,56 @@ Provider replies and resource observations were scripted; this gate does not establish automatic desktop recovery, active-tool interruption, actual-IQ3_S recovery or computer reboot continuation. +## Automatic desktop recovery of a pending tool operation + +`DesktopComposition` now detects the pending-effect worker-loss boundary itself +and runs the verified export, reconcile and replacement sequence described in the +[video recipe](video-tool-recipe.md#automatic-desktop-recovery). Only three runtime +modules changed from the published pending-effect increment (`composition.py`, +`desktop_controls.py`, `task_handover.py`); the +[source-preservation record](../provenance/automatic-tool-recovery/source-lineage.json) +binds them and three modified test modules to `9242434`. + +The first complete local suite run found **2 errors in 1,595 tests**: two existing +launch tests used a bare `object()` as the tool host, and the new binding-reuse +guard calls `compiler_host.lifecycle()`. Production hosts always provide it. The +test doubles were corrected to report a bound host, tests only; that failed run is +retained. The complete suite then passed **1,595 tests with no failures, errors or +skips** in 867.39 seconds, and all 16 remaining workflow commands passed, including +90 provenance tests ([local validation record](../provenance/automatic-tool-recovery/local-validation-2026-11-08.json)). + +### Installed SDK automatic recovery + +The [installed SDK trial](../provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json) +ran the installed composition with the real Hermes SDK and one real NVENC +operation. The first worker exited with code 86 after the host finished the video +and before the task journal recorded it. With no operator action, the composition +ran `reconcile` (exit 0, no provider call) and `worker-2` (exit 0), which resumed +the same task under the same deadline with a fresh owner and completed it after two +cumulative provider requests. Independent closed review passed **21 checks** with +**48 original case files unchanged**. Provider replies and resource observations +were scripted. + +### Physical IQ3_S foreground recovery + +The [physical trial](../provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json) +repeated this with real IQ3_S inference on the default desktop and unchanged quality +settings. An Explorer `This PC` refresh (F5) during the first request drained and +parked inference. After return, the host ran the fixed NVENC clip, the first worker +exited with code 86, and the desktop host reconciled and launched `worker-2` +automatically. A second native owner served the second request, which consumed +the reconciled video feedback; the task was accepted and stopped normally. The +independent closed audit passed **49 checks** with all original files unchanged +(166 foreground samples, one action, one video operation, empty job accounting). + +Two earlier attempts are retained as failures. In the first, the F5 came 1.9 s into +the request, so the 60 s input-idle window expired 2 s before `parked` and the driver +timed out. In the second, cold parking correctly refused to start because the engine +active pointer still named the owner terminated by that timeout. The independent +observer again failed at shutdown (`three_connection_errors_after_response`); that +diagnosis remains a separate gate. This is one bounded workflow, not repeated +reliability, performance, restart or reboot evidence. + ## Original video CI timeout and correction The [original hosted CI record](../provenance/video-tool/hosted-ci-timeout-2026-11-02.json) diff --git a/docs/video-tool-recipe.md b/docs/video-tool-recipe.md index c6008e6..f9529f0 100644 --- a/docs/video-tool-recipe.md +++ b/docs/video-tool-recipe.md @@ -173,6 +173,35 @@ host-reporting pattern also draws on [Temporal's asynchronous activity completion](https://docs.temporal.io/develop/python/activities/asynchronous-activity); it is implemented through the existing local task journal and host ownership. +### Automatic desktop recovery + +The desktop composition now performs that same sequence itself. When the +retained worker exits non-zero while its checkpoint holds one pending effect +and the live tool host has the exact joined terminal result, `DesktopComposition` +joins the worker, exports and saves `CompilerHost.terminal_receipt`, retires the +old binding and runs the ordinary `run_recovery reconcile` command as a retained +child. It then requires a cleanly repaired `ready` checkpoint and launches one +replacement worker through a narrow recovered-state lifecycle entry, which keeps +the task, native session, original deadline, request counters, reopen context and +durable execution budget. The replacement is committed into the current handover +episode, so Stop and cleanup still join every child. + +Recovery is refused after a clean or Stop-requested exit, when the host is +closing or faulted, without a pending effect or bound host, within 5 seconds of +the original deadline, at the attempt limit, or for a second recovery of the same +worker and operation. A failed reconcile or unclean checkpoint stops recovery +rather than dispatching around it. The bounded-restart shape follows +[Erlang/OTP supervision principles](https://www.erlang.org/doc/system/sup_princ.html). + +The [local validation record](../provenance/automatic-tool-recovery/local-validation-2026-11-08.json) +retains 41 focused component tests, the complete 1,595-test core suite, all 16 +remaining workflow commands, package verification and an isolated installation. +The [installed SDK trial](../provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json) +(scripted provider) and the [physical IQ3_S foreground trial](../provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json) +both completed with automatic recovery after worker exit 86 and passed independent +closed reviews of 21 and 49 checks. Loss of the host, interruption during a running +tool, application restart and reboot remain separate gates. + ## Sources and licensing The commands and media oracle reuse this repository's `protected_video` work; diff --git a/provenance/automatic-tool-recovery/README.md b/provenance/automatic-tool-recovery/README.md index 1cccd52..367a9ab 100644 --- a/provenance/automatic-tool-recovery/README.md +++ b/provenance/automatic-tool-recovery/README.md @@ -2,9 +2,10 @@ The source-lineage record binds automatic desktop tool recovery to the exact published pending-effect increment at `9242434991f2c2c21461061d61d898649299b847`. -Six complete predecessor Git blobs preserve the three changed runtime modules -(`composition.py`, `desktop_controls.py`, `task_handover.py`), one modified -desktop composition test module, and two source-lineage files. They remain data +Eight complete predecessor Git blobs preserve the three changed runtime modules +(`composition.py`, `desktop_controls.py`, `task_handover.py`), three modified +test modules (desktop composition, plus the compiler and video launch tests +whose inert host doubles now expose `lifecycle()`), and two source-lineage files. They remain data under this repository's Apache-2.0 license; tests do not import or execute the retained copies. @@ -23,3 +24,20 @@ This preserves the meaning of historical evidence. It does not qualify the new automatic recovery behavior: component checks, the complete local suite, distribution, installed-agent recovery and physical IQ3_S tests remain separate gates. + +## Evidence records + +- [local-validation-2026-11-08.json](local-validation-2026-11-08.json): 41 + focused component tests, the complete 1,595-test local core suite, all 16 + remaining workflow commands (including 90 provenance tests), 77-module package + verification and a fresh isolated host/worker installation. The first full + core attempt (two test-double errors) is retained as failed. +- [installed-worker-loss-2026-11-08.json](installed-worker-loss-2026-11-08.json): + installed Hermes SDK trial in which the desktop composition automatically + recovered a pending NVENC operation after worker exit 86; scripted provider. + Independent closed review passed 21 checks with 48 case files unchanged. +- [physical-foreground-recovery-2026-11-08.json](physical-foreground-recovery-2026-11-08.json): + one real IQ3_S task with an Explorer foreground action, parked inference, one + NVENC operation, worker loss and automatic recovery. Independent closed audit + passed 49 checks with all original files unchanged. Two earlier unsuccessful + attempts and the separate observer shutdown failure are retained. diff --git a/provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json b/provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json new file mode 100644 index 0000000..3040567 --- /dev/null +++ b/provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json @@ -0,0 +1,285 @@ +{ + "artifact": { + "job_accounting_empty": true, + "media": { + "codec_name": "h264", + "frames": 90, + "height": 720, + "width": 1280 + }, + "recipe_id": "nvenc-testclip.v1", + "sha256": "8425abb7459b478c239771eefb6e8202e1f01c466775ce30d5878b124c98e7cc", + "video_children_joined_zero": true + }, + "core_result": { + "actual_video_operations": 1, + "completed_tool_redispatched": false, + "composition": "Installed DesktopComposition automatic branch; no explicit operator reconcile", + "cumulative_provider_requests": 2, + "fresh_owner_after_join": true, + "native_turn_leases_after_completion": 0, + "operation_reconciled_completed_in_history": true, + "phases": [ + "tool_running", + "ready", + "accepted" + ], + "process_sequence": [ + { + "exit_code": 86, + "label": "worker" + }, + { + "exit_code": 0, + "label": "reconcile" + }, + { + "exit_code": 0, + "label": "worker-2" + } + ], + "reconcile_provider_calls": 0, + "replacement_provider_calls": 1, + "replacement_resumed_and_completed": true, + "same_task_and_original_deadline": true + }, + "date": "2026-11-08", + "design_references": [ + { + "title": "Erlang/OTP supervision principles", + "url": "https://www.erlang.org/doc/system/sup_princ.html", + "use": "Supervisor-owned bounded restart of a failed worker with explicit restart intensity; design reference only." + }, + { + "title": "Temporal asynchronous activity completion", + "url": "https://docs.temporal.io/develop/python/activities/asynchronous-activity", + "use": "Separate exact host-retained external completion from worker lifetime; design reference only." + }, + { + "title": "AWS durable-execution idempotency", + "url": "https://docs.aws.amazon.com/durable-execution/patterns/best-practices/idempotency/", + "use": "Do not redispatch an uncertain side effect; design reference only." + }, + { + "title": "Temporal activity idempotency", + "url": "https://docs.temporal.io/activity-definition#idempotency", + "use": "Recognize a durable completed effect rather than dispatching it again." + }, + { + "title": "LangGraph functional durable execution", + "url": "https://docs.langchain.com/oss/python/langgraph/functional-api", + "use": "Separate persisted effect results from continuation after worker loss." + }, + { + "title": "Pinned Hermes native conversation projection", + "url": "https://github.com/NousResearch/hermes-agent/blob/14e1e43a23761b369641208e204365cd6cb9399d/hermes_state_messages.py#L1545-L1638", + "use": "Reuse the supported reader, including derived tool-call identity." + } + ], + "documentation": "docs/video-tool-recipe.md", + "environment": { + "hermes_revision": "14e1e43a23761b369641208e204365cd6cb9399d", + "hermes_source": "https://github.com/NousResearch/hermes-agent", + "host_python": { + "sha256": "5f7b89a612c9b8af1d6456cdfcd1dbe5ca630849e79aebced9bee9a6694952ec", + "version": "3.11.9" + }, + "model_configuration": { + "actual_inference": false, + "model": "qwen3.8-flash-next-iq3_s-adaptive-media-beta", + "quality": { + "experimental_speed_projection": false, + "max_tokens": 10923, + "reasoning_budget_tokens": 8192, + "reasoning_effort": "high", + "strata_tune": { + "pcie_frac": 0.37 + } + } + }, + "platform": "Windows x64", + "tools": { + "ffmpeg": { + "sha256": "09948d4cdd0650da6ff5a87577469f2a218dc2615ae379f8f734d24c49de0f73", + "version": "ffmpeg version 8.1.1-full_build-www.gyan.dev Copyright (c) 2000-2026 the FFmpeg developers" + }, + "ffprobe": { + "sha256": "a6618e99bb58869ded3c6f37b53aa1a8d701c3591dbb7b5b317d47369c112be2", + "version": "ffprobe version 8.1.1-full_build-www.gyan.dev Copyright (c) 2007-2026 the FFmpeg developers" + } + }, + "worker_python": { + "sha256": "716fbcb56cb6753dd9ea9f7acb3f988e9c4ba65ab492cb32b9bcd0ab81938737", + "version": "3.14.7" + } + }, + "evidence_sha256": { + "identity": "63f599071fa5827ffefbedaf2ad8b5a8dbf4b53aed1af5320056def9ccbca028", + "installation": "4af07b15431f750063f44b10c2ba279996b9d1bd1710762471f42403669b35cb", + "package": "971f779bad97a26f58d519f84958539090941f2ecf105fbd4ec378b68a793232", + "profile": "2e22d83f291b692c1a6af600ab9f05a36f84741db1f8a60a48112fd5385885bf", + "sdk_closed_review": "9ff7d8545e12533f123db755bd6df2ecc2085564a9eab856960bbb5900868cfe", + "sdk_closure": "3efcf53af513ee2b605db3eddca34fb95ff6f6640a38548abaa8b53ed750cb02", + "sdk_first_worker_loss": "908c9d02ccd5ac097fb0c11dd822f9c56e94bb72198e9ef0189eab3821e83998" + }, + "feature": { + "budget_reset": false, + "dispatch_during_reconciliation": false, + "entry_point": "DesktopComposition automatic recovery branch before the ordinary worker-exit verifier", + "explicit_reconcile_preserved": true, + "new_endpoint": false, + "preserved_across_replacement": [ + "task", + "native session", + "original deadline", + "request and attempt counters", + "reopen context", + "durable execution budget" + ], + "refusals": [ + "clean or Stop-requested exit", + "host closing or faulted", + "no pending effect", + "host not bound", + "less than 5 seconds before the original deadline", + "attempt limit reached", + "second recovery for the same worker and operation", + "reconcile failure or unclean repaired checkpoint" + ], + "sequence": [ + "join the retained worker", + "export CompilerHost.terminal_receipt for the original operation and save it durably", + "retire the old host binding", + "run the ordinary run_recovery reconcile command as a retained child with no provider call", + "refresh the runtime and require a cleanly repaired ready checkpoint", + "launch one replacement worker through a narrow recovered-state lifecycle entry", + "commit the replacement into the current handover episode and re-bind the tool host" + ], + "supported_recipe_kinds": [ + "compiler", + "video" + ], + "trigger": "Retained worker exits non-zero while its checkpoint holds one pending effect and the live tool host holds the exact joined terminal result." + }, + "independent_review": { + "case_files": 48, + "case_files_unchanged": true, + "checks": 21, + "original_database_sql_opened": false, + "passed": true, + "review_sha256": "9ff7d8545e12533f123db755bd6df2ecc2085564a9eab856960bbb5900868cfe", + "runtime_locations_compared": 4, + "runtime_modules_per_location": 77, + "status": "passed", + "workload_replayed": false + }, + "limitations": [ + "Provider answers and resource observations are scripted. No actual model inference or quality claim.", + "Only an effect whose terminal result is retained by the live tool host with confirmed child joins is recovered. Unknown or in-flight effects remain fenced.", + "Intentional worker process loss at this declared boundary only; arbitrary crash points, whole-host loss and reboot are not qualified.", + "The fixed silent clip is a tool/lifecycle check, not H3, ComfyUI, audio synchronization or media realism qualification." + ], + "related_records": [ + "provenance/pending-tool-recovery/installed-worker-loss-2026-11-02.json", + "provenance/automatic-tool-recovery/local-validation-2026-11-08.json" + ], + "schema": "adaptive.automatic-tool-recovery.installed-worker-loss-evidence.v1", + "scope": "Automatic desktop recovery of a pending video operation after intentional worker process loss; real installed Hermes SDK and one actual NVENC video, scripted provider and resource observations.", + "source_identity": { + "base_revision": "9242434991f2c2c21461061d61d898649299b847", + "changed_runtime_modules_from_base": [ + "desktop/composition.py", + "desktop/desktop_controls.py", + "desktop/task_handover.py" + ], + "installed_identity_sha256": "63f599071fa5827ffefbedaf2ad8b5a8dbf4b53aed1af5320056def9ccbca028", + "note": "Source after the base revision is identified by this exact runtime map and the retained source lineage; the publication commit is recorded separately.", + "published_revision": null, + "runtime_map_hash_serialization": "UTF-8 json.dumps(map, sort_keys=True, ensure_ascii=False, allow_nan=False), default separators", + "runtime_map_sha256": "f2735f97f1027cf7720820a2f9197a4ece610bf19d1f0019c35f79fbfe7c7373", + "runtime_paths_relative_to": "src/adaptive_inference", + "runtime_sha256": { + "__init__.py": "1d906cca6d5581290d73292cce949dcaf74001de67bda63b903dfb2e70edacba", + "__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "adapters.py": "378ee41c1bdbb16873e31752d704e656a70009862d41b36898d30f57a36dfc58", + "cli.py": "a13d6c2c52445884f2ee9897eb7fadb75188339be4472db98bd760c18bce5ecf", + "client_routing.py": "d57178cfbd5666e75aef311f1362cf8486f48085c6f4995861b4fbd0cb7e41d1", + "config.py": "a6800a081a1e14a46ee18235dd21c1d1a714e6ef4e879840cd2768c590fde669", + "context_budget.py": "cd16b57bc105a179e94dbc9e429c02b901bcd8602cfaf04c9cf8776e9d799ccf", + "control_mailbox.py": "12e6ceaef36b41099422040fb23db0a9bccf96a6afb799f0e301a64bccb95888", + "costs.py": "9e8fcd06d39af57392f191b5d5701aa828f134e5141c7e298aa62e0c232224e1", + "demand.py": "76a9fbb26128c1e458348a37f0d15c36f6a9a5ae2d0f89b15d372bd2d1e394ad", + "demo.py": "5a3b1ff743bee6c45f85df8417bedbc18657f2c59f3e894f1f3298a480b22b93", + "desktop/__init__.py": "6169073159f3c0e506752f54aa51b40b058f71ed058d5272e160aa4a61ee7daa", + "desktop/activity.py": "3282dc37b493ba3fd347e279b074fa3dfd5d2c1ea5eaba8131dc568d97ec0750", + "desktop/cold_start.py": "1264f34910a0d619c6502c7565e66abbfe7ec642391a461390271b8e0799bf68", + "desktop/compiler_bridge.py": "a33613e13995c098488144557a87ee4b34b1ed3f4342610b3a43fa8923945299", + "desktop/compiler_host.py": "95a7d000bea96f527996e360eadb7324aaafd583506d4587dae89288d47ab3d2", + "desktop/compiler_profile.py": "872b8babf3fe26828ce50422485489fdde35bd59f010ff063be66ef253e323fc", + "desktop/compiler_recipe.py": "12b31514327a7c168c915b3a321ee1c17bf8446b49181eed6bed2bced55b01d2", + "desktop/compiler_template.py": "bf7a3123eb147caded7700c7f67e0c072cfe915239fa3a5cb004e7e1772f7f85", + "desktop/composition.py": "a8688f7dadceac26747e139a8d2e53e3a7a1dd5bdb71751049ab72698650165f", + "desktop/continuation_host.py": "9bd258aa8b1a2516e80cd94641f2a9fce3584c5b8aa484badf9854dfc121d7c7", + "desktop/control_client.py": "ba950c73328e084329fdb29ab78fdfe86cfed1715a792cdbf9872a74e8c75919", + "desktop/control_transport.py": "280d8a55793c241b3ec30e253b758de42386f3329bb6477ba47537f9ca264c6b", + "desktop/desktop_controls.py": "89eeb900ce8bcdb079d5400f09408a56e1df15383d58ec74d405a027795469c9", + "desktop/execution_budget.py": "77ad74ddbf0eaa89379fbecb305574d5b70cc32f9e63934a14f439ed2c744728", + "desktop/foreground_policy.py": "35781fb869c03220c6f48c3e138083b15bbcf7e40187edbfe546ad4da09b7faa", + "desktop/gui_launch.py": "c93e837ae15092de1a7814f766731ef38dc118f3e8a50280ee7658b91966355b", + "desktop/gui_session.py": "f232609a01f11977d7f011a8b1c7f2b7c2a0c3ca697766b63ec0873ec9f4cc64", + "desktop/hermes/__init__.py": "43830f7aa3deb6001eb8aa8646fc515f5d8d74a26f50590984c8278e4db67532", + "desktop/hermes/admission.py": "6f12fd6ad21fa194cb337590cf0bbd71b1d394e6a2be0cd79dfdd634af5e0f07", + "desktop/hermes/compiler_client.py": "fb0372c6b16738cdb1b0c8d527765e8b7f60a038513950988c84d40b741bf8a1", + "desktop/hermes/compiler_evidence.py": "8b2144e5b1634d8e2bc132711d948308ebc62320e98f9129c44817848a806d56", + "desktop/hermes/compiler_recovery.py": "03d19056b0c2f73679754aa3eab215d3fb2728b22c71a5ab5e1c327ad6a8c114", + "desktop/hermes/compiler_worker.py": "8d18ef60025318804e81f848546e1c3e0e84f61cf4a09fe7bfd1c759b243d91e", + "desktop/hermes/continuation.py": "d3b396bba2ed794b7304e3bebb888e2d8d51568de846fbe6549363b42d6de5c6", + "desktop/hermes/continuation_worker.py": "6871ecbfaec346ccd658d957b5f593f0dc242db17c1b13b93ab603f693967b67", + "desktop/hermes/control_bridge.py": "7a8e0aa672efeb1c6f676446e0f3b6886fd778f5d0f6c185a87d005a0256a3db", + "desktop/hermes/native_reader.py": "1f51ea1180ef0d7ad52359430831084de7cde45e21e758068fee5b9a93079464", + "desktop/hermes/pause_activity.py": "3cffa8eae39b956de3d873e82593a052c2682aad5f06b36a8ff845080baf0e5d", + "desktop/hermes/recovery.py": "c6f07435ae29f4e2542af585ace107f7952583ed3c0735cf6df2f3a0df438f37", + "desktop/hermes/report_contract.py": "d86aa6ee0f18912400aa80d96c343128af08ae80e7d8c367ee9c8733ea5d69de", + "desktop/hermes/report_validator.py": "378dd445a088a075d300fc85b86160f3b96790ee046adecf926f9e59e8e7d9af", + "desktop/hermes/run_recovery.py": "3d78a3f1f78e024d04b0020e3702c784dfc3723a802bdb87c9cbca25ec827fd3", + "desktop/hermes/sdk_pins.py": "538a5ea8651e1106c7c535ccd73b33b2f56de6c736d926ce6b92c5fb6eb075ce", + "desktop/hermes/service_stop.py": "235b61ad248083f8f0536767ad79473955d96ad1b40189b73c9cfe71439a51c9", + "desktop/installation.py": "13e13d532bedb68f150d1096767c98796de83f74dce54021d286c6c18c521e85", + "desktop/load_feasibility.py": "23e94f3ee770b065fb323f30bb104d8d5732bf710bd3cc9b2a27daa746014dda", + "desktop/mailbox_runtime.py": "aad1c3b383f6954be89ebe6c9f2e680cd286c672d7bb26bd7027402e0a7d6b72", + "desktop/native_owner_adapter.py": "5a79c48911c798310104be07ebb978f9ad9fe4f7ab000c328afd992f75415977", + "desktop/owned_job.py": "90b59dbdae226419fc9680c961c21dfa9e061df08512a7a8089d6f6d11b2ccc4", + "desktop/persistent_host.py": "acff567d5eb6f2c569588bfa684363907897bdb66be8008bc20c10d3ca7c0011", + "desktop/profile.py": "caac6f07fc87ccbf51d97003a0d8cb209d21e04bfc00acda6a5c9e05b2d3fbbc", + "desktop/recipe_dispatch.py": "3a4813e156448305e7880c3cc3ac3b82e992b34a0e05b7f6b74f09bc9e2c6f4b", + "desktop/retained_ui.py": "749b947510377518c452ae7a70fa21c32f27c3f10225f6eb90d87ef03406f5cb", + "desktop/run_gui.py": "eb67c9bac75b72bcc545c234b28c54b8c9c5fbfb3c2183f99e63a47abb3a6ef0", + "desktop/saved_tasks.py": "75703584b3c4cda42aed6de0e4645ea75b2a3b1e2e81c702e919f626cc8ec1b7", + "desktop/serial_listener.py": "048e8ab061e175a978f755801b9a27e3daa292c1dc7dabe3c8a8763be81199b2", + "desktop/startup_pipe.py": "0617c527461b4c437474a947a497300e4c28c06d11f8bb43f1e67fc51a159fae", + "desktop/task_admission.py": "2ca007580bca43678c529599b6296ff4b97c15e210b9daeb651a8f2c4f12b25d", + "desktop/task_handover.py": "8fd90d86635a3b5968a39d32b20e54e67b17c8dc9bb4a90b9fcdcabff97f01b4", + "desktop/ui_intent.py": "ff67208a3890656bc76717ac18c8d0c02a19dc2e09b11b81d88fc1f63c0a29d1", + "desktop/ui_progress.py": "466deda7ba15d73ea779abf3b1b2e690e92217950b15afdbdf8acf9cda3d2de9", + "desktop/video_profile.py": "2ac53c40937e60d984e2903d643d2ee83372760756d1afedf5406b71644b41e4", + "desktop/video_recipe.py": "b255d3f939aa3a1e29449bc397e39b50859fcc0cf8a10201859b2ec2dc50bcff", + "desktop/video_template.py": "1a74d771d83eed358196467857e11c5986cfc6c196b6d172c614224db311a835", + "desktop/window_marker.py": "e2613ef662993be88c332657e4e53c598fdc006634bf8fabf9e0d0a9765c0d36", + "desktop/windows_host_base.py": "ad08c31435bc4da77b43c68321a7c08130ab1c4f7bc688c71191c6497abc97bd", + "execution_clock.py": "c9d04e8e029d4d88830560482e2437cab3d06ef140a942d16543c3be95642cff", + "feedback.py": "c76eba0110659ae67f4a0efd2ca21aff0987af7e9095998e4db9382de20826c3", + "gateway.py": "b2009a8a776bcea4dfc3bedac5296c930ae2211bb8e29d7ccdf76a227fefdef8", + "handoff.py": "6cd7f593698f6d72fc3ac16d9a541436f65b85103bed2420372364262fcb4db1", + "resource_policy.py": "53768d4b95b589bb76d1788a9ccceefe60c05b332b1b1f43dece6b04bfec212a", + "resources.py": "a72e376e4dfc6021e61756812f93275af20c28ec1438faf5eb381a491d3db4d9", + "scheduler.py": "c7f962bd9ef8e44587cadcc02a41fae8a00de7fc15b105848dcdfc4862bd5b4f", + "task_prepare.py": "9cddd6e09b3f5435da08316f8d9ccdfbcff67a336f6f74b420108c4c856223b2", + "task_spec.py": "2975c0c4489ac863502189777699d13d7b2b8aca6d19191247fd860ed4a0ce76", + "task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6" + }, + "sdist_sha256": "efd17ce8a5c588878c995857c18b135b12f777c392de1bd48b242947ab1f94f9", + "source_lineage_sha256": "ccf7907107cebdac3f3c9f25ac39710ff352c0179737f961ca86f4ff85bfab26", + "wheel_sha256": "401977bfb0541069752ac386ddc6aaaafad23ab095afc74dc7613ef80481b6a7" + }, + "status": "passed_at_declared_scope" +} diff --git a/provenance/automatic-tool-recovery/local-validation-2026-11-08.json b/provenance/automatic-tool-recovery/local-validation-2026-11-08.json new file mode 100644 index 0000000..9d8927c --- /dev/null +++ b/provenance/automatic-tool-recovery/local-validation-2026-11-08.json @@ -0,0 +1,597 @@ +{ + "date": "2026-11-08", + "design_references": [ + { + "title": "Erlang/OTP supervision principles", + "url": "https://www.erlang.org/doc/system/sup_princ.html", + "use": "Supervisor-owned bounded restart of a failed worker with explicit restart intensity; design reference only." + }, + { + "title": "Temporal asynchronous activity completion", + "url": "https://docs.temporal.io/develop/python/activities/asynchronous-activity", + "use": "Separate exact host-retained external completion from worker lifetime; design reference only." + }, + { + "title": "AWS durable-execution idempotency", + "url": "https://docs.aws.amazon.com/durable-execution/patterns/best-practices/idempotency/", + "use": "Do not redispatch an uncertain side effect; design reference only." + }, + { + "title": "Temporal activity idempotency", + "url": "https://docs.temporal.io/activity-definition#idempotency", + "use": "Recognize a durable completed effect rather than dispatching it again." + }, + { + "title": "LangGraph functional durable execution", + "url": "https://docs.langchain.com/oss/python/langgraph/functional-api", + "use": "Separate persisted effect results from continuation after worker loss." + }, + { + "title": "Pinned Hermes native conversation projection", + "url": "https://github.com/NousResearch/hermes-agent/blob/14e1e43a23761b369641208e204365cd6cb9399d/hermes_state_messages.py#L1545-L1638", + "use": "Reuse the supported reader, including derived tool-call identity." + } + ], + "documentation": "docs/video-tool-recipe.md", + "evidence_sha256": { + "components": "810744b2490f5412de456073e0083d9110c0932032839560d9fcb32bdfadd156", + "core": "4bf8099beb7f3ea4b07d4bb2c33dfebacc4577c40c45650e437fef279b7ca0c6", + "core_failed_attempt": "bfc867de354b8d542e53f9eede74e6da51537f0b4e288009224e9ba1b7e37150", + "core_sources": "d749cfab88d9feb88772b09785da9c213bb10f47668803269eb91906713ca8c6", + "identity": "63f599071fa5827ffefbedaf2ad8b5a8dbf4b53aed1af5320056def9ccbca028", + "installation": "4af07b15431f750063f44b10c2ba279996b9d1bd1710762471f42403669b35cb", + "package": "971f779bad97a26f58d519f84958539090941f2ecf105fbd4ec378b68a793232", + "workflow": "26690afbf84473150314a083b807ef58a6a5c96770d223090250eb1f2d2571c8", + "workflow_before_correction": "8576a14493f45c626a2d313a83ee86247d0ebaa0658a73156dadd3adad64baca", + "workflow_closure": "c532434ae3f3b63b8985251c88f98a16f5474c0f189b6b735cf22d66df6c26e2" + }, + "feature": { + "budget_reset": false, + "dispatch_during_reconciliation": false, + "entry_point": "DesktopComposition automatic recovery branch before the ordinary worker-exit verifier", + "explicit_reconcile_preserved": true, + "new_endpoint": false, + "preserved_across_replacement": [ + "task", + "native session", + "original deadline", + "request and attempt counters", + "reopen context", + "durable execution budget" + ], + "refusals": [ + "clean or Stop-requested exit", + "host closing or faulted", + "no pending effect", + "host not bound", + "less than 5 seconds before the original deadline", + "attempt limit reached", + "second recovery for the same worker and operation", + "reconcile failure or unclean repaired checkpoint" + ], + "sequence": [ + "join the retained worker", + "export CompilerHost.terminal_receipt for the original operation and save it durably", + "retire the old host binding", + "run the ordinary run_recovery reconcile command as a retained child with no provider call", + "refresh the runtime and require a cleanly repaired ready checkpoint", + "launch one replacement worker through a narrow recovered-state lifecycle entry", + "commit the replacement into the current handover episode and re-bind the tool host" + ], + "supported_recipe_kinds": [ + "compiler", + "video" + ], + "trigger": "Retained worker exits non-zero while its checkpoint holds one pending effect and the live tool host holds the exact joined terminal result." + }, + "limitations": [ + "Local Windows/Python 3.11 validation does not establish the hosted OS/Python matrix.", + "The first full core attempt had two test-double errors; it remains failed and retained. The correction changed tests only.", + "Component and installation success alone do not qualify installed SDK recovery, inference, foreground behavior or reboot." + ], + "related_records": [ + "provenance/automatic-tool-recovery/source-lineage.json", + "provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json", + "provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json" + ], + "schema": "adaptive.automatic-tool-recovery.local-validation-evidence.v1", + "scope": "Exact local source tests, workflow commands, distribution verification and isolated installation; installed SDK and physical evidence are recorded separately.", + "source_identity": { + "base_revision": "9242434991f2c2c21461061d61d898649299b847", + "changed_runtime_modules_from_base": [ + "desktop/composition.py", + "desktop/desktop_controls.py", + "desktop/task_handover.py" + ], + "installed_identity_sha256": "63f599071fa5827ffefbedaf2ad8b5a8dbf4b53aed1af5320056def9ccbca028", + "note": "Source after the base revision is identified by this exact runtime map and the retained source lineage; the publication commit is recorded separately.", + "published_revision": null, + "runtime_map_hash_serialization": "UTF-8 json.dumps(map, sort_keys=True, ensure_ascii=False, allow_nan=False), default separators", + "runtime_map_sha256": "f2735f97f1027cf7720820a2f9197a4ece610bf19d1f0019c35f79fbfe7c7373", + "runtime_paths_relative_to": "src/adaptive_inference", + "runtime_sha256": { + "__init__.py": "1d906cca6d5581290d73292cce949dcaf74001de67bda63b903dfb2e70edacba", + "__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "adapters.py": "378ee41c1bdbb16873e31752d704e656a70009862d41b36898d30f57a36dfc58", + "cli.py": "a13d6c2c52445884f2ee9897eb7fadb75188339be4472db98bd760c18bce5ecf", + "client_routing.py": "d57178cfbd5666e75aef311f1362cf8486f48085c6f4995861b4fbd0cb7e41d1", + "config.py": "a6800a081a1e14a46ee18235dd21c1d1a714e6ef4e879840cd2768c590fde669", + "context_budget.py": "cd16b57bc105a179e94dbc9e429c02b901bcd8602cfaf04c9cf8776e9d799ccf", + "control_mailbox.py": "12e6ceaef36b41099422040fb23db0a9bccf96a6afb799f0e301a64bccb95888", + "costs.py": "9e8fcd06d39af57392f191b5d5701aa828f134e5141c7e298aa62e0c232224e1", + "demand.py": "76a9fbb26128c1e458348a37f0d15c36f6a9a5ae2d0f89b15d372bd2d1e394ad", + "demo.py": "5a3b1ff743bee6c45f85df8417bedbc18657f2c59f3e894f1f3298a480b22b93", + "desktop/__init__.py": "6169073159f3c0e506752f54aa51b40b058f71ed058d5272e160aa4a61ee7daa", + "desktop/activity.py": "3282dc37b493ba3fd347e279b074fa3dfd5d2c1ea5eaba8131dc568d97ec0750", + "desktop/cold_start.py": "1264f34910a0d619c6502c7565e66abbfe7ec642391a461390271b8e0799bf68", + "desktop/compiler_bridge.py": "a33613e13995c098488144557a87ee4b34b1ed3f4342610b3a43fa8923945299", + "desktop/compiler_host.py": "95a7d000bea96f527996e360eadb7324aaafd583506d4587dae89288d47ab3d2", + "desktop/compiler_profile.py": "872b8babf3fe26828ce50422485489fdde35bd59f010ff063be66ef253e323fc", + "desktop/compiler_recipe.py": "12b31514327a7c168c915b3a321ee1c17bf8446b49181eed6bed2bced55b01d2", + "desktop/compiler_template.py": "bf7a3123eb147caded7700c7f67e0c072cfe915239fa3a5cb004e7e1772f7f85", + "desktop/composition.py": "a8688f7dadceac26747e139a8d2e53e3a7a1dd5bdb71751049ab72698650165f", + "desktop/continuation_host.py": "9bd258aa8b1a2516e80cd94641f2a9fce3584c5b8aa484badf9854dfc121d7c7", + "desktop/control_client.py": "ba950c73328e084329fdb29ab78fdfe86cfed1715a792cdbf9872a74e8c75919", + "desktop/control_transport.py": "280d8a55793c241b3ec30e253b758de42386f3329bb6477ba47537f9ca264c6b", + "desktop/desktop_controls.py": "89eeb900ce8bcdb079d5400f09408a56e1df15383d58ec74d405a027795469c9", + "desktop/execution_budget.py": "77ad74ddbf0eaa89379fbecb305574d5b70cc32f9e63934a14f439ed2c744728", + "desktop/foreground_policy.py": "35781fb869c03220c6f48c3e138083b15bbcf7e40187edbfe546ad4da09b7faa", + "desktop/gui_launch.py": "c93e837ae15092de1a7814f766731ef38dc118f3e8a50280ee7658b91966355b", + "desktop/gui_session.py": "f232609a01f11977d7f011a8b1c7f2b7c2a0c3ca697766b63ec0873ec9f4cc64", + "desktop/hermes/__init__.py": "43830f7aa3deb6001eb8aa8646fc515f5d8d74a26f50590984c8278e4db67532", + "desktop/hermes/admission.py": "6f12fd6ad21fa194cb337590cf0bbd71b1d394e6a2be0cd79dfdd634af5e0f07", + "desktop/hermes/compiler_client.py": "fb0372c6b16738cdb1b0c8d527765e8b7f60a038513950988c84d40b741bf8a1", + "desktop/hermes/compiler_evidence.py": "8b2144e5b1634d8e2bc132711d948308ebc62320e98f9129c44817848a806d56", + "desktop/hermes/compiler_recovery.py": "03d19056b0c2f73679754aa3eab215d3fb2728b22c71a5ab5e1c327ad6a8c114", + "desktop/hermes/compiler_worker.py": "8d18ef60025318804e81f848546e1c3e0e84f61cf4a09fe7bfd1c759b243d91e", + "desktop/hermes/continuation.py": "d3b396bba2ed794b7304e3bebb888e2d8d51568de846fbe6549363b42d6de5c6", + "desktop/hermes/continuation_worker.py": "6871ecbfaec346ccd658d957b5f593f0dc242db17c1b13b93ab603f693967b67", + "desktop/hermes/control_bridge.py": "7a8e0aa672efeb1c6f676446e0f3b6886fd778f5d0f6c185a87d005a0256a3db", + "desktop/hermes/native_reader.py": "1f51ea1180ef0d7ad52359430831084de7cde45e21e758068fee5b9a93079464", + "desktop/hermes/pause_activity.py": "3cffa8eae39b956de3d873e82593a052c2682aad5f06b36a8ff845080baf0e5d", + "desktop/hermes/recovery.py": "c6f07435ae29f4e2542af585ace107f7952583ed3c0735cf6df2f3a0df438f37", + "desktop/hermes/report_contract.py": "d86aa6ee0f18912400aa80d96c343128af08ae80e7d8c367ee9c8733ea5d69de", + "desktop/hermes/report_validator.py": "378dd445a088a075d300fc85b86160f3b96790ee046adecf926f9e59e8e7d9af", + "desktop/hermes/run_recovery.py": "3d78a3f1f78e024d04b0020e3702c784dfc3723a802bdb87c9cbca25ec827fd3", + "desktop/hermes/sdk_pins.py": "538a5ea8651e1106c7c535ccd73b33b2f56de6c736d926ce6b92c5fb6eb075ce", + "desktop/hermes/service_stop.py": "235b61ad248083f8f0536767ad79473955d96ad1b40189b73c9cfe71439a51c9", + "desktop/installation.py": "13e13d532bedb68f150d1096767c98796de83f74dce54021d286c6c18c521e85", + "desktop/load_feasibility.py": "23e94f3ee770b065fb323f30bb104d8d5732bf710bd3cc9b2a27daa746014dda", + "desktop/mailbox_runtime.py": "aad1c3b383f6954be89ebe6c9f2e680cd286c672d7bb26bd7027402e0a7d6b72", + "desktop/native_owner_adapter.py": "5a79c48911c798310104be07ebb978f9ad9fe4f7ab000c328afd992f75415977", + "desktop/owned_job.py": "90b59dbdae226419fc9680c961c21dfa9e061df08512a7a8089d6f6d11b2ccc4", + "desktop/persistent_host.py": "acff567d5eb6f2c569588bfa684363907897bdb66be8008bc20c10d3ca7c0011", + "desktop/profile.py": "caac6f07fc87ccbf51d97003a0d8cb209d21e04bfc00acda6a5c9e05b2d3fbbc", + "desktop/recipe_dispatch.py": "3a4813e156448305e7880c3cc3ac3b82e992b34a0e05b7f6b74f09bc9e2c6f4b", + "desktop/retained_ui.py": "749b947510377518c452ae7a70fa21c32f27c3f10225f6eb90d87ef03406f5cb", + "desktop/run_gui.py": "eb67c9bac75b72bcc545c234b28c54b8c9c5fbfb3c2183f99e63a47abb3a6ef0", + "desktop/saved_tasks.py": "75703584b3c4cda42aed6de0e4645ea75b2a3b1e2e81c702e919f626cc8ec1b7", + "desktop/serial_listener.py": "048e8ab061e175a978f755801b9a27e3daa292c1dc7dabe3c8a8763be81199b2", + "desktop/startup_pipe.py": "0617c527461b4c437474a947a497300e4c28c06d11f8bb43f1e67fc51a159fae", + "desktop/task_admission.py": "2ca007580bca43678c529599b6296ff4b97c15e210b9daeb651a8f2c4f12b25d", + "desktop/task_handover.py": "8fd90d86635a3b5968a39d32b20e54e67b17c8dc9bb4a90b9fcdcabff97f01b4", + "desktop/ui_intent.py": "ff67208a3890656bc76717ac18c8d0c02a19dc2e09b11b81d88fc1f63c0a29d1", + "desktop/ui_progress.py": "466deda7ba15d73ea779abf3b1b2e690e92217950b15afdbdf8acf9cda3d2de9", + "desktop/video_profile.py": "2ac53c40937e60d984e2903d643d2ee83372760756d1afedf5406b71644b41e4", + "desktop/video_recipe.py": "b255d3f939aa3a1e29449bc397e39b50859fcc0cf8a10201859b2ec2dc50bcff", + "desktop/video_template.py": "1a74d771d83eed358196467857e11c5986cfc6c196b6d172c614224db311a835", + "desktop/window_marker.py": "e2613ef662993be88c332657e4e53c598fdc006634bf8fabf9e0d0a9765c0d36", + "desktop/windows_host_base.py": "ad08c31435bc4da77b43c68321a7c08130ab1c4f7bc688c71191c6497abc97bd", + "execution_clock.py": "c9d04e8e029d4d88830560482e2437cab3d06ef140a942d16543c3be95642cff", + "feedback.py": "c76eba0110659ae67f4a0efd2ca21aff0987af7e9095998e4db9382de20826c3", + "gateway.py": "b2009a8a776bcea4dfc3bedac5296c930ae2211bb8e29d7ccdf76a227fefdef8", + "handoff.py": "6cd7f593698f6d72fc3ac16d9a541436f65b85103bed2420372364262fcb4db1", + "resource_policy.py": "53768d4b95b589bb76d1788a9ccceefe60c05b332b1b1f43dece6b04bfec212a", + "resources.py": "a72e376e4dfc6021e61756812f93275af20c28ec1438faf5eb381a491d3db4d9", + "scheduler.py": "c7f962bd9ef8e44587cadcc02a41fae8a00de7fc15b105848dcdfc4862bd5b4f", + "task_prepare.py": "9cddd6e09b3f5435da08316f8d9ccdfbcff67a336f6f74b420108c4c856223b2", + "task_spec.py": "2975c0c4489ac863502189777699d13d7b2b8aca6d19191247fd860ed4a0ce76", + "task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6" + }, + "sdist_sha256": "efd17ce8a5c588878c995857c18b135b12f777c392de1bd48b242947ab1f94f9", + "source_lineage_sha256": "ccf7907107cebdac3f3c9f25ac39710ff352c0179737f961ca86f4ff85bfab26", + "wheel_sha256": "401977bfb0541069752ac386ddc6aaaafad23ab095afc74dc7613ef80481b6a7" + }, + "status": "passed_at_declared_scope", + "tested_source_snapshot": { + "files_sha256": { + "src/adaptive_inference/__init__.py": "1d906cca6d5581290d73292cce949dcaf74001de67bda63b903dfb2e70edacba", + "src/adaptive_inference/__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "src/adaptive_inference/adapters.py": "378ee41c1bdbb16873e31752d704e656a70009862d41b36898d30f57a36dfc58", + "src/adaptive_inference/cli.py": "a13d6c2c52445884f2ee9897eb7fadb75188339be4472db98bd760c18bce5ecf", + "src/adaptive_inference/client_routing.py": "d57178cfbd5666e75aef311f1362cf8486f48085c6f4995861b4fbd0cb7e41d1", + "src/adaptive_inference/config.py": "a6800a081a1e14a46ee18235dd21c1d1a714e6ef4e879840cd2768c590fde669", + "src/adaptive_inference/context_budget.py": "cd16b57bc105a179e94dbc9e429c02b901bcd8602cfaf04c9cf8776e9d799ccf", + "src/adaptive_inference/control_mailbox.py": "12e6ceaef36b41099422040fb23db0a9bccf96a6afb799f0e301a64bccb95888", + "src/adaptive_inference/costs.py": "9e8fcd06d39af57392f191b5d5701aa828f134e5141c7e298aa62e0c232224e1", + "src/adaptive_inference/demand.py": "76a9fbb26128c1e458348a37f0d15c36f6a9a5ae2d0f89b15d372bd2d1e394ad", + "src/adaptive_inference/demo.py": "5a3b1ff743bee6c45f85df8417bedbc18657f2c59f3e894f1f3298a480b22b93", + "src/adaptive_inference/desktop/__init__.py": "6169073159f3c0e506752f54aa51b40b058f71ed058d5272e160aa4a61ee7daa", + "src/adaptive_inference/desktop/activity.py": "3282dc37b493ba3fd347e279b074fa3dfd5d2c1ea5eaba8131dc568d97ec0750", + "src/adaptive_inference/desktop/cold_start.py": "1264f34910a0d619c6502c7565e66abbfe7ec642391a461390271b8e0799bf68", + "src/adaptive_inference/desktop/compiler_bridge.py": "a33613e13995c098488144557a87ee4b34b1ed3f4342610b3a43fa8923945299", + "src/adaptive_inference/desktop/compiler_host.py": "95a7d000bea96f527996e360eadb7324aaafd583506d4587dae89288d47ab3d2", + "src/adaptive_inference/desktop/compiler_profile.py": "872b8babf3fe26828ce50422485489fdde35bd59f010ff063be66ef253e323fc", + "src/adaptive_inference/desktop/compiler_recipe.py": "12b31514327a7c168c915b3a321ee1c17bf8446b49181eed6bed2bced55b01d2", + "src/adaptive_inference/desktop/compiler_template.py": "bf7a3123eb147caded7700c7f67e0c072cfe915239fa3a5cb004e7e1772f7f85", + "src/adaptive_inference/desktop/composition.py": "a8688f7dadceac26747e139a8d2e53e3a7a1dd5bdb71751049ab72698650165f", + "src/adaptive_inference/desktop/continuation_host.py": "9bd258aa8b1a2516e80cd94641f2a9fce3584c5b8aa484badf9854dfc121d7c7", + "src/adaptive_inference/desktop/control_client.py": "ba950c73328e084329fdb29ab78fdfe86cfed1715a792cdbf9872a74e8c75919", + "src/adaptive_inference/desktop/control_transport.py": "280d8a55793c241b3ec30e253b758de42386f3329bb6477ba47537f9ca264c6b", + "src/adaptive_inference/desktop/desktop_controls.py": "89eeb900ce8bcdb079d5400f09408a56e1df15383d58ec74d405a027795469c9", + "src/adaptive_inference/desktop/execution_budget.py": "77ad74ddbf0eaa89379fbecb305574d5b70cc32f9e63934a14f439ed2c744728", + "src/adaptive_inference/desktop/foreground_policy.py": "35781fb869c03220c6f48c3e138083b15bbcf7e40187edbfe546ad4da09b7faa", + "src/adaptive_inference/desktop/gui_launch.py": "c93e837ae15092de1a7814f766731ef38dc118f3e8a50280ee7658b91966355b", + "src/adaptive_inference/desktop/gui_session.py": "f232609a01f11977d7f011a8b1c7f2b7c2a0c3ca697766b63ec0873ec9f4cc64", + "src/adaptive_inference/desktop/hermes/__init__.py": "43830f7aa3deb6001eb8aa8646fc515f5d8d74a26f50590984c8278e4db67532", + "src/adaptive_inference/desktop/hermes/admission.py": "6f12fd6ad21fa194cb337590cf0bbd71b1d394e6a2be0cd79dfdd634af5e0f07", + "src/adaptive_inference/desktop/hermes/compiler_client.py": "fb0372c6b16738cdb1b0c8d527765e8b7f60a038513950988c84d40b741bf8a1", + "src/adaptive_inference/desktop/hermes/compiler_evidence.py": "8b2144e5b1634d8e2bc132711d948308ebc62320e98f9129c44817848a806d56", + "src/adaptive_inference/desktop/hermes/compiler_recovery.py": "03d19056b0c2f73679754aa3eab215d3fb2728b22c71a5ab5e1c327ad6a8c114", + "src/adaptive_inference/desktop/hermes/compiler_worker.py": "8d18ef60025318804e81f848546e1c3e0e84f61cf4a09fe7bfd1c759b243d91e", + "src/adaptive_inference/desktop/hermes/continuation.py": "d3b396bba2ed794b7304e3bebb888e2d8d51568de846fbe6549363b42d6de5c6", + "src/adaptive_inference/desktop/hermes/continuation_worker.py": "6871ecbfaec346ccd658d957b5f593f0dc242db17c1b13b93ab603f693967b67", + "src/adaptive_inference/desktop/hermes/control_bridge.py": "7a8e0aa672efeb1c6f676446e0f3b6886fd778f5d0f6c185a87d005a0256a3db", + "src/adaptive_inference/desktop/hermes/native_reader.py": "1f51ea1180ef0d7ad52359430831084de7cde45e21e758068fee5b9a93079464", + "src/adaptive_inference/desktop/hermes/pause_activity.py": "3cffa8eae39b956de3d873e82593a052c2682aad5f06b36a8ff845080baf0e5d", + "src/adaptive_inference/desktop/hermes/recovery.py": "c6f07435ae29f4e2542af585ace107f7952583ed3c0735cf6df2f3a0df438f37", + "src/adaptive_inference/desktop/hermes/report_contract.py": "d86aa6ee0f18912400aa80d96c343128af08ae80e7d8c367ee9c8733ea5d69de", + "src/adaptive_inference/desktop/hermes/report_validator.py": "378dd445a088a075d300fc85b86160f3b96790ee046adecf926f9e59e8e7d9af", + "src/adaptive_inference/desktop/hermes/run_recovery.py": "3d78a3f1f78e024d04b0020e3702c784dfc3723a802bdb87c9cbca25ec827fd3", + "src/adaptive_inference/desktop/hermes/sdk_pins.py": "538a5ea8651e1106c7c535ccd73b33b2f56de6c736d926ce6b92c5fb6eb075ce", + "src/adaptive_inference/desktop/hermes/service_stop.py": "235b61ad248083f8f0536767ad79473955d96ad1b40189b73c9cfe71439a51c9", + "src/adaptive_inference/desktop/installation.py": "13e13d532bedb68f150d1096767c98796de83f74dce54021d286c6c18c521e85", + "src/adaptive_inference/desktop/load_feasibility.py": "23e94f3ee770b065fb323f30bb104d8d5732bf710bd3cc9b2a27daa746014dda", + "src/adaptive_inference/desktop/mailbox_runtime.py": "aad1c3b383f6954be89ebe6c9f2e680cd286c672d7bb26bd7027402e0a7d6b72", + "src/adaptive_inference/desktop/native_owner_adapter.py": "5a79c48911c798310104be07ebb978f9ad9fe4f7ab000c328afd992f75415977", + "src/adaptive_inference/desktop/owned_job.py": "90b59dbdae226419fc9680c961c21dfa9e061df08512a7a8089d6f6d11b2ccc4", + "src/adaptive_inference/desktop/persistent_host.py": "acff567d5eb6f2c569588bfa684363907897bdb66be8008bc20c10d3ca7c0011", + "src/adaptive_inference/desktop/profile.py": "caac6f07fc87ccbf51d97003a0d8cb209d21e04bfc00acda6a5c9e05b2d3fbbc", + "src/adaptive_inference/desktop/recipe_dispatch.py": "3a4813e156448305e7880c3cc3ac3b82e992b34a0e05b7f6b74f09bc9e2c6f4b", + "src/adaptive_inference/desktop/retained_ui.py": "749b947510377518c452ae7a70fa21c32f27c3f10225f6eb90d87ef03406f5cb", + "src/adaptive_inference/desktop/run_gui.py": "eb67c9bac75b72bcc545c234b28c54b8c9c5fbfb3c2183f99e63a47abb3a6ef0", + "src/adaptive_inference/desktop/saved_tasks.py": "75703584b3c4cda42aed6de0e4645ea75b2a3b1e2e81c702e919f626cc8ec1b7", + "src/adaptive_inference/desktop/serial_listener.py": "048e8ab061e175a978f755801b9a27e3daa292c1dc7dabe3c8a8763be81199b2", + "src/adaptive_inference/desktop/startup_pipe.py": "0617c527461b4c437474a947a497300e4c28c06d11f8bb43f1e67fc51a159fae", + "src/adaptive_inference/desktop/task_admission.py": "2ca007580bca43678c529599b6296ff4b97c15e210b9daeb651a8f2c4f12b25d", + "src/adaptive_inference/desktop/task_handover.py": "8fd90d86635a3b5968a39d32b20e54e67b17c8dc9bb4a90b9fcdcabff97f01b4", + "src/adaptive_inference/desktop/ui_intent.py": "ff67208a3890656bc76717ac18c8d0c02a19dc2e09b11b81d88fc1f63c0a29d1", + "src/adaptive_inference/desktop/ui_progress.py": "466deda7ba15d73ea779abf3b1b2e690e92217950b15afdbdf8acf9cda3d2de9", + "src/adaptive_inference/desktop/video_profile.py": "2ac53c40937e60d984e2903d643d2ee83372760756d1afedf5406b71644b41e4", + "src/adaptive_inference/desktop/video_recipe.py": "b255d3f939aa3a1e29449bc397e39b50859fcc0cf8a10201859b2ec2dc50bcff", + "src/adaptive_inference/desktop/video_template.py": "1a74d771d83eed358196467857e11c5986cfc6c196b6d172c614224db311a835", + "src/adaptive_inference/desktop/window_marker.py": "e2613ef662993be88c332657e4e53c598fdc006634bf8fabf9e0d0a9765c0d36", + "src/adaptive_inference/desktop/windows_host_base.py": "ad08c31435bc4da77b43c68321a7c08130ab1c4f7bc688c71191c6497abc97bd", + "src/adaptive_inference/execution_clock.py": "c9d04e8e029d4d88830560482e2437cab3d06ef140a942d16543c3be95642cff", + "src/adaptive_inference/feedback.py": "c76eba0110659ae67f4a0efd2ca21aff0987af7e9095998e4db9382de20826c3", + "src/adaptive_inference/gateway.py": "b2009a8a776bcea4dfc3bedac5296c930ae2211bb8e29d7ccdf76a227fefdef8", + "src/adaptive_inference/handoff.py": "6cd7f593698f6d72fc3ac16d9a541436f65b85103bed2420372364262fcb4db1", + "src/adaptive_inference/resource_policy.py": "53768d4b95b589bb76d1788a9ccceefe60c05b332b1b1f43dece6b04bfec212a", + "src/adaptive_inference/resources.py": "a72e376e4dfc6021e61756812f93275af20c28ec1438faf5eb381a491d3db4d9", + "src/adaptive_inference/scheduler.py": "c7f962bd9ef8e44587cadcc02a41fae8a00de7fc15b105848dcdfc4862bd5b4f", + "src/adaptive_inference/task_prepare.py": "9cddd6e09b3f5435da08316f8d9ccdfbcff67a336f6f74b420108c4c856223b2", + "src/adaptive_inference/task_spec.py": "2975c0c4489ac863502189777699d13d7b2b8aca6d19191247fd860ed4a0ce76", + "src/adaptive_inference/task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6", + "tests/admission_activity_lineage.py": "6f68c3a34071b01939a9cd9daea9f031b3f28b40b3565fca13c0adaacea49228", + "tests/ast_preservation.py": "0961a958c54975ff8eee14359015e6c221896b3cef91cb2bddc42333d7829b18", + "tests/automatic_tool_recovery_lineage.py": "817799a32fe254db53c0aef5e2254d5a10a47b9b769d64291c211966b79757c1", + "tests/compiler_lineage.py": "efb1343b31727ea4cc7845c40a00e30cb89b14c148cf6c4df67013a0a3f8e28e", + "tests/continuation_cpu_child.py": "8df26213c640ac0cfc057714315c8bb6e97b4232b4af668bfbf7733bd85d94b6", + "tests/continuation_diagnostics_lineage.py": "d00a10fc050b6bb71c7271c872b5640135960b564fd3493d461c5bf3be19340f", + "tests/continuation_lineage.py": "9b0b36aecb833dc318e620c8175a228b87da5b2ee66667105b6395845c7bccc1", + "tests/continuation_timeout_diagnostics.py": "f57c0ec235134d8a1057504d01f834cffad4d913426bb77cbdfa6be9fa89f903", + "tests/continuation_timeout_lineage.py": "81de8bdcff663fe84d3f248a4f69e56b33c3fd6410d19e3b1722f92d85165917", + "tests/desktop_gui_support/intent_lineage.py": "4d43e464ca7fe0a43f196cc4124b92490c0f75f924afab86a46dfc24c4c92966", + "tests/desktop_native_support/planner_source/memory_reader.py": "d3c36b74a6040b8c743bb7362275fb7fc6c7a22f8aaed38e94798ab193167e86", + "tests/desktop_native_support/planner_source/owner_engine.py": "d0b4174d3b1aff944d749c6094e2953399bf895f31001aa2f8a1180fba18a4f4", + "tests/desktop_native_support/planner_source/sealed_memory_policy.py": "7372fc3078380e3545830ed68de66934780e34391f6f1ad7a4c99df89d1d51fc", + "tests/desktop_support/__init__.py": "235aa7f9813b470478cc5309d395b0b4bd7a2c8d6b81214ef5fb44d12e5e599e", + "tests/desktop_support/policy.py": "35781fb869c03220c6f48c3e138083b15bbcf7e40187edbfe546ad4da09b7faa", + "tests/execution_clock_lineage.py": "986dadeee0a9feff4860b12ae09ab170f2dec281d83c8941e37657c50642ad4b", + "tests/paused_expiry_lineage.py": "49bc724e68c8e851f744c1258b609b82f0c358967fdfeb76759b06f394afc3c0", + "tests/pending_tool_lineage.py": "2727e96a5280c0e24b1b2c5428bbff4a8d18cec7d628dd951bf1d99645b80cb4", + "tests/recipe_reconciliation_lineage.py": "fd297aeea09c5080dd28cb3a373abda4a910b3dea642a849f1cd2e8cebc3c96e", + "tests/recovery_capability_support.py": "71589cb8b192714566fc1446dabe9301bdbffee7175760b981f735c0afa1223a", + "tests/safe_wait_fixture.py": "1b4ad2f7f38f9f08cae85d34b6b395b75dacb3ea2c8b5a4d3b5911f94952d792", + "tests/task_contract_lineage.py": "d1b822b7c1a8738a86e0b121ebe7e0c7a07137e01e90b115681cacea157dff64", + "tests/task_file_alias_lineage.py": "798ba1fad6d66f28e2fd424ccb6dbca936396ed7efe8dd2f4e5ff6f6e683df2f", + "tests/task_handover_lineage.py": "3bdc0af1c0b71acb82a68f9b842868fd8e57535e69bf2ae771113556817ae743", + "tests/task_spec_support.py": "59c38306f1bcb1437038cf4bd24a1bf0f0e97bf69cc57985294a969c9dfa79cb", + "tests/test_adapters.py": "4c86020e8e2812b5c0a02a776ec95768ce5ad7716a098be874009d4f21fe68fb", + "tests/test_admission_activity_lineage.py": "93282b45ff751718fffd152084a5a628fbf326af19d6c20cb24ffbf65f25262b", + "tests/test_admission_wait_continuation_storage.py": "f388a5194cf42d460f994a8c07dba46b2d082785fdb2fc59117f59c228344a60", + "tests/test_admission_wait_continuation_worker.py": "57d2202a7c3e899521b8a71d9ca07231ca7c0afc49e7eef744b0cf0116fcdd74", + "tests/test_admission_wait_expiry.py": "d953e4adff7d2e4cc5b75cb3bafa76a45bf5dbe81e62a83fd79112777099309d", + "tests/test_ast_preservation.py": "c122cf3270ddb3781aac401e80fbbecacaf974803b9214feb0cf09afc07eaa69", + "tests/test_automatic_tool_recovery_lineage.py": "2dea3671bcc09a8fee02b346a547a1b5813bc15131aa9b3a3ae0581b0ef4d99f", + "tests/test_cli.py": "d43418742fa159814ba6e229f1e3ff78599821687cec22fb5be5d6bedcefb771", + "tests/test_client_routing.py": "77ca8f1a1d362e19b012090e6ecfa7a9734b8416d6a6a9c9c7732b7fa05c1be9", + "tests/test_compiler_bridge.py": "1add5c0263d14d494842b3cbf28bdf0d262e4e0b6d8ed9d22853265d857db3c5", + "tests/test_compiler_client.py": "28ada40a394da69330ab5b76a01aad28b504607de4d68cb88237c4e24b5c3aae", + "tests/test_compiler_composition.py": "e2804afb1df37841fdd466cf54bef9e08e009d5fba2694f2b3227d0c151191c0", + "tests/test_compiler_evidence.py": "f49d8afe18c7be1a3ce36ba72b91674a1dbd9cd0ed04612f0774251d3d78a4aa", + "tests/test_compiler_host.py": "01bf791daf9af0848f2e0f7224d171100eea9b443cc0b813a502f006f857555e", + "tests/test_compiler_lineage.py": "e0ee3adb15ec70c372cf09b2b7921a62266f0c8ca704bdca2143a92fcf2fb229", + "tests/test_compiler_recipe.py": "cf7662e4e2974a1d6190862733d5c5a77c1aeb274b082331f16c2169639ff642", + "tests/test_compiler_recovery.py": "768d2b208ebe655e20915f449671fc0a4af62c19d6c635060efd17d2e42b3511", + "tests/test_compiler_task_spec.py": "5bdb2cfffca65a962b11913ba5a3bdf8a462e18e66f5d028c93630854dc7283f", + "tests/test_compiler_template.py": "30f0fdd430d3b02711ad42cbaee7af989ad9a9d268690380590fab2e1daa571c", + "tests/test_compiler_wire_boundary.py": "f31b3300e581fd6a64480f16c0acd5f1dfd7cfb21878e57df48679dd2f07ccb8", + "tests/test_compiler_worker.py": "73889e833156060ad65262b6797e682be9e307c29f99c635eb2d6f9cb4108423", + "tests/test_context_budget.py": "5242798b1c523f9a348d20d7d48819c0052c8ef47d1199f19642ffb6e3ddb60c", + "tests/test_continuation_chain.py": "c836c44faa92c40d2f9bbd3a41159cebbe5fafe579080e6ffaff8a75e126770b", + "tests/test_continuation_diagnostics_lineage.py": "2bd03e2d6956d2a40718e6861afb9f91c503f0f0620d68c411c6ebd586abdf8a", + "tests/test_continuation_host.py": "4858cf5d589dd3960ccf16f16896c8c6c085672d8c6aa4425c37184f02810bbf", + "tests/test_continuation_lineage.py": "9644ead6fc86d61b923d3bbe78984542add01816cbd82c505818c04b421a2bab", + "tests/test_continuation_observation.py": "eb377962df08cc3049c8aa9311ada117f3480cd52bdab2eda1f75922cdbb66b4", + "tests/test_continuation_protocol.py": "f2cbe50938e8fb3135d72833017e6d9add347ad76a139a663ac487593221e926", + "tests/test_continuation_readiness.py": "b2b5bba16fff464fcd42dc09bb8cf4cf991c92682ed01823765556bb2c08a440", + "tests/test_continuation_storage.py": "b2def3c19429c1733c35d497c1d617df554d6c0f91983c423be64a8e0ed9ad23", + "tests/test_continuation_timeout_diagnostics.py": "400bb2a43132940c4900d3be80cf6966625ca4368b1eec45847721cbfc5f237c", + "tests/test_continuation_timeout_lineage.py": "d3e368b685e46facd950eb3cbf9486d1dd6fe6620ef08d4d9ee11daa675877e4", + "tests/test_continuation_worker.py": "d3bc078658b5bce213a5bb9d4f57336b4c5ebc42638717dc99890e00d11fbf1c", + "tests/test_control_mailbox.py": "e6352613bd631883d618a27537b21869122ef686e60295e86f4143cbb7fce920", + "tests/test_control_mailbox_snapshot.py": "7496e28eb169968a96474812c2e2ca821d7c4d5e8562921215a731ba9673076f", + "tests/test_core.py": "713380f1f3338bcd5ccc2284c318cbd810f65af82f4c35166ed8ac57362258c0", + "tests/test_demand.py": "e88377d7239fdf29c046515c2b395bd89ac83306483f3d3f7a0f01792f888872", + "tests/test_desktop_admission_activity.py": "761975f00aa126027c12124a6a88c322524877f5693b275f5835823314a3cbbf", + "tests/test_desktop_ambiguous_start.py": "68569ed408aca73bc336ed2bd7ecd18891867efff9eba92f86047656a18c9317", + "tests/test_desktop_automatic_recovery.py": "64665523cc1e54e47577bab4653a3d00e830cc58ef64b3f5b46fd7727051699c", + "tests/test_desktop_composition.py": "29efb54d499c69c00d44ef5b5660c9aa7c747ccdb0ef109c9c431d0787b35df1", + "tests/test_desktop_control_client.py": "bb7efc826978c838391642efa9ba360a5d5df8f895f1e622540581160eacff68", + "tests/test_desktop_control_intent.py": "dad0926b462f537233c00243dace31bd8c933b4dc18a92cd75da9067cd0a2e73", + "tests/test_desktop_control_intent_lineage.py": "efc05bfae64bc23c769c9c846280b4fd9481541aef5a1402220a6d1e4fcbfc1c", + "tests/test_desktop_control_transport.py": "b7d944a92f2b3c50ed948c1d72b8842c680a49af208a95e1425fa329d2d7cb5e", + "tests/test_desktop_expiry_composition.py": "6c70cf05667986381edc06f8779246d335264f06f8312f853f8e5e4048915570", + "tests/test_desktop_expiry_controls.py": "94d817cf047c91791303c6e108e0161138f31a7e5476c7a7f7ae019b4f875fcb", + "tests/test_desktop_expiry_worker.py": "179d1dc3b3cfdcbe284978f21bd2439b6823ca964739bbb88c57451f9d29637b", + "tests/test_desktop_gui_installation.py": "e2df5291e1296bf7a0e23f264f5e9b6ae921f5148dd08019d8cc5ae8e1c0ad2b", + "tests/test_desktop_gui_render.py": "b31dc791a4e32ade8b7103c1c75980084a6e22ded148b83535f0bf9e27ecadd2", + "tests/test_desktop_hermes_installation.py": "bedd5ab75cf5a3eff6ea073f59832d60d663604aaa240e02c41122e689c46a2f", + "tests/test_desktop_installation.py": "1c4a0130e08902db83f58f3f86e79659b1f230de70878c3371e2829d5e6804c5", + "tests/test_desktop_lifecycle.py": "addae0018a741a7165946edd0f71f8bde5de43a3866412ae99f6879fa081fcfd", + "tests/test_desktop_mailbox_runtime.py": "02cb14272823904f022307a5757f3a13daa9fe68bf734d6e6238554d57268697", + "tests/test_desktop_native_owner.py": "883b99e6b5257930f3152f1608bc725d81482897231bd3d3d0970b1d18747c1d", + "tests/test_desktop_owned_launch.py": "548c0891dcc1725219cd2e4c77049b0b3f7b11269053e91e9a0586e37f3a597a", + "tests/test_desktop_owned_stdio.py": "7a9932ed00d010507f9a521798d88dfe96b1e7060d57c984ba086dee6e65c5fc", + "tests/test_desktop_pause_activity.py": "a62882efd2a7b7acfae74fa9df5109ced630f83773b54f144a7658f885e94cb5", + "tests/test_desktop_paused_expiry.py": "72ba7af9375163465186640d6cb1065765e6f3c68eef70b8580a60a791a3f9aa", + "tests/test_desktop_persistent_host.py": "6461882654efc7a9b936d0a44e6b0191139836236ae2914ef00857cc0d1ae7d3", + "tests/test_desktop_recovery_capabilities.py": "158dbb4fa145cf915e48fb543386b6d41a2a9f8d3e4de9bd3f0c9a7116b983c9", + "tests/test_desktop_report_feedback.py": "cff5bc678bb85cd0281ba80a58f8b30e56549ce2054a486fe920a190bcf446c2", + "tests/test_desktop_report_hold.py": "0694783f096b3173c3223514e6fe6b885ef05a4bdb327374c1ce4ed5800f0b6f", + "tests/test_desktop_retained_ui.py": "512bc9554651ed3c263b6a4202b29d3334f6966656fdd76e45b6aab03cb6a252", + "tests/test_desktop_saved_tasks.py": "abbf75c14b13bbd201ca3fbecd7094f72ed9ad82eb0bde0e8489585e1b410564", + "tests/test_desktop_serial_listener.py": "053a0efa5060798099983e99852d922a1ded68f5d489478ca8d210c1ebc8a3f7", + "tests/test_desktop_setup.py": "fe62c305c7074b373a2ad08155865f2740d9951ea5e25ad6cfc8eeb448f947a8", + "tests/test_desktop_source_lineage.py": "303b8be5d581c936d7183fe48261e619625fbd953f71aee17a14ad1983d5a75a", + "tests/test_desktop_startup_acquisition.py": "b37546ab7e7c0b1b7df3d37b7886a25d8daafe7ea2dd19ea739df7c9d402de62", + "tests/test_desktop_ui_intent.py": "4e5bf4704bfe0f4ce6c039d23acfc30ec3c1299f7e4137d981118e36e5bd8779", + "tests/test_desktop_ui_progress.py": "0cb9025da99232ddab49b9688167782c549448b5940a76456b758046646c584f", + "tests/test_distribution_checks.py": "35f1d5c5da751141422fe0f545dc5d197bb34a123b73e46bac0bab71345b42dc", + "tests/test_execution_budget.py": "2308ea5c50fcae261da2d58e67e49c976ff56eca7076bf1f9b900e23aca56433", + "tests/test_execution_clock.py": "5fd7fe388cc437089f7702d1e643c60483e83a7b9d69b744eeb268bde8a2f8d4", + "tests/test_execution_clock_lineage.py": "c969946509367ad1b167f0492861d85ce573a358a36ee808ef9778235da19c25", + "tests/test_gateway_regressions.py": "99ce58efd78800e491543f7e6128a24e6a6f35a159b82ec7ebe174623f8b72f3", + "tests/test_generation_options.py": "c8f2297140580b78303d86bfa75dbcede27f9652b194b44b32caf66fa4b30431", + "tests/test_handoff.py": "db9725043a720b29e3af4b809c3af4d92bba540febea94515de52d5505c7d8f7", + "tests/test_handoff_demand.py": "2045eb00fe4e12761bed32a040ed28231cd35f38b4e81ffeee2ef3981d98288c", + "tests/test_handover_saved_tasks.py": "53bfffa3b6e3fc06c6fcd1e615635d0f9a93fe53fa11cf485f40dfa9dcbd4920", + "tests/test_immediate_admission.py": "0eb75d3e01cbb220c04b03c077b87e1ba7e05dff42cdc73a44a8c446cc36ccdc", + "tests/test_installed_desktop_qualification.py": "1ea4eb0fb8ae3784a3043c42b17d067a10ba3530e7bbf943226798a44b568e36", + "tests/test_openclaw_config.py": "dfb4cc1bc481bd2fa6cfadd230fa4a99c157ca1533eb5f26a1429ce5d34604ed", + "tests/test_pause_observation_lineage.py": "b0acbc024b51fd6fcadece68d8a06f204531dc28cb1ca80fef76cd59fba584a0", + "tests/test_paused_execution.py": "5746905286c0643c13a199b4a50a0340234130c92329882e5d429d9ef086b5ed", + "tests/test_paused_expiry_lineage.py": "a2128bc8bdab1f375fbdf8247b10cf71a01588ff5aff21a1c0caed141f071c2b", + "tests/test_paused_observation_race.py": "aa41e7bce723c88b0de777d875d6dc76cc5e1995e2b2672555ca77714da300e7", + "tests/test_pending_recipe_reconciliation.py": "c778563d6cf3890c77b3bc12f66fc95191e24a2cea6d8097889ccfcb3b868a42", + "tests/test_pending_tool_lineage.py": "b63114b5896666122d4bebd30d346356c3b50489e0c7a7ae674cf9d9ecdcfa89", + "tests/test_persistent_feedback.py": "8bf8520dd4a564d2a172e52aa732c27f7e87931d7c230fc311bf99a400cdcf45", + "tests/test_prebegin_deadline_edges.py": "c878e8339344032996d48eb4bf90e3a1dcbb836906c755695f2f25a9b2b389e6", + "tests/test_reboot_saved_tasks.py": "71661640de44fb0a4db519de39460f3772c5d0f550db80dad00967c775b1aff7", + "tests/test_reboot_worker.py": "9390ccc2f3a5bd0b66ef4a83c3d69eab7c91c9d8f4bd031340e9fddf7b446cf1", + "tests/test_recipe_reconciliation.py": "e45616abc1f48a91787c6c6e294100401a09646d176c4f3ad65f3509c01faf8a", + "tests/test_recipe_reconciliation_lineage.py": "ac6562d3ec47c3703aefd426fe02d3b5adeceaca036859de23309f3d86f65ebc", + "tests/test_resource_integration.py": "0f174caa6fe5be9c11658114f45abb7c36922c598c4e5e14f1f3845246f6d85a", + "tests/test_resource_policy.py": "7bb50909c986ed959d35cf46ee3efafa0d8f21eb718349e63825d43279cdb9b3", + "tests/test_restart_demo.py": "0f78c045ad1bdcb9729d2e7cc17acb6bd94f00f0e6cbb5d76b535557ca4aecc4", + "tests/test_runtime_timing.py": "8f995d57600b530198020d7fbe6b4f55064e77bf9ff49f49f842ce7d4358802f", + "tests/test_runtime_timing_review.py": "77ff919951906d8ee6b3883bc52e7e00e744295847a45c57192515187f79c964", + "tests/test_safe_wait_lineage.py": "968b0a70de21f13a7a14c3d655460742ffebd7fb7fdea85a3dd0c6d139890911", + "tests/test_sdk_rebuild_pins.py": "7fe955041aa54c6dc841daa639b9718392d491012096cd373b2ea72102ed9e31", + "tests/test_task_admission.py": "52662f4136d655bc37124a65c56187d05c88b0d47ef9bfa4d2793785ecd87447", + "tests/test_task_contract_lineage.py": "6c3c0c8cdeedb5aeee762cc811ada8269d28be11a7cc4088d8b3e0ab473561f8", + "tests/test_task_file_alias_lineage.py": "13231bedd85541c887aaf93ccfa49386a4b45b1d738d0eb5205cea03ba10ceb9", + "tests/test_task_file_alias_review.py": "7cf62a539e3e33e3b5682bef2195bf1fa2c6114229166403ac09c3c1985a7838", + "tests/test_task_file_aliases.py": "9b1f5f7175c85b9adb58537058fd4b6c562be2e1bad3e9c3252371f810d15884", + "tests/test_task_handover.py": "cb423d2d72d0b163174f0d5902d41489a5dbe79647aff2519c2f2bcf067263df", + "tests/test_task_handover_lineage.py": "bc7f4f49276df93cdad2bc160ec049bc8a9a6235c7cae412ffe6c5e45cb20f96", + "tests/test_task_handover_protocol.py": "c8f1747d2159dc7c02201a3944f5655adcc7af621b38ada7de63e900170a5a9a", + "tests/test_task_preparation_lineage.py": "669ed83471487167829bf2ce3c3dcdc72f62f0c099b301801d36a87a2316a7c0", + "tests/test_task_prepare.py": "451e44c412fbfa87b8f3aa38ce3917018c73856991c5b47240085e6ef025f101", + "tests/test_task_spec.py": "682d10e7b1e4d77a7af170e5aff7611b63134dfeeb8a166379ced182188fc999", + "tests/test_task_spec_recovery.py": "458c0e2a92c5a286194392e31d4df4413dc78a8fcc5bf372115bad916003e035", + "tests/test_task_spec_worker.py": "633ac717efb0e21b601cfea3c0397acd212680ad8cd611d90870588d468fbf20", + "tests/test_task_store.py": "7a59289ee59254befa36201ca70dc9074bcadde859ea7a3056c42b5fa966c06e", + "tests/test_task_store_review.py": "7a697104bb2fa8c5546bb6aea48852d7682e28214c2653b94de0fe9c78bda087", + "tests/test_task_store_verification.py": "d7518ece8db9e668abdeaf3c311d626582ca2ec2d0bc926effdc35da3815a9de", + "tests/test_thread_pause_lineage.py": "3a2f43352e956a4eed5cdb095c65f75514b2a7b3715cfc00a54e8934e3d9272a", + "tests/test_video_adaptive_audit.py": "3660c835bdfc4b9f9c340789956e803839a7a10291c68f3a451f9a158a95322b", + "tests/test_video_adaptive_observation.py": "7c26abb6b121bc280dd88e2a82f1e481225bcbe0127b16daf8a02e86bf57a5b2", + "tests/test_video_coexistence.py": "4ad43757a6f7c5aefcdc0686cbfa26d24873fcf6675a2fa5cba873119a4806c4", + "tests/test_video_composition.py": "c48c626de34fdf3d6e2ea07d5d908a894af9106c5c40f5e1fc9a41fefb8c5f04", + "tests/test_video_dispatch.py": "e11b31646656f9d5d2632dd6b84f0940c5b3694cbc63092818ac3cfabfc645ea", + "tests/test_video_lineage.py": "fd93d29e915c400690c603c1990ae3528c8a245173be639041d28cf0eb70dcba", + "tests/test_video_recipe.py": "5950ba021e2788e208e9c44c61e32304d342c4bebc4d6ccbd215e6bc103c49ca", + "tests/test_video_saved_tasks.py": "a7fa6fc47e10c17ab163f92db2c8f432eb64d58f549c2c883b37c06ffdbcac92", + "tests/test_video_task_spec.py": "0144a5fd1adbb125dccf7734087f36dfcec2e4e257de04340e96dec5b8dc9e71", + "tests/test_wait_continuation_ui.py": "0b9fdfcf1d972fde8034af730344c8f1beaf5d72d689e5407a5e09f3a32b5c62", + "tests/thread_pause_lineage.py": "c3e0e881475093975350f33adcbfd855b19da32e4b2705a0e203140717139325", + "tests/video_lineage.py": "971470d6fd3bb1f6c2c7c1b5b8c46c6844540651e9db63e251ae5322825b45c3" + }, + "manifest_sha256": "d749cfab88d9feb88772b09785da9c213bb10f47668803269eb91906713ca8c6", + "paths_relative_to": "repository root" + }, + "validation": { + "components": { + "child_joined": true, + "log_sha256": "6c9b7743ca7b6b9e274f8fc51443864d0c92d99e56e84824460508dc13441b4e", + "modules": [ + "test_compiler_composition", + "test_video_composition", + "test_desktop_automatic_recovery", + "test_desktop_composition", + "test_automatic_tool_recovery_lineage" + ], + "passed": true, + "tests": 41 + }, + "core": { + "child_joined": true, + "elapsed_seconds": 867.39, + "errors": 0, + "failures": 0, + "forced_cleanup": false, + "includes_all_lineage_tests": true, + "log_sha256": "a1ebfca7109111f2e8644e8c5be60cfa70fe3d6fc6f02dcceb8a48a2ba3b9cd5", + "passed": true, + "skipped": 0, + "tests": 1595 + }, + "hosted_ci_claim": false, + "installation": { + "commands_joined": 7, + "isolated_copies": true, + "passed": true, + "roles": [ + "host", + "worker" + ], + "runtime_modules_per_role": 77 + }, + "package": { + "note": "Built before the tests-only correction; runtime modules are identical. Final distribution check is repeated at publication.", + "passed": true, + "scope": "built source payload and runtime modules/licenses match local files; no runtime/model test", + "sdist_sha256": "efd17ce8a5c588878c995857c18b135b12f777c392de1bd48b242947ab1f94f9", + "source_files_verified": 1416, + "version": "0.4.0a1", + "wheel_modules_verified": 77, + "wheel_sha256": "401977bfb0541069752ac386ddc6aaaafad23ab095afc74dc7613ef80481b6a7" + }, + "platform": "Windows", + "preserved_failed_core_attempt": { + "change": "Test doubles now expose lifecycle() reporting a bound host. Tests only; all 77 runtime modules unchanged. Both test modules are added to the source lineage.", + "errors": 2, + "failures": 0, + "finding": "Two existing launch tests used a bare object() as the tool host; the new reuse guard calls compiler_host.lifecycle().", + "log_sha256": "e29e79f396dc857446d891144f1c2cad5a1b23460938348b0fa766a221bf5b0a", + "passed": false, + "tests": 1595 + }, + "python": "3.11.9", + "remaining_workflow": { + "commands": [ + { + "exit_code": 0, + "joined": true, + "label": "provenance-complete", + "log_sha256": "ea060c83bccb6e59c64d697d1c536819a257579c9bd2c3562986564624c051d1" + }, + { + "exit_code": 0, + "joined": true, + "label": "hermes-boundary", + "log_sha256": "d76d538b040481ab6ffb4b7c30ae1d7e6a71daf64b3cf6916d790420645ab07a" + }, + { + "exit_code": 0, + "joined": true, + "label": "foreground-fixtures", + "log_sha256": "72fc8db58ce14380741108deef72f2c85a1849668e48034555704d71089fed08" + }, + { + "exit_code": 0, + "joined": true, + "label": "native-fixtures", + "log_sha256": "113c65c4b7ea7da97c6f3706421fd573035a1eb0baf4abae60dfe82f41247ba9" + }, + { + "exit_code": 0, + "joined": true, + "label": "arithmetic-feedback", + "log_sha256": "e5e1819e280d5af4cd5aff45bb5ffe885511e0cd588e8717328cc8f16970a62a" + }, + { + "exit_code": 0, + "joined": true, + "label": "shutdown-receipts", + "log_sha256": "826de1cb04d990157d0b556548327db813e2feb93e55d5d2850b510974a7115c" + }, + { + "exit_code": 0, + "joined": true, + "label": "historical-native-sources", + "log_sha256": "6434d94ccde97a1664ae9d7e780225db8b33dbb81411bb5f3af61cf7e75d97b7" + }, + { + "exit_code": 0, + "joined": true, + "label": "sanitizer", + "log_sha256": "3b992a31d5472853d7d26bb5c0292e3979438a4ac5405d0da16abbc04c8234da" + }, + { + "exit_code": 0, + "joined": true, + "label": "reference-sources", + "log_sha256": "3d2b3dda7fb7bf23b77745b7abd19134ca17116e865e9edd00341818719ca6ea" + }, + { + "exit_code": 0, + "joined": true, + "label": "client-transports", + "log_sha256": "1ffe3530989328c54a02839804673ecca5388166a95c834d33d0370800537ffe" + }, + { + "exit_code": 0, + "joined": true, + "label": "strata-verify-only", + "log_sha256": "2780603be4c952661d193e01b7219feea318e356a511d9700724bac3aba6ea96" + }, + { + "exit_code": 0, + "joined": true, + "label": "hermes-verify-only", + "log_sha256": "5585ff7be8c4f90b98d93519105b080879083349f5bfddc620597519dd7ac7fb" + }, + { + "exit_code": 0, + "joined": true, + "label": "demo", + "log_sha256": "e5e97c1e33ae077f2193a76b273d2a3a63bb58fe743f47f58949ddff159a353b" + }, + { + "exit_code": 0, + "joined": true, + "label": "profiles", + "log_sha256": "cf0b68d1d56d3a05f846b0a7862df30b913b962713693a83e44be674034dfb96" + }, + { + "exit_code": 0, + "joined": true, + "label": "offline-source", + "log_sha256": "af8e28bdd5c620aff959a4487978d156f21da99338934374656c9fffa62f4fc1" + }, + { + "exit_code": 0, + "joined": true, + "label": "hermes-current-source", + "log_sha256": "527eb377ca81a272d57a2e6280125ea058fb686d5ae3a6b9592cc969eda56d79" + } + ], + "passed": true, + "pre_correction_run_also_passed": true, + "source_and_dependencies_unchanged": true + } + } +} diff --git a/provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json b/provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json new file mode 100644 index 0000000..5dbe958 --- /dev/null +++ b/provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json @@ -0,0 +1,344 @@ +{ + "contract": { + "budget": { + "execution_seconds": 900, + "max_admission_attempts": 32, + "max_attempts": 8, + "max_requests": 8 + }, + "foreground_policy": "cooperative-owned-foreground.v1", + "original_task_deadline_unchanged": true, + "profile_sha256": "2e22d83f291b692c1a6af600ab9f05a36f84741db1f8a60a48112fd5385885bf", + "task_spec": "examples/tasks/video-testclip/task.json", + "task_spec_file_sha256": "8154b215c9cb526053a96ef89fbd12c8bd194058f89a0976c41b2e11d19ad65e", + "task_spec_sha256": "e5ebf412ec13e689815f46b42b12ea5e50377a5caf702f1e7a55ee40f1959ab8" + }, + "core_result": { + "artifact": { + "audio": false, + "bytes": 1051294, + "decode_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "media": { + "codec_name": "h264", + "frames": 90, + "height": 720, + "width": 1280 + }, + "probe_sha256": "b07166007a1b411e3db7bb3b97cf446a11fb5e20bb6178754a525f8494f72f23", + "sha256": "8425abb7459b478c239771eefb6e8202e1f01c466775ce30d5878b124c98e7cc" + }, + "completed_tool_redispatched": false, + "controls": { + "other": 0, + "resume": 1, + "stop": 1 + }, + "distinct_native_owners": 2, + "foreground": { + "actions": 1, + "activity_samples": 166 + }, + "model_requests": 2, + "process_exits": { + "reconcile": 0, + "supervisor": 0, + "worker": 86, + "worker-2": 0 + }, + "resource_sequence": [ + "one observed Explorer F5 during the first real IQ3_S request", + "foreground policy drains that request and fully releases the first native owner (parked)", + "actual return to the controller", + "fresh capacity checked and exclusive video grant", + "fixed NVENC encode, probe and decode completed by the tool host", + "first worker exits with code 86 before effect journal or native feedback", + "host exports terminal_receipt, retires the binding and runs reconcile automatically", + "replacement worker launched under the original deadline", + "distinct native owner restored", + "second real request consumed the reconciled video feedback", + "task accepted and observed normal Stop" + ], + "restoration_during_video_grant": false, + "video_job_accounting": { + "active": 0, + "limit_terminated": 0, + "total": 9 + }, + "video_operations": 1 + }, + "date": "2026-11-08", + "design_references": [ + { + "title": "Erlang/OTP supervision principles", + "url": "https://www.erlang.org/doc/system/sup_princ.html", + "use": "Supervisor-owned bounded restart of a failed worker with explicit restart intensity; design reference only." + }, + { + "title": "Temporal asynchronous activity completion", + "url": "https://docs.temporal.io/develop/python/activities/asynchronous-activity", + "use": "Separate exact host-retained external completion from worker lifetime; design reference only." + }, + { + "title": "AWS durable-execution idempotency", + "url": "https://docs.aws.amazon.com/durable-execution/patterns/best-practices/idempotency/", + "use": "Do not redispatch an uncertain side effect; design reference only." + }, + { + "title": "Temporal activity idempotency", + "url": "https://docs.temporal.io/activity-definition#idempotency", + "use": "Recognize a durable completed effect rather than dispatching it again." + }, + { + "title": "LangGraph functional durable execution", + "url": "https://docs.langchain.com/oss/python/langgraph/functional-api", + "use": "Separate persisted effect results from continuation after worker loss." + }, + { + "title": "Pinned Hermes native conversation projection", + "url": "https://github.com/NousResearch/hermes-agent/blob/14e1e43a23761b369641208e204365cd6cb9399d/hermes_state_messages.py#L1545-L1638", + "use": "Reuse the supported reader, including derived tool-call identity." + } + ], + "documentation": "docs/video-tool-recipe.md", + "environment": { + "hermes_revision": "14e1e43a23761b369641208e204365cd6cb9399d", + "hermes_source": "https://github.com/NousResearch/hermes-agent", + "host_python": { + "sha256": "5f7b89a612c9b8af1d6456cdfcd1dbe5ca630849e79aebced9bee9a6694952ec", + "version": "3.11.9" + }, + "model_configuration": { + "actual_inference": true, + "model": "qwen3.8-flash-next-iq3_s-adaptive-media-beta", + "quality": { + "experimental_speed_projection": false, + "max_tokens": 10923, + "reasoning_budget_tokens": 8192, + "reasoning_effort": "high", + "strata_tune": { + "pcie_frac": 0.37 + } + } + }, + "platform": "Windows x64", + "tools": { + "ffmpeg": { + "sha256": "09948d4cdd0650da6ff5a87577469f2a218dc2615ae379f8f734d24c49de0f73", + "version": "ffmpeg version 8.1.1-full_build-www.gyan.dev Copyright (c) 2000-2026 the FFmpeg developers" + }, + "ffprobe": { + "sha256": "a6618e99bb58869ded3c6f37b53aa1a8d701c3591dbb7b5b317d47369c112be2", + "version": "ffprobe version 8.1.1-full_build-www.gyan.dev Copyright (c) 2007-2026 the FFmpeg developers" + } + }, + "worker_python": { + "sha256": "716fbcb56cb6753dd9ea9f7acb3f988e9c4ba65ab492cb32b9bcd0ab81938737", + "version": "3.14.7" + } + }, + "evidence_sha256": { + "identity": "63f599071fa5827ffefbedaf2ad8b5a8dbf4b53aed1af5320056def9ccbca028", + "physical_attempt_01": "5e122883efae18d4eb7b827b8fb0089534b8d7ab6357cbcd2ed25de6ec478b8f", + "physical_attempt_02": "4fff1f577a452cdae2f825be464c17800faaab3ac9e6162eb9dca92c16cdeb75", + "physical_audit": "448a83578da3731db425264fc406ae9ef258c64cae6d8d44955233800615ae37", + "physical_first_worker_loss": "3cbcf0132718be467498053ebc22b164ba23c26cd71677aa9b74b0b4b05c6633", + "physical_replacement_result": "482233cffc5ad7495b4172566649882e37c1d7d7db0ad47bf7b10dae8055921f", + "physical_root_closure": "17882546406aa0338e486efee09b9c919b1c5f1ea89605bae11f47387c418d27", + "physical_video_receipt": "09b68fc82ce210289633b6aa4b992e55b3f904f90d876e121356bfcf29618101", + "profile": "2e22d83f291b692c1a6af600ab9f05a36f84741db1f8a60a48112fd5385885bf" + }, + "independent_audit": { + "audit_sha256": "448a83578da3731db425264fc406ae9ef258c64cae6d8d44955233800615ae37", + "check_names": [ + "Retained normal app supervisor and verified first-worker fault wrapper pins", + "Actual independent observer source", + "Retained prepare/worker/reconcile/worker-2 joins with exits [0, 86, 0, 0]", + "Actual app GUI listener native Stop and joins", + "First lost worker wrote no completion receipt", + "Distinct bootstrap and controller-ready records for initial and replacement workers", + "Single native session across worker loss and replacement", + "Exact public video task and original allowance", + "Exact deployment and foreground policy", + "Exact previously selected FFmpeg and FFprobe pins", + "Accepted same task/session without unknown effect", + "Actual installed SDK replacement worker model quality and native completion", + "One distinct real fixed video operation without re-execution", + "First-worker loss witness, host terminal_receipt export, and reconcile receipt match", + "Original request and attempt bounds preserved across worker loss", + "Same-instance completed metrics capture", + "Complete wire gateway history", + "Real serialized request and gateway completion 3af173c8dae3411cb39f2767734e1cd8", + "Real serialized request and gateway completion 6aa3011fea774478afd4c24c65131b49", + "Every provider request has a durable generation", + "Durable request equals actual wire and completed provider", + "Durable request equals actual wire and completed provider", + "Untruncated cold controller event ledger", + "Exact fixed video preparation 5d8a6183da6941f190e58e764a85f22a", + "Actual video recipe receipt identity 5d8a6183da6941f190e58e764a85f22a", + "Joined bounded video children 5d8a6183da6941f190e58e764a85f22a", + "Exact NVENC encode probe decode commands and successful exits 5d8a6183da6941f190e58e764a85f22a", + "Exact one native intent and durable reconciled operation 5d8a6183da6941f190e58e764a85f22a", + "Actual reconciled video feedback in replacement worker provider request 5d8a6183da6941f190e58e764a85f22a", + "Native release then single tool grant and completed cleanup 5d8a6183da6941f190e58e764a85f22a", + "Fresh independent tool capacity 5d8a6183da6941f190e58e764a85f22a", + "No model restoration while video owns resource grant 5d8a6183da6941f190e58e764a85f22a", + "Bounded actual video and probe 5d8a6183da6941f190e58e764a85f22a", + "Actual H264 1280x720 90-frame artifact and error-free decode 5d8a6183da6941f190e58e764a85f22a", + "One accepted actual video artifact", + "Original 900-second allowance and durable deadline unchanged across worker loss and replacement", + "Task and mailbox released", + "Exact pre-Resume readiness capture", + "Observed Resume and Stop match actual retained control identity", + "One recorded foreground action for this F5 trial", + "Real first-request foreground hold and policy handoff", + "Exact installed runtime source map", + "Every distinct native owner release has a closed result", + "Actual native owner and all recorded exits 0602b21d30eb4198aa1aace11ea0cf30", + "Actual native owner and all recorded exits fcbcfc658b15431b97335eaad5b7557c", + "Fresh native owner after each release", + "Real subsequent requests use a newly verified runtime generation", + "Copied base database bytes unchanged", + "All explicit audit input bytes unchanged" + ], + "checks": 49, + "closed_database_copies": true, + "original_files_unchanged": true, + "passed": true, + "workload_replayed": false + }, + "limits": [ + "One bounded foreground/IQ3_S/fixed-video workflow with one intentional worker loss; not repeated reliability or performance evidence.", + "The independent observer shutdown still failed; observer shutdown diagnosis remains a separate gate.", + "No application restart, reboot, sleep/wake, active-tool interruption or whole-host loss qualification.", + "Synthetic silent clip only; no H3/ComfyUI generation, visual realism, audio or named autonomous media-test qualification.", + "Model contents were not rehashed; native asset metadata and the unchanged IQ3_S quality settings were retained.", + "External FFmpeg build is identified, not distributed." + ], + "observer": { + "closure_passed": false, + "end_reason": "three_connection_errors_after_response", + "exit_code": 2, + "note": "The independent GET-only observer again failed at shutdown, as in earlier physical trials; task and cleanup evidence come from the closed audit." + }, + "preserved_previous_attempts": [ + { + "attempt": "automatic-physical-run-01", + "change": "Wait about 20 s into the first request before the foreground action.", + "outcome": "operator_timing_window_missed", + "sha256": "5e122883efae18d4eb7b827b8fb0089534b8d7ab6357cbcd2ed25de6ec478b8f", + "summary": "The workflow completed end to end, but the scripted F5 was sent 1.9 s into the first request; request plus release took 63 s, so the 60 s input-idle window expired 2 s before parked and the driver timed out." + }, + { + "attempt": "automatic-physical-run-02", + "change": "Restore the active pointer to the last cleanly stopped owner; the terminated owner record was retained untouched.", + "outcome": "cold_parking_refused_stale_active_pointer_from_run_01_timeout", + "sha256": "4fff1f577a452cdae2f825be464c17800faaab3ac9e6162eb9dca92c16cdeb75", + "summary": "Cold parking failed closed before any task launched because the engine active pointer still named the native owner terminated by the run-01 supervisor timeout." + } + ], + "related_records": [ + "provenance/video-tool/physical-foreground-video-2026-11-02.json", + "provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json", + "provenance/automatic-tool-recovery/local-validation-2026-11-08.json", + "examples/tasks/video-testclip/README.md" + ], + "schema": "adaptive.automatic-tool-recovery.physical-evidence.v1", + "scope": "One Windows IQ3_S video task responds to an actual foreground Explorer action, parks inference, runs one fixed NVENC operation, loses its first worker (exit 86) before journal completion, and is recovered automatically by the desktop host with a replacement worker that consumes the durable feedback and completes.", + "source_identity": { + "base_revision": "9242434991f2c2c21461061d61d898649299b847", + "changed_runtime_modules_from_base": [ + "desktop/composition.py", + "desktop/desktop_controls.py", + "desktop/task_handover.py" + ], + "installed_identity_sha256": "63f599071fa5827ffefbedaf2ad8b5a8dbf4b53aed1af5320056def9ccbca028", + "note": "Source after the base revision is identified by this exact runtime map and the retained source lineage; the publication commit is recorded separately.", + "published_revision": null, + "runtime_map_hash_serialization": "UTF-8 json.dumps(map, sort_keys=True, ensure_ascii=False, allow_nan=False), default separators", + "runtime_map_sha256": "f2735f97f1027cf7720820a2f9197a4ece610bf19d1f0019c35f79fbfe7c7373", + "runtime_paths_relative_to": "src/adaptive_inference", + "runtime_sha256": { + "__init__.py": "1d906cca6d5581290d73292cce949dcaf74001de67bda63b903dfb2e70edacba", + "__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", + "adapters.py": "378ee41c1bdbb16873e31752d704e656a70009862d41b36898d30f57a36dfc58", + "cli.py": "a13d6c2c52445884f2ee9897eb7fadb75188339be4472db98bd760c18bce5ecf", + "client_routing.py": "d57178cfbd5666e75aef311f1362cf8486f48085c6f4995861b4fbd0cb7e41d1", + "config.py": "a6800a081a1e14a46ee18235dd21c1d1a714e6ef4e879840cd2768c590fde669", + "context_budget.py": "cd16b57bc105a179e94dbc9e429c02b901bcd8602cfaf04c9cf8776e9d799ccf", + "control_mailbox.py": "12e6ceaef36b41099422040fb23db0a9bccf96a6afb799f0e301a64bccb95888", + "costs.py": "9e8fcd06d39af57392f191b5d5701aa828f134e5141c7e298aa62e0c232224e1", + "demand.py": "76a9fbb26128c1e458348a37f0d15c36f6a9a5ae2d0f89b15d372bd2d1e394ad", + "demo.py": "5a3b1ff743bee6c45f85df8417bedbc18657f2c59f3e894f1f3298a480b22b93", + "desktop/__init__.py": "6169073159f3c0e506752f54aa51b40b058f71ed058d5272e160aa4a61ee7daa", + "desktop/activity.py": "3282dc37b493ba3fd347e279b074fa3dfd5d2c1ea5eaba8131dc568d97ec0750", + "desktop/cold_start.py": "1264f34910a0d619c6502c7565e66abbfe7ec642391a461390271b8e0799bf68", + "desktop/compiler_bridge.py": "a33613e13995c098488144557a87ee4b34b1ed3f4342610b3a43fa8923945299", + "desktop/compiler_host.py": "95a7d000bea96f527996e360eadb7324aaafd583506d4587dae89288d47ab3d2", + "desktop/compiler_profile.py": "872b8babf3fe26828ce50422485489fdde35bd59f010ff063be66ef253e323fc", + "desktop/compiler_recipe.py": "12b31514327a7c168c915b3a321ee1c17bf8446b49181eed6bed2bced55b01d2", + "desktop/compiler_template.py": "bf7a3123eb147caded7700c7f67e0c072cfe915239fa3a5cb004e7e1772f7f85", + "desktop/composition.py": "a8688f7dadceac26747e139a8d2e53e3a7a1dd5bdb71751049ab72698650165f", + "desktop/continuation_host.py": "9bd258aa8b1a2516e80cd94641f2a9fce3584c5b8aa484badf9854dfc121d7c7", + "desktop/control_client.py": "ba950c73328e084329fdb29ab78fdfe86cfed1715a792cdbf9872a74e8c75919", + "desktop/control_transport.py": "280d8a55793c241b3ec30e253b758de42386f3329bb6477ba47537f9ca264c6b", + "desktop/desktop_controls.py": "89eeb900ce8bcdb079d5400f09408a56e1df15383d58ec74d405a027795469c9", + "desktop/execution_budget.py": "77ad74ddbf0eaa89379fbecb305574d5b70cc32f9e63934a14f439ed2c744728", + "desktop/foreground_policy.py": "35781fb869c03220c6f48c3e138083b15bbcf7e40187edbfe546ad4da09b7faa", + "desktop/gui_launch.py": "c93e837ae15092de1a7814f766731ef38dc118f3e8a50280ee7658b91966355b", + "desktop/gui_session.py": "f232609a01f11977d7f011a8b1c7f2b7c2a0c3ca697766b63ec0873ec9f4cc64", + "desktop/hermes/__init__.py": "43830f7aa3deb6001eb8aa8646fc515f5d8d74a26f50590984c8278e4db67532", + "desktop/hermes/admission.py": "6f12fd6ad21fa194cb337590cf0bbd71b1d394e6a2be0cd79dfdd634af5e0f07", + "desktop/hermes/compiler_client.py": "fb0372c6b16738cdb1b0c8d527765e8b7f60a038513950988c84d40b741bf8a1", + "desktop/hermes/compiler_evidence.py": "8b2144e5b1634d8e2bc132711d948308ebc62320e98f9129c44817848a806d56", + "desktop/hermes/compiler_recovery.py": "03d19056b0c2f73679754aa3eab215d3fb2728b22c71a5ab5e1c327ad6a8c114", + "desktop/hermes/compiler_worker.py": "8d18ef60025318804e81f848546e1c3e0e84f61cf4a09fe7bfd1c759b243d91e", + "desktop/hermes/continuation.py": "d3b396bba2ed794b7304e3bebb888e2d8d51568de846fbe6549363b42d6de5c6", + "desktop/hermes/continuation_worker.py": "6871ecbfaec346ccd658d957b5f593f0dc242db17c1b13b93ab603f693967b67", + "desktop/hermes/control_bridge.py": "7a8e0aa672efeb1c6f676446e0f3b6886fd778f5d0f6c185a87d005a0256a3db", + "desktop/hermes/native_reader.py": "1f51ea1180ef0d7ad52359430831084de7cde45e21e758068fee5b9a93079464", + "desktop/hermes/pause_activity.py": "3cffa8eae39b956de3d873e82593a052c2682aad5f06b36a8ff845080baf0e5d", + "desktop/hermes/recovery.py": "c6f07435ae29f4e2542af585ace107f7952583ed3c0735cf6df2f3a0df438f37", + "desktop/hermes/report_contract.py": "d86aa6ee0f18912400aa80d96c343128af08ae80e7d8c367ee9c8733ea5d69de", + "desktop/hermes/report_validator.py": "378dd445a088a075d300fc85b86160f3b96790ee046adecf926f9e59e8e7d9af", + "desktop/hermes/run_recovery.py": "3d78a3f1f78e024d04b0020e3702c784dfc3723a802bdb87c9cbca25ec827fd3", + "desktop/hermes/sdk_pins.py": "538a5ea8651e1106c7c535ccd73b33b2f56de6c736d926ce6b92c5fb6eb075ce", + "desktop/hermes/service_stop.py": "235b61ad248083f8f0536767ad79473955d96ad1b40189b73c9cfe71439a51c9", + "desktop/installation.py": "13e13d532bedb68f150d1096767c98796de83f74dce54021d286c6c18c521e85", + "desktop/load_feasibility.py": "23e94f3ee770b065fb323f30bb104d8d5732bf710bd3cc9b2a27daa746014dda", + "desktop/mailbox_runtime.py": "aad1c3b383f6954be89ebe6c9f2e680cd286c672d7bb26bd7027402e0a7d6b72", + "desktop/native_owner_adapter.py": "5a79c48911c798310104be07ebb978f9ad9fe4f7ab000c328afd992f75415977", + "desktop/owned_job.py": "90b59dbdae226419fc9680c961c21dfa9e061df08512a7a8089d6f6d11b2ccc4", + "desktop/persistent_host.py": "acff567d5eb6f2c569588bfa684363907897bdb66be8008bc20c10d3ca7c0011", + "desktop/profile.py": "caac6f07fc87ccbf51d97003a0d8cb209d21e04bfc00acda6a5c9e05b2d3fbbc", + "desktop/recipe_dispatch.py": "3a4813e156448305e7880c3cc3ac3b82e992b34a0e05b7f6b74f09bc9e2c6f4b", + "desktop/retained_ui.py": "749b947510377518c452ae7a70fa21c32f27c3f10225f6eb90d87ef03406f5cb", + "desktop/run_gui.py": "eb67c9bac75b72bcc545c234b28c54b8c9c5fbfb3c2183f99e63a47abb3a6ef0", + "desktop/saved_tasks.py": "75703584b3c4cda42aed6de0e4645ea75b2a3b1e2e81c702e919f626cc8ec1b7", + "desktop/serial_listener.py": "048e8ab061e175a978f755801b9a27e3daa292c1dc7dabe3c8a8763be81199b2", + "desktop/startup_pipe.py": "0617c527461b4c437474a947a497300e4c28c06d11f8bb43f1e67fc51a159fae", + "desktop/task_admission.py": "2ca007580bca43678c529599b6296ff4b97c15e210b9daeb651a8f2c4f12b25d", + "desktop/task_handover.py": "8fd90d86635a3b5968a39d32b20e54e67b17c8dc9bb4a90b9fcdcabff97f01b4", + "desktop/ui_intent.py": "ff67208a3890656bc76717ac18c8d0c02a19dc2e09b11b81d88fc1f63c0a29d1", + "desktop/ui_progress.py": "466deda7ba15d73ea779abf3b1b2e690e92217950b15afdbdf8acf9cda3d2de9", + "desktop/video_profile.py": "2ac53c40937e60d984e2903d643d2ee83372760756d1afedf5406b71644b41e4", + "desktop/video_recipe.py": "b255d3f939aa3a1e29449bc397e39b50859fcc0cf8a10201859b2ec2dc50bcff", + "desktop/video_template.py": "1a74d771d83eed358196467857e11c5986cfc6c196b6d172c614224db311a835", + "desktop/window_marker.py": "e2613ef662993be88c332657e4e53c598fdc006634bf8fabf9e0d0a9765c0d36", + "desktop/windows_host_base.py": "ad08c31435bc4da77b43c68321a7c08130ab1c4f7bc688c71191c6497abc97bd", + "execution_clock.py": "c9d04e8e029d4d88830560482e2437cab3d06ef140a942d16543c3be95642cff", + "feedback.py": "c76eba0110659ae67f4a0efd2ca21aff0987af7e9095998e4db9382de20826c3", + "gateway.py": "b2009a8a776bcea4dfc3bedac5296c930ae2211bb8e29d7ccdf76a227fefdef8", + "handoff.py": "6cd7f593698f6d72fc3ac16d9a541436f65b85103bed2420372364262fcb4db1", + "resource_policy.py": "53768d4b95b589bb76d1788a9ccceefe60c05b332b1b1f43dece6b04bfec212a", + "resources.py": "a72e376e4dfc6021e61756812f93275af20c28ec1438faf5eb381a491d3db4d9", + "scheduler.py": "c7f962bd9ef8e44587cadcc02a41fae8a00de7fc15b105848dcdfc4862bd5b4f", + "task_prepare.py": "9cddd6e09b3f5435da08316f8d9ccdfbcff67a336f6f74b420108c4c856223b2", + "task_spec.py": "2975c0c4489ac863502189777699d13d7b2b8aca6d19191247fd860ed4a0ce76", + "task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6" + }, + "sdist_sha256": "efd17ce8a5c588878c995857c18b135b12f777c392de1bd48b242947ab1f94f9", + "source_lineage_sha256": "ccf7907107cebdac3f3c9f25ac39710ff352c0179737f961ca86f4ff85bfab26", + "wheel_sha256": "401977bfb0541069752ac386ddc6aaaafad23ab095afc74dc7613ef80481b6a7" + }, + "status": "bounded_foreground_iq3s_automatic_recovery_verified_observer_shutdown_failed" +} diff --git a/provenance/automatic-tool-recovery/predecessor/tests/test_compiler_composition.py b/provenance/automatic-tool-recovery/predecessor/tests/test_compiler_composition.py new file mode 100644 index 0000000..1da3615 --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/tests/test_compiler_composition.py @@ -0,0 +1,124 @@ +"""Compiler launch/Stop integration with inert tools and retained lifecycle fakes.""" +import asyncio +import hashlib +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest + +from adaptive_inference.desktop.composition import DesktopComposition +from adaptive_inference.desktop.desktop_controls import DesktopControlLifecycle +from adaptive_inference.desktop.control_transport import PreparedTask +from adaptive_inference.desktop.profile import WindowsProfile +from adaptive_inference.desktop.compiler_profile import verify_settings +from adaptive_inference.desktop.task_handover import HostTaskHandover +from adaptive_inference.task_spec import TaskSpec +from test_compiler_task_spec import compiler_data +from test_desktop_installation import profile_value +from test_desktop_lifecycle import Host, Worker, GUI + + +class CompilerLaunchTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.c = object.__new__(DesktopComposition) + self.c.compiler_host = object(); self.c.compiler_bindings = {} + self.c.host_epoch = 'host-1'; self.c.host = SimpleNamespace(gateway=SimpleNamespace(instance='gateway-1')) + self.spec = TaskSpec(compiler_data()) + self.plan = {'task_spec': self.spec.data, 'task_spec_sha256': self.spec.sha256} + + def test_launch_binding_contains_original_identity_without_secret_on_argv(self): + args = self.c.compiler_launch_args(self.root, self.plan, execution_id='task', task_id='task', + deadline=900., diagnostics=self.root) + path = Path(args[1]); raw = path.read_bytes(); data = json.loads(raw) + self.assertEqual(args[3], hashlib.sha256(raw).hexdigest()) + self.assertNotIn(data['capability'], args) + self.assertEqual(data['task_deadline'], 900.) + self.assertEqual(data['execution_id'], 'task') + with self.assertRaises(RuntimeError): + self.c.compiler_launch_args(self.root, self.plan, execution_id='task', task_id='task', + deadline=1800., diagnostics=self.root) + self.assertEqual(path.read_bytes(), raw) + + def test_file_only_launch_has_no_compiler_binding(self): + self.assertEqual(self.c.compiler_launch_args(self.root, {'task_id': 'orders-report'}, + execution_id='task', task_id='task', deadline=900., diagnostics=self.root), []) + self.assertEqual(list(self.root.iterdir()), []) + + def test_explicit_profile_pins_tools_and_keeps_fixed_resource_limits(self): + value = profile_value(self.root) + compiler = {'schema': 'adaptive.compiler-settings.v1', 'environment': {'PATH': str(self.root)}} + for key, name in [('cmake','cmake.exe'), ('ctest','ctest.exe'), ('compiler','clang-cl.exe'), + ('ninja','ninja.exe'), ('linker','lld-link.exe')]: + path = self.root/name; path.write_bytes(b'inert-'+key.encode()) + compiler[key] = {'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} + self.assertIsNone(WindowsProfile.from_dict(value).compiler) + configured = WindowsProfile.from_dict({**value, 'compiler': compiler}).compiler + verify_settings(configured) + self.assertEqual((configured.parallel_jobs, configured.job_memory_mib, configured.max_seconds), (2,2048,90)) + configured.compiler.path.write_bytes(b'changed') + with self.assertRaises(ValueError): verify_settings(configured) + + +class CompilerStopTests(unittest.IsolatedAsyncioTestCase): + async def test_handover_failure_cleanup_joins_compiler_before_native_close(self): + release, entered = asyncio.Event(), asyncio.Event() + operation = asyncio.create_task(release.wait()) + events = [] + class Compiler: + def interrupt(self): events.append('interrupt') + async def close(self, **_): entered.set() + def obligations(self): + return ({'close_task': None, 'operations': ({'task': operation, 'runner': None},)},) + def lifecycle(self): return {'bridge': {'closed': operation.done()}} + class Native: + async def close(self, **_): + assert operation.done(); events.append('native_closed'); return {'closed': True} + composition = SimpleNamespace(compiler_host=Compiler(), handles=[], host=Native(), result={}) + owner = HostTaskHandover(composition); owner.fence = lambda: None + cleanup = asyncio.create_task(owner.failure_cleanup()) + try: + await asyncio.wait_for(entered.wait(), 2) + self.assertNotIn('native_closed', events) + self.assertFalse(cleanup.done()) + release.set(); await asyncio.wait_for(asyncio.shield(cleanup), 2) + self.assertIn('native_closed', events) + self.assertTrue(all(task.done() for task in owner._failure_drains)) + finally: + release.set(); await asyncio.wait_for(asyncio.shield(cleanup), 2) + await operation + + async def test_stop_interrupts_and_joins_compiler_before_worker_stop(self): + worker, gui = Worker(), GUI() + host = Host(worker, gui) + entered, released = asyncio.Event(), asyncio.Event() + events = [] + class Compiler: + def interrupt(self): events.append('compiler_interrupt') + async def close(self, *, timeout): + entered.set(); await released.wait(); events.append('compiler_joined') + return {'bridge': {'closed': True, 'unknown': 0}} + host.compiler_operations = Compiler() + original = worker.request_stop + def stop(cap): events.append('worker_stop'); original(cap) + worker.request_stop = stop + facade = DesktopControlLifecycle(host, PreparedTask('host','task','objective'), + worker_stop=worker, worker_capability=object(), gui=gui) + work = asyncio.create_task(facade.cooperative_stop()) + try: + await asyncio.wait_for(entered.wait(), 2) + self.assertEqual(worker.requests, 0) + self.assertEqual(events, ['compiler_interrupt']) + released.set(); worker.release.set(); gui.release.set() + result = await asyncio.wait_for(asyncio.shield(work), 2) + self.assertLess(events.index('compiler_joined'), events.index('worker_stop')) + self.assertTrue(result['worker_joined']); self.assertTrue(result['host_closed']) + self.assertFalse(result['uncertain']) + finally: + released.set(); worker.release.set(); gui.release.set() + await asyncio.wait_for(asyncio.shield(work), 2) + + +if __name__ == '__main__': unittest.main() diff --git a/provenance/automatic-tool-recovery/predecessor/tests/test_video_composition.py b/provenance/automatic-tool-recovery/predecessor/tests/test_video_composition.py new file mode 100644 index 0000000..e289b69 --- /dev/null +++ b/provenance/automatic-tool-recovery/predecessor/tests/test_video_composition.py @@ -0,0 +1,50 @@ +"""Selected video launch authority with inert bindings; no native execution.""" +import hashlib +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest + +from adaptive_inference.desktop.composition import DesktopComposition +from adaptive_inference.task_spec import TaskSpec +from test_video_task_spec import video_data + + +class VideoLaunchTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.c = object.__new__(DesktopComposition) + self.c.compiler_host = object(); self.c.compiler_bindings = {}; self.c.compiler_kind = 'video' + self.c.host_epoch = 'host-1'; self.c.host = SimpleNamespace(gateway=SimpleNamespace(instance='gateway-1')) + spec = TaskSpec(video_data()) + self.plan = {'task_spec': spec.data, 'task_spec_sha256': spec.sha256} + + def test_video_binding_is_private_and_cannot_renew_original_deadline(self): + args = self.c.compiler_launch_args(self.root, self.plan, execution_id='task', task_id='task', + deadline=900., diagnostics=self.root) + path = Path(args[1]); raw = path.read_bytes(); binding = json.loads(raw) + self.assertEqual(binding['schema'], 'adaptive.video-worker-binding.v1') + self.assertEqual(binding['task_deadline'], 900.) + self.assertEqual(args[3], hashlib.sha256(raw).hexdigest()) + self.assertNotIn(binding['capability'], args) + with self.assertRaises(RuntimeError): + self.c.compiler_launch_args(self.root, self.plan, execution_id='task', task_id='task', + deadline=1800., diagnostics=self.root) + self.assertEqual(path.read_bytes(), raw) + + def test_video_task_cannot_select_compiler_deployment(self): + self.c.compiler_kind = 'compiler' + with self.assertRaisesRegex(RuntimeError, 'configured useful-tool deployment'): + self.c.compiler_launch_args(self.root, self.plan, execution_id='task', task_id='task', + deadline=900., diagnostics=self.root) + self.assertEqual(list(self.root.iterdir()), []) + + def test_file_only_task_does_not_get_video_binding(self): + self.assertEqual(self.c.compiler_launch_args(self.root, {'task_id': 'orders-report'}, + execution_id='task', task_id='task', deadline=900., diagnostics=self.root), []) + self.assertEqual(list(self.root.iterdir()), []) + + +if __name__ == '__main__': unittest.main() diff --git a/provenance/automatic-tool-recovery/source-lineage.json b/provenance/automatic-tool-recovery/source-lineage.json index 8b7b52d..67b8070 100644 --- a/provenance/automatic-tool-recovery/source-lineage.json +++ b/provenance/automatic-tool-recovery/source-lineage.json @@ -107,6 +107,13 @@ "successor_ast_sha256": "c6fc69456720937c5dbac32f04b99d1b11d6cce8559faf9d60e258d075f14011", "successor_sha256": "2727e96a5280c0e24b1b2c5428bbff4a8d18cec7d628dd951bf1d99645b80cb4" }, + "tests/test_compiler_composition.py": { + "predecessor_ast_sha256": "4707818af0c3545480496b9f4603ff04ac1e325fc526263a2cc4677b931762c4", + "predecessor_sha256": "802a5d33dc9cd602bf3d5c00a58e8297b41b89b44b6221d6716131234843f481", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/tests/test_compiler_composition.py", + "successor_ast_sha256": "f56362c97284cbd90282096c8e95dc45abfb3569cfd9d2002b52d7cd1d98645f", + "successor_sha256": "e2804afb1df37841fdd466cf54bef9e08e009d5fba2694f2b3227d0c151191c0" + }, "tests/test_desktop_composition.py": { "predecessor_ast_sha256": "040422da37507cfb592cab3fa3f24bf10e5307f129e28850921f022416b6874f", "predecessor_sha256": "b84ea2ce1b5e8484387dbe1fc5d0557dfb62589452442ea3e48b08ab2f50f1c0", @@ -120,6 +127,13 @@ "predecessor_source": "provenance/automatic-tool-recovery/predecessor/tests/test_pending_tool_lineage.py", "successor_ast_sha256": "d279f1055c39474b0a08f7d6a314542c1a3cb4ed6775e745020a7fd1261250fa", "successor_sha256": "b63114b5896666122d4bebd30d346356c3b50489e0c7a7ae674cf9d9ecdcfa89" + }, + "tests/test_video_composition.py": { + "predecessor_ast_sha256": "6637ea1f568e45ed940147ec2c21513fbc0b37bff07457708d33c2be4e4676a4", + "predecessor_sha256": "31c4c6ee8a629055ec217d80cf577540e51b32e104effb1d64ee7a0afac25d4b", + "predecessor_source": "provenance/automatic-tool-recovery/predecessor/tests/test_video_composition.py", + "successor_ast_sha256": "9d556d3f234cc7473c239bb52a7ec0c860038ac899d3050dfe43d580faef9f73", + "successor_sha256": "c48c626de34fdf3d6e2ea07d5d908a894af9106c5c40f5e1fc9a41fefb8c5f04" } }, "license": "Apache-2.0", diff --git a/tests/test_compiler_composition.py b/tests/test_compiler_composition.py index 1da3615..f308692 100644 --- a/tests/test_compiler_composition.py +++ b/tests/test_compiler_composition.py @@ -24,7 +24,7 @@ def setUp(self): temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup) self.root = Path(temporary.name).resolve() self.c = object.__new__(DesktopComposition) - self.c.compiler_host = object(); self.c.compiler_bindings = {} + self.c.compiler_host = SimpleNamespace(lifecycle=lambda: {'bound': True}); self.c.compiler_bindings = {} self.c.host_epoch = 'host-1'; self.c.host = SimpleNamespace(gateway=SimpleNamespace(instance='gateway-1')) self.spec = TaskSpec(compiler_data()) self.plan = {'task_spec': self.spec.data, 'task_spec_sha256': self.spec.sha256} diff --git a/tests/test_video_composition.py b/tests/test_video_composition.py index e289b69..361bbdc 100644 --- a/tests/test_video_composition.py +++ b/tests/test_video_composition.py @@ -16,7 +16,7 @@ def setUp(self): temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup) self.root = Path(temporary.name).resolve() self.c = object.__new__(DesktopComposition) - self.c.compiler_host = object(); self.c.compiler_bindings = {}; self.c.compiler_kind = 'video' + self.c.compiler_host = SimpleNamespace(lifecycle=lambda: {'bound': True}); self.c.compiler_bindings = {}; self.c.compiler_kind = 'video' self.c.host_epoch = 'host-1'; self.c.host = SimpleNamespace(gateway=SimpleNamespace(instance='gateway-1')) spec = TaskSpec(video_data()) self.plan = {'task_spec': spec.data, 'task_spec_sha256': spec.sha256} From 08053a00d92c065b88bda3ce14f80a7cfe078f0f Mon Sep 17 00:00:00 2001 From: Midhat Nashar <35053211+midhatn@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:29:38 +0300 Subject: [PATCH 3/3] Replace fixed sleep in recovery harness test with polling loop On virtualized Windows CI runners, a fixed 50ms sleep was too short under load, causing test_it_m2_01_automatic_recovery_compiler_harness to fail before interact() completed worker replacement. Replace with a bounded polling loop (up to 2.0s, polling every 10ms) matching adjacent tests. Updates source lineage and the three sanitized evidence records with the new lineage SHA. All 77 runtime files remain identical. --- .../installed-worker-loss-2026-11-08.json | 2 +- .../automatic-tool-recovery/local-validation-2026-11-08.json | 2 +- .../physical-foreground-recovery-2026-11-08.json | 2 +- provenance/automatic-tool-recovery/source-lineage.json | 4 ++-- tests/test_desktop_composition.py | 5 ++++- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json b/provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json index 3040567..3e3960d 100644 --- a/provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json +++ b/provenance/automatic-tool-recovery/installed-worker-loss-2026-11-08.json @@ -278,7 +278,7 @@ "task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6" }, "sdist_sha256": "efd17ce8a5c588878c995857c18b135b12f777c392de1bd48b242947ab1f94f9", - "source_lineage_sha256": "ccf7907107cebdac3f3c9f25ac39710ff352c0179737f961ca86f4ff85bfab26", + "source_lineage_sha256": "8707639da53e139967a5b6762550aca758402aa0dab053e50caa146780243881", "wheel_sha256": "401977bfb0541069752ac386ddc6aaaafad23ab095afc74dc7613ef80481b6a7" }, "status": "passed_at_declared_scope" diff --git a/provenance/automatic-tool-recovery/local-validation-2026-11-08.json b/provenance/automatic-tool-recovery/local-validation-2026-11-08.json index 9d8927c..8085866 100644 --- a/provenance/automatic-tool-recovery/local-validation-2026-11-08.json +++ b/provenance/automatic-tool-recovery/local-validation-2026-11-08.json @@ -189,7 +189,7 @@ "task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6" }, "sdist_sha256": "efd17ce8a5c588878c995857c18b135b12f777c392de1bd48b242947ab1f94f9", - "source_lineage_sha256": "ccf7907107cebdac3f3c9f25ac39710ff352c0179737f961ca86f4ff85bfab26", + "source_lineage_sha256": "8707639da53e139967a5b6762550aca758402aa0dab053e50caa146780243881", "wheel_sha256": "401977bfb0541069752ac386ddc6aaaafad23ab095afc74dc7613ef80481b6a7" }, "status": "passed_at_declared_scope", diff --git a/provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json b/provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json index 5dbe958..b036174 100644 --- a/provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json +++ b/provenance/automatic-tool-recovery/physical-foreground-recovery-2026-11-08.json @@ -337,7 +337,7 @@ "task_store.py": "7bf559ea55e0c19155515f321f7505b4d9275648333bcf2c983c2c280bd2e2e6" }, "sdist_sha256": "efd17ce8a5c588878c995857c18b135b12f777c392de1bd48b242947ab1f94f9", - "source_lineage_sha256": "ccf7907107cebdac3f3c9f25ac39710ff352c0179737f961ca86f4ff85bfab26", + "source_lineage_sha256": "8707639da53e139967a5b6762550aca758402aa0dab053e50caa146780243881", "wheel_sha256": "401977bfb0541069752ac386ddc6aaaafad23ab095afc74dc7613ef80481b6a7" }, "status": "bounded_foreground_iq3s_automatic_recovery_verified_observer_shutdown_failed" diff --git a/provenance/automatic-tool-recovery/source-lineage.json b/provenance/automatic-tool-recovery/source-lineage.json index 67b8070..711d244 100644 --- a/provenance/automatic-tool-recovery/source-lineage.json +++ b/provenance/automatic-tool-recovery/source-lineage.json @@ -118,8 +118,8 @@ "predecessor_ast_sha256": "040422da37507cfb592cab3fa3f24bf10e5307f129e28850921f022416b6874f", "predecessor_sha256": "b84ea2ce1b5e8484387dbe1fc5d0557dfb62589452442ea3e48b08ab2f50f1c0", "predecessor_source": "provenance/automatic-tool-recovery/predecessor/tests/test_desktop_composition.py", - "successor_ast_sha256": "76f9c2f7f3f43eea819956480347c4263b99bfc70cf55508c8715f8050b7ae5a", - "successor_sha256": "29efb54d499c69c00d44ef5b5660c9aa7c747ccdb0ef109c9c431d0787b35df1" + "successor_ast_sha256": "a59cdd585527f714e72cf734227b9466ff11de52dff4e3ced68523f16f2e86d5", + "successor_sha256": "8aae72acbe6e265a29d60281604fd7ccff3933e9dcd0da79e5532fc55fbc32a3" }, "tests/test_pending_tool_lineage.py": { "predecessor_ast_sha256": "e1789a3af4289c067a8233c636c60473b4d95012f512421d33814276cffe1e4c", diff --git a/tests/test_desktop_composition.py b/tests/test_desktop_composition.py index e617503..25ce631 100644 --- a/tests/test_desktop_composition.py +++ b/tests/test_desktop_composition.py @@ -400,7 +400,10 @@ def fake_replace(expected, cb): interact_task = asyncio.create_task(c.interact()) # Let interact() run through worker_1 joining and recovery - await asyncio.sleep(0.05) + for _ in range(200): + if c.worker is worker_2: + break + await asyncio.sleep(0.01) self.assertIn((501, op_id), c.recovery_records) self.assertIs(c.worker, worker_2)