From 36d006a1ee059ab83c70f3cddbc93221244ae943 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:19:27 +0800 Subject: [PATCH 01/16] Read independent ACK sources with the native receipt writer --- .../network/open-repair-offer-client.ts | 281 ++++++++++++++++++ clients/typescript/network/package.json | 1 + docs/OPEN_ACK_RECOVERY.md | 26 ++ scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 2 +- scripts/run_open_network_ci.py | 4 +- ...est_open_repair_offer_client_typescript.py | 119 ++++++++ 7 files changed, 431 insertions(+), 4 deletions(-) create mode 100644 clients/typescript/network/open-repair-offer-client.ts create mode 100644 tests/test_open_repair_offer_client_typescript.py diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts new file mode 100644 index 0000000..3be1f55 --- /dev/null +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -0,0 +1,281 @@ +/** Native receipt-writer preflight over independently authorized empty sources. + * This reads original source evidence only; it does not authorize or upload a receipt. + */ +import {performance} from 'node:perf_hooks'; +import {isProxy} from 'node:util/types'; +import {OpenHTTPTransport,endpoint} from './open-transport.ts'; +import {RepairBudget,RepairError,buildNewWire,objectFields,rawRef,u53,LocalRawResolver,parseNewWire} from './open-repair-wire.ts'; +import type {RepairPolicy,RawRef,RawOriginal} from './open-repair-wire.ts'; +import {verifyAckOfferBootstrapOriginal} from './open-repair-bound.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import type {AckOwnerRecoveryOptions,RecoveredAckOwnerEmptyProof} from './open-repair-client.ts'; +import {verifyAckEmptySourceEvent} from './open-repair-empty.ts'; +import type {AuthenticatedAckEmptySourceEvent} from './open-repair-empty.ts'; +import {parseOriginalControl,verifySourceNodeOriginal,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {verifyBootstrapProofResponse,makeBootstrapChildRequest} from './open-repair-proof.ts'; +import {statusScope,verifyStatusOriginal,authenticateStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; + +type Obj=Record; +const clock=()=>performance.now()/1000; +export interface AckOfferOptions { + readonly targetNodeEntry:unknown;readonly expectedTarget:unknown;readonly expectedAckSlot:unknown; + readonly expectedOwner:unknown;readonly expectedMessageId:string;readonly expectedEnvelopeRef:unknown; + readonly rootEntry:unknown;readonly writeEntry:unknown;readonly bootstrapEntry:unknown; + readonly knownStatuses?:readonly unknown[];readonly archiveStatuses?:readonly unknown[];readonly timeout?:number; +} +export type PreparedAckOffer=RecoveredAckOwnerEmptyProof; +interface Obligation{role:string|null;kind:string;subject:unknown;revision:number;signer:Obj;scope_id:string;mask?:number;} +function fail(code:string):never{throw new RepairError(code);} +function fields(value:unknown,names:readonly string[]):Obj{return objectFields(value,names);} +function options(value:unknown,required:readonly string[],optional:readonly string[]):Obj{ + if(value===null||typeof value!=='object'||isProxy(value))fail('repair_unknown_fields'); + const names=Reflect.ownKeys(value);if(names.some(name=>typeof name!=='string'||![...required,...optional].includes(name)))fail('repair_unknown_fields'); + return fields(value,[...required,...optional.filter(name=>Object.hasOwn(value,name))]); +} +function same(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(a===null||b===null||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(name=>Object.hasOwn(b,name)&&same((a as Obj)[name],(b as Obj)[name])); +} +function refKey(ref:RawRef):string{return `${ref.namespace}:${ref.key}:${ref.raw_sha256}:${ref.size}`;} +function scopeKey(item:Obj):string{return item.scope_kind+':'+item.scope_id;} +function checkedStatus(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{ + const signed=fields(parseOriginalControl(raw,policy,budget).value,['payload','proof']); + const payload=fields(signed.payload,['schema_version','kind','signing_key','scope_key','revision','issued_at','valid_until','entries']); + if(!Array.isArray(payload.entries)||payload.entries.length<1||payload.entries.length>16)fail('repair_invalid_status'); + for(const item of payload.entries)fields(item,['scope_kind','scope_id','minimum_document_revision','status','operation_mask']); + return payload; +} +function snapshotEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget,legacy=false):RawOriginal{ + const held=fields(value,['raw','ref']),ref=rawRef(held.ref),document=legacy?parseOriginalControl(held.raw,policy,budget):parseNewWire(held.raw,policy,budget); + const raw=document.raw;if(ref.namespace!=='meta'||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + return {raw,ref}; +} +export class AckOfferClient{ + readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #limits:Obj; + readonly #statusObserver?: (item:AuthenticatedStatusOriginal)=>void; + readonly #transport:OpenHTTPTransport;readonly #allowLoopback:boolean;readonly #clock:()=>number;readonly #ownTransport:boolean; + constructor(identity:unknown,encryptionIdentity:unknown,value:AckOwnerRecoveryOptions={}){ + const args=options(value,[],['policy','limitPolicy','allowLoopback','transport','clock','statusObserver']); + const meter=new RepairBudget(args.policy??DEFAULT_REPAIR_CLIENT_POLICY);this.#policy=meter.policy; + const held=buildNewWire({identity,encryption:encryptionIdentity,limits:args.limitPolicy??DEFAULT_REPAIR_CLIENT_LIMITS},this.#policy,meter).value as Obj; + this.#identity=fields(held.identity,['schema_version','algorithm','key_id','public_key','private_key']); + this.#encryption=fields(held.encryption,['schema_version','algorithm','key_id','public_key','private_key']); + this.#limits=fields(held.limits,Object.keys(DEFAULT_REPAIR_CLIENT_LIMITS));for(const amount of Object.values(this.#limits))u53(amount,1); + this.#subject=buildNewWire({signing_key:{schema_version:'universal-memory-public-key/v1',algorithm:this.#identity.algorithm, + key_id:this.#identity.key_id,public_key:this.#identity.public_key},encryption_key:{schema_version:'memory-vault-network-encryption-key/v1', + algorithm:this.#encryption.algorithm,key_id:this.#encryption.key_id,public_key:this.#encryption.public_key}},this.#policy,meter).value as Obj; + if(args.allowLoopback!==undefined&&typeof args.allowLoopback!=='boolean')fail('repair_invalid_policy'); + if(args.clock!==undefined&&typeof args.clock!=='function')fail('repair_invalid_policy'); + if(args.statusObserver!==undefined&&typeof args.statusObserver!=='function')fail('repair_invalid_policy'); + this.#allowLoopback=args.allowLoopback??false;this.#clock=args.clock??(()=>Date.now()/1000); + this.#transport=args.transport??new OpenHTTPTransport({allow_loopback:this.#allowLoopback});this.#ownTransport=args.transport===undefined; + this.#statusObserver=args.statusObserver; + Object.freeze(this); + } + close():void{if(this.#ownTransport)this.#transport.close();} + #now():number{return u53(Math.floor(this.#clock()));} + async preflight(baseUrl:string,value:AckOfferOptions):Promise{ + const args=options(value,['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId', + 'expectedEnvelopeRef','rootEntry','writeEntry','bootstrapEntry'],['knownStatuses','archiveStatuses','timeout']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + const policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),deadline=clock()+timeout; + const expected=buildNewWire({target:args.expectedTarget,slot:args.expectedAckSlot,owner:args.expectedOwner, + message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const history=checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget); + const setup=verifyAckOfferBootstrapOriginal(args.bootstrapEntry,{root:args.rootEntry,write:args.writeEntry}, + {expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject, + expectedMessageId:expected.message,expectedEnvelopeRef:expected.envelope,at:started,limitPolicy:this.#limits,policy,budget}); + const nodeEntry=snapshotEntry(args.targetNodeEntry,policy,budget,true),nodePayload=checkedNodePayload(nodeEntry.raw,policy,budget); + const node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:expected.target.signing_key, + expectedStorageEpoch:nodePayload.storage_epoch,at:started,policy,budget}); + if(!same(endpoint(baseUrl,this.#allowLoopback),endpoint(node.payload.base_url,this.#allowLoopback)))fail('repair_proof_mismatch'); + const grant=setup.originals.bootstrap.payload as Obj; + const known=checks.known(history,checks.obligations(setup.originals,expected.slot,budget),expected.slot.root_key,expected.target,budget,true); + const expiry=Math.min(started+Math.min(60,Math.max(1,Math.floor(timeout))),grant.probe_until,grant.proof_until,grant.expires_at, + node.payload.expires_at as number,setup.originals.root.payload.expires_at as number,setup.originals.write.payload.expires_at as number); + if(expiry<=started)fail('repair_access_expired'); + const binding={expectedSubject:this.#subject,expectedTarget:expected.target,targetStorageEpoch:node.payload.storage_epoch as string, + bootstrapGrantSha256:setup.originals.bootstrap.ref.raw_sha256,selector:grant.selector,policy,budget,consumer:'ack_offer' as const}; + let requests=0,wireBytes=0,proofBytes=0; + const request=async(body:Uint8Array,child=false):Promise=>{ + if(requests>=grant.limits.max_requests||clock()>=deadline)fail('repair_over_budget');requests++; + const reply=await this.#transport.requestRepair(baseUrl,body,deadline,{child});wireBytes+=body.length+reply.length;return reply; + }; + const outgoing=await makeBootstrapProbe(this.#identity,{...binding,at:started,expiresAt:expiry}); + if(outgoing.original.raw.length>grant.limits.max_probe_bytes)fail('repair_over_budget'); + const challengeRaw=await request(outgoing.original.raw),challengeDigest=budget.hash(challengeRaw); + const challenge={raw:challengeRaw,ref:rawRef({namespace:'meta',key:challengeDigest,raw_sha256:challengeDigest,size:challengeRaw.length})}; + const challengeValue=fields(parseNewWire(challengeRaw,policy,budget).value,['payload','proof']); + const answer=await solveBootstrapChallenge({raw:outgoing.original.raw,ref:outgoing.original.ref},challenge, + {...binding,signer:this.#identity,encryptionIdentity:this.#encryption,targetNonce:outgoing.nonce,at:this.#now(), + expiresAt:Math.min(expiry,u53((challengeValue.payload as Obj)?.expires_at))}); + const response=await request(answer.raw),held=verifyBootstrapProofResponse(response,{expectedSubject:this.#subject,expectedTarget:expected.target, + targetStorageEpoch:node.payload.storage_epoch as string,selector:grant.selector,bootstrapGrantRef:setup.originals.bootstrap.ref, + probeRef:outgoing.original.ref,challengeRef:challenge.ref,answerRef:answer.ref,at:this.#now(),maxProofItems:grant.limits.max_proof_items, + maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'empty',consumer:'ack_offer',policy,budget}); + proofBytes=response.length+held.handle.raw.length+held.manifest.raw.length; + // Reuse only exact already-held originals. Verification below still checks + // the complete independently authenticated source and both history events. + const available=new Map(); + for(const item of [...Object.values(setup.originals),...history,nodeEntry] as RawOriginal[])available.set(refKey(item.ref),item.raw); + const originals=new Map(),roles=new Map(); + for(const item of (held.manifest.value as Obj).children){ + const reference=rawRef(item.ref),key=refKey(reference);roles.set(item.role,[...(roles.get(item.role)??[]),reference]);if(originals.has(key))continue; + if(reference.size>policy.max_document_bytes||proofBytes+reference.size>grant.limits.max_proof_bytes)fail('repair_over_budget'); + let assembled:Uint8Array; + if(available.has(key)){assembled=available.get(key)!;budget.input(assembled.length);} + else{ + const chunks:Uint8Array[]=[];let offset=0; + while(offset=(held.handle.payload.expires_at as number)||clock()>=deadline)fail('repair_access_expired'); + const entry=(role:string):RawOriginal=>originals.get(refKey(roles.get(role)![0]))!; + const resolver=new LocalRawResolver(policy,budget); + for(const reference of roles.get('history.raw_pack')!){const item=originals.get(refKey(reference))!; + if(!same(resolver.put(reference.namespace,reference.key,item.raw).ref,reference))fail('repair_ref_mismatch');} + const source=verifyAckEmptySourceEvent(entry('history.ack_empty'),resolver,entry('ack.empty_custody'),{ + expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedTarget:expected.target, + targetStorageEpoch:node.payload.storage_epoch as string,expectedReceiptWriter:this.#subject, + expectedMessageId:expected.message,expectedEnvelopeRef:expected.envelope,limitPolicy:this.#limits,policy,budget}); + this.#emptyHead(entry('ack.head'),source,expected.target,budget); + const prior=source.predecessor; + for(const item of source.manifest.roles){const actual=originals.get(refKey(item.original.ref)); + if(!same(roles.get(item.role),[item.original.ref])||!actual||!Buffer.from(actual.raw).equals(Buffer.from(item.original.raw)))fail('repair_proof_mismatch');} + if(!same(prior.resources.originals.root.ref,setup.originals.root.ref)||!same(source.authorities.originals.write.ref,setup.originals.write.ref)|| + !same(source.authorities.originals.bootstrap.ref,setup.originals.bootstrap.ref))fail('repair_proof_mismatch'); + const usedPacks=new Set();for(const manifest of [source.manifest,prior.manifest]) + for(const row of (manifest.manifest.value as Obj).roles)usedPacks.add(refKey(rawRef(row.pack_ref))); + const suppliedPacks=roles.get('history.raw_pack')!.map(refKey); + if(suppliedPacks.length!==usedPacks.size||suppliedPacks.some(key=>!usedPacks.has(key)))fail('repair_unused_pack'); + const current=checks.current(source,roles,originals,expected.target,budget,known),now=this.#now(); + if(now>=Math.min(expiry,held.handle.payload.expires_at as number,source.read_until,...current.map(item=>item.payload.valid_until as number))||clock()>=deadline)fail('repair_access_expired'); + const saved=Object.freeze([...originals.values()].map(item=>Object.freeze({ref:item.ref,get raw(){return Uint8Array.from(item.raw);}}))); + return Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:saved, + metrics:Object.freeze({requests,wire_bytes:wireBytes,proof_bytes:proofBytes,...budget.snapshot()})}); + } + #emptyHead(entry:RawOriginal,source:AuthenticatedAckEmptySourceEvent,target:Obj,budget:RepairBudget):void{ + const input=snapshotEntry(entry,this.#policy,budget),signed=fields(parseNewWire(input.raw,this.#policy,budget).value,['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key','ack_slot','generation','observed_at','retain_until','state','root_authority_ref','grant_ref','binding_ref']); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!=='ack.head'||p.state!=='empty'||u53(p.generation,1)!==1|| + !same(p.ack_slot,source.custody.payload.ack_slot)||p.observed_at!==source.stored_at||p.retain_until!==source.retain_until|| + ['root_authority_ref','grant_ref','binding_ref'].some(name=>!same(p[name],source.custody.payload[name])))fail('repair_ack_empty_mismatch'); + verifyBoundedControlSignature(p,signed.proof,target.signing_key,budget); + } +} +class OfferStatusChecks{ + readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #clock:()=>number; + readonly #statusObserver?: (item:AuthenticatedStatusOriginal)=>void; + constructor(owner:Obj,policy:RepairPolicy,now:()=>number,observer?: (item:AuthenticatedStatusOriginal)=>void){ + this.#subject=owner;this.#policy=policy;this.#clock=now;this.#statusObserver=observer; + } + #now():number{return u53(Math.floor(this.#clock()));} + obligations(originals:Obj,slot:Obj,budget:RepairBudget):Obligation[]{ + const result:Obligation[]=[]; + for(const [name,role,kind,mask] of [['root','current.status.ack_root','ack.root_authority',3], + ['write','current.status.ack_write','ack.write_grant',1],['bootstrap','current.status.ack_offer_bootstrap','bootstrap.grant',3]] as const){ + const item=originals[name],subject={authority_kind:kind,authority_sha256:item.ref.raw_sha256}; + result.push({role,kind:'authority',subject,revision:item.payload.revision,signer:this.#subject.signing_key,mask, + scope_id:statusScope(slot.root_key,'authority',subject,this.#policy,budget)}); + } + result.push({role:'current.status.ack_slot',kind:'ack_slot',subject:slot,revision:originals.root.payload.revision, + signer:this.#subject.signing_key,mask:3,scope_id:statusScope(slot.root_key,'ack_slot',slot,this.#policy,budget)});return result; + } + history(known:unknown,archive:unknown,budget:RepairBudget):RawOriginal[]{ + // Both sets remain signed evidence. Archive entries carry old conflict + // witnesses as well as revocation/floor observations; no summary is trusted. + const merged=new Map(); + for(const [values,maximum] of [[known,16],[archive,32]] as const){ + if(!Array.isArray(values)||isProxy(values))fail('repair_invalid_status'); + if(values.length>maximum)fail(maximum===16?'repair_invalid_status':'repair_status_history_capacity'); + const descriptors=Object.getOwnPropertyDescriptors(values); + for(let index=0;index=32)fail('repair_status_history_capacity');merged.set(key,item); + } + } + return [...merged.values()]; + } + known(values:unknown,obligations:Obligation[],root:Obj,target:Obj,budget:RepairBudget,deferredAuthorities=false):AuthenticatedStatusOriginal[]{ + // Copy raw entries before the first asynchronous operation. + if(!Array.isArray(values)||isProxy(values)||values.length>32)fail('repair_invalid_status'); + const descriptors=Object.getOwnPropertyDescriptors(values);for(let index=0;indexthis.#now()+30)fail('repair_status_mismatch');let signer:Obj,allowed:Obj[]; + if(issuer===this.#subject.signing_key.key_id){signer=this.#subject.signing_key;allowed=obligations.map(v=>({scope_kind:v.kind,scope_id:v.scope_id})); + if(deferredAuthorities){const scopes=new Map(allowed.map(item=>[scopeKey(item),item])); + for(const item of payload.entries)if(item.scope_kind==='authority')scopes.set(scopeKey(item),{scope_kind:item.scope_kind,scope_id:item.scope_id});allowed=[...scopes.values()];} + if(issuer===target.signing_key.key_id)allowed.push(...payload.entries.filter((v:Obj)=>v.scope_kind==='resource').map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})));} + else if(issuer===target.signing_key.key_id){signer=target.signing_key; + if(payload.entries.some((v:Obj)=>v.scope_kind!=='resource'))fail('repair_status_disclosure'); + allowed=payload.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id}));} + else fail('repair_status_mismatch'); + checked.push(authenticateStatusOriginal(item,{expectedRoot:root,expectedSigningKey:signer,at:issued,allowedScopes:allowed,policy:this.#policy,budget},this.#statusObserver)); + } + // Activation's historical owner READ grant is checked against the exact + // recovered history below; it cannot revoke the writer's independent ADMIT. + this.#floors(checked,[],obligations,true,deferredAuthorities);return checked; + } + #floors(known:readonly AuthenticatedStatusOriginal[],current:readonly AuthenticatedStatusOriginal[],obligations:Obligation[],probePhase=false,deferredAuthorities=false):void{ + const seen=new Map(),floors=new Map(),required=new Map(obligations.map(v=>[v.signer.key_id+':'+v.kind+':'+v.scope_id,v])); + for(const observed of [...known,...current]){ + const payload=observed.payload as Obj,issuer=payload.scope_key.issuer_key_id,revision=payload.revision,id=issuer+':'+revision; + if(seen.has(id)&&seen.get(id)!==observed.canonical_sha256)fail('repair_status_conflict');seen.set(id,observed.canonical_sha256); + for(const entry of payload.entries){const key=issuer+':'+scopeKey(entry),wanted=required.get(key); + let mask=wanted?.mask??2; + if(probePhase&&wanted&&['current.status.ack_root','current.status.ack_offer_bootstrap'].includes(wanted.role??''))mask|=8; + if(deferredAuthorities&&!wanted&&issuer===this.#subject.signing_key.key_id&&entry.scope_kind==='authority')mask=0; + if(entry.status==='revoked'&&(entry.operation_mask&mask))fail('repair_authority_revoked'); + if(wanted&&mask&&entry.minimum_document_revision>wanted.revision)fail('repair_status_revision'); + floors.set(key,[...(floors.get(key)??[]),[revision,entry.minimum_document_revision]]); + } + } + for(const values of floors.values()){let minimum=0;for(const [,floor] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(floorv[0])))fail('repair_status_rollback');}} + } + current(source:AuthenticatedAckEmptySourceEvent,roles:Map,originals:Map,target:Obj, + budget:RepairBudget,known:AuthenticatedStatusOriginal[]):AuthenticatedStatusOriginal[]{ + const prior=source.predecessor,slot=source.custody.payload.ack_slot as Obj,root=slot.root_key,policy=this.#policy; + const obligations=this.obligations(source.authorities.originals,slot,budget),active=prior.resources.originals.active.payload as Obj; + obligations.push({role:'current.status.ack_resource',kind:'resource',subject:active.resource,revision:active.reservation_generation, + signer:target.signing_key,mask:3,scope_id:statusScope(root,'resource',active.resource,policy,budget)}); + for(const [item,kind] of [[prior.resources.originals.read,'ack.read_grant'],[prior.bootstrap.originals.bootstrap,'bootstrap.grant']] as const){ + const subject={authority_kind:kind,authority_sha256:item.ref.raw_sha256}; + obligations.push({role:null,kind:'authority',subject,revision:item.payload.revision as number,signer:this.#subject.signing_key,mask:0, + scope_id:statusScope(root,'authority',subject,policy,budget)}); + } + const permitted=new Set(obligations.map(v=>v.signer.key_id+':'+v.kind+':'+v.scope_id)); + for(const observed of known){const p=observed.payload as Obj; + if(p.entries.some((v:Obj)=>!permitted.has(p.scope_key.issuer_key_id+':'+scopeKey(v))))fail('repair_status_disclosure');} + const groups=new Map(); + for(const item of obligations){if(item.role===null)continue;const ref=roles.get(item.role)![0],key=refKey(ref),group=groups.get(key)??{ref,signer:item.signer,required:[]}; + if(!same(group.signer,item.signer))fail('repair_proof_mismatch'); + group.required.push({scope_kind:item.kind,scope_id:item.scope_id,document_revision:item.revision,operation_mask:item.mask});groups.set(key,group);} + const checked:AuthenticatedStatusOriginal[]=[]; + for(const [key,group] of groups){ + const allowed=obligations.filter(v=>same(v.signer,group.signer)),entry=originals.get(key)!; + checked.push(verifyStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:group.signer,at:this.#now(), + allowedScopes:allowed.map(v=>({scope_kind:v.kind,scope_id:v.scope_id})),required:group.required,policy,budget},this.#statusObserver)); + } + this.#floors([...prior.statuses,...source.statuses,...known],checked,obligations); + if(this.#now()>=Math.min(...['admit_until','read_until','retain_until'].map(n=>active.windows[n])))fail('repair_access_expired'); + return checked; + } +} +function checkedNodePayload(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{return fields(parseOriginalControl(raw,policy,budget).value,['payload','proof']).payload;} diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index 08c8aed..e456d82 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -36,6 +36,7 @@ "./open-delivery-client": "./open-delivery-client.ts", "./open-repair-admin": "./open-repair-admin.ts", "./open-repair-client": "./open-repair-client.ts", + "./open-repair-offer-client": "./open-repair-offer-client.ts", "./open-repair-wire": "./open-repair-wire.ts", "./open-repair-bound": "./open-repair-bound.ts", "./open-repair-empty": "./open-repair-empty.ts", diff --git a/docs/OPEN_ACK_RECOVERY.md b/docs/OPEN_ACK_RECOVERY.md index df2fd31..b782b41 100644 --- a/docs/OPEN_ACK_RECOVERY.md +++ b/docs/OPEN_ACK_RECOVERY.md @@ -1029,3 +1029,29 @@ same queue. Completed jobs stay local on subsequent runs. Use `list` to inspect the queue or `remove --job-id selected-message` to stop retaining a job. Removing it preserves the copy preparation journal and remote storage. Keep the private request and exported result outside public repositories. + + +## Native writer preflight (development after alpha.0.34) + +The native TypeScript `AckOfferClient` in `open-repair-offer-client.ts` can read +and authenticate a message-bound empty ACK source directly over HTTP. It checks +the source's two keys, both storage history events, the caller's exact original +owner root, write grant and offer bootstrap, and current ADMIT/READ/DISCOVER +permissions. Retained status originals keep revocations and revision floors in +force. A historical owner READ revocation does not remove the recipient's +separately granted ADMIT permission. + +Construct the client with the receipt writer's signing and encryption identities, +using the same bounded policy and transport options as `AckOwnerRecoveryClient`. +Call `preflight(baseUrl, options)` with `targetNodeEntry`, `expectedTarget`, +`expectedAckSlot`, `expectedOwner`, `expectedMessageId`, `expectedEnvelopeRef`, +`rootEntry`, `writeEntry`, and `bootstrapEntry`. Raw entries are `{raw: Uint8Array, +ref: RawRef}` containing independently held signed originals. Optional +`knownStatuses` and `archiveStatuses` retain original status evidence; `timeout` +is a positive deadline of at most sixty seconds. + +The result contains the authenticated empty source, bounded proof, current +statuses, original bytes and actual request/work counts. This preflight uploads +no receipt and grants no upload authority. The complete native saved-receipt +upload and restart journal remain subsequent integration work; Python's existing +receipt-return workflow remains the available complete path. diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 89c5409..93f2dca 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -145,7 +145,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 40312f5..874dfa7 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -243,7 +243,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 1ec261b..580f751 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -47,7 +47,7 @@ "repair_http", "repair_client", "repair_client_typescript", "repair_admin", "repair_admin_typescript", "repair_runtime_review", "repair_bound", "repair_bound_typescript", "repair_empty", "repair_empty_typescript", "repair_empty_review", "repair_empty_access", "repair_empty_http", "repair_empty_client", "repair_empty_client_typescript", - "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( + "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( "tests.test_open_repair_stage", "tests.test_open_repair_index", "tests.test_open_repair_index_access", "tests.test_open_repair_index_journal", "tests.test_open_repair_index_state", "tests.test_open_repair_index_http", "tests.test_open_repair_index_client", "tests.test_open_repair_index_recovery", "tests.test_open_repair_index_admin", @@ -245,7 +245,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-capacity.ts", - "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", + "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", "agent.ts", "peer.ts", "io.ts", "crypto.ts", "nodes.ts")] diff --git a/tests/test_open_repair_offer_client_typescript.py b/tests/test_open_repair_offer_client_typescript.py new file mode 100644 index 0000000..2c34fe3 --- /dev/null +++ b/tests/test_open_repair_offer_client_typescript.py @@ -0,0 +1,119 @@ +"""Native writer reads the independently authorized source over actual HTTP.""" +import hashlib +import json +import subprocess +import unittest + +from memory_vault import canonical_bytes +from memory_vault_open_repair_offer_access import RepairAckOfferAccess +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_repair_probe_typescript as probe_ts +from tests import test_open_repair_proof_typescript as proof_ts +from tests.open_repair_ack_fixtures import signed_entry +from tests.test_open_repair_empty_http import EmptyHTTPFixture + +DRIVER=r""" +import child from 'node:child_process'; +import crypto from 'node:crypto'; +import {syncBuiltinESMExports} from 'node:module'; +let subprocessCalls=0,nativeChecks=0; +const deny=()=>{subprocessCalls++;throw Error('native offer must not delegate');}; +for(const name of ['spawn','spawnSync','exec','execSync','execFile','execFileSync','fork'])child[name]=deny; +const originalVerify=crypto.verify;crypto.verify=(...args)=>{nativeChecks++;return originalVerify(...args);};syncBuiltinESMExports(); +const {AckOfferClient}=await import('./open-repair-offer-client.ts'); +const {OpenHTTPTransport}=await import('./open-transport.ts'); +const chunks=[];let size=0;for await(const chunk of process.stdin){size+=chunk.length;if(size>1048576)throw Error('fixture limit');chunks.push(chunk);} +const input=JSON.parse(Buffer.concat(chunks).toString('utf8')),decode=item=>({raw:Buffer.from(item.raw,'base64'),ref:item.ref}); +const transport=new OpenHTTPTransport({allow_loopback:true}),calls=[],request=transport.requestRepair.bind(transport); +transport.requestRepair=async(base,raw,...rest)=>{calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);return request(base,raw,...rest);}; +const client=new AckOfferClient(input.signing,input.encryption,{policy:input.policy,limitPolicy:input.limits,transport, + allowLoopback:true,clock:()=>input.now}); +const options={...input.options}; +for(const name of ['targetNodeEntry','rootEntry','writeEntry','bootstrapEntry'])options[name]=decode(options[name]); +for(const name of ['knownStatuses','archiveStatuses'])if(options[name])options[name]=options[name].map(decode); +try{ + const result=await client.preflight(input.base,options); + const original=result.originals[0],before=Buffer.from(original.raw).toString('hex');original.raw.fill(0); + process.stdout.write(JSON.stringify({ok:true,metrics:result.metrics,consumer:result.proof.handle.payload.consumer, + write:result.source.authorities.originals.write.ref,bootstrap:result.source.authorities.originals.bootstrap.ref, + custody:Buffer.from(result.source.custody.raw).toString('base64'),originals:result.originals.length, + immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls})); +}catch(error){process.stdout.write(JSON.stringify({ok:false,code:error.code??'untyped_error',detail:error.code?undefined:String(error.stack),calls,nativeChecks,subprocessCalls}));} +finally{client.close();transport.close();} +""" + + +class NativeOfferClientTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.host=EmptyHTTPFixture(self) + self.f=self.host.f + + def call(self,**extra): + f=self.f;e=f['expected'];h=self.host + return dict(base=h.http.base,signing=probe_ts.private_signing(f['signers']['writer']), + encryption=f['encryption']['writer'].private_document(),now=2_000_000_007, + policy=proof_ts.POLICY,limits=e['limit_policy'],options=dict( + targetNodeEntry=probe_ts.encoded(f['entries']['descriptor']),expectedTarget=e['expected_target'], + expectedAckSlot=e['expected_ack_slot'],expectedOwner=e['expected_owner'], + expectedMessageId=h.expected['expected_message_id'],expectedEnvelopeRef=h.expected['expected_envelope_ref'], + rootEntry=probe_ts.encoded(f['entries']['root']),writeEntry=probe_ts.encoded(h.write),bootstrapEntry=probe_ts.encoded(h.offer))|extra) + + def native(self,value): + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(value).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=40) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]) + value=json.loads(result.stdout);self.assertEqual(value['subprocessCalls'],0);return value + + def revoke(self,kind,original,mask): + root=self.f['expected']['expected_ack_slot']['root_key'] + scope=hashlib.sha256(canonical_bytes(dict(kind='authority',root_key=root, + authority_kind=kind,authority_sha256=original['ref']['raw_sha256']))).hexdigest() + payload=json.loads(self.host.expected['current_statuses'][0]['raw'])['payload'];payload['revision']=3 + for item in payload['entries']: + if item['scope_id']==scope:item.update(status='revoked',operation_mask=mask) + return signed_entry(payload,self.f['signers']['owner'],'synthetic-native-offer-revocation') + + def test_native_complete_source_read_after_restart(self): + self.host.http.restart();result=self.native(self.call()) + self.assertTrue(result['ok'],result);self.assertTrue(result['immutable']) + self.assertEqual(result['consumer'],'ack_offer') + self.assertEqual(result['write'],self.host.write['ref']);self.assertEqual(result['bootstrap'],self.host.offer['ref']) + self.assertEqual(result['metrics']['requests'],12);self.assertEqual(result['originals'],12) + self.assertEqual(result['nativeChecks'],result['metrics']['signature_checks']) + self.assertEqual(result['calls'][:2],['bootstrap.probe','bootstrap.answer']) + self.assertTrue(all(kind=='bootstrap.proof_child_request' for kind in result['calls'][2:])) + + def test_writer_admit_revocation_refuses_before_any_private_probe(self): + revoked=self.revoke('ack.write_grant',self.host.write,1) + result=self.native(self.call(knownStatuses=[probe_ts.encoded(revoked)])) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_authority_revoked');self.assertEqual(result['calls'],[]) + + def test_historical_owner_read_revocation_does_not_revoke_writer_admit(self): + revoked=self.revoke('ack.read_grant',self.f['entries']['read'],2) + gate=RepairAckOfferAccess(self.host.source.state);gate.initialize() + prepared=gate.prepare(self.host.source.resource_id,action='proof',current_statuses=[revoked,self.host.expected['current_statuses'][1]]) + with self.host.source.state._transaction():self.assertTrue(gate.check_locked(prepared).allowed) + result=self.native(self.call(knownStatuses=[probe_ts.encoded(revoked)])) + self.assertTrue(result['ok'],result);self.assertEqual(result['metrics']['requests'],12) + + def test_root_discover_revocation_and_archived_write_revocation_refuse_before_probe(self): + cases=[('ack.root_authority',self.f['entries']['root'],8,'knownStatuses'), + ('ack.write_grant',self.host.write,1,'archiveStatuses')] + for kind,original,mask,field in cases: + with self.subTest(kind=kind): + revoked=self.revoke(kind,original,mask) + result=self.native(self.call(**{field:[probe_ts.encoded(revoked)]})) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_authority_revoked');self.assertEqual(result['calls'],[]) + + def test_message_binding_mismatch_refuses_before_network(self): + result=self.native(self.call(expectedMessageId='msg_'+'f'*64)) + self.assertFalse(result['ok']);self.assertEqual(result['calls'],[]) + self.assertNotEqual(result['code'],'untyped_error') + + +if __name__=='__main__':unittest.main() From 6d36f3f19e50ce1af2e2b3cdd4a30ca8c78ca7f7 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:33:05 +0800 Subject: [PATCH 02/16] Upload native saved receipts and retain interoperable recovery journals --- .../network/open-repair-occupied.ts | 9 +- .../network/open-repair-offer-client.ts | 168 +++++++++++++++++- docs/OPEN_ACK_RECOVERY.md | 19 +- scripts/run_open_network_ci.py | 2 +- .../test_open_repair_put_client_typescript.py | 107 +++++++++++ 5 files changed, 294 insertions(+), 11 deletions(-) create mode 100644 tests/test_open_repair_put_client_typescript.py diff --git a/clients/typescript/network/open-repair-occupied.ts b/clients/typescript/network/open-repair-occupied.ts index 40a8e6d..e40d601 100644 --- a/clients/typescript/network/open-repair-occupied.ts +++ b/clients/typescript/network/open-repair-occupied.ts @@ -123,14 +123,14 @@ function floors(previous:readonly AuthenticatedStatusOriginal[],current:readonly for(const observed of previous){const p=observed.payload as Obj;for(const item of p.entries){const scope=p.scope_key.issuer_key_id+':'+scopeKey(item),held=prior.get(scope)??[0,0];prior.set(scope,[Math.max(held[0],p.revision),Math.max(held[1],item.minimum_document_revision)]);}} for(const observed of current){const p=observed.payload as Obj;for(const item of p.entries){const held=prior.get(p.scope_key.issuer_key_id+':'+scopeKey(item));if(held&&(p.revision!same(p[name],c[name])))mismatch();return head; } + +// The native writer uses the same receipt, consent, window and historical +// checks as the independently verified occupied source response. +export {inputs as verifyAckReceiptInputs, obligations as ackReceiptObligations, + windows as ackReceiptWindows, floors as checkAckReceiptHistoryFloors}; diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 3be1f55..1577c22 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -2,6 +2,7 @@ * This reads original source evidence only; it does not authorize or upload a receipt. */ import {performance} from 'node:perf_hooks'; +import {randomBytes} from 'node:crypto'; import {isProxy} from 'node:util/types'; import {OpenHTTPTransport,endpoint} from './open-transport.ts'; import {RepairBudget,RepairError,buildNewWire,objectFields,rawRef,u53,LocalRawResolver,parseNewWire} from './open-repair-wire.ts'; @@ -12,11 +13,14 @@ import type {AckOwnerRecoveryOptions,RecoveredAckOwnerEmptyProof} from './open-r import {verifyAckEmptySourceEvent} from './open-repair-empty.ts'; import type {AuthenticatedAckEmptySourceEvent} from './open-repair-empty.ts'; import {parseOriginalControl,verifySourceNodeOriginal,verifyBoundedControlSignature} from './open-repair-original.ts'; -import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge,signBoundedBootstrapOriginal} from './open-repair-probe.ts'; import {verifyBootstrapProofResponse,makeBootstrapChildRequest} from './open-repair-proof.ts'; import {statusScope,verifyStatusOriginal,authenticateStatusOriginal} from './open-repair-status.ts'; import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +import {verifyAckReceiptInputs,ackReceiptObligations,ackReceiptWindows,checkAckReceiptHistoryFloors, + verifyAckOccupiedSourceEvent,verifyAckOccupiedHead} from './open-repair-occupied.ts'; +import type {AuthenticatedAckOccupiedSourceEvent} from './open-repair-occupied.ts'; type Obj=Record; const clock=()=>performance.now()/1000; export interface AckOfferOptions { @@ -77,10 +81,13 @@ export class AckOfferClient{ close():void{if(this.#ownTransport)this.#transport.close();} #now():number{return u53(Math.floor(this.#clock()));} async preflight(baseUrl:string,value:AckOfferOptions):Promise{ + return this.#preflight(baseUrl,value,new RepairBudget(this.#policy)); + } + async #preflight(baseUrl:string,value:AckOfferOptions,budget:RepairBudget):Promise{ const args=options(value,['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId', 'expectedEnvelopeRef','rootEntry','writeEntry','bootstrapEntry'],['knownStatuses','archiveStatuses','timeout']); const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); - const policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),deadline=clock()+timeout; + const policy=this.#policy,started=this.#now(),deadline=clock()+timeout; const expected=buildNewWire({target:args.expectedTarget,slot:args.expectedAckSlot,owner:args.expectedOwner, message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); @@ -164,6 +171,114 @@ export class AckOfferClient{ return Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:saved, metrics:Object.freeze({requests,wire_bytes:wireBytes,proof_bytes:proofBytes,...budget.snapshot()})}); } + async put(baseUrl:string,receiptEntry:unknown,disclosureEntry:unknown,putEntry:unknown,value:AckReceiptPutOptions):Promise{ + const preflightNames=['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', + 'rootEntry','writeEntry','bootstrapEntry']; + const args=options(value,[...preflightNames,'currentStatuses','readUntil','retainUntil'], + ['knownStatuses','archiveStatuses','knownDisclosureStatuses','timeout','journal']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + if(args.journal!==undefined&&typeof args.journal!=='function')fail('repair_invalid_context'); + const started=clock(),policy=this.#policy,budget=new RepairBudget(policy); + const expected=buildNewWire({owner:args.expectedOwner,target:args.expectedTarget,message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; + const readUntil=u53(args.readUntil),retainUntil=u53(args.retainUntil); + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const retained=checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget),disclosureHistory=checks.history(args.knownDisclosureStatuses??[],[],budget); + if(!Array.isArray(args.currentStatuses)||isProxy(args.currentStatuses)||args.currentStatuses.length<1||args.currentStatuses.length>8)fail('repair_invalid_status'); + const currentEntries=checks.history(args.currentStatuses,[],budget); + // Snapshot private upload originals before the first asynchronous read. + const receipt=snapshotReceiptEntry(receiptEntry,policy,budget),disclosure=snapshotEntry(disclosureEntry,policy,budget,true),put=snapshotEntry(putEntry,policy,budget,true); + const preflight=Object.fromEntries(preflightNames.map(name=>[name,args[name]])); + preflight.archiveStatuses=retained;preflight.timeout=timeout; + const prepared=await this.#preflight(baseUrl,preflight as unknown as AckOfferOptions,budget),source=prepared.source; + const inputs=verifyAckReceiptInputs(source,receipt,disclosure,put,expected.owner,this.#subject,this.#now(),policy,budget); + if(!same((inputs.disclosure.payload as Obj).bootstrap_return.roles,['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt']))fail('repair_disclosure_permission'); + const windows=ackReceiptWindows(source,inputs,this.#now()); + if(!(this.#now()=expiry||clock()>=started+timeout)fail('repair_access_expired'); + if(budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); + budget.output(16);const useId='use_'+randomBytes(16).toString('hex'); + const use=signBoundedBootstrapOriginal({schema_version:'memory-vault-open-repair/v1',kind:'bootstrap.use',signing_key:this.#subject.signing_key, + issued_at:this.#now(),expires_at:expiry,use_id:useId,subject:handle.subject,target:handle.target,target_storage_epoch:handle.target_storage_epoch, + consumer:'ack_offer',operation:'ack.put',bootstrap_probe_ref:handle.probe_ref,bootstrap_manifest_ref:prepared.proof.manifest_ref, + request_ref:inputs.put.ref},this.#identity,policy,budget); + const packet=buildNewWire({schema_version:'memory-vault-open-repair/v1',kind:'ack.put_request',use:encodePutEntry(use,budget), + receipt:encodePutEntry(inputs.receipt,budget),disclosure:encodePutEntry(inputs.disclosure,budget),put:encodePutEntry(inputs.put,budget), + current_statuses:checked.map(item=>encodePutEntry(item,budget)),read_until:readUntil,retain_until:retainUntil},policy,budget); + const grant=source.authorities.originals.bootstrap.payload as Obj; + const uploadBytes=packet.raw.length+use.raw.length+[...Object.values(inputs),...checked].reduce((sum,item)=>sum+item.raw.length,0); + if(packet.raw.length>65536||prepared.metrics.requests+1>grant.limits.max_requests||prepared.metrics.proof_bytes+uploadBytes>grant.limits.max_proof_bytes)fail('repair_over_budget'); + if(args.journal){ + const digest=budget.hash(packet.raw),journal=buildNewWire({schema_version:'memory-vault-ack-put-journal/v1', + packet:encodePutEntry({raw:packet.raw,ref:rawRef({namespace:'meta',key:digest,raw_sha256:digest,size:packet.raw.length})},budget), + handle:encodePutEntry(prepared.proof.handle,budget),manifest:encodePutEntry({raw:prepared.proof.manifest.raw,ref:prepared.proof.manifest_ref},budget), + originals:prepared.originals.map(item=>encodePutEntry(item,budget)),retained:retained.map(item=>encodePutEntry(item,budget)), + disclosure_statuses:disclosureHistory.map(item=>encodePutEntry(item,budget))},policy,budget); + await journalBeforeDeadline(args.journal,'request',journal.raw,started+timeout); + } + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); + const response=await this.#transport.requestRepair(baseUrl,packet.raw,started+timeout); + if(response.length===0||response.length>65536)fail('repair_invalid_response'); + const body=fields(parseNewWire(response,policy,budget).value,['schema_version','kind','use_ref','manifest','commit','head','packs']); + if(body.schema_version!=='memory-vault-open-repair/v1'||body.kind!=='ack.put_response'||!same(rawRef(body.use_ref),use.ref))fail('repair_put_mismatch'); + const manifest=decodePutEntry(body.manifest,policy,budget),commit=decodePutEntry(body.commit,policy,budget),head=decodePutEntry(body.head,policy,budget); + if(!Array.isArray(body.packs)||body.packs.length<1||body.packs.length>policy.max_entries)fail('repair_invalid_pack'); + const packs:RawOriginal[]=body.packs.map((item:unknown)=>decodePutEntry(item,policy,budget)); + const transferred=prepared.metrics.proof_bytes+uploadBytes+response.length+[manifest,commit,head,...packs].reduce((sum,item)=>sum+item.raw.length,0); + if(transferred>grant.limits.max_proof_bytes)fail('repair_over_budget'); + const originals=new Map(prepared.originals.map(item=>[refKey(item.ref),item])),resolver=new LocalRawResolver(policy,budget); + for(const item of (prepared.proof.manifest.value as Obj).children)if(item.role==='history.raw_pack'){ + const reference=rawRef(item.ref),raw=originals.get(refKey(reference))!.raw; + if(!same(resolver.put(reference.namespace,reference.key,raw).ref,reference))fail('repair_ref_mismatch'); + } + const wanted=new Set((parseNewWire(manifest.raw,policy,budget).value as Obj).roles.map((item:Obj)=>refKey(rawRef(item.pack_ref)))); + if(packs.length!==wanted.size||new Set(packs.map(item=>refKey(item.ref))).size!==wanted.size||packs.some(item=>!wanted.has(refKey(item.ref))))fail('repair_unused_pack'); + for(const item of packs)if(!same(resolver.put(item.ref.namespace,item.ref.key,item.raw).ref,item.ref))fail('repair_ref_mismatch'); + const complete=verifyAckOccupiedSourceEvent(manifest,resolver,commit,{expectedAckSlot:slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject, + expectedMessageId:expected.message,expectedEnvelopeRef:expected.envelope,expectedTarget:expected.target, + targetStorageEpoch:handle.target_storage_epoch,limitPolicy:this.#limits,policy,budget}); + verifyAckOccupiedHead(head,complete,{expectedTarget:expected.target,policy,budget}); + if(!same(complete.predecessor.custody.ref,source.custody.ref)||complete.read_until!==readUntil||complete.retain_until!==retainUntil|| + Object.entries(inputs).some(([name,item])=>!same(complete.inputs[name as keyof typeof inputs].ref,item.ref)|| + !Buffer.from(complete.inputs[name as keyof typeof inputs].raw).equals(Buffer.from(item.raw))))fail('repair_put_mismatch'); + checkAckReceiptHistoryFloors(checked,complete.statuses); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); + for(const item of [manifest,commit,head,...packs,...complete.manifest.roles.map(item=>item.original)])originals.set(refKey(item.ref),item); + if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); + return Object.freeze({source:complete,originals:Object.freeze([...originals.values()].map(item=>Object.freeze({ref:item.ref,get raw(){return Uint8Array.from(item.raw);}}))), + metrics:Object.freeze({requests:prepared.metrics.requests+1,transfer_bytes:transferred,...budget.snapshot()})}); + } + #putStatuses(entries:unknown,retained:RawOriginal[],duties:Obj[],source:AuthenticatedAckEmptySourceEvent,owner:Obj,budget:RepairBudget, + retainedAuthorities:RawOriginal[],currentAuthorities:readonly AuthenticatedStatusOriginal[]):AuthenticatedStatusOriginal[]{ + if(!Array.isArray(entries)||isProxy(entries)||entries.length<1||entries.length>8)fail('repair_invalid_status'); + if(retained.length>16)fail('repair_status_history_capacity'); + const authenticated:AuthenticatedStatusOriginal[]=[],old:AuthenticatedStatusOriginal[]=[],root=(source.custody.payload.ack_slot as Obj).root_key; + for(const [index,value] of [...entries,...retained,...retainedAuthorities].entries()){ + const item=snapshotEntry(value,this.#policy,budget,true),payload=checkedStatus(item.raw,this.#policy,budget),issuer=payload.scope_key.issuer_key_id; + const allowed=duties.filter(duty=>duty.signer.key_id===issuer); + if(!allowed.length||(index>=entries.length&&indexthis.#now()+30)fail('repair_status_mismatch'); + const checked=authenticateStatusOriginal(item,{expectedRoot:root,expectedSigningKey:allowed[0].signer,at, + allowedScopes:allowed.map(duty=>({scope_kind:duty.scope_kind,scope_id:duty.scope_id})),policy:this.#policy,budget},this.#statusObserver); + (index(item.payload as Obj).scope_key.issuer_key_id===duty.signer.key_id? + (item.payload as Obj).entries.filter((entry:Obj)=>entry.scope_kind===duty.scope_kind&&entry.scope_id===duty.scope_id).map((entry:Obj)=>({revision:item.payload.revision,entry})):[]); + if(!observations.length)fail('repair_status_missing'); + observations.sort((a,b)=>b.revision-a.revision);const entry=observations[0].entry; + if(entry.status!=='active'||(entry.operation_mask&duty.mask)!==duty.mask)fail('repair_authority_revoked'); + if(entry.minimum_document_revision>duty.revision)fail('repair_status_revision'); + } + const obligations=duties.map(duty=>({kind:duty.scope_kind,scope_id:duty.scope_id,signer:duty.signer,mask:duty.mask,revision:duty.revision,role:duty.role,subject:null})); + new OfferStatusChecks(owner,this.#policy,this.#clock,this.#statusObserver).floors( + [...source.predecessor.statuses,...source.statuses,...currentAuthorities,...old],authenticated,obligations); + return authenticated; + } #emptyHead(entry:RawOriginal,source:AuthenticatedAckEmptySourceEvent,target:Obj,budget:RepairBudget):void{ const input=snapshotEntry(entry,this.#policy,budget),signed=fields(parseNewWire(input.raw,this.#policy,budget).value,['payload','proof']); const p=fields(signed.payload,['schema_version','kind','signing_key','ack_slot','generation','observed_at','retain_until','state','root_authority_ref','grant_ref','binding_ref']); @@ -229,9 +344,9 @@ class OfferStatusChecks{ } // Activation's historical owner READ grant is checked against the exact // recovered history below; it cannot revoke the writer's independent ADMIT. - this.#floors(checked,[],obligations,true,deferredAuthorities);return checked; + this.floors(checked,[],obligations,true,deferredAuthorities);return checked; } - #floors(known:readonly AuthenticatedStatusOriginal[],current:readonly AuthenticatedStatusOriginal[],obligations:Obligation[],probePhase=false,deferredAuthorities=false):void{ + floors(known:readonly AuthenticatedStatusOriginal[],current:readonly AuthenticatedStatusOriginal[],obligations:Obligation[],probePhase=false,deferredAuthorities=false):void{ const seen=new Map(),floors=new Map(),required=new Map(obligations.map(v=>[v.signer.key_id+':'+v.kind+':'+v.scope_id,v])); for(const observed of [...known,...current]){ const payload=observed.payload as Obj,issuer=payload.scope_key.issuer_key_id,revision=payload.revision,id=issuer+':'+revision; @@ -273,9 +388,52 @@ class OfferStatusChecks{ checked.push(verifyStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:group.signer,at:this.#now(), allowedScopes:allowed.map(v=>({scope_kind:v.kind,scope_id:v.scope_id})),required:group.required,policy,budget},this.#statusObserver)); } - this.#floors([...prior.statuses,...source.statuses,...known],checked,obligations); + this.floors([...prior.statuses,...source.statuses,...known],checked,obligations); if(this.#now()>=Math.min(...['admit_until','read_until','retain_until'].map(n=>active.windows[n])))fail('repair_access_expired'); return checked; } } function checkedNodePayload(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{return fields(parseOriginalControl(raw,policy,budget).value,['payload','proof']).payload;} + +export interface AckReceiptPutOptions extends AckOfferOptions { + readonly currentStatuses:readonly unknown[];readonly readUntil:number;readonly retainUntil:number; + readonly knownDisclosureStatuses?:readonly unknown[]; + readonly journal?:(kind:'request'|'response',raw:Uint8Array)=>void|Promise; +} +export interface CommittedAckReceipt { + readonly source:AuthenticatedAckOccupiedSourceEvent;readonly originals:readonly RawOriginal[]; + readonly metrics:Readonly; +} +export class AckReceiptClient extends AckOfferClient { + constructor(identity:unknown,encryptionIdentity:unknown,value:AckOwnerRecoveryOptions={}){ + const args=options(value,[],['policy','limitPolicy','allowLoopback','transport','clock','statusObserver']); + super(identity,encryptionIdentity,{...args,policy:args.policy??{...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:96}}); + } +} +function snapshotReceiptEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const held=fields(value,['raw','ref']),ref=rawRef(held.ref); + if(ref.size>4096)fail('repair_put_too_large'); + const parsed=parseOriginalControl(held.raw,policy,budget),raw=parsed.raw; + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; +} +function encodePutEntry(item:RawOriginal,budget:RepairBudget):Obj{ + const ref=rawRef(item.ref),raw=item.raw;budget.input(raw.length); + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + budget.output(Math.ceil(raw.length*4/3));return {ref,raw_base64url:Buffer.from(raw).toString('base64url')}; +} +function decodePutEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const item=fields(value,['raw_base64url','ref']),ref=rawRef(item.ref),encoded=item.raw_base64url; + if(ref.namespace!=='meta'||ref.size>policy.max_document_bytes||typeof encoded!=='string'||encoded.length>Math.ceil(ref.size*4/3)||!/^[A-Za-z0-9_-]*$/.test(encoded))fail('repair_put_too_large'); + budget.output(ref.size);const raw=Buffer.from(encoded,'base64url'); + if(raw.length!==ref.size||raw.toString('base64url')!==encoded||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; +} + +async function journalBeforeDeadline(callback:NonNullable,kind:'request'|'response',raw:Uint8Array,deadline:number):Promise{ + const remaining=deadline-clock();if(remaining<=0)fail('repair_access_expired'); + let timer:ReturnType|undefined; + try{ + await Promise.race([Promise.resolve().then(()=>callback(kind,raw)),new Promise((_,reject)=>{ + timer=setTimeout(()=>reject(new RepairError('repair_access_expired')),Math.ceil(remaining*1000)); + })]); + }finally{if(timer!==undefined)clearTimeout(timer);} +} diff --git a/docs/OPEN_ACK_RECOVERY.md b/docs/OPEN_ACK_RECOVERY.md index b782b41..b0fa7b9 100644 --- a/docs/OPEN_ACK_RECOVERY.md +++ b/docs/OPEN_ACK_RECOVERY.md @@ -1052,6 +1052,19 @@ is a positive deadline of at most sixty seconds. The result contains the authenticated empty source, bounded proof, current statuses, original bytes and actual request/work counts. This preflight uploads -no receipt and grants no upload authority. The complete native saved-receipt -upload and restart journal remain subsequent integration work; Python's existing -receipt-return workflow remains the available complete path. +no receipt and grants no upload authority. `AckReceiptClient` from the same module additionally uploads the existing saved +receipt under its independent disclosure and put consent, then authenticates the +returned occupied storage history. It uses one finite work budget for preflight, +upload and response verification, with a default limit of 96 signature checks. + +Call `put(baseUrl, receiptEntry, disclosureEntry, putEntry, options)` with the +preflight options plus `currentStatuses`, `readUntil` and `retainUntil`. Optional +`knownDisclosureStatuses` retain prior recipient consent observations. A +`journal(kind, raw)` callback may persist the exact `request` journal before any +private receipt upload and the verified `response` afterward. The callback is +awaited; failure to store the request stops the upload. Journal data contains +private original records and belongs in the participant's private durable store. +The journal format is compatible with the existing Python receipt recovery +client. Native prepared-return reuse, native journal replay and automatic Agent +integration remain subsequent work; Python remains the complete automated +receipt-return path. diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 580f751..30bca3f 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -47,7 +47,7 @@ "repair_http", "repair_client", "repair_client_typescript", "repair_admin", "repair_admin_typescript", "repair_runtime_review", "repair_bound", "repair_bound_typescript", "repair_empty", "repair_empty_typescript", "repair_empty_review", "repair_empty_access", "repair_empty_http", "repair_empty_client", "repair_empty_client_typescript", - "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( + "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_client_typescript", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( "tests.test_open_repair_stage", "tests.test_open_repair_index", "tests.test_open_repair_index_access", "tests.test_open_repair_index_journal", "tests.test_open_repair_index_state", "tests.test_open_repair_index_http", "tests.test_open_repair_index_client", "tests.test_open_repair_index_recovery", "tests.test_open_repair_index_admin", diff --git a/tests/test_open_repair_put_client_typescript.py b/tests/test_open_repair_put_client_typescript.py new file mode 100644 index 0000000..feacf16 --- /dev/null +++ b/tests/test_open_repair_put_client_typescript.py @@ -0,0 +1,107 @@ +"""Native recipient uploads its actual receipt and checks the returned storage event.""" +import base64 +import json +import unittest +from unittest.mock import patch + +from tests import test_open_repair_offer_client_typescript as offer_ts +from tests import test_open_repair_put_client as put_py +from tests import test_open_repair_probe_typescript as probe_ts +from tests.test_open_repair_occupied import receipt_inputs + +DRIVER=offer_ts.DRIVER.replace("const {AckOfferClient}=", "const {AckReceiptClient}=").replace('new AckOfferClient(', 'new AckReceiptClient(') +DRIVER=DRIVER.replace("calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);","const packet=JSON.parse(Buffer.from(raw).toString());calls.push(packet.payload?.kind??packet.kind);") +DRIVER=DRIVER.replace("const input=JSON.parse", "const journal=[];\nconst input=JSON.parse") +DRIVER=DRIVER.replace("return request(base,raw,...rest);", """const response=await request(base,raw,...rest); + if(input.corruptReply&&calls.at(-1)==='ack.put_request'){ + const body=JSON.parse(Buffer.from(response).toString());body.commit.ref.raw_sha256='0'.repeat(64); + const {canonicalBytes}=await import('./crypto.ts');return canonicalBytes(body); + }return response;""") +DRIVER=DRIVER.replace("['knownStatuses','archiveStatuses']", "['knownStatuses','archiveStatuses','knownDisclosureStatuses','currentStatuses']") +DRIVER=DRIVER.replace('const result=await client.preflight(input.base,options);', """options.journal=async(kind,raw)=>{ + journal.push({kind,raw:Buffer.from(raw).toString('base64'),calls:[...calls]}); + if(input.failJournal&&kind==='request')throw Error('synthetic journal unavailable'); + if(input.blockJournal&&kind==='request')await new Promise(()=>{}); + }; + const result=await client.put(input.base,decode(input.receipt),decode(input.disclosure),decode(input.put),options);""") +start=DRIVER.index(' process.stdout.write(JSON.stringify({ok:true') +end=DRIVER.index('\n}catch(error)',start) +DRIVER=DRIVER[:start]+''' process.stdout.write(JSON.stringify({ok:true,metrics:result.metrics, + receipt:Buffer.from(result.source.inputs.receipt.raw).toString('base64'),commit:result.source.commit.ref, + immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls,journal}));'''+DRIVER[end:] +DRIVER=DRIVER.replace('calls,nativeChecks,subprocessCalls}));}', 'calls,nativeChecks,subprocessCalls,journal}));}') + + +class NativePutClientTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + offer_ts.NativeOfferClientTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.case=put_py.RepairPutClientTests();self.addCleanup(self.case.doCleanups) + if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage': + # One original finite grant funds both the upload and exact replay. + with patch.object(put_py.offer_fixture.RepairOfferClientTests,'fixture_limits',dict(proof_limit=524288),create=True):self.case.setUp() + else:self.case.setUp() + self.host=self.case.host;self.f=self.host.f + + def call(self,inputs=None,**extra): + value=offer_ts.NativeOfferClientTests.call(self) + receipt,disclosure,put,options=inputs or self.case.inputs + value.update(receipt=probe_ts.encoded(receipt),disclosure=probe_ts.encoded(disclosure),put=probe_ts.encoded(put),now=2_000_000_008,**extra) + value['policy']=dict(value['policy'],max_signature_checks=96) + value['options'].update(currentStatuses=[probe_ts.encoded(item) for item in options['current_statuses']],readUntil=options['read_until'],retainUntil=options['retain_until']) + return value + + native=offer_ts.NativeOfferClientTests.native + + def test_native_upload_saves_exact_receipt_and_records_request_before_network(self): + self.host.http.restart();result=self.native(self.call()) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'});self.assertTrue(result['immutable']) + self.assertEqual(result['receipt'],probe_ts.encoded(self.case.inputs[0])['raw']) + self.assertEqual(result['metrics']['requests'],13) + self.assertEqual(result['nativeChecks'],result['metrics']['signature_checks']) + self.assertEqual([item['kind'] for item in result['journal']],['request','response']) + self.assertNotIn('ack.put_request',result['journal'][0]['calls']);self.assertEqual(result['calls'][-1],'ack.put_request') + self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'occupied') + + def test_legacy_disclosure_does_not_send_receipt(self): + from memory_vault_open_repair_occupied import RETURN_ROLES_LEGACY + inputs=receipt_inputs(self.case.fixture,bootstrap_roles=RETURN_ROLES_LEGACY) + result=self.native(self.call(inputs));self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_disclosure_permission') + self.assertNotIn('ack.put_request',result['calls']) + + def test_original_signature_budget_covers_preflight_and_upload_together(self): + value=self.call();value['policy']['max_signature_checks']=64 + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_over_budget') + self.assertNotIn('ack.put_request',result['calls']) + + def test_journal_failure_stops_before_private_upload(self): + result=self.native(self.call(failJournal=True));self.assertFalse(result['ok']) + self.assertNotIn('ack.put_request',result['calls']) + self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'empty') + + def test_unfinished_journal_obeys_original_deadline_without_upload(self): + value=self.call(blockJournal=True);value['options']['timeout']=8 + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + self.assertNotIn('ack.put_request',result['calls']) + + def test_bad_returned_commit_cannot_claim_verified_storage(self): + result=self.native(self.call(corruptReply=True));self.assertFalse(result['ok']) + self.assertEqual(result['code'],'repair_ref_mismatch') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'occupied') + # The exact native journal is interoperable with the existing durable + # Python recovery path; no second probe or replacement put is created. + receipt,disclosure,put,options=self.case.inputs + recovered=self.case.client().resume(self.host.http.base,base64.b64decode(result['journal'][0]['raw']), + receipt,disclosure,put,**options,**self.case.case.args) + self.assertEqual(recovered.source.inputs['receipt'].raw,receipt['raw']) + self.assertEqual(recovered.metrics['requests'],1) + self.assertEqual(self.host.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + + +if __name__=='__main__':unittest.main() From 83011b902727e965b02d38d127caa4738194083e Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:44:16 +0800 Subject: [PATCH 03/16] Stop ACK upload when current permission expires during journal persistence --- .../network/open-repair-offer-client.ts | 2 +- memory_vault_open_repair_put_client.py | 5 ++- .../test_open_repair_put_client_typescript.py | 38 ++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 1577c22..6a2a96f 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -197,7 +197,7 @@ export class AckOfferClient{ const slot=source.custody.payload.ack_slot as Obj,duties=ackReceiptObligations(source,inputs,expected.owner,this.#subject,expected.target,slot,policy,budget); const checked=this.#putStatuses(currentEntries,disclosureHistory,duties,source,expected.owner,budget,retained,prepared.current_statuses); const handle=prepared.proof.handle.payload as Obj; - const expiry=Math.min(handle.expires_at,windows.admitUntil,this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); + const expiry=Math.min(handle.expires_at,windows.admitUntil,...checked.map(item=>item.payload.valid_until as number),this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); if(budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); budget.output(16);const useId='use_'+randomBytes(16).toString('hex'); diff --git a/memory_vault_open_repair_put_client.py b/memory_vault_open_repair_put_client.py index 7ea2354..2d36f15 100644 --- a/memory_vault_open_repair_put_client.py +++ b/memory_vault_open_repair_put_client.py @@ -94,7 +94,8 @@ def put(self, base_url, receipt_entry, disclosure_entry, put_entry, *, current_s checked = self._put_statuses(current_statuses,known_disclosure_statuses,duties,source,owner,budget, retained_authorities=retained,current_authorities=prepared.current_statuses) handle = prepared.proof.handle.payload - expiry = min(handle["expires_at"],admit_until,self._now()+max(1,int(timeout-(time.monotonic()-started)))) + expiry = min(handle["expires_at"],admit_until,*(item.payload["valid_until"] for item in checked), + self._now()+max(1,int(timeout-(time.monotonic()-started)))) if self._now() >= expiry or time.monotonic()-started >= timeout: _fail("repair_access_expired") # Reserve enough of this same local meter for the full occupied @@ -118,6 +119,8 @@ def put(self, base_url, receipt_entry, disclosure_entry, put_entry, *, current_s if _journal is not None: journal = self._journal_record(packet.raw,prepared,retained,known_disclosure_statuses,budget) _journal("request",journal) + if self._now() >= expiry or time.monotonic()-started >= timeout: + _fail("repair_access_expired") response = self.transport.request_repair(base_url,packet.raw,deadline=started+timeout) result=self._finish(response,prepared,inputs,checked,owner,target,use,read_until,retain_until, preflight_options,started,timeout,expiry,upload_bytes,budget) diff --git a/tests/test_open_repair_put_client_typescript.py b/tests/test_open_repair_put_client_typescript.py index feacf16..1f04b4a 100644 --- a/tests/test_open_repair_put_client_typescript.py +++ b/tests/test_open_repair_put_client_typescript.py @@ -8,6 +8,7 @@ from tests import test_open_repair_put_client as put_py from tests import test_open_repair_probe_typescript as probe_ts from tests.test_open_repair_occupied import receipt_inputs +from tests.open_repair_ack_fixtures import signed_entry DRIVER=offer_ts.DRIVER.replace("const {AckOfferClient}=", "const {AckReceiptClient}=").replace('new AckOfferClient(', 'new AckReceiptClient(') DRIVER=DRIVER.replace("calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);","const packet=JSON.parse(Buffer.from(raw).toString());calls.push(packet.payload?.kind??packet.kind);") @@ -20,6 +21,7 @@ DRIVER=DRIVER.replace("['knownStatuses','archiveStatuses']", "['knownStatuses','archiveStatuses','knownDisclosureStatuses','currentStatuses']") DRIVER=DRIVER.replace('const result=await client.preflight(input.base,options);', """options.journal=async(kind,raw)=>{ journal.push({kind,raw:Buffer.from(raw).toString('base64'),calls:[...calls]}); + if(input.advanceOnJournal&&kind==='request')input.now=input.advanceOnJournal; if(input.failJournal&&kind==='request')throw Error('synthetic journal unavailable'); if(input.blockJournal&&kind==='request')await new Promise(()=>{}); }; @@ -41,7 +43,7 @@ def setUpClass(cls): def setUp(self): self.case=put_py.RepairPutClientTests();self.addCleanup(self.case.doCleanups) if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage': - # One original finite grant funds both the upload and exact replay. + # One original finite grant funds the two actual client exchanges. with patch.object(put_py.offer_fixture.RepairOfferClientTests,'fixture_limits',dict(proof_limit=524288),create=True):self.case.setUp() else:self.case.setUp() self.host=self.case.host;self.f=self.host.f @@ -82,6 +84,40 @@ def test_journal_failure_stops_before_private_upload(self): self.assertNotIn('ack.put_request',result['calls']) self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'empty') + def short_statuses(self,case=None): + case=case or self.case + signers={signer.key_id:signer for signer in case.host.f['signers'].values()} + result=[] + for index,item in enumerate(case.inputs[3]['current_statuses']): + payload=json.loads(item['raw'])['payload'];payload['revision']+=1;payload['valid_until']=2_000_000_009 + result.append(signed_entry(payload,signers[payload['signing_key']['key_id']],f'synthetic-short-status-{index}')) + return result + + def test_status_expiry_during_journal_write_stops_native_and_python_private_upload(self): + # The receiving service still has a live clock; the clients must refuse + # before transmitting even if that remote service would accept the put. + statuses=self.short_statuses();value=self.call(advanceOnJournal=2_000_000_009) + value['options']['currentStatuses']=[probe_ts.encoded(item) for item in statuses] + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + self.assertNotIn('ack.put_request',result['calls']) + from memory_vault_open_repair_wire import RepairWireError + # Each implementation uses its own original finite source; the first + # preflight must not consume the second implementation's test budget. + other=put_py.RepairPutClientTests();self.addCleanup(other.doCleanups);other.setUp() + receipt,disclosure,put,options=other.inputs;client=other.client() + def journal(kind,raw): + if kind=='request':client.clock=lambda:2_000_000_009 + calls=[];request=client.transport.request_repair + def observed(base,raw,**options): + body=json.loads(raw);calls.append(body.get('kind',body.get('payload',{}).get('kind'))) + return request(base,raw,**options) + with patch.object(client.transport,'request_repair',side_effect=observed),self.assertRaisesRegex(RepairWireError,'repair_access_expired'): + client.put(other.host.http.base,receipt,disclosure,put,**(options|dict(current_statuses=self.short_statuses(other))),**other.case.args,_journal=journal) + self.assertNotIn('ack.put_request',calls) + for host in (self.host,other.host): + self.assertEqual(host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'empty') + def test_unfinished_journal_obeys_original_deadline_without_upload(self): value=self.call(blockJournal=True);value['options']['timeout']=8 result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') From 5c2594d323730bb656c6245f4ee681a74be513bc Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:02:34 +0800 Subject: [PATCH 04/16] Replay native receipt journals across restarts without renewing authorization --- .../network/open-repair-offer-client.ts | 200 ++++++++++++++++-- .../test_open_repair_put_client_typescript.py | 103 ++++++++- 2 files changed, 281 insertions(+), 22 deletions(-) diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 6a2a96f..9be5f38 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -1,5 +1,5 @@ -/** Native receipt-writer preflight over independently authorized empty sources. - * This reads original source evidence only; it does not authorize or upload a receipt. +/** Native receipt-writer preflight, consent-bound upload and exact journal replay. + * Preflight alone reads evidence; upload requires the independent receipt consent. */ import {performance} from 'node:perf_hooks'; import {randomBytes} from 'node:crypto'; @@ -30,6 +30,10 @@ export interface AckOfferOptions { readonly knownStatuses?:readonly unknown[];readonly archiveStatuses?:readonly unknown[];readonly timeout?:number; } export type PreparedAckOffer=RecoveredAckOwnerEmptyProof; +const OFFER_FIELDS=['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', + 'rootEntry','writeEntry','bootstrapEntry'] as const; +interface CachedAckReturn {context:string;prepared:PreparedAckOffer;budget:RepairBudget;started:number;timeout:number;retained:RawOriginal[];} + interface Obligation{role:string|null;kind:string;subject:unknown;revision:number;signer:Obj;scope_id:string;mask?:number;} function fail(code:string):never{throw new RepairError(code);} function fields(value:unknown,names:readonly string[]):Obj{return objectFields(value,names);} @@ -57,6 +61,7 @@ function snapshotEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget,leg return {raw,ref}; } export class AckOfferClient{ + #preparedReturn:CachedAckReturn|undefined;#preparedGeneration=0; readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #limits:Obj; readonly #statusObserver?: (item:AuthenticatedStatusOriginal)=>void; readonly #transport:OpenHTTPTransport;readonly #allowLoopback:boolean;readonly #clock:()=>number;readonly #ownTransport:boolean; @@ -78,14 +83,32 @@ export class AckOfferClient{ this.#statusObserver=args.statusObserver; Object.freeze(this); } - close():void{if(this.#ownTransport)this.#transport.close();} + close():void{this.#preparedReturn=undefined;this.#preparedGeneration++;if(this.#ownTransport)this.#transport.close();} #now():number{return u53(Math.floor(this.#clock()));} async preflight(baseUrl:string,value:AckOfferOptions):Promise{ return this.#preflight(baseUrl,value,new RepairBudget(this.#policy)); } + async prepareReturn(baseUrl:string,value:AckOfferOptions):Promise{ + this.#preparedReturn=undefined;const generation=++this.#preparedGeneration; + const args=options(value,OFFER_FIELDS,['knownStatuses','archiveStatuses','timeout']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + const started=clock(),budget=new RepairBudget(this.#policy),context=this.#offerContext(baseUrl,args,budget); + const owner=buildNewWire(args.expectedOwner,this.#policy,budget).value as Obj; + const retained=new OfferStatusChecks(owner,this.#policy,this.#clock,this.#statusObserver).history(args.knownStatuses??[],args.archiveStatuses??[],budget); + const selected={...args,knownStatuses:[],archiveStatuses:retained} as unknown as AckOfferOptions; + const prepared=await this.#preflight(baseUrl,selected,budget); + if(generation!==this.#preparedGeneration)fail('repair_invalid_context'); + if(clock()>=started+timeout)fail('repair_access_expired'); + this.#preparedReturn={context,prepared,budget,started,timeout,retained};return prepared; + } + #offerContext(baseUrl:string,args:Obj,budget:RepairBudget):string{ + const selected:Obj={base_url:baseUrl}; + for(const name of OFFER_FIELDS)selected[name]=['targetNodeEntry','rootEntry','writeEntry','bootstrapEntry'].includes(name)? + snapshotEntry(args[name],this.#policy,budget,true).ref:args[name]; + return budget.hash(buildNewWire(selected,this.#policy,budget).raw); + } async #preflight(baseUrl:string,value:AckOfferOptions,budget:RepairBudget):Promise{ - const args=options(value,['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId', - 'expectedEnvelopeRef','rootEntry','writeEntry','bootstrapEntry'],['knownStatuses','archiveStatuses','timeout']); + const args=options(value,OFFER_FIELDS,['knownStatuses','archiveStatuses','timeout']); const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); const policy=this.#policy,started=this.#now(),deadline=clock()+timeout; const expected=buildNewWire({target:args.expectedTarget,slot:args.expectedAckSlot,owner:args.expectedOwner, @@ -172,24 +195,36 @@ export class AckOfferClient{ metrics:Object.freeze({requests,wire_bytes:wireBytes,proof_bytes:proofBytes,...budget.snapshot()})}); } async put(baseUrl:string,receiptEntry:unknown,disclosureEntry:unknown,putEntry:unknown,value:AckReceiptPutOptions):Promise{ - const preflightNames=['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', - 'rootEntry','writeEntry','bootstrapEntry']; + const preflightNames=OFFER_FIELDS; const args=options(value,[...preflightNames,'currentStatuses','readUntil','retainUntil'], ['knownStatuses','archiveStatuses','knownDisclosureStatuses','timeout','journal']); - const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + let timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); if(args.journal!==undefined&&typeof args.journal!=='function')fail('repair_invalid_context'); - const started=clock(),policy=this.#policy,budget=new RepairBudget(policy); + const cached=this.#preparedReturn;this.#preparedReturn=undefined;this.#preparedGeneration++; + const started=cached?.started??clock(),policy=this.#policy,budget=cached?.budget??new RepairBudget(policy); + if(cached){ + timeout=Math.min(timeout,cached.timeout); + if(clock()>=started+timeout)fail('repair_access_expired'); + if(cached.context!==this.#offerContext(baseUrl,args,budget))fail('repair_proof_mismatch'); + } const expected=buildNewWire({owner:args.expectedOwner,target:args.expectedTarget,message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; const readUntil=u53(args.readUntil),retainUntil=u53(args.retainUntil); const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); - const retained=checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget),disclosureHistory=checks.history(args.knownDisclosureStatuses??[],[],budget); + const retained=mergeRetained(cached?.retained??[],checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget)), + disclosureHistory=checks.history(args.knownDisclosureStatuses??[],[],budget); if(!Array.isArray(args.currentStatuses)||isProxy(args.currentStatuses)||args.currentStatuses.length<1||args.currentStatuses.length>8)fail('repair_invalid_status'); const currentEntries=checks.history(args.currentStatuses,[],budget); // Snapshot private upload originals before the first asynchronous read. const receipt=snapshotReceiptEntry(receiptEntry,policy,budget),disclosure=snapshotEntry(disclosureEntry,policy,budget,true),put=snapshotEntry(putEntry,policy,budget,true); const preflight=Object.fromEntries(preflightNames.map(name=>[name,args[name]])); preflight.archiveStatuses=retained;preflight.timeout=timeout; - const prepared=await this.#preflight(baseUrl,preflight as unknown as AckOfferOptions,budget),source=prepared.source; + const prepared=cached?.prepared??await this.#preflight(baseUrl,preflight as unknown as AckOfferOptions,budget),source=prepared.source; + // The cached proof does not renew its node descriptor or permit a changed + // source. Recheck the same exact original before private upload. + const nodeEntry=snapshotEntry(args.targetNodeEntry,policy,budget,true); + const sourceNode=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:expected.target.signing_key, + expectedStorageEpoch:prepared.proof.handle.payload.target_storage_epoch,at:this.#now(),policy,budget}); + if(!same(endpoint(baseUrl,this.#allowLoopback),endpoint(sourceNode.payload.base_url,this.#allowLoopback)))fail('repair_proof_mismatch'); const inputs=verifyAckReceiptInputs(source,receipt,disclosure,put,expected.owner,this.#subject,this.#now(),policy,budget); if(!same((inputs.disclosure.payload as Obj).bootstrap_return.roles,['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt']))fail('repair_disclosure_permission'); const windows=ackReceiptWindows(source,inputs,this.#now()); @@ -197,7 +232,7 @@ export class AckOfferClient{ const slot=source.custody.payload.ack_slot as Obj,duties=ackReceiptObligations(source,inputs,expected.owner,this.#subject,expected.target,slot,policy,budget); const checked=this.#putStatuses(currentEntries,disclosureHistory,duties,source,expected.owner,budget,retained,prepared.current_statuses); const handle=prepared.proof.handle.payload as Obj; - const expiry=Math.min(handle.expires_at,windows.admitUntil,...checked.map(item=>item.payload.valid_until as number),this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); + const expiry=Math.min(handle.expires_at,sourceNode.payload.expires_at as number,windows.admitUntil,...checked.map(item=>item.payload.valid_until as number),this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); if(budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); budget.output(16);const useId='use_'+randomBytes(16).toString('hex'); @@ -221,6 +256,14 @@ export class AckOfferClient{ } if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); const response=await this.#transport.requestRepair(baseUrl,packet.raw,started+timeout); + const result=this.#finish(response,{prepared,inputs,checked,expected,use,readUntil,retainUntil,started,timeout,expiry,uploadBytes,budget}); + if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired');return result; + } + #finish(response:Uint8Array,context:PutFinishContext):CommittedAckReceipt{ + const {prepared,inputs,checked,expected,use,readUntil,retainUntil,started,timeout,expiry,uploadBytes,budget}=context; + const source=prepared.source,policy=this.#policy,slot=source.custody.payload.ack_slot as Obj, + handle=prepared.proof.handle.payload as Obj,grant=source.authorities.originals.bootstrap.payload as Obj; if(response.length===0||response.length>65536)fail('repair_invalid_response'); const body=fields(parseNewWire(response,policy,budget).value,['schema_version','kind','use_ref','manifest','commit','head','packs']); if(body.schema_version!=='memory-vault-open-repair/v1'||body.kind!=='ack.put_response'||!same(rawRef(body.use_ref),use.ref))fail('repair_put_mismatch'); @@ -247,10 +290,119 @@ export class AckOfferClient{ checkAckReceiptHistoryFloors(checked,complete.statuses); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); for(const item of [manifest,commit,head,...packs,...complete.manifest.roles.map(item=>item.original)])originals.set(refKey(item.ref),item); - if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); return Object.freeze({source:complete,originals:Object.freeze([...originals.values()].map(item=>Object.freeze({ref:item.ref,get raw(){return Uint8Array.from(item.raw);}}))), - metrics:Object.freeze({requests:prepared.metrics.requests+1,transfer_bytes:transferred,...budget.snapshot()})}); + metrics:Object.freeze({requests:context.replay?1:prepared.metrics.requests+1,transfer_bytes:transferred,...budget.snapshot()})}); + } + async resume(baseUrl:string,journalRaw:Uint8Array,receiptEntry:unknown,disclosureEntry:unknown,putEntry:unknown,value:AckReceiptPutOptions):Promise{ + const args=options(value,[...OFFER_FIELDS,'currentStatuses','readUntil','retainUntil'], + ['knownStatuses','archiveStatuses','knownDisclosureStatuses','timeout','journal']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + if(args.journal!==undefined&&typeof args.journal!=='function')fail('repair_invalid_context'); + this.#preparedReturn=undefined;this.#preparedGeneration++; + const started=clock(),policy=this.#policy,budget=new RepairBudget(policy); + const journal=fields(parseNewWire(journalRaw,policy,budget).value,['schema_version','packet','handle','manifest','originals','retained','disclosure_statuses']); + if(journal.schema_version!=='memory-vault-ack-put-journal/v1')fail('repair_invalid_context'); + for(const [name,maximum] of [['originals',policy.max_entries],['retained',32],['disclosure_statuses',16]] as const) + if(!Array.isArray(journal[name])||journal[name].length>maximum)fail('repair_over_budget'); + const packet=decodePutEntry(journal.packet,policy,budget);if(packet.raw.length>65536)fail('repair_put_too_large'); + const body=fields(parseNewWire(packet.raw,policy,budget).value,['schema_version','kind','use','receipt','disclosure','put','current_statuses','read_until','retain_until']); + if(body.schema_version!=='memory-vault-open-repair/v1'||body.kind!=='ack.put_request')fail('repair_put_mismatch'); + const useEntry=decodePutEntry(body.use,policy,budget),signed=fields(parseNewWire(useEntry.raw,policy,budget).value,['payload','proof']); + const use=fields(signed.payload,['schema_version','kind','signing_key','issued_at','expires_at','use_id','subject','target','target_storage_epoch', + 'consumer','operation','bootstrap_probe_ref','bootstrap_manifest_ref','request_ref']); + if(this.#now()>=u53(use.expires_at))fail('repair_reconciliation_required'); + verifyBoundedControlSignature(use,signed.proof,this.#subject.signing_key,budget); + const expected=buildNewWire({owner:args.expectedOwner,target:args.expectedTarget,slot:args.expectedAckSlot, + message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const retained=mergeRetained(journal.retained.map((item:unknown)=>decodePutEntry(item,policy,budget)), + checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget)); + const prepared=this.#restore(baseUrl,journal,use,args,expected,retained,budget),source=prepared.source,handle=prepared.proof.handle.payload as Obj; + const dual=(keys:Obj)=>({signing_key_id:keys.signing_key.key_id,encryption_key_id:keys.encryption_key.key_id}); + if(use.schema_version!=='memory-vault-open-repair/v1'||use.kind!=='bootstrap.use'||!same(use.signing_key,this.#subject.signing_key)|| + use.consumer!=='ack_offer'||use.operation!=='ack.put'||!same(use.subject,dual(this.#subject))||!same(use.target,dual(expected.target))|| + use.target_storage_epoch!==handle.target_storage_epoch||!same(rawRef(use.bootstrap_probe_ref),rawRef(handle.probe_ref))|| + !same(rawRef(use.bootstrap_manifest_ref),prepared.proof.manifest_ref))fail('repair_put_mismatch'); + if(!(u53(use.issued_at)<=this.#now()&&use.expires_at-use.issued_at>=1&&use.expires_at-use.issued_at<=60)|| + typeof use.use_id!=='string'||!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(use.use_id))fail('repair_invalid_probe'); + const decoded={receipt:decodePutEntry(body.receipt,policy,budget,true),disclosure:decodePutEntry(body.disclosure,policy,budget),put:decodePutEntry(body.put,policy,budget)}; + for(const [name,value] of [['receipt',receiptEntry],['disclosure',disclosureEntry],['put',putEntry]] as const){ + const item=name==='receipt'?snapshotReceiptEntry(value,policy,budget):snapshotEntry(value,policy,budget,true); + if(!same(item.ref,decoded[name].ref)||!Buffer.from(item.raw).equals(Buffer.from(decoded[name].raw)))fail('repair_put_mismatch'); + } + const readUntil=u53(args.readUntil),retainUntil=u53(args.retainUntil); + if(!same(rawRef(use.request_ref),decoded.put.ref)||body.read_until!==readUntil||body.retain_until!==retainUntil)fail('repair_put_mismatch'); + const inputs=verifyAckReceiptInputs(source,decoded.receipt,decoded.disclosure,decoded.put,expected.owner,this.#subject,this.#now(),policy,budget); + if(!same((inputs.disclosure.payload as Obj).bootstrap_return.roles,['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt']))fail('repair_disclosure_permission'); + const windows=ackReceiptWindows(source,inputs,this.#now()),grant=source.authorities.originals.bootstrap.payload as Obj; + if(!(this.#now()Math.min(handle.expires_at,windows.admitUntil,grant.upload_until))fail('repair_put_mismatch'); + if(!Array.isArray(body.current_statuses)||body.current_statuses.length<1||body.current_statuses.length>8)fail('repair_invalid_status'); + const packetStatuses=body.current_statuses.map((item:unknown)=>decodePutEntry(item,policy,budget)); + const disclosureHistory=mergeRetained(journal.disclosure_statuses.map((item:unknown)=>decodePutEntry(item,policy,budget)), + checks.history(args.knownDisclosureStatuses??[],[],budget)); + if(disclosureHistory.length>16)fail('repair_status_history_capacity'); + const duties=ackReceiptObligations(source,inputs,expected.owner,this.#subject,expected.target,source.custody.payload.ack_slot as Obj,policy,budget); + const current=this.#putStatuses(args.currentStatuses,disclosureHistory,duties,source,expected.owner,budget,retained,prepared.current_statuses); + const checked=this.#putStatuses(packetStatuses,[],duties,source,expected.owner,budget,[],[...prepared.current_statuses,...current]); + const uploadBytes=packet.raw.length+useEntry.raw.length+[...Object.values(inputs),...checked].reduce((sum,item)=>sum+item.raw.length,0); + if(prepared.metrics.proof_bytes+uploadBytes>grant.limits.max_proof_bytes||budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); + const node=snapshotEntry(args.targetNodeEntry,policy,budget,true),nodePayload=checkedNodePayload(node.raw,policy,budget); + const expiry=Math.min(use.expires_at,nodePayload.expires_at,...checked.map(item=>item.payload.valid_until as number),...current.map(item=>item.payload.valid_until as number)); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_reconciliation_required'); + // Every byte, nonce and consent is the original retained carrier. A fresh + // local read budget cannot renew the source's original use or resource. + const response=await this.#transport.requestRepair(baseUrl,packet.raw,started+timeout); + const result=this.#finish(response,{prepared,inputs,checked,expected,use:useEntry,readUntil,retainUntil,started,timeout,expiry,uploadBytes,budget,replay:true}); + if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired');return result; + } + #restore(baseUrl:string,journal:Obj,use:Obj,args:Obj,expected:Obj,retained:RawOriginal[],budget:RepairBudget):PreparedAckOffer{ + const policy=this.#policy,setup=verifyAckOfferBootstrapOriginal(args.bootstrapEntry,{root:args.rootEntry,write:args.writeEntry}, + {expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject,expectedMessageId:expected.message, + expectedEnvelopeRef:expected.envelope,at:this.#now(),limitPolicy:this.#limits,policy,budget}); + const nodeEntry=snapshotEntry(args.targetNodeEntry,policy,budget,true),nodePayload=checkedNodePayload(nodeEntry.raw,policy,budget); + const node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:expected.target.signing_key,expectedStorageEpoch:nodePayload.storage_epoch,at:this.#now(),policy,budget}); + if(!same(endpoint(baseUrl,this.#allowLoopback),endpoint(node.payload.base_url,this.#allowLoopback)))fail('repair_proof_mismatch'); + const handle=decodePutEntry(journal.handle,policy,budget),manifest=decodePutEntry(journal.manifest,policy,budget); + const hp=fields(parseNewWire(handle.raw,policy,budget).value,['payload','proof']).payload as Obj,grant=setup.originals.bootstrap.payload as Obj; + const handleEncoded=encodePutEntry(handle,budget),manifestEncoded=encodePutEntry(manifest,budget); + const response=buildNewWire({schema_version:'memory-vault-open-repair/v1',kind:'bootstrap.proof_response', + handle_raw_base64url:handleEncoded.raw_base64url,manifest_raw_base64url:manifestEncoded.raw_base64url},policy,budget); + const held=verifyBootstrapProofResponse(response.raw,{expectedSubject:this.#subject,expectedTarget:expected.target, + targetStorageEpoch:node.payload.storage_epoch as string,selector:grant.selector,bootstrapGrantRef:setup.originals.bootstrap.ref, + probeRef:rawRef(use.bootstrap_probe_ref),challengeRef:rawRef(hp.challenge_ref),answerRef:rawRef(hp.answer_ref),at:this.#now(), + maxProofItems:grant.limits.max_proof_items,maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'empty',consumer:'ack_offer',policy,budget}); + if(!same(held.handle.ref,handle.ref)||!same(held.manifest_ref,manifest.ref))fail('repair_ref_mismatch'); + const originals=new Map(),roles=new Map(); + for(const encoded of journal.originals){const item=decodePutEntry(encoded,policy,budget),key=refKey(item.ref);if(originals.has(key))fail('repair_invalid_context');originals.set(key,item);} + for(const item of (held.manifest.value as Obj).children){const reference=rawRef(item.ref);roles.set(item.role,[...(roles.get(item.role)??[]),reference]);} + const required=new Set([...roles.values()].flat().map(refKey)); + if(originals.size!==required.size||[...originals.keys()].some(key=>!required.has(key)))fail('repair_proof_mismatch'); + const resolver=new LocalRawResolver(policy,budget); + for(const reference of roles.get('history.raw_pack')!){const item=originals.get(refKey(reference))!; + if(!same(resolver.put(reference.namespace,reference.key,item.raw).ref,reference))fail('repair_ref_mismatch');} + const entry=(role:string):RawOriginal=>originals.get(refKey(roles.get(role)![0]))!; + const source=verifyAckEmptySourceEvent(entry('history.ack_empty'),resolver,entry('ack.empty_custody'),{ + expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject,expectedMessageId:expected.message, + expectedEnvelopeRef:expected.envelope,expectedTarget:expected.target,targetStorageEpoch:node.payload.storage_epoch as string,limitPolicy:this.#limits,policy,budget}); + this.#emptyHead(entry('ack.head'),source,expected.target,budget); + for(const item of source.manifest.roles){const actual=originals.get(refKey(item.original.ref)); + if(!same(roles.get(item.role),[item.original.ref])||!actual||!Buffer.from(actual.raw).equals(Buffer.from(item.original.raw)))fail('repair_proof_mismatch');} + for(const [name,item] of [['root',source.predecessor.resources.originals.root],['write',source.authorities.originals.write],['bootstrap',source.authorities.originals.bootstrap]] as const) + if(!same(item.ref,setup.originals[name].ref)||!Buffer.from(item.raw).equals(Buffer.from(setup.originals[name].raw)))fail('repair_proof_mismatch'); + const packs=new Set();for(const historical of [source.manifest,source.predecessor.manifest]) + for(const item of (historical.manifest.value as Obj).roles)packs.add(refKey(rawRef(item.pack_ref))); + const supplied=roles.get('history.raw_pack')!.map(refKey); + if(packs.size!==supplied.length||supplied.some(key=>!packs.has(key)))fail('repair_unused_pack'); + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const known=checks.known(retained,checks.obligations(setup.originals,expected.slot,budget),expected.slot.root_key,expected.target,budget,true); + const current=checks.current(source,roles,originals,expected.target,budget,known); + const transferred=response.raw.length+handle.raw.length+manifest.raw.length+[...originals.values()].reduce((sum,item)=>sum+item.raw.length,0); + if(transferred>grant.limits.max_proof_bytes)fail('repair_over_budget'); + return Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:Object.freeze([...originals.values()]), + metrics:Object.freeze({requests:0,wire_bytes:0,proof_bytes:transferred,...budget.snapshot()})}); } #putStatuses(entries:unknown,retained:RawOriginal[],duties:Obj[],source:AuthenticatedAckEmptySourceEvent,owner:Obj,budget:RepairBudget, retainedAuthorities:RawOriginal[],currentAuthorities:readonly AuthenticatedStatusOriginal[]):AuthenticatedStatusOriginal[]{ @@ -421,9 +573,9 @@ function encodePutEntry(item:RawOriginal,budget:RepairBudget):Obj{ if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); budget.output(Math.ceil(raw.length*4/3));return {ref,raw_base64url:Buffer.from(raw).toString('base64url')}; } -function decodePutEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ +function decodePutEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget,receipt=false):RawOriginal{ const item=fields(value,['raw_base64url','ref']),ref=rawRef(item.ref),encoded=item.raw_base64url; - if(ref.namespace!=='meta'||ref.size>policy.max_document_bytes||typeof encoded!=='string'||encoded.length>Math.ceil(ref.size*4/3)||!/^[A-Za-z0-9_-]*$/.test(encoded))fail('repair_put_too_large'); + if((!receipt&&ref.namespace!=='meta')||ref.size>(receipt?4096:policy.max_document_bytes)||typeof encoded!=='string'||encoded.length>Math.ceil(ref.size*4/3)||!/^[A-Za-z0-9_-]*$/.test(encoded))fail('repair_put_too_large'); budget.output(ref.size);const raw=Buffer.from(encoded,'base64url'); if(raw.length!==ref.size||raw.toString('base64url')!==encoded||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; } @@ -437,3 +589,19 @@ async function journalBeforeDeadline(callback:NonNullable(); + for(const item of [...first,...second]){ + const key=refKey(item.ref),old=merged.get(key); + if(old&&!Buffer.from(old.raw).equals(Buffer.from(item.raw)))fail('repair_ref_conflict'); + merged.set(key,item);if(merged.size>32)fail('repair_status_history_capacity'); + } + return [...merged.values()]; +} + +interface PutFinishContext { + prepared:PreparedAckOffer;inputs:AuthenticatedAckOccupiedSourceEvent['inputs'];checked:AuthenticatedStatusOriginal[]; + expected:Obj;use:RawOriginal;readUntil:number;retainUntil:number;started:number;timeout:number;expiry:number; + uploadBytes:number;budget:RepairBudget;replay?:boolean; +} diff --git a/tests/test_open_repair_put_client_typescript.py b/tests/test_open_repair_put_client_typescript.py index 1f04b4a..e3ec36f 100644 --- a/tests/test_open_repair_put_client_typescript.py +++ b/tests/test_open_repair_put_client_typescript.py @@ -12,8 +12,9 @@ DRIVER=offer_ts.DRIVER.replace("const {AckOfferClient}=", "const {AckReceiptClient}=").replace('new AckOfferClient(', 'new AckReceiptClient(') DRIVER=DRIVER.replace("calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);","const packet=JSON.parse(Buffer.from(raw).toString());calls.push(packet.payload?.kind??packet.kind);") -DRIVER=DRIVER.replace("const input=JSON.parse", "const journal=[];\nconst input=JSON.parse") -DRIVER=DRIVER.replace("return request(base,raw,...rest);", """const response=await request(base,raw,...rest); +DRIVER=DRIVER.replace("const input=JSON.parse", "const journal=[],sentPackets=[];\nconst input=JSON.parse") +DRIVER=DRIVER.replace("return request(base,raw,...rest);", """if(calls.at(-1)==='ack.put_request')sentPackets.push(Buffer.from(raw).toString('base64')); + const response=await request(base,raw,...rest); if(input.corruptReply&&calls.at(-1)==='ack.put_request'){ const body=JSON.parse(Buffer.from(response).toString());body.commit.ref.raw_sha256='0'.repeat(64); const {canonicalBytes}=await import('./crypto.ts');return canonicalBytes(body); @@ -25,13 +26,22 @@ if(input.failJournal&&kind==='request')throw Error('synthetic journal unavailable'); if(input.blockJournal&&kind==='request')await new Promise(()=>{}); }; - const result=await client.put(input.base,decode(input.receipt),decode(input.disclosure),decode(input.put),options);""") + if(input.prepare){ + const selected=Object.fromEntries(['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', + 'rootEntry','writeEntry','bootstrapEntry','knownStatuses','archiveStatuses','timeout'].filter(name=>Object.hasOwn(options,name)).map(name=>[name,options[name]])); + if(input.prepareTimeout)selected.timeout=input.prepareTimeout; + await client.prepareReturn(input.base,selected); + if(input.changedBase)input.base=input.changedBase; + if(input.newKnownAfterPrepare)options.knownStatuses=input.newKnownAfterPrepare.map(decode); + if(input.waitAfterPrepare)await new Promise(resolve=>setTimeout(resolve,input.waitAfterPrepare)); + } + const result=input.resumeJournal?await client.resume(input.base,Buffer.from(input.resumeJournal,'base64'),decode(input.receipt),decode(input.disclosure),decode(input.put),options):await client.put(input.base,decode(input.receipt),decode(input.disclosure),decode(input.put),options);""") start=DRIVER.index(' process.stdout.write(JSON.stringify({ok:true') end=DRIVER.index('\n}catch(error)',start) DRIVER=DRIVER[:start]+''' process.stdout.write(JSON.stringify({ok:true,metrics:result.metrics, receipt:Buffer.from(result.source.inputs.receipt.raw).toString('base64'),commit:result.source.commit.ref, - immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls,journal}));'''+DRIVER[end:] -DRIVER=DRIVER.replace('calls,nativeChecks,subprocessCalls}));}', 'calls,nativeChecks,subprocessCalls,journal}));}') + immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls,journal,sentPackets}));'''+DRIVER[end:] +DRIVER=DRIVER.replace('calls,nativeChecks,subprocessCalls}));}', 'calls,nativeChecks,subprocessCalls,journal,sentPackets}));}') class NativePutClientTests(unittest.TestCase): @@ -42,7 +52,7 @@ def setUpClass(cls): def setUp(self): self.case=put_py.RepairPutClientTests();self.addCleanup(self.case.doCleanups) - if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage': + if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage' or 'resume' in self._testMethodName: # One original finite grant funds the two actual client exchanges. with patch.object(put_py.offer_fixture.RepairOfferClientTests,'fixture_limits',dict(proof_limit=524288),create=True):self.case.setUp() else:self.case.setUp() @@ -68,6 +78,35 @@ def test_native_upload_saves_exact_receipt_and_records_request_before_network(se self.assertNotIn('ack.put_request',result['journal'][0]['calls']);self.assertEqual(result['calls'][-1],'ack.put_request') self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'occupied') + def test_prepared_return_reuses_actual_preflight_and_original_work_meter(self): + result=self.native(self.call(prepare=True)) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'}) + self.assertEqual(result['calls'].count('bootstrap.probe'),1) + self.assertEqual(result['metrics']['requests'],13) + self.assertEqual(result['metrics']['signature_checks'],result['nativeChecks']) + self.assertGreater(result['metrics']['signature_checks'],64) + + def test_prepared_return_rejects_changed_source_before_network(self): + result=self.native(self.call(prepare=True,changedBase='http://127.0.0.1:1')) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_proof_mismatch') + self.assertEqual(len(result['calls']),12);self.assertNotIn('ack.put_request',result['calls']) + + def test_prepared_return_does_not_reset_local_signature_allowance(self): + value=self.call(prepare=True);value['policy']['max_signature_checks']=64 + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_over_budget') + self.assertEqual(result['calls'].count('bootstrap.probe'),1);self.assertNotIn('ack.put_request',result['calls']) + + def test_prepared_return_keeps_original_deadline(self): + result=self.native(self.call(prepare=True,prepareTimeout=8,waitAfterPrepare=8000)) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') + self.assertEqual(len(result['calls']),12);self.assertNotIn('ack.put_request',result['calls']) + + def test_new_revocation_after_preparation_stops_private_upload(self): + revoked=self.case.case.revoke('ack.write_grant',self.host.write) + result=self.native(self.call(prepare=True,newKnownAfterPrepare=[probe_ts.encoded(revoked)])) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_authority_revoked') + self.assertEqual(len(result['calls']),12);self.assertNotIn('ack.put_request',result['calls']) + def test_legacy_disclosure_does_not_send_receipt(self): from memory_vault_open_repair_occupied import RETURN_ROLES_LEGACY inputs=receipt_inputs(self.case.fixture,bootstrap_roles=RETURN_ROLES_LEGACY) @@ -124,6 +163,58 @@ def test_unfinished_journal_obeys_original_deadline_without_upload(self): self.assertEqual([item['kind'] for item in result['journal']],['request']) self.assertNotIn('ack.put_request',result['calls']) + def pending_native(self): + result=self.native(self.call(corruptReply=True)) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_ref_mismatch') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + return result['journal'][0]['raw'] + + def test_resume_exact_native_journal_after_client_and_service_restart(self): + journal=self.pending_native();self.host.http.restart() + result=self.native(self.call(resumeJournal=journal)) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'}) + self.assertEqual(result['calls'],['ack.put_request']) + packet=json.loads(base64.b64decode(journal))['packet']['raw_base64url'] + self.assertEqual(base64.b64decode(result['sentPackets'][0]),base64.urlsafe_b64decode(packet+'='*((-len(packet))%4))) + self.assertEqual(result['metrics']['requests'],1) + self.assertEqual(result['nativeChecks'],result['metrics']['signature_checks']) + self.assertEqual(result['receipt'],probe_ts.encoded(self.case.inputs[0])['raw']) + self.assertEqual([item['kind'] for item in result['journal']],['response']) + self.assertEqual(self.host.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + def test_resume_rejects_changed_source_message_and_raw_journal_without_network(self): + journal=self.pending_native() + values=[] + value=self.call(resumeJournal=journal);value['base']='http://127.0.0.1:1';values.append(value) + value=self.call(resumeJournal=journal);value['options']['expectedMessageId']='msg_'+'f'*64;values.append(value) + value=self.call(resumeJournal=journal);body=json.loads(base64.b64decode(journal)) + body['packet']['ref']['raw_sha256']='0'*64 + value['resumeJournal']=base64.b64encode(json.dumps(body,sort_keys=True,separators=(',',':')).encode()).decode();values.append(value) + for value in values: + with self.subTest(): + result=self.native(value);self.assertFalse(result['ok']) + self.assertNotEqual(result['code'],'untyped_error');self.assertEqual(result['calls'],[]) + + def test_resume_python_journal_in_native_client(self): + journal=[];receipt,disclosure,put,options=self.case.inputs + self.case.client().put(self.host.http.base,receipt,disclosure,put,**options,**self.case.case.args,_journal=lambda kind,raw:journal.append((kind,raw))) + self.host.http.restart() + result=self.native(self.call(resumeJournal=base64.b64encode(journal[0][1]).decode())) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'}) + self.assertEqual(result['calls'],['ack.put_request']) + self.assertEqual(self.host.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + def test_resume_expired_original_use_requires_reconciliation_without_network(self): + journal=self.pending_native();value=self.call(resumeJournal=journal);value['now']+=61 + result=self.native(value);self.assertFalse(result['ok']) + self.assertEqual(result['code'],'repair_reconciliation_required');self.assertEqual(result['calls'],[]) + + def test_resume_refuses_new_revocation_before_network(self): + journal=self.pending_native();revoked=self.case.case.revoke('ack.write_grant',self.host.write) + value=self.call(resumeJournal=journal);value['options']['knownStatuses']=[probe_ts.encoded(revoked)] + result=self.native(value);self.assertFalse(result['ok']) + self.assertEqual(result['code'],'repair_authority_revoked');self.assertEqual(result['calls'],[]) + def test_bad_returned_commit_cannot_claim_verified_storage(self): result=self.native(self.call(corruptReply=True));self.assertFalse(result['ok']) self.assertEqual(result['code'],'repair_ref_mismatch') From e383f65122cb6248874b1e3ba59b8ed738b13ebf Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:10:59 +0800 Subject: [PATCH 05/16] Return actual saved receipts through the native Agent and shared durable journal --- clients/typescript/network/open-ack-client.ts | 18 ++ clients/typescript/network/open-client.ts | 4 +- .../network/open-delivery-client.ts | 19 ++ .../typescript/network/open-repair-empty.ts | 4 + .../network/open-repair-offer-client.ts | 4 + .../typescript/network/open-repair-receipt.ts | 175 ++++++++++++++++++ clients/typescript/network/package.json | 1 + docs/OPEN_ACK_PROVISIONING.md | 30 ++- docs/OPEN_ACK_RECOVERY.md | 14 +- memory_vault.py | 2 +- packaging/CLIENT_README.md | 17 +- packaging/PROTOCOL_README.md | 17 +- packaging/RELEASE_NOTES.md | 19 +- packaging/REVIEW_README.md | 15 +- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 2 +- scripts/run_open_network_ci.py | 4 +- tests/test_open_repair_receipt_typescript.py | 93 ++++++++++ 18 files changed, 416 insertions(+), 24 deletions(-) create mode 100644 clients/typescript/network/open-repair-receipt.ts create mode 100644 tests/test_open_repair_receipt_typescript.py diff --git a/clients/typescript/network/open-ack-client.ts b/clients/typescript/network/open-ack-client.ts index 6b3a853..e5f1222 100644 --- a/clients/typescript/network/open-ack-client.ts +++ b/clients/typescript/network/open-ack-client.ts @@ -5,6 +5,8 @@ import {endpoint} from './open-transport.ts'; import {envelopeRef} from './open-delivery-control.ts'; import {DiscoveredAckRecoveryClient} from './open-ack-discovery.ts'; import {OpenProviderClient} from './open-provider-client.ts'; +import {AckReceiptClient} from './open-repair-offer-client.ts'; +import {SavedAckReceiptPublisher,SAVED_ACK_REQUEST_FIELDS} from './open-repair-receipt.ts'; import {OriginalAckStatusJournal} from './open-ack-status.ts'; import type {DatabaseSync} from 'node:sqlite'; import {canonicalBytes,document,objectFields,sha256,validateSigningIdentity,decodeBase64url} from './crypto.ts'; @@ -73,6 +75,22 @@ class Journal{ if(!saved)fail('repair_setup_journal_capacity'); } } +export async function returnSavedReceipt(participant:OpenParticipant,encryption:EncryptionIdentityDocument, + delivery:OpenDeliveryClient,invitation:unknown):Promise{ + const v=objectFields(document(invitation as any,65536),['schema_version','action','repair_profile','base_url','request']); + if(v.schema_version!==ACK_CONNECT_SCHEMA||v.action!=='return_receipt'||typeof v.repair_profile!=='string'||!Object.hasOwn(PROFILES,v.repair_profile))fail('open_invalid_ack_request'); + const request=objectFields(v.request,SAVED_ACK_REQUEST_FIELDS) as Obj; + const decoded={...request};for(const name of SAVED_ACK_REQUEST_FIELDS.filter(n=>n.endsWith('_entry')))decoded[name]=decode(request[name]); + if(!Array.isArray(request.current_statuses)||request.current_statuses.length>7)fail('open_invalid_ack_request'); + decoded.current_statuses=request.current_statuses.map(decode); + const client=new AckReceiptClient(participant.identity,encryption,{limitPolicy:PROFILES[v.repair_profile as keyof typeof PROFILES], + transport:participant.transport,allowLoopback:participant.transport.allow_loopback}); + try{ + const result=await new SavedAckReceiptPublisher(delivery,client).publishSaved(v.base_url as string,decoded); + return {state:'retained_at_ack_source',message_id:request.message_id,receipt_ref:result.source.inputs.receipt.ref, + commit_ref:result.source.commit.ref,from_local_history:result.from_local_history,network_accessed:!result.from_local_history}; + }finally{client.close();} +} export async function recoverReceipt(participant:OpenParticipant,encryption:EncryptionIdentityDocument, delivery:OpenDeliveryClient,invitation:unknown,deadline?:number):Promise{ const parsed=document(invitation as any,65536),routed=parsed.action==='recover_routed_receipt',discovered=routed||parsed.action==='recover_discovered_receipt'; diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index d1738a0..082ab86 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -9,7 +9,7 @@ import {OpenDeliveryClient} from './open-delivery-client.ts'; import {OpenParticipant} from './open-participant.ts'; import {OpenContactClient,CONNECT_SCHEMA} from './open-contact-client.ts'; import type {SignedNode} from './open-control.ts'; -import {ACK_CONNECT_SCHEMA,recoverReceipt,recoverPrepared} from './open-ack-client.ts'; +import {ACK_CONNECT_SCHEMA,recoverReceipt,recoverPrepared,returnSavedReceipt} from './open-ack-client.ts'; export const OPEN_CLIENT_CONFIG='memory-vault-open-client-config/v1'; function fail(code:string):never{throw new NetworkError(code);} export class OpenNetworkClient{ @@ -37,7 +37,7 @@ export class OpenNetworkClient{ async connect(invitation?:unknown,requestId?:string):Promise>{ if(invitation!=null){ if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA) - return this.result(await recoverReceipt(this.participant,this.encryption,this.delivery(),invitation)); + return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); if(typeof invitation!=='object'||Array.isArray(invitation)||(invitation as any).schema_version!==CONNECT_SCHEMA)fail('open_private_invitation_unsupported'); const result=await new OpenContactClient(this.participant,this.encryption).dispatch(invitation,requestId); return this.result({...result,profile:'open-routing-v1',network_accessed:true,open_messaging_supported:true}); diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 5589efa..d69c7b8 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -371,6 +371,25 @@ export class OpenDeliveryClient{ db.prepare('UPDATE open_delivery_inbox SET receipt=? WHERE message_id=?').run(encoded,messageId);return receipt; })); } + /** Read the actually saved inbox receipt for an explicitly requested return. + * The caller cannot substitute receipt bytes or a saved-state flag. */ + savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):{raw:Uint8Array;ref:Obj}{ + if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); + const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); + const keys=OpenDeliveryClient.keys(document(row.session,MAX_INBOX_SESSION_BYTES) as Obj); + if(sha256(row.envelope)!==row.envelope_sha256||row.sender!==owner.signing_key.key_id||!same(envelopeRef(row.envelope),expectedEnvelope)|| + !same(keys,{sender_signing_key:owner.signing_key,sender_encryption_key:owner.encryption_key, + recipient_signing_key:validateSigningIdentity(this.participant.identity),recipient_encryption_key:validateEncryptionIdentity(this.encryption)}))fail('repair_saved_tuple_mismatch'); + const result=document(row.result,MAX_RESULT_BYTES) as Obj; + if(result.state!=='validated_saved'||result.message_id!==messageId||result.sender_key_id!==row.sender|| + !['message','memory_transfer'].includes(result.content_kind))fail('repair_receipt_not_saved'); + this.savedReceipt(messageId);const saved=this.inbox(messageId); + if(!saved||saved.phase!=='saved'||saved.sender!==row.sender||!raw(saved.envelope).equals(raw(row.envelope))||!raw(saved.body).equals(raw(row.body)))fail('repair_saved_tuple_mismatch'); + const bytes=Buffer.from(saved.receipt); + verifyRecipientReceipt(document(bytes,4096),{recipient_signing_key:validateSigningIdentity(this.participant.identity), + sender_key_id:row.sender,message_id:messageId,envelope_ref:expectedEnvelope} as any); + const digest=sha256(bytes);return {raw:bytes,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:bytes.length}}; + } private async sendReceipt(messageId:string,budget:DeliveryBudget,node?:SignedNode){ const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('open_delivery_not_saved');if(row.receipt_sent)return; const receipt=this.savedReceipt(messageId);verifyRecipientReceipt(receipt,{recipient_signing_key:validateSigningIdentity(this.participant.identity), diff --git a/clients/typescript/network/open-repair-empty.ts b/clients/typescript/network/open-repair-empty.ts index 9ffa874..83adc4d 100644 --- a/clients/typescript/network/open-repair-empty.ts +++ b/clients/typescript/network/open-repair-empty.ts @@ -140,3 +140,7 @@ export function verifyAckEmptySourceEvent(manifestEntry:unknown,resolver:LocalRa const observed=statuses(roles,obligations(prior,authorities,owner,target,ackSlot,policy,budget),ackSlot.root_key,at,policy,budget);historyFloors(prior.statuses,observed); return Object.freeze({manifest,custody,predecessor:prior,binding,authorities,statuses:Object.freeze(observed),stored_at:at,read_until:event.read_until,retain_until:event.retain_until}); } + +export function verifyAckBindingOriginal(value:unknown,target:Record,policy:RepairPolicy,budget:RepairBudget):AuthenticatedRepairOriginal{ + return signed(value,target,'ack.binding',policy,budget); +} diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 9be5f38..c1e90f4 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -84,6 +84,10 @@ export class AckOfferClient{ Object.freeze(this); } close():void{this.#preparedReturn=undefined;this.#preparedGeneration++;if(this.#ownTransport)this.#transport.close();} + get subject():Readonly{return this.#subject;} + get policy():RepairPolicy{return this.#policy;} + get limits():Readonly{return this.#limits;} + get timestamp():number{return this.#now();} #now():number{return u53(Math.floor(this.#clock()));} async preflight(baseUrl:string,value:AckOfferOptions):Promise{ return this.#preflight(baseUrl,value,new RepairBudget(this.#policy)); diff --git a/clients/typescript/network/open-repair-receipt.ts b/clients/typescript/network/open-repair-receipt.ts new file mode 100644 index 0000000..af24445 --- /dev/null +++ b/clients/typescript/network/open-repair-receipt.ts @@ -0,0 +1,175 @@ +/** Explicitly return an actually saved inbox receipt. Shares the protected, + * finite Python journal; a local historical retry is never a fresh network claim. */ +import type {DatabaseSync} from 'node:sqlite'; +import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {AckReceiptClient} from './open-repair-offer-client.ts'; +import type {CommittedAckReceipt} from './open-repair-offer-client.ts'; +import {RepairBudget,RepairError,buildNewWire,parseNewWire,objectFields,rawRef,u53,LocalRawResolver} from './open-repair-wire.ts'; +import type {RawOriginal,RepairPolicy} from './open-repair-wire.ts'; +import {parseOriginalControl,verifyOriginalControl} from './open-repair-original.ts'; +import {verifyAckOfferBootstrapOriginal} from './open-repair-bound.ts'; +import {verifyAckBindingOriginal} from './open-repair-empty.ts'; +import {verifyAckOccupiedSourceEvent} from './open-repair-occupied.ts'; +import type {AuthenticatedAckOccupiedSourceEvent} from './open-repair-occupied.ts'; +import {signBoundedBootstrapOriginal} from './open-repair-probe.ts'; +import {statusScope} from './open-repair-status.ts'; +import {validateSigningIdentity,validateEncryptionIdentity} from './crypto.ts'; +import {transaction} from './io.ts'; +type Obj=Record; +export const SAVED_ACK_REQUEST_FIELDS=['message_id','envelope_ref','ack_slot','owner','target','target_node_entry', + 'root_entry','write_entry','bootstrap_entry','binding_entry','current_statuses','read_until','retain_until'] as const; +const MAX_RECORDS=16,MAX_BYTES=1048576; +const key=(r:Obj)=>`${r.namespace}:${r.key}:${r.raw_sha256}:${r.size}`; +function fail(code:string):never{throw new RepairError(code);} +function equal(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(n=>Object.hasOwn(b,n)&&equal((a as Obj)[n],(b as Obj)[n])); +} +function sameEntry(a:RawOriginal,b:RawOriginal):boolean{return equal(a.ref,b.ref)&&Buffer.from(a.raw).equals(Buffer.from(b.raw));} +export interface PublishedSavedAck{readonly source:AuthenticatedAckOccupiedSourceEvent;readonly originals:readonly RawOriginal[];readonly from_local_history:boolean;} +export class SavedAckReceiptPublisher{ + readonly #delivery:OpenDeliveryClient;readonly #client:AckReceiptClient;readonly #policy:RepairPolicy; + constructor(delivery:OpenDeliveryClient,client:AckReceiptClient){ + if(!(delivery instanceof OpenDeliveryClient))fail('repair_saved_client_required'); + if(!equal(client.subject,{signing_key:validateSigningIdentity(delivery.participant.identity),encryption_key:validateEncryptionIdentity(delivery.encryption)}))fail('repair_saved_identity_mismatch'); + this.#delivery=delivery;this.#client=client;this.#policy=client.policy; + this.#db(db=>transaction(db,()=>{ + db.exec(`CREATE TABLE IF NOT EXISTS open_repair_saved_ack_roots(root_digest TEXT PRIMARY KEY,revision INTEGER NOT NULL); + CREATE TABLE IF NOT EXISTS open_repair_saved_acks(slot_digest TEXT PRIMARY KEY,root_digest TEXT NOT NULL, + request_digest TEXT NOT NULL,issued_at INTEGER NOT NULL,status_revision INTEGER NOT NULL, + originals BLOB,result BLOB,attempted INTEGER NOT NULL DEFAULT 0);`); + const columns=new Set((db.prepare('PRAGMA table_info(open_repair_saved_acks)').all() as Obj[]).map(r=>r.name)); + if(!columns.has('attempted')){db.exec('ALTER TABLE open_repair_saved_acks ADD COLUMN attempted INTEGER NOT NULL DEFAULT 0'); + db.exec('UPDATE open_repair_saved_acks SET attempted=1 WHERE originals IS NOT NULL AND result IS NULL');} + for(const name of ['put_journal','put_response'])if(!columns.has(name))db.exec('ALTER TABLE open_repair_saved_acks ADD COLUMN '+name+' BLOB'); + })); + } + #db(operation:(db:DatabaseSync)=>T):T{return this.#delivery.participant.contactStorage(operation);} + #entry(value:unknown,budget:RepairBudget):RawOriginal{ + const e=objectFields(value,['raw','ref']),ref=rawRef(e.ref),raw=parseOriginalControl(e.raw,this.#policy,budget).raw; + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; + } + #request(value:unknown,budget:RepairBudget):Obj{ + const v=objectFields(value,SAVED_ACK_REQUEST_FIELDS),scalars=SAVED_ACK_REQUEST_FIELDS.filter(n=>!n.endsWith('_entry')&&n!=='current_statuses'); + const r={...buildNewWire(Object.fromEntries(scalars.map(n=>[n,v[n]])),this.#policy,budget).value as Obj}; + for(const n of SAVED_ACK_REQUEST_FIELDS.filter(n=>n.endsWith('_entry')))r[n]=this.#entry(v[n],budget); + if(!Array.isArray(v.current_statuses)||v.current_statuses.length<1||v.current_statuses.length>7)fail('repair_invalid_status'); + r.current_statuses=v.current_statuses.map((e:unknown)=>this.#entry(e,budget));return r; + } + #decodeEntries(raw:Uint8Array,budget:RepairBudget):Record{ + const v=objectFields(parseNewWire(raw,this.#policy,budget).value,['receipt','disclosure','put','status']); + return Object.fromEntries(Object.entries(v).map(([name,item])=>{ + const e=objectFields(item,['raw','ref']);if(typeof e.raw!=='string')fail('repair_saved_corrupt'); + return [name,this.#entry({raw:Buffer.from(e.raw,'utf8'),ref:e.ref},budget)]; + })); + } + #archive(result:CommittedAckReceipt,budget:RepairBudget):Uint8Array{ + const originals=[...result.originals].sort((a,b)=>(a.ref.namespaceb.ref.namespace?1:a.ref.keyb.ref.key?1:0)); + const raw=buildNewWire({manifest_ref:result.source.commit.payload.historical_manifest_ref,commit_ref:result.source.commit.ref, + originals:originals.map(e=>({ref:e.ref,raw_base64url:Buffer.from(e.raw).toString('base64url')}))},this.#policy,budget).raw; + if(raw.length>MAX_BYTES)fail('repair_saved_capacity');return raw; + } + #restore(raw:Uint8Array,r:Obj,entries:Record,budget:RepairBudget):PublishedSavedAck{ + const value=objectFields(parseNewWire(raw,this.#policy,budget).value,['manifest_ref','commit_ref','originals']); + if(!Array.isArray(value.originals)||value.originals.length<1||value.originals.length>128)fail('repair_saved_corrupt'); + const resolver=new LocalRawResolver(this.#policy,budget),originals=new Map(); + for(const item of value.originals){ + const encoded=objectFields(item,['ref','raw_base64url']),ref=rawRef(encoded.ref); + if(typeof encoded.raw_base64url!=='string'||!/^[A-Za-z0-9_-]*$/.test(encoded.raw_base64url))fail('repair_saved_corrupt'); + budget.output(ref.size);const body=Buffer.from(encoded.raw_base64url,'base64url'); + if(body.toString('base64url')!==encoded.raw_base64url||body.length!==ref.size||budget.hash(body)!==ref.raw_sha256||originals.has(key(ref)))fail('repair_saved_corrupt'); + originals.set(key(ref),{raw:body,ref});if(!equal(resolver.put(ref.namespace,ref.key,body).ref,ref))fail('repair_saved_corrupt'); + } + const manifest=originals.get(key(rawRef(value.manifest_ref))),commit=originals.get(key(rawRef(value.commit_ref))); + if(!manifest||!commit)fail('repair_saved_corrupt'); + const node=r.target_node_entry,nodeValue=parseOriginalControl(node.raw,this.#policy,budget).value as Obj; + const checked=verifyOriginalControl(node.raw,{expectedSigningKey:r.target.signing_key,expectedSchema:'memory-vault-open-control/v1',expectedKind:'node', + at:nodeValue.payload.issued_at,policy:this.#policy,budget}); + const source=verifyAckOccupiedSourceEvent(manifest,resolver,commit,{expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedReceiptWriter:this.#client.subject, + expectedMessageId:r.message_id,expectedEnvelopeRef:r.envelope_ref,expectedTarget:r.target,targetStorageEpoch:checked.payload.storage_epoch as string, + limitPolicy:this.#client.limits,policy:this.#policy,budget}); + if(['receipt','disclosure','put'].some(name=>!sameEntry(source.inputs[name as keyof typeof source.inputs],entries[name])))fail('repair_saved_corrupt'); + return Object.freeze({source,originals:Object.freeze([...originals.values()].map(e=>Object.freeze({ref:e.ref,get raw(){return Uint8Array.from(e.raw);}}))),from_local_history:true}); + } + async publishSaved(baseUrl:string,value:unknown,timeout=30):Promise{ + if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + const client=this.#client,policy=this.#policy,budget=new RepairBudget(policy),r=this.#request(value,budget); + const receipt=this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; + const receiptPayload=(parseOriginalControl(receipt.raw,policy,budget).value as Obj).payload; + const authorities=verifyAckOfferBootstrapOriginal(r.bootstrap_entry,{root:r.root_entry,write:r.write_entry}, + {expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedReceiptWriter:client.subject,expectedMessageId:r.message_id, + expectedEnvelopeRef:r.envelope_ref,at:now,limitPolicy:client.limits,policy,budget}); + const binding=verifyAckBindingOriginal(r.binding_entry,r.target,policy,budget),{root,write,bootstrap:grant}=authorities.originals; + if(!equal(binding.payload.ack_slot,r.ack_slot)||!equal(binding.payload.root_authority_ref,root.ref)||!equal(binding.payload.grant_ref,write.ref)|| + binding.payload.bound_at>receiptPayload.saved_at||!(receiptPayload.saved_at<=now&&nowroot.payload.windows.read_until||r.retain_until>Math.min(binding.payload.retain_until as number,write.payload.windows.retain_until,root.payload.windows.retain_until))fail('repair_saved_tuple_mismatch'); + const hash=(v:unknown)=>budget.hash(buildNewWire(v,policy,budget).raw),rootDigest=hash(r.ack_slot.root_key),slotDigest=hash(r.ack_slot); + const requestDigest=hash({slot:r.ack_slot,receipt:receipt.ref,...Object.fromEntries(['owner','target','read_until','retain_until'].map(n=>[n,r[n]])), + ...Object.fromEntries(['root_entry','write_entry','bootstrap_entry','binding_entry'].map(n=>[n,r[n].ref]))}); + let saved=this.#db(db=>transaction(db,()=>{ + let row=db.prepare('SELECT * FROM open_repair_saved_acks WHERE slot_digest=?').get(slotDigest) as Obj|undefined; + if(!row){ + if((db.prepare('SELECT count(*) AS n FROM open_repair_saved_acks').get() as Obj).n>=MAX_RECORDS)fail('repair_saved_capacity'); + const prior=db.prepare('SELECT revision FROM open_repair_saved_ack_roots WHERE root_digest=?').get(rootDigest) as Obj|undefined; + const revision=u53((prior?.revision??0)+1,1); + db.prepare('INSERT INTO open_repair_saved_ack_roots VALUES(?,?) ON CONFLICT(root_digest) DO UPDATE SET revision=excluded.revision').run(rootDigest,revision); + db.prepare('INSERT INTO open_repair_saved_acks(slot_digest,root_digest,request_digest,issued_at,status_revision) VALUES(?,?,?,?,?)').run(slotDigest,rootDigest,requestDigest,now,revision); + row=db.prepare('SELECT * FROM open_repair_saved_acks WHERE slot_digest=?').get(slotDigest) as Obj; + } + if(row.root_digest!==rootDigest||row.request_digest!==requestDigest)fail('repair_saved_retry_mismatch');return row; + })); + if(saved.originals===null){ + const issued=u53(saved.issued_at),retain=r.retain_until,read=r.read_until,expiry=Math.min(retain,write.payload.expires_at,grant.payload.upload_until,issued+300); + if(expiry<=issued)fail('repair_access_expired'); + const sign=(payload:Obj)=>signBoundedBootstrapOriginal(payload,this.#delivery.participant.identity as unknown as Obj,policy,budget); + const common={schema_version:'memory-vault-open-repair/v1',signing_key:client.subject.signing_key}; + const disclosure=sign({...common,kind:'ack.disclosure',issued_at:issued,expires_at:retain,consent_id:'consent_'+slotDigest, + ack_slot:r.ack_slot,root_authority_ref:root.ref,grant_ref:write.ref,receipt_ref:receipt.ref,recipient:r.ack_slot.receipt_writer, + owner:r.ack_slot.root_key.owner,allowed_roles:['ack.disclosure','ack.put','historical.status.ack_disclosure','recipient.receipt'],operation_mask:67, + consent_until:retain,bootstrap_return:{subject:r.ack_slot.root_key.owner,consumer:'ack_owner', + roles:['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt'],until:read},revision:1}); + const put=sign({...common,kind:'ack.put',issued_at:issued,expires_at:expiry,put_id:'put_'+slotDigest,ack_slot:r.ack_slot, + grant_ref:write.ref,binding_ref:binding.ref,receipt_ref:receipt.ref,disclosure_ref:disclosure.ref,operation:'receipt.put'}); + const scope=statusScope(r.ack_slot.root_key,'authority',{authority_kind:'ack.disclosure',authority_sha256:disclosure.ref.raw_sha256},policy,budget); + const status=sign({schema_version:'memory-vault-open-authority/v1',kind:'authority.status',signing_key:client.subject.signing_key, + scope_key:{root_key:r.ack_slot.root_key,issuer_key_id:client.subject.signing_key.key_id},revision:saved.status_revision, + issued_at:issued,valid_until:Math.min(retain,issued+3600),entries:[{scope_kind:'authority',scope_id:scope,minimum_document_revision:1,status:'active',operation_mask:67}]}); + const encoded=buildNewWire(Object.fromEntries(Object.entries({receipt,disclosure,put,status}).map(([name,e])=>[name,{raw:Buffer.from(e.raw).toString('utf8'),ref:e.ref}])),policy,budget).raw; + if(encoded.length>65536)fail('repair_saved_capacity'); + saved=this.#db(db=>transaction(db,()=>{ + db.prepare('UPDATE open_repair_saved_acks SET originals=? WHERE slot_digest=? AND request_digest=? AND originals IS NULL').run(encoded,slotDigest,requestDigest); + return db.prepare('SELECT * FROM open_repair_saved_acks WHERE slot_digest=?').get(slotDigest) as Obj; + })); + } + const entries=this.#decodeEntries(saved.originals,budget);if(!sameEntry(entries.receipt,receipt))fail('repair_saved_corrupt'); + if(saved.result!==null)return this.#restore(saved.result,r,entries,budget); + const pending=this.#db(db=>db.prepare('SELECT attempted,put_journal FROM open_repair_saved_acks WHERE slot_digest=? AND request_digest=?').get(slotDigest,requestDigest)) as Obj|undefined; + if(!pending||(pending.attempted&&pending.put_journal===null))fail('repair_saved_reconciliation_required'); + const resume=pending.attempted?Buffer.from(pending.put_journal):null; + const journal=(phase:'request'|'response',raw:Uint8Array)=>{ + if(!raw.length||raw.length>MAX_BYTES)fail('repair_saved_capacity');const column=phase==='request'?'put_journal':'put_response'; + this.#db(db=>transaction(db,()=>{ + const held=db.prepare('SELECT originals,put_journal,put_response,result FROM open_repair_saved_acks WHERE slot_digest=? AND request_digest=?').get(slotDigest,requestDigest) as Obj|undefined; + if(!held||!Buffer.from(held.originals).equals(Buffer.from(saved.originals)))fail('repair_saved_corrupt'); + if(held[column]!==null&&!Buffer.from(held[column]).equals(Buffer.from(raw)))fail('repair_saved_retry_mismatch'); + if(phase==='response'&&held.put_journal===null)fail('repair_saved_corrupt'); + if(Object.entries(held).reduce((n,[name,v])=>n+(name!==column&&v!==null?(v as Uint8Array).length:0),0)+raw.length>MAX_BYTES)fail('repair_saved_capacity'); + db.prepare('UPDATE open_repair_saved_acks SET '+column+'=?,attempted=1 WHERE slot_digest=?').run(raw,slotDigest); + })); + }; + const options={currentStatuses:[...r.current_statuses,entries.status],readUntil:r.read_until,retainUntil:r.retain_until, + targetNodeEntry:r.target_node_entry,expectedTarget:r.target,expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedMessageId:r.message_id, + expectedEnvelopeRef:r.envelope_ref,rootEntry:r.root_entry,writeEntry:r.write_entry,bootstrapEntry:r.bootstrap_entry,timeout,journal}; + const result=resume===null?await client.put(baseUrl,entries.receipt,entries.disclosure,entries.put,options): + await client.resume(baseUrl,resume,entries.receipt,entries.disclosure,entries.put,options); + const archive=this.#archive(result,budget); + this.#db(db=>transaction(db,()=>{ + const current=db.prepare('SELECT originals,result,put_journal,put_response FROM open_repair_saved_acks WHERE slot_digest=? AND request_digest=?').get(slotDigest,requestDigest) as Obj|undefined; + if(!current||!Buffer.from(current.originals).equals(Buffer.from(saved.originals)))fail('repair_saved_corrupt'); + if(current.result!==null&&!Buffer.from(current.result).equals(Buffer.from(archive)))fail('repair_saved_retry_mismatch'); + if(['originals','put_journal','put_response'].reduce((n,k)=>n+(current[k]?.length??0),0)+archive.length>MAX_BYTES)fail('repair_saved_capacity'); + db.prepare('UPDATE open_repair_saved_acks SET result=? WHERE slot_digest=?').run(archive,slotDigest); + })); + return Object.freeze({source:result.source,originals:result.originals,from_local_history:false}); + } +} diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index e456d82..b2d36c5 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -37,6 +37,7 @@ "./open-repair-admin": "./open-repair-admin.ts", "./open-repair-client": "./open-repair-client.ts", "./open-repair-offer-client": "./open-repair-offer-client.ts", + "./open-repair-receipt": "./open-repair-receipt.ts", "./open-repair-wire": "./open-repair-wire.ts", "./open-repair-bound": "./open-repair-bound.ts", "./open-repair-empty": "./open-repair-empty.ts", diff --git a/docs/OPEN_ACK_PROVISIONING.md b/docs/OPEN_ACK_PROVISIONING.md index e92a5f9..2e01b37 100644 --- a/docs/OPEN_ACK_PROVISIONING.md +++ b/docs/OPEN_ACK_PROVISIONING.md @@ -516,6 +516,32 @@ or `ack_recovery_error` when that attempt is refused or unavailable; the ordinar send retains its existing storage/recipient receipt distinction. This adds owner receipt recovery from the selected original ACK source. ACK -source preparation, receipt return, cold-mailbox reception and ACK replica -recovery still require their Python workflows. It does not automatically select +source preparation, cold-mailbox reception and ACK replica recovery still require +their Python workflows; native receipt return is described below. It does not automatically select nodes, grant access, import memory or run a Python subprocess. + + +## Native saved receipt return (development after alpha.0.34) + +The native TypeScript Agent now accepts the existing `connect/return_receipt` +invitation shown above. The recipient uses the same private exported preparation +and `receipt` or `receipt-index` profile. It reads its protected delivery inbox, +requires an actually validated and saved message or memory transfer, and checks +the exact sender, recipient, envelope and source binding before returning the +original receipt. Receiving alone does not grant this disclosure permission. + +The recipient signs the existing bounded return consent and persists it before +private upload. The exact upload journal is durable before transmission. A +restart, or switching between Python and native clients, reuses those originals +and the same upload carrier while its original use remains valid. An expired +use needs independent reconciliation. A completed retry reauthenticates stored +history without contacting the network and sets `from_local_history: true` and +`network_accessed: false`; this is not a fresh source availability claim. + +Both clients share the existing SQLite journal, limited to sixteen saved ACK +records and one MiB per record. Conflicting retry inputs, changed inbox data, +failed journal persistence and insufficient original permissions stop the return. +The original delivery node may be offline: the sender can independently recover +this exact receipt from the authorized ACK source and match its original send. +This is the explicit original-source return path; automatic native mailbox and +replica orchestration remains unfinished. diff --git a/docs/OPEN_ACK_RECOVERY.md b/docs/OPEN_ACK_RECOVERY.md index b0fa7b9..ebc913c 100644 --- a/docs/OPEN_ACK_RECOVERY.md +++ b/docs/OPEN_ACK_RECOVERY.md @@ -1065,6 +1065,14 @@ private receipt upload and the verified `response` afterward. The callback is awaited; failure to store the request stops the upload. Journal data contains private original records and belongs in the participant's private durable store. The journal format is compatible with the existing Python receipt recovery -client. Native prepared-return reuse, native journal replay and automatic Agent -integration remain subsequent work; Python remains the complete automated -receipt-return path. +client. `prepareReturn(baseUrl, options)` saves one preflight for the next +matching `put` on that client. It retains the original work meter and deadline; +changed destinations, newer revocations and elapsed permissions stop upload. +`resume(baseUrl, journalRaw, receiptEntry, disclosureEntry, putEntry, options)` +reauthenticates the retained originals and replays the exact saved carrier. It +never creates another probe, consent or use. An expired use reports +`repair_reconciliation_required`; it does not silently renew authority. + +The native Agent now connects this writer to its actual saved inbox through +`connect/return_receipt`. See [saved receipt return](OPEN_ACK_PROVISIONING.md#native-saved-receipt-return-development-after-alpha034). +Preparation and complete mailbox/replica orchestration still use Python. diff --git a/memory_vault.py b/memory_vault.py index cc1b0bf..0c1e1f5 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.34" +VERSION = "0.28.0-alpha.0.35" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 33d7691..1bc2989 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 — authorized full client +# Memory Vault v0.28.0-alpha.0.35 — authorized full client + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them @@ -125,7 +136,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.34 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.35 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 0a81004..15e70c4 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 — independent protocol +# Memory Vault v0.28.0-alpha.0.35 — independent protocol + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them @@ -142,7 +153,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.34; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.35; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index 91bb699..f35a776 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 — directory republication and routed receipt recovery +# Memory Vault v0.28.0-alpha.0.35 — native saved receipt return and restart recovery + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them @@ -151,7 +162,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.34/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.34/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 5c3eff4..6648812 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 independent review kit +# Memory Vault v0.28.0-alpha.0.35 independent review kit + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 93f2dca..268b2c4 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -145,7 +145,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 874dfa7..089dde7 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -243,7 +243,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 30bca3f..a2c633b 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -47,7 +47,7 @@ "repair_http", "repair_client", "repair_client_typescript", "repair_admin", "repair_admin_typescript", "repair_runtime_review", "repair_bound", "repair_bound_typescript", "repair_empty", "repair_empty_typescript", "repair_empty_review", "repair_empty_access", "repair_empty_http", "repair_empty_client", "repair_empty_client_typescript", - "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_client_typescript", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( + "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_client_typescript", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_receipt_typescript", "repair_occupied_client_typescript")) + ( "tests.test_open_repair_stage", "tests.test_open_repair_index", "tests.test_open_repair_index_access", "tests.test_open_repair_index_journal", "tests.test_open_repair_index_state", "tests.test_open_repair_index_http", "tests.test_open_repair_index_client", "tests.test_open_repair_index_recovery", "tests.test_open_repair_index_admin", @@ -245,7 +245,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-capacity.ts", - "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", + "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", "agent.ts", "peer.ts", "io.ts", "crypto.ts", "nodes.ts")] diff --git a/tests/test_open_repair_receipt_typescript.py b/tests/test_open_repair_receipt_typescript.py new file mode 100644 index 0000000..e8929fb --- /dev/null +++ b/tests/test_open_repair_receipt_typescript.py @@ -0,0 +1,93 @@ +"""Real saved inbox to native durable ACK return, with the delivery node offline.""" +import json +import subprocess +import unittest +from unittest.mock import patch +from tests import test_open_repair_receipt as saved_py +from tests import test_open_ack_typescript_agent as agent_ts + +DRIVER=agent_ts.DRIVER.replace("const agent=new Agent", "Date.now=()=>2_000_000_008_000;\nconst agent=new Agent") +DRIVER=DRIVER.replace("if(name==='requestNode'", """if(name==='requestRepair'&&input.corruptReply&&JSON.parse(Buffer.from(args[1]).toString()).kind==='ack.put_request'){ + const body=JSON.parse(Buffer.from(result).toString()); + if(body.kind==='ack.put_response'){body.commit.ref.raw_sha256='0'.repeat(64);return Buffer.from(JSON.stringify(body));} + } + if(name==='requestNode'""") + +class NativeSavedReceiptTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + agent_ts.NativeAckAgentTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.case=saved_py.SavedReceiptPublicationTests();self.addCleanup(self.case.doCleanups);self.case.setUp() + self.case.send(memory=True);self.case.save_without_old_receipt_upload() + def encode(e):return dict(raw=e['raw'].decode(),ref=e['ref']) + self.request={n:encode(v) if n.endswith('_entry') else [encode(e) for e in v] if n=='current_statuses' else v for n,v in self.case.request.items()} + self.invitation=dict(schema_version='memory-vault-open-ack-connect/v1',action='return_receipt',repair_profile='receipt',base_url=self.case.ack.http.base,request=self.request) + + def native(self,*,no_network=False,corrupt=False,agent=None,invitation=None): + agent=agent or self.case.delivery.b + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config), + requests=[dict(op='connect',invitation=invitation or self.invitation)],no_network=no_network,corruptReply=corrupt)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=45) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-3000:]) + output=json.loads(result.stdout);self.assertEqual(output['subprocessCalls'],0) + if no_network:self.assertEqual(output['calls'],[]) + return output + + def test_native_saved_memory_return_and_shared_completed_history(self): + result=self.native();returned=result['results'][0];self.assertTrue(returned['ok'],returned) + self.assertEqual(returned['result']['state'],'retained_at_ack_source');self.assertFalse(returned['result']['from_local_history']) + again=self.native(no_network=True)['results'][0];self.assertTrue(again['ok'],again) + self.assertTrue(again['result']['from_local_history']);self.assertEqual(again['result']['commit_ref'],returned['result']['commit_ref']) + publisher=self.case.publisher() + with patch.object(publisher.client,'put',side_effect=AssertionError('completed history accessed network')): + result=publisher.publish_saved(self.case.ack.http.base,self.case.request) + self.assertTrue(result.from_local_history);self.assertEqual(result.source.commit.ref.as_dict(),returned['result']['commit_ref']) + self.assertEqual(self.case.delivery.host.processes,{}) + self.case.ack.http.restart();f=self.case.ack.f + def encode(e):return dict(raw=e['raw'].decode(),ref=e['ref']) + request=dict(target_node_entry=encode(f['entries']['descriptor']),expected_target=f['expected']['expected_target'], + expected_ack_slot=f['expected']['expected_ack_slot'],root_entry=encode(f['entries']['root']),read_entry=encode(f['entries']['read']), + bootstrap_entry=encode(f['entries']['bootstrap']),expected_receipt_writer=self.case.ack.expected['expected_receipt_writer'], + expected_message_id=self.case.message_id,expected_envelope_ref=self.case.envelope_ref) + invitation=dict(schema_version=self.invitation['schema_version'],action='recover_receipt',repair_profile='receipt',base_url=self.case.ack.http.base,request=request) + recovered=self.native(agent=self.case.delivery.a,invitation=invitation)['results'][0] + self.assertTrue(recovered['ok'],recovered);self.assertTrue(recovered['result']['endpoint_validated']) + self.assertFalse(recovered['result']['acknowledgement_pending']) + self.assertEqual(recovered['result']['commit_ref'],returned['result']['commit_ref']) + + def test_native_resumes_pending_python_return(self): + publisher=self.case.publisher();exchange=publisher.client.transport.request_repair + def lose_response(base,raw,**options): + result=exchange(base,raw,**options) + if json.loads(raw).get('kind')=='ack.put_request':raise RuntimeError('synthetic lost committed response') + return result + with patch.object(publisher.client.transport,'request_repair',side_effect=lose_response),self.assertRaisesRegex(RuntimeError,'synthetic lost'): + publisher.publish_saved(self.case.ack.http.base,self.case.request) + self.case.ack.http.restart() + result=self.native();returned=result['results'][0];self.assertTrue(returned['ok'],returned) + self.assertEqual(len(result['calls']),1) + self.assertEqual(self.case.ack.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + def test_unsaved_inbox_and_changed_envelope_refuse_before_network(self): + self.request['envelope_ref']=dict(self.request['envelope_ref'],raw_sha256='0'*64) + result=self.native(no_network=True)['results'][0];self.assertFalse(result['ok']) + self.assertEqual(result['error']['code'],'repair_saved_tuple_mismatch') + self.request['envelope_ref']=self.case.request['envelope_ref'] + with self.case.delivery.b._network() as network: + with network.participant.state.db() as db:db.execute("UPDATE open_delivery_inbox SET phase='staged'") + result=self.native(no_network=True)['results'][0];self.assertFalse(result['ok']) + self.assertEqual(result['error']['code'],'repair_receipt_not_saved') + + def test_native_restarts_and_replays_pending_put_without_new_consent(self): + result=self.native(corrupt=True)['results'][0];self.assertFalse(result['ok']);self.assertEqual(result['error']['code'],'repair_ref_mismatch') + self.case.ack.http.restart() + result=self.native();returned=result['results'][0];self.assertTrue(returned['ok'],returned) + self.assertEqual(len(result['calls']),1) + self.assertEqual(self.case.ack.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + self.assertTrue(self.native(no_network=True)['results'][0]['result']['from_local_history']) + +if __name__=='__main__':unittest.main() From 2479cf16498752c6e3dc1b563c4beaa4e29cc9f8 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:13:44 +0800 Subject: [PATCH 06/16] Align alpha35 plugin version and include native writer review fixtures --- plugins/memory-vault-client/.codex-plugin/plugin.json | 2 +- scripts/build_release.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index 1006795..6f856be 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.34", + "version": "0.28.0-alpha.0.35", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" diff --git a/scripts/build_release.py b/scripts/build_release.py index 089dde7..298113c 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -110,6 +110,7 @@ "tests/test_open_repair_offer_access.py", "tests/test_open_repair_offer_http.py", "tests/test_open_repair_offer_client.py", + "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_offer_http_typescript.py", "tests/test_open_repair_occupied.py", "tests/test_open_repair_occupied_typescript.py", @@ -117,9 +118,11 @@ "tests/test_open_repair_occupied_client.py", "tests/test_open_repair_put_http.py", "tests/test_open_repair_put_client.py", + "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_put_recovery.py", "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", + "tests/test_open_repair_receipt_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", From 751978ab3d6461c71f9263387cbdb94babdfb7bb Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:29:29 +0800 Subject: [PATCH 07/16] Authenticate native mailbox owner controls and persisted resource activation --- .../network/open-repair-mailbox-authority.ts | 191 ++++++++++++++++++ ...pen_repair_mailbox_authority_typescript.py | 118 +++++++++++ 2 files changed, 309 insertions(+) create mode 100644 clients/typescript/network/open-repair-mailbox-authority.ts create mode 100644 tests/test_open_repair_mailbox_authority_typescript.py diff --git a/clients/typescript/network/open-repair-mailbox-authority.ts b/clients/typescript/network/open-repair-mailbox-authority.ts new file mode 100644 index 0000000..ac1e013 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-authority.ts @@ -0,0 +1,191 @@ +/** Authenticate retained mailbox owner controls and initial node history. + * Resource-offer signatures, live status and source custody are separate checks. + * No authority is created and no network or local storage is accessed here. */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,RawOriginal} from './open-repair-wire.ts'; +import type {AuthenticatedRepairOriginal} from './open-repair-resource.ts'; +import {originalPublicDescriptor,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {knownHistoricalRoles} from './open-repair-history.ts'; +import {emptyMailboxState} from './open-repair-mailbox-range.ts'; +type Obj=Record; +const COMMON=['schema_version','kind','signing_key']; +const FIELDS={ + maintenance:'issued_at expires_at root_key authority_id slot_key sender recipient maintainers operation_mask allowed_roles budget windows max_delegate_depth max_destinations_per_job max_concurrent_jobs revision', + read:'issued_at expires_at grant_id root_key slot_key reader serving_authority_id operation_mask budget windows revision', + slot:'issued_at expires_at revision slot_key feed_ref sender recipient data_resource_ref data_resource_offer_ref metadata_resource_ref metadata_resource_offer_ref read_grant_ref maintenance_root_ref max_appends max_live_items budget windows', + bootstrap:'issued_at expires_at grant_id revision owner subject root_key consumer selector parent_authority_ref caller_authority_ref probe_until proof_until upload_until probe_profile response_profile upload_roles limits', + activation:'issued_at expires_at activation_id subject target_node_key_id target_storage_epoch root_key scope resource_offer_refs authority_refs', +}; +const KINDS={maintenance:'mailbox.maintenance_root',read:'mailbox.read_grant',slot:'mailbox.slot',bootstrap:'bootstrap.grant',activation:'resource.activation'}; +const BUDGET=['max_live_bytes','max_meta_bytes','max_items','max_requests','max_pending','max_replay_records','max_jobs','max_job_bytes']; +const WINDOWS=['admit_until','read_until','copy_until','publish_until','retain_until']; +const PARENT_CAPS:Record={max_probe_bytes:['max_meta_bytes','max_job_bytes'],max_proof_bytes:['max_meta_bytes','max_job_bytes'], + max_proof_items:['max_items'],max_signature_checks:['max_requests'],max_requests:['max_requests'],max_pending:['max_pending'], + max_replay_records:['max_replay_records'],max_concurrent_handles:['max_pending','max_jobs'],max_candidate_attempts:['max_requests','max_jobs']}; +const ROLES=new Set([...knownHistoricalRoles,'bootstrap.grant','mailbox_root_service_v1','selected_slot_service_v1','ack_owner_service_v1','ack_offer_service_v1']); +const UPLOAD=new Set(['mailbox.slot','mailbox.read_grant','mailbox.maintenance_root','bootstrap.grant']); +const CONTROL_NAMES=['maintenance','read','slot','bootstrap'] as const; +export interface MailboxAuthorityOptions{ + readonly expectedSlot:unknown;readonly expectedOwner:unknown;readonly expectedTarget:unknown;readonly targetStorageEpoch:string; + readonly limitPolicy:unknown;readonly at:number;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +export type AuthenticatedMailboxControls=Readonly>; +function fail(code='repair_mailbox_activation_mismatch'):never{throw new RepairError(code);} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function same(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(n=>Object.hasOwn(b,n)&&same((a as Obj)[n],(b as Obj)[n])); +} +function pattern(value:unknown,re:RegExp):string{if(typeof value!=='string'||re.exec(value)?.[0]!==value)fail();return value;} +const opaque=(v:unknown)=>pattern(v,/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/); +function dual(value:unknown):Obj{const v=fields(value,['signing_key_id','encryption_key_id']);pattern(v.signing_key_id,/^ed25519_[0-9a-f]{64}$/);pattern(v.encryption_key_id,/^x25519_[0-9a-f]{64}$/);return v;} +function dualKey(value:unknown,budget:RepairBudget):Obj{const v=fields(value,['signing_key','encryption_key']);originalPublicDescriptor(v.signing_key,budget);originalPublicDescriptor(v.encryption_key,budget,true);return {signing_key_id:v.signing_key.key_id,encryption_key_id:v.encryption_key.key_id};} +function limits(value:unknown):Obj{const v=fields(value,Object.keys(PARENT_CAPS));for(const n of Object.keys(PARENT_CAPS))u53(v[n],1);return v;} +function ordered(value:unknown,get:(v:any)=>string,check:(v:any)=>unknown):void{ + if(!Array.isArray(value))fail();let prior:string|undefined;for(const item of value){check(item);const next=get(item);if(prior!==undefined&&next<=prior)fail();prior=next;} +} +function roles(value:unknown):void{ordered(value,v=>v,v=>{if(typeof v!=='string'||!ROLES.has(v))fail();});} +function original(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const v=fields(value,['raw','ref']),ref=rawRef(v.ref),raw=parseNewWire(v.raw,policy,budget).raw; + if(ref.namespace!=='meta'||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; +} +function authenticate(value:unknown,name:keyof typeof FIELDS,expected:Obj,policy:RepairPolicy,budget:RepairBudget):AuthenticatedRepairOriginal{ + const held=original(value,policy,budget),signed=fields(parseNewWire(held.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,...FIELDS[name].split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==KINDS[name]||u53(p.issued_at)>=u53(p.expires_at))fail(); + if(!(p.issued_at<=expected.at&&expected.at|undefined,policy:RepairPolicy,budget:RepairBudget):void{ + const key=e.slot,root=key.root_key,owner=dualKey(e.owner,budget),target=dualKey(e.target,budget),now=e.at; + const p=Object.fromEntries(Object.entries(held).map(([n,v])=>[n,(v as AuthenticatedRepairOriginal).payload])); + const {maintenance:m,read:r,slot:s,bootstrap:g,activation:a}=p; + if(!same(root.owner,owner)||!same(key.writer,target)||key.writer_storage_epoch!==e.epoch)fail(); + for(const name of ['maintenance','read','slot']){ + const v=p[name];emptyMailboxState(v.slot_key,policy,budget);u53(v.revision); + fields(v.budget,BUDGET);fields(v.windows,WINDOWS);for(const n of BUDGET)u53(v.budget[n]); + for(const n of WINDOWS)if(!(v.issued_at127)fail(); + for(const v of [m,s]){dual(v.sender);if(!same(dual(v.recipient),owner))fail();} + dual(r.reader);for(const v of [m.authority_id,r.grant_id,r.serving_authority_id])opaque(v); + if(!same(r.reader,owner)||!same(s.sender,m.sender)||r.serving_authority_id!==m.authority_id||!same(s.read_grant_ref,held.read.ref)||!same(s.maintenance_root_ref,held.maintenance.ref))fail(); + fields(s.feed_ref,['namespace','key']);if(s.feed_ref.namespace!=='feed')fail();pattern(s.feed_ref.key,/^[0-9a-f]{64}$/); + if(!(1<=u53(s.max_live_items)&&s.max_live_items<=u53(s.max_appends,1)&&s.max_appends<=65536&&s.max_live_items<=s.budget.max_items)||u53(m.max_delegate_depth)!==2)fail(); + u53(m.max_destinations_per_job);u53(m.max_concurrent_jobs);ordered(m.maintainers,v=>v.signing_key_id+':'+v.encryption_key_id,dual);roles(m.allowed_roles); + if((m.operation_mask&10)!==10||(r.operation_mask&2)!==2||(r.operation_mask&~m.operation_mask)!==0|| + !['bootstrap.grant','selected_slot_service_v1'].every(v=>m.allowed_roles.includes(v))||BUDGET.some(n=>r.budget[n]>m.budget[n])||WINDOWS.some(n=>r.windows[n]>m.windows[n])||r.expires_at>m.expires_at)fail(); + if(offers)for(const [name,purpose] of [['data','mailbox_data'],['metadata','feed_metadata']]){ + const offer=offers[name],v=(parseNewWire(offer.raw,policy,budget).value as Obj).payload,intent=v.intent; + if(!same(s[name+'_resource_ref'],v.resource)||!same(s[name+'_resource_offer_ref'],offer.ref)||intent.purpose!==purpose||!same(intent.root_key,root)|| + !same(intent.owner,e.owner)||!same(intent.target,e.target)||v.issued_at>Math.min(m.issued_at,r.issued_at,s.issued_at)||Math.min(...Object.values(intent.windows) as number[])<=now)fail(); + } + fields(g.selector,['root_key_sha256','slot_key_sha256','feed_ref','slot_sha256','read_grant_sha256','maintenance_root_sha256']);opaque(g.grant_id);u53(g.revision); + const selector={root_key_sha256:budget.hash(buildNewWire(root,policy,budget).raw),slot_key_sha256:budget.hash(buildNewWire(key,policy,budget).raw),feed_ref:s.feed_ref, + slot_sha256:held.slot.ref.raw_sha256,read_grant_sha256:held.read.ref.raw_sha256,maintenance_root_sha256:held.maintenance.ref.raw_sha256}; + if(!same(g.owner,owner)||!same(g.subject,owner)||!same(g.root_key,root)||g.consumer!=='mailbox_feed'||g.probe_profile!=='opaque_v1'||g.response_profile!=='selected_slot_service_v1'|| + !same(g.parent_authority_ref,held.maintenance.ref)||!same(g.caller_authority_ref,held.read.ref)||!same(g.selector,selector))fail(); + roles(g.upload_roles);if(g.upload_roles.some((v:string)=>!UPLOAD.has(v)||!m.allowed_roles.includes(v)))fail();limits(g.limits); + for(const [n,parents] of Object.entries(PARENT_CAPS))if(g.limits[n]>e.limits[n]||[m,r].some(v=>parents.some(k=>g.limits[n]>v.budget[k])))fail(); + const activated=offers?a.issued_at:now; + if(!(m.issued_at<=r.issued_at&&r.issued_at<=s.issued_at&&s.issued_at<=g.issued_at&&g.issued_at<=activated&&activated<=now&&g.expires_at<=Math.min(m.expires_at,r.expires_at)))fail(); + for(const n of ['probe_until','proof_until','upload_until'])if(!(now[m,r].some(v=>['read_until','retain_until'].some(w=>g[n]>v.windows[w]))))fail(); + if(!offers)return; + opaque(a.activation_id);const compare=(x:Obj,y:Obj)=>{for(const n of ['namespace','key','raw_sha256','size']){if(x[n]y[n])return 1;}return 0;}; + const refs=Object.values(offers).map(v=>v.ref).sort(compare),authorities=['maintenance','read','slot'].map(n=>({role:KINDS[n as keyof typeof KINDS],ref:held[n].ref})); + if(!same(a.subject,e.owner)||a.target_node_key_id!==e.target.signing_key.key_id||a.target_storage_epoch!==key.writer_storage_epoch||!same(a.root_key,root)|| + !same(a.scope,{kind:'mailbox_slot',slot_key:key,slot_ref:held.slot.ref})||!same(a.resource_offer_refs,refs)||!same(a.authority_refs,authorities))fail(); +} +export function verifyMailboxFeedBootstrap(entries:unknown,options:MailboxAuthorityOptions):AuthenticatedMailboxControls{ + const {expected:e,policy,budget}=context(options),v=fields(entries,CONTROL_NAMES),held:Obj={}; + for(const name of CONTROL_NAMES)held[name]=authenticate(v[name],name,e,policy,budget); + chain(held,e,undefined,policy,budget);return Object.freeze(held) as AuthenticatedMailboxControls; +} +export function verifyMailboxSlotOwnerInputs(entries:unknown,offers:unknown,options:MailboxAuthorityOptions):Readonly>{ + const {expected:e,policy,budget}=context(options),v=fields(entries,Object.keys(FIELDS)),held:Obj={},supplied=fields(offers,['data','metadata']); + for(const name of Object.keys(FIELDS) as (keyof typeof FIELDS)[])held[name]=authenticate(v[name],name,e,policy,budget); + const frozen={data:original(supplied.data,policy,budget),metadata:original(supplied.metadata,policy,budget)}; + chain(held,e,frozen,policy,budget);return Object.freeze(held) as Readonly>; +} +export function verifyMailboxGenesis(entries:unknown,options:{expectedSlot:unknown;expectedSigningKey:unknown;committedAt:number;at:number;policy:RepairPolicy;budget:RepairBudget}):Readonly>{ + const a=fields(options,['expectedSlot','expectedSigningKey','committedAt','at','policy','budget']),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,committed:a.committedAt,at:a.at},policy,budget).value as Obj; + u53(e.committed);u53(e.at);const initial=emptyMailboxState(e.slot,policy,budget),v=fields(entries,['checkpoint','head']),held:Obj={}; + for(const name of ['checkpoint','head']){ + const item=original(v[name],policy,budget),signed=fields(parseNewWire(item.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,'slot_key','committed_at','retain_until',...(name==='checkpoint'?Object.keys(initial):['checkpoint_ref','count','range_root_ref','catalog_generation'])]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==(name==='checkpoint'?'mailbox.checkpoint':'mailbox.feed_head')||!same(p.slot_key,e.slot)||p.committed_at!==e.committed||!(u53(p.committed_at)<=e.at&&e.at!same(p[k],value)))fail();} + else if(!same(p.checkpoint_ref,held.checkpoint.ref)||u53(p.count)!==0||p.range_root_ref!==null||u53(p.catalog_generation)!==0||p.retain_until!==held.checkpoint.payload.retain_until)fail(); + held[name]=Object.freeze({ref:item.ref,payload:p,get raw(){budget.output(item.raw.length);return Uint8Array.from(item.raw);}}); + } + return Object.freeze(held) as Readonly>; +} + +export interface MailboxResourceOptions{ + readonly expectedRoot:unknown;readonly expectedOwner:unknown;readonly expectedTarget:unknown;readonly targetStorageEpoch:string; + readonly expectedPurpose:string;readonly expectedScope:unknown;readonly expectedAuthorityRefs:unknown;readonly expectedOfferRefs:unknown; + readonly at:number;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +/** Scope and authority refs must come from the independently checked owner chain. */ +export function verifyMailboxResourceInputs(entries:unknown,options:MailboxResourceOptions):Readonly>{ + const a=fields(options,['expectedRoot','expectedOwner','expectedTarget','targetStorageEpoch','expectedPurpose','expectedScope','expectedAuthorityRefs','expectedOfferRefs','at','policy','budget']); + const {policy,budget}=a,e=buildNewWire({root:a.expectedRoot,owner:a.expectedOwner,target:a.expectedTarget,epoch:a.targetStorageEpoch,purpose:a.expectedPurpose, + scope:a.expectedScope,authorities:a.expectedAuthorityRefs,offers:a.expectedOfferRefs,at:a.at},policy,budget).value as Obj; + const mismatch=()=>fail('repair_resource_mismatch'); + const root=(v:unknown):Obj=>{const r=fields(v,['owner','root_kind','anchor_ref','owner_epoch','root_id']);dual(r.owner); + const anchor=fields(r.anchor_ref,['namespace','key']);if(r.root_kind!=='mailbox'||anchor.namespace!=='anchor')mismatch(); + pattern(anchor.key,/^[0-9a-f]{64}$/);opaque(r.owner_epoch);opaque(r.root_id);return r;}; + root(e.root);opaque(e.epoch);u53(e.at);const owner=dualKey(e.owner,budget),target=dualKey(e.target,budget); + if(!same(e.root.owner,owner)||!['anchor_catalog','feed_metadata','mailbox_data'].includes(e.purpose))mismatch(); + const shapes={allocate:'issued_at expires_at request_id target_node_key_id target_storage_epoch intent intent_sha256', + offer:'issued_at reservation_until offer_id allocation_request_ref intent intent_sha256 resource target_encryption_key reservation_generation budget windows', + activation:FIELDS.activation,active:'activated_at offer_ref activation_ref resource reservation_generation root_key purpose budget windows'}; + const v=fields(entries,Object.keys(shapes)),checked:Obj={}; + for(const name of Object.keys(shapes) as (keyof typeof shapes)[]){ + const item=original(v[name],policy,budget),signed=fields(parseNewWire(item.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,...shapes[name].split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!=='resource.'+name)fail('repair_invalid_resource'); + verifyBoundedControlSignature(p,signed.proof,(name==='offer'||name==='active'?e.target:e.owner).signing_key,budget); + checked[name]=Object.freeze({ref:item.ref,payload:p,get raw(){budget.output(item.raw.length);return Uint8Array.from(item.raw);}}); + } + const [allocate,offer,activation,active]=['allocate','offer','activation','active'].map(n=>checked[n].payload); + for(const p of [allocate,activation]){ + if(u53(p.issued_at)>=u53(p.expires_at))fail('repair_invalid_resource'); + if(p.target_node_key_id!==target.signing_key_id||p.target_storage_epoch!==e.epoch)mismatch(); + } + opaque(allocate.request_id);opaque(activation.activation_id);opaque(offer.offer_id); + const amounts=(v:unknown)=>{const p=fields(v,BUDGET);for(const n of BUDGET)u53(p[n]);}; + const windows=(v:unknown,issued?:number)=>{const p=fields(v,WINDOWS);for(const n of WINDOWS){u53(p[n]);if(p[n]>p.retain_until||(issued!==undefined&&p[n]<=issued))fail('repair_invalid_resource');}}; + for(const p of [allocate,offer]){ + const intent=fields(p.intent,['kind','allocation_id','root_key','owner','target','target_storage_epoch','purpose','budget','windows']); + if(intent.kind!=='resource.owner_intent'||intent.purpose!==e.purpose)fail('repair_invalid_resource'); + opaque(intent.allocation_id);opaque(intent.target_storage_epoch);root(intent.root_key);dualKey(intent.owner,budget);dualKey(intent.target,budget);amounts(intent.budget);windows(intent.windows); + if(p.intent_sha256!==budget.hash(buildNewWire(intent,policy,budget).raw))mismatch(); + } + const intent=allocate.intent; + if(!same(offer.intent,intent)||!same(intent.root_key,e.root)||!same(intent.owner,e.owner)||!same(intent.target,e.target)||intent.target_storage_epoch!==e.epoch||intent.purpose!==e.purpose)mismatch(); + windows(intent.windows,allocate.issued_at); + const resource=fields(offer.resource,['node_key_id','storage_epoch','lease_id','resource_id']);pattern(resource.node_key_id,/^ed25519_[0-9a-f]{64}$/); + for(const n of ['storage_epoch','lease_id','resource_id'])opaque(resource[n]); + if(resource.node_key_id!==target.signing_key_id||resource.storage_epoch!==e.epoch||!same(offer.target_encryption_key,e.target.encryption_key)|| + !same(rawRef(offer.allocation_request_ref),checked.allocate.ref)||!same(rawRef(active.offer_ref),checked.offer.ref)||!same(rawRef(active.activation_ref),checked.activation.ref))mismatch(); + if(!Array.isArray(e.offers)||!same(activation.subject,e.owner)||!same(activation.root_key,e.root)||!same(activation.scope,e.scope)|| + !same(activation.authority_refs,e.authorities)||!same(activation.resource_offer_refs,e.offers)||!e.offers.some((ref:unknown)=>same(ref,checked.offer.ref))||!same(active.root_key,e.root)||active.purpose!==e.purpose)mismatch(); + amounts(offer.budget);windows(offer.windows);u53(offer.reservation_generation,1); + if(['budget','windows'].some(n=>!same(offer[n],intent[n]))||['resource','reservation_generation','budget','windows'].some(n=>!same(active[n],offer[n])))mismatch(); + const issued=u53(offer.issued_at),reserved=u53(offer.reservation_until),activated=u53(active.activated_at); + if(!(allocate.issued_at<=issued&&issued>; +} diff --git a/tests/test_open_repair_mailbox_authority_typescript.py b/tests/test_open_repair_mailbox_authority_typescript.py new file mode 100644 index 0000000..1219836 --- /dev/null +++ b/tests/test_open_repair_mailbox_authority_typescript.py @@ -0,0 +1,118 @@ +"""Original Python mailbox owner controls consumed by the native reader.""" +import copy +import json +import subprocess +import unittest +from dataclasses import asdict +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_repair_mailbox_activation as activation +from tests.test_open_repair_probe_typescript import encoded +from tests.open_repair_ack_fixtures import signed_entry +from memory_vault_open_repair_mailbox_activation import verify_mailbox_feed_bootstrap,verify_mailbox_slot_owner_inputs,verify_mailbox_genesis +from memory_vault_open_repair_wire import RepairBudget,RepairWireError + +DRIVER=runtime.DRIVER.split('const {Agent}',1)[0]+r''' +const {RepairBudget}=await import('./open-repair-wire.ts'); +const api=await import('./open-repair-mailbox-authority.ts'); +const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk); +const v=JSON.parse(Buffer.concat(chunks).toString()),decode=e=>({raw:Buffer.from(e.raw,'base64'),ref:e.ref}), + entries=Object.fromEntries(Object.entries(v.entries).map(([k,e])=>[k,decode(e)])),budget=new RepairBudget(v.policy),options={...v.options,policy:v.policy,budget}; +try{ + const result=v.operation==='resource'?api.verifyMailboxResourceInputs(entries,options):v.operation==='genesis'?api.verifyMailboxGenesis(entries,options):v.operation==='bootstrap'?api.verifyMailboxFeedBootstrap(entries,options): + api.verifyMailboxSlotOwnerInputs(entries,Object.fromEntries(Object.entries(v.offers).map(([k,e])=>[k,decode(e)])),options); + const first=Object.values(result)[0],before=Buffer.from(first.raw).toString('hex');first.raw.fill(0); + process.stdout.write(JSON.stringify({ok:true,refs:Object.fromEntries(Object.entries(result).map(([k,e])=>[k,e.ref])), + immutable:before===Buffer.from(first.raw).toString('hex'),metrics:budget.snapshot(),subprocessCalls})); +}catch(e){process.stdout.write(JSON.stringify({ok:false,code:e.code??'untyped_error',detail:e.code?undefined:String(e.stack),metrics:budget.snapshot(),subprocessCalls}));} +''' + +class NativeMailboxAuthorityTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.case=activation.MailboxActivationTests();self.case.setUp();self.addCleanup(self.case.doCleanups) + self.entries=self.case.setup_entries();self.policy=self.case.source.policy + self.options=dict(expectedSlot=self.case.slot_key,expectedOwner=self.case.owner,expectedTarget=self.case.source.target, + targetStorageEpoch=self.case.slot_key['writer_storage_epoch'],limitPolicy=self.case.source.limits,at=self.case.now) + self.offers={n:self.case.offers[p] for n,p in [('data','mailbox_data'),('metadata','feed_metadata')]} + + def native(self,entries=None,*,operation='owner',options=None,policy=None): + selected=entries or self.entries + if operation=='bootstrap':selected={k:v for k,v in selected.items() if k!='activation'} + value=dict(operation=operation,entries={k:encoded(v) for k,v in selected.items()},offers={k:encoded(v) for k,v in self.offers.items()}, + options=options or self.options,policy=policy or asdict(self.policy)) + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(value).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=25) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout) + self.assertEqual(out['subprocessCalls'],0);return out + + def python(self,entries,*,bootstrap=False,at=None): + c=self.case;args=dict(expected_slot=c.slot_key,expected_owner=c.owner,expected_target=c.source.target, + target_storage_epoch=c.slot_key['writer_storage_epoch'],limit_policy=c.source.limits,at=at or c.now, + policy=self.policy,budget=RepairBudget(self.policy)) + if bootstrap:return verify_mailbox_feed_bootstrap({k:v for k,v in entries.items() if k!='activation'},**args) + return verify_mailbox_slot_owner_inputs(entries,offers=self.offers,**args) + + def test_exact_owner_chain_and_bootstrap_keep_original_refs(self): + for operation in ['owner','bootstrap']: + native=self.native(operation=operation);self.assertTrue(native['ok'],native);self.assertTrue(native['immutable']) + checked=self.python(self.entries,bootstrap=operation=='bootstrap') + self.assertEqual(native['refs'],{k:v.ref.as_dict() for k,v in checked.items()}) + self.assertEqual(native['metrics']['signature_checks'],5 if operation=='owner' else 4) + + def test_owner_signed_wrong_reader_is_not_authorized(self): + entries=self.case.setup_entries(dict(read=dict(reader=self.case.dual(self.case.source.target)))) + for bootstrap in [False,True]: + with self.assertRaises(RepairWireError):self.python(entries,bootstrap=bootstrap) + result=self.native(entries,operation='bootstrap' if bootstrap else 'owner');self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + + def test_changed_scope_or_budget_never_repairs_signed_parent_chain(self): + for name,field,value in [('activation','target_storage_epoch','changed_epoch'),('slot','max_live_items',65537),('bootstrap','proof_until',self.case.now), + ('maintenance','allowed_roles',['bootstrap.grant','bootstrap.grant'])]: + with self.subTest(name=name): + entries=copy.deepcopy(self.entries);p=json.loads(entries[name]['raw'])['payload'];p[field]=value + entries[name]=signed_entry(p,self.case.f['signers']['owner'],'synthetic_mutated_owner') + with self.assertRaises(RepairWireError):self.python(entries) + result=self.native(entries);self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + + def test_invalid_reference_expiry_and_signature_budget_refuse(self): + entries=copy.deepcopy(self.entries);entries['slot']['ref']['raw_sha256']='0'*64 + result=self.native(entries);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_ref_mismatch') + result=self.native(options=dict(self.options,at=self.case.now+600));self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_resource_expired') + result=self.native(policy=dict(asdict(self.policy),max_signature_checks=4));self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_over_budget') + + def test_actual_durable_resource_chain_matches_owner_scope_and_original_limits(self): + from memory_vault_open_repair_resource import verify_mailbox_resource_inputs + c=self.case;activated=c.activate(self.entries);p=json.loads(self.entries['activation']['raw'])['payload'] + for name,purpose in [('data','mailbox_data'),('metadata','feed_metadata')]: + row=c.db.execute('SELECT allocation,allocation_ref FROM open_repair_mailbox_resources WHERE purpose=?',(purpose,)).fetchone() + entries=dict(allocate=dict(raw=bytes(row[0]),ref=json.loads(row[1])),offer=self.offers[name],activation=self.entries['activation'],active=activated[name]) + options=dict(expectedRoot=c.root,expectedOwner=c.owner,expectedTarget=c.source.target,targetStorageEpoch=c.slot_key['writer_storage_epoch'], + expectedPurpose=purpose,expectedScope=p['scope'],expectedAuthorityRefs=p['authority_refs'],expectedOfferRefs=p['resource_offer_refs'],at=c.now) + checked=verify_mailbox_resource_inputs(entries,expected_root=c.root,expected_owner=c.owner,expected_target=c.source.target, + target_storage_epoch=c.slot_key['writer_storage_epoch'],expected_purpose=purpose,expected_scope=p['scope'],expected_authority_refs=p['authority_refs'], + expected_offer_refs=p['resource_offer_refs'],at=c.now,policy=self.policy,budget=RepairBudget(self.policy)) + result=self.native(entries,operation='resource',options=options);self.assertTrue(result['ok'],result);self.assertTrue(result['immutable']) + self.assertEqual(result['refs'],{k:v.ref.as_dict() for k,v in checked.items()}) + result=self.native(entries,operation='resource',options=dict(options,expectedPurpose='anchor_catalog')) + self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + changed=json.loads(entries['active']['raw'])['payload'];changed['budget']['max_items']+=1 + wrong=dict(entries,active=signed_entry(changed,c.f['signers']['target'],'synthetic_inflated_resource')) + result=self.native(wrong,operation='resource',options=options);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_resource_mismatch') + + def test_actual_node_genesis_requires_empty_count_and_exact_commit(self): + c=self.case;activated=c.activate(self.entries);entries={n:activated[n] for n in ('checkpoint','head')} + options=dict(expectedSlot=c.slot_key,expectedSigningKey=c.source.identity.public_descriptor(),committedAt=c.now,at=c.now) + result=self.native(entries,operation='genesis',options=options);self.assertTrue(result['ok'],result);self.assertTrue(result['immutable']) + self.assertEqual(result['refs'],{n:e['ref'] for n,e in entries.items()}) + for field,value in [('count',1),('catalog_generation',1),('range_root_ref',entries['checkpoint']['ref'])]: + p=json.loads(entries['head']['raw'])['payload'];p[field]=value + wrong=dict(entries,head=signed_entry(p,c.f['signers']['target'],'synthetic_nonempty_genesis')) + result=self.native(wrong,operation='genesis',options=options);self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + with self.assertRaises(RepairWireError):verify_mailbox_genesis(wrong,expected_slot=c.slot_key,expected_signing_key=c.source.identity.public_descriptor(), + committed_at=c.now,at=c.now,policy=self.policy,budget=RepairBudget(self.policy)) + +if __name__=='__main__':unittest.main() From 21d59cbdb41636bbb033b5f1b03b272145ef597f Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:44:47 +0800 Subject: [PATCH 08/16] Authenticate native mailbox admission and complete retained feed history --- .../network/open-repair-mailbox-feed.ts | 129 +++++++++++++++++ .../network/open-repair-mailbox-member.ts | 136 ++++++++++++++++++ ...t_open_repair_mailbox_member_typescript.py | 105 ++++++++++++++ 3 files changed, 370 insertions(+) create mode 100644 clients/typescript/network/open-repair-mailbox-feed.ts create mode 100644 clients/typescript/network/open-repair-mailbox-member.ts create mode 100644 tests/test_open_repair_mailbox_member_typescript.py diff --git a/clients/typescript/network/open-repair-mailbox-feed.ts b/clients/typescript/network/open-repair-mailbox-feed.ts new file mode 100644 index 0000000..c460f46 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-feed.ts @@ -0,0 +1,129 @@ +/** Original feed custody and complete retained prefix. This authenticates + * ciphertext layout; recipient decryption and current access remain separate. */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,LocalRawResolver,RawOriginal} from './open-repair-wire.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import {verifyBoundedControlSignature} from './open-repair-original.ts'; +import {verifyMailboxMemberInputs} from './open-repair-mailbox-member.ts'; +import {emptyMailboxState,appendMailboxState,verifyMailboxInclusion} from './open-repair-mailbox-range.ts'; +import {verifyStatusOriginal} from './open-repair-status.ts'; +import {validateJwe} from './crypto.ts'; +type Obj=Record; +const COMMON=['schema_version','kind','signing_key']; +const SLOT_INPUT='mailbox.slot mailbox.read_grant mailbox.maintenance_root bootstrap.mailbox_feed resource.data_allocate resource.metadata_allocate resource.data_offer resource.metadata_offer resource.slot_activation resource.data_active resource.metadata_active source.descriptor'.split(' '); +export interface MailboxFeedOptions{readonly expectedSlot:unknown;readonly expectedOwner:unknown;readonly expectedSender:unknown;readonly expectedTarget:unknown;readonly limitPolicy:unknown;readonly policy:RepairPolicy;readonly budget:RepairBudget;} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function fail(code='repair_mailbox_activation_mismatch'):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +const refKey=(v:unknown)=>{const r=rawRef(v);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +function parse(v:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const entry=fields(v,['raw','ref']),ref=rawRef(entry.ref),raw=parseNewWire(entry.raw,policy,budget).raw; + if(ref.namespace!=='meta'||ref.size!==raw.length||ref.raw_sha256!==budget.hash(raw))fail('repair_ref_mismatch');return {raw,ref}; +} +export function verifyMailboxFeedSourceEvent(manifestEntry:unknown,resolver:LocalRawResolver,custodyEntry:unknown,options:MailboxFeedOptions):Readonly{ + const args=fields(options,['expectedSlot','expectedOwner','expectedSender','expectedTarget','limitPolicy','policy','budget']),{policy,budget}=args; + const e=buildNewWire({slot:args.expectedSlot,owner:args.expectedOwner,sender:args.expectedSender,target:args.expectedTarget,limits:args.limitPolicy},policy,budget).value as Obj,key=e.slot; + emptyMailboxState(key,policy,budget); + const custody=parse(custodyEntry,policy,budget),signed=fields(parseNewWire(custody.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,...'root_key slot_key slot_ref feed_head_ref covered_interval subtree historical_manifest_ref resource_refs stored_at read_until retain_until'.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!=='feed.custody'||!same(p.slot_key,key)||!same(p.root_key,key.root_key))fail(); + verifyBoundedControlSignature(p,signed.proof,e.target.signing_key,budget); + const at=u53(p.stored_at);if(!(at(),wanted=new Set(); + for(const row of resolved.roles){const id=row.role+'|'+refKey(row.original.ref);if(byRef.has(id))fail();byRef.set(id,row);} + const get=(role:string,ref:unknown):RawOriginal=>{const id=role+'|'+refKey(ref),row=byRef.get(id);if(!row)fail('repair_original_missing');wanted.add(id); + const raw=row.original.raw;if(raw.length!==row.original.ref.size||budget.hash(raw)!==row.original.ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref:row.original.ref};}; + const event=(role:string,ref:unknown,kind:string,names:string):Obj=>{const entry=get(role,ref),signed=fields(parseNewWire(entry.raw,policy,budget).value,['payload','proof']); + const value=fields(signed.payload,[...COMMON,'slot_key',...names.split(' ')]); + if(value.schema_version!=='memory-vault-open-repair/v1'||value.kind!==kind||!same(value.slot_key,key))fail();verifyBoundedControlSignature(value,signed.proof,e.target.signing_key,budget);return value;}; + const HEAD='checkpoint_ref count range_root_ref catalog_generation committed_at retain_until',CHECKPOINT='slot_binding count leaf_root frontier committed_at retain_until'; + const head=event('feed.head',m.feed_head_ref,'mailbox.feed_head',HEAD),checkpoint=event('feed.checkpoint',head.checkpoint_ref,'mailbox.checkpoint',CHECKPOINT),count=u53(head.count,1); + if(!same(m.covered_interval,{start:0,end:count})||!same(m.subtree,{root_ref:head.range_root_ref,parent_path_refs:[]})||checkpoint.count!==count||head.committed_at!==checkpoint.committed_at||head.retain_until!==checkpoint.retain_until||!(u53(head.committed_at)<=at&&at{ + const entry=get(role,ref),context={schema_version:'memory-vault-open-repair/v1',kind,slot_key:key,plaintext_sha256:budget.hash(raw),plaintext_size:raw.length,...extra}; + let value;try{value=validateJwe(entry.raw,{context});}catch{fail();} + if(value.recipients.length!==1||value.recipients[0].header?.kid!==e.owner.encryption_key.key_id)fail();}; + const predecessors=new Map(resolved.predecessors.map(v=>[budget.hash(v.manifest.raw),v])),members=new Map(),setups:Obj[]=[],deadlines=[head.retain_until]; + for(const member of m.members){ + const sequence=u53(member.sequence),manifest=get('history.member',member.historical_manifest_ref),child=predecessors.get(manifest.ref.raw_sha256); + if(!child)fail();const childValue=child.manifest.value as Obj; + const core=event('member.core',member.admission_core_ref,'admission.core','core_id sequence message_id envelope_ref attempt_ref historical_manifest_ref data_resource_ref metadata_resource_ref accepted_at object_until enum_until'); + if(core.sequence!==sequence||['message_id','envelope_ref','historical_manifest_ref'].some(n=>!same(core[n],member[n]))||['message_id','envelope_ref','attempt_ref'].some(n=>!same(core[n],childValue[n]))||!(u53(core.accepted_at)<=at&&at!same(link[n],core[n])))fail(); + sealed('member.sealed_core',link.sealed_core_ref,get('member.core',member.admission_core_ref).raw,'admission.sealed_core',{sequence}); + const cp=event('member.checkpoint',link.checkpoint_ref,'mailbox.checkpoint',CHECKPOINT); + if(cp.count!==sequence+1||!(core.accepted_at<=u53(cp.committed_at)&&cp.committed_at<=head.committed_at)||u53(cp.retain_until)[n,cp[n]])),sequence,link.sealed_core_ref.raw_sha256,link.inclusion_path,key,policy,budget); + const custody=event('member.custody',member.source_custody_ref,'message.custody','root_key message_id envelope_ref admission_link_ref checkpoint_ref feed_head_ref resource_refs stored_at object_until enum_until'); + if(!same(custody.root_key,key.root_key)||!same(custody.admission_link_ref,member.admission_link_ref)||!same(custody.checkpoint_ref,link.checkpoint_ref)||custody.stored_at!==cp.committed_at||!same(custody.resource_refs,{data:core.data_resource_ref,metadata:core.metadata_resource_ref})||['message_id','envelope_ref','object_until','enum_until'].some(n=>!same(custody[n],core[n])))fail(); + const oldHead=event('member.head',custody.feed_head_ref,'mailbox.feed_head',HEAD); + if(!same(oldHead.checkpoint_ref,link.checkpoint_ref)||oldHead.count!==sequence+1||oldHead.committed_at!==cp.committed_at||oldHead.retain_until!==cp.retain_until)fail(); + if(members.has(sequence))fail();members.set(sequence,{sequence,admission_link_ref:member.admission_link_ref,sealed_core_ref:link.sealed_core_ref}); + } + if(members.size!==count||predecessors.size!==count)fail(); + let state=emptyMailboxState(key,policy,budget); + const walk=(ref:unknown,level:number,start:number,end:number):void=>{ + const role=level>=0?'range.index':'range.repair_page',entry=get(role,ref),value=fields(parseNewWire(entry.raw,policy,budget).value,['schema_version','kind','slot_key','start','end',...(level>=0?['level','children']:['entries','sealed_page_ref'])]); + if(value.schema_version!=='memory-vault-open-repair/v1'||value.kind!==role||!same(value.slot_key,key)||u53(value.start)!==start||u53(value.end)!==end)fail(); + if(level>=0){ + if(u53(value.level)!==level||!Array.isArray(value.children)||value.children.length<1||value.children.length>16)fail();let cursor=start;const span=16**(level+1); + for(const row of value.children){fields(row,['start','end','ref']);const stop=Math.min(cursor+span,end);if(u53(row.start)!==cursor||u53(row.end)!==stop||cursor>=stop)fail();walk(row.ref,level-1,cursor,stop);cursor=stop;} + if(cursor!==end)fail(); + }else{ + if(end-start<1||end-start>16||!same(value.entries,Array.from({length:end-start},(_,i)=>members.get(start+i))))fail(); + const plain=buildNewWire({schema_version:'memory-vault-open-repair/v1',kind:'range.private_page',slot_key:key,start,end,entries:value.entries},policy,budget).raw; + sealed('range.sealed_page',value.sealed_page_ref,plain,'range.sealed_page',{start,end}); + for(const row of value.entries)state=appendMailboxState(state,row.sealed_core_ref.raw_sha256,key,policy,budget); + } + }; + let level=0;while(count>16**(level+2))level++;walk(head.range_root_ref,level,0,count); + if(Object.keys(state).some(n=>!same((state as unknown as Obj)[n],checkpoint[n])))fail(); + const obligations=new Map(),allowed=new Map(); + const dutyKey=(v:Obj)=>[v.signer.key_id,v.scope_kind,v.scope_id].join(':'); + for(const setup of setups){ + for(const duty of setup.obligations){const id=dutyKey(duty);allowed.set(id,duty);if(!['destination','data_resource'].includes(duty.role))obligations.set(id,{...duty,operation_mask:duty.role==='read'?2:duty.role==='bootstrap'?10:66});} + for(const n of ['slot','read','maintenance','bootstrap']){const v=setup.originals[n].payload;deadlines.push(v.expires_at);if(v.windows)deadlines.push(v.windows.read_until,v.windows.retain_until);} + const v=setup.resources.metadata.active.payload;deadlines.push(v.windows.read_until,v.windows.retain_until); + } + const covered=new Set(),statuses:Obj[]=[],revisions=new Map(),cache=new Map(); + for(const [id,{role,original:entry}] of byRef){ + if(!role.startsWith('historical.status.'))continue; + const raw=entry.raw,preview=fields(parseNewWire(raw,policy,budget).value,['payload','proof']).payload; + if(!preview||!preview.signing_key||!Array.isArray(preview.entries))fail('repair_invalid_status');const issuer=preview.signing_key.key_id; + const present=new Set(preview.entries.map((v:Obj)=>{if(!v||typeof v.scope_kind!=='string'||typeof v.scope_id!=='string')fail('repair_invalid_status');return v.scope_kind+':'+v.scope_id;})); + const included=([...obligations] as [string,Obj][]).filter(([,v])=>v.signer.key_id===issuer&&present.has(v.scope_kind+':'+v.scope_id)),matches=included.filter(([,v])=>role==='historical.status.'+v.role); + if(!matches.length)fail(); + const context={expectedRoot:key.root_key,expectedSigningKey:matches[0][1].signer,at,allowedScopes:[...allowed.values()].filter(v=>v.signer.key_id===issuer).map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id})), + required:included.map(([,v])=>({scope_kind:v.scope_kind,scope_id:v.scope_id,document_revision:v.document_revision,operation_mask:v.operation_mask}))}; + const contextRaw=buildNewWire(context,policy,budget).raw,cacheKey=refKey(entry.ref)+':'+budget.hash(contextRaw);let observed=cache.get(cacheKey); + if(!observed){observed=verifyStatusOriginal({raw,ref:entry.ref},{...context,policy,budget});budget.retain(raw.length+contextRaw.length);cache.set(cacheKey,observed);} + const revision=issuer+':'+observed.payload.revision;if(revisions.has(revision)&&revisions.get(revision)!==observed.canonical_sha256)fail('repair_status_conflict'); + revisions.set(revision,observed.canonical_sha256);for(const [identity] of matches)covered.add(identity);statuses.push(observed);wanted.add(id); + } + if(covered.size!==obligations.size||[...obligations.keys()].some(id=>!covered.has(id)))fail('repair_status_missing'); + const floors=new Map(); + for(const observed of [...statuses,...setups.flatMap(v=>v.statuses)]){ + const issuer=observed.payload.scope_key.issuer_key_id,revision=observed.payload.revision,id=issuer+':'+revision; + if(revisions.has(id)&&revisions.get(id)!==observed.canonical_sha256)fail('repair_status_conflict');revisions.set(id,observed.canonical_sha256); + for(const row of observed.payload.entries){const identity=[issuer,row.scope_kind,row.scope_id].join(':'),required=obligations.get(identity); + if(required&&row.status==='revoked'&&(row.operation_mask&required.operation_mask))fail('repair_authority_revoked'); + if(required&&row.minimum_document_revision>required.document_revision)fail('repair_status_revision'); + const held=floors.get(identity)??[];held.push([revision,row.minimum_document_revision]);floors.set(identity,held);} + } + for(const values of floors.values()){let min=0;for(const [,floor] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(floorv[0]>observed.payload.revision))fail('repair_status_rollback');} + if(wanted.size!==byRef.size||at>=Math.min(...deadlines))fail(); + const graph=Object.freeze({head,checkpoint,members:Object.freeze(setups),statuses:Object.freeze(statuses),obligations:Object.freeze([...obligations.values()].map(Object.freeze)),retain_until:Math.min(...deadlines),metadata_resource:setups[0].resources.metadata.active.payload.resource}); + if(!same(p.resource_refs,{metadata:graph.metadata_resource,dependencies:[]})||p.retain_until>graph.retain_until)fail(); + return Object.freeze({manifest:resolved,graph,custody:Object.freeze({ref:custody.ref,payload:p,get raw(){budget.output(custody.raw.length);return Uint8Array.from(custody.raw);}}),read_until:p.read_until,retain_until:p.retain_until}); +} diff --git a/clients/typescript/network/open-repair-mailbox-member.ts b/clients/typescript/network/open-repair-mailbox-member.ts new file mode 100644 index 0000000..aea9499 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-member.ts @@ -0,0 +1,136 @@ +/** Authenticate a retained mailbox member at its original admission time. + * Current permission, source custody and ciphertext are verified by the reader. + */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,LocalRawResolver,RawOriginal} from './open-repair-wire.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import type {DraftHistoryInputs} from './open-repair-history.ts'; +import {originalPublicDescriptor,verifyContactOriginals,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {verifyMailboxSlotOwnerInputs,verifyMailboxResourceInputs} from './open-repair-mailbox-authority.ts'; +import {verifyAckOfferBootstrapOriginal} from './open-repair-bound.ts'; +import {statusScope,verifyStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +type Obj=Record; +const ACK=['ack.root_authority','ack.write_grant','bootstrap.ack_offer','historical.status.ack_root','historical.status.ack_write','historical.status.ack_offer_bootstrap']; +const BASE=('mailbox.slot mailbox.read_grant mailbox.maintenance_root bootstrap.mailbox_feed resource.data_allocate resource.metadata_allocate resource.data_offer resource.metadata_offer resource.slot_activation resource.data_active resource.metadata_active source.descriptor historical.status.slot historical.status.read historical.status.maintenance historical.status.bootstrap contact.request delivery.attempt message.disclosure contact.policy contact.decision contact.store_grant contact.knock_lease contact.delivery_lease delivery.destination historical.status.disclosure historical.status.destination historical.status.data_resource historical.status.metadata_resource').split(' '); +const CAPS='max_live_bytes max_meta_bytes max_items max_requests max_pending max_replay_records max_jobs max_job_bytes'.split(' '); +const WINDOWS='admit_until read_until copy_until publish_until retain_until'.split(' '); +const INDEX_LIMITS={max_probe_bytes:8192,max_proof_bytes:4194304,max_proof_items:64,max_signature_checks:4096,max_requests:256,max_pending:8,max_replay_records:256,max_concurrent_handles:8,max_candidate_attempts:8}; +export interface MailboxMemberOptions{ + readonly expectedSlot:unknown;readonly expectedOwner:unknown;readonly expectedSender:unknown;readonly expectedTarget:unknown; + readonly targetStorageEpoch:string;readonly acceptedAt:number;readonly limitPolicy:unknown;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +function fail(code='repair_mailbox_activation_mismatch'):never{throw new RepairError(code);} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function same(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(n=>Object.hasOwn(b,n)&&same((a as Obj)[n],(b as Obj)[n])); +} +function dual(v:unknown,budget:RepairBudget):Obj{const k=fields(v,['signing_key','encryption_key']);originalPublicDescriptor(k.signing_key,budget);originalPublicDescriptor(k.encryption_key,budget,true);return {signing_key_id:k.signing_key.key_id,encryption_key_id:k.encryption_key.key_id};} +function opaque(v:unknown):void{if(typeof v!=='string'||/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.exec(v)?.[0]!==v)fail();} +function roleSet(roles:Obj,names:string[]):boolean{return same(Object.keys(roles).sort(),[...names].sort());} +function setup(resolved:DraftHistoryInputs,e:Obj,policy:RepairPolicy,budget:RepairBudget):Obj{ + const m=resolved.manifest.value as Obj,roles:Record=Object.create(null); + if(m.variant!=='mailbox_member'||!same(m.slot_key,e.slot)||!same(m.root_key,e.slot.root_key)||resolved.predecessors.length)fail(); + for(const row of resolved.roles){if(Object.hasOwn(roles,row.role))fail();roles[row.role]=row.original;} + const get=(role:string)=>{if(!Object.hasOwn(roles,role))fail('repair_original_missing');return {raw:roles[role].raw,ref:roles[role].ref};}; + const entries=Object.fromEntries(Object.entries({slot:'mailbox.slot',read:'mailbox.read_grant',maintenance:'mailbox.maintenance_root',bootstrap:'bootstrap.mailbox_feed',activation:'resource.slot_activation'}).map(([n,r])=>[n,get(r)])); + const resources=Object.fromEntries(['data','metadata'].map(n=>[n,Object.fromEntries(['allocate','offer','active'].map(p=>[p,get('resource.'+n+'_'+p)]))])); + const active=['data','metadata'].map(n=>fields(parseNewWire(resources[n].active.raw,policy,budget).value,['payload','proof']).payload); + const activated=u53(active[0]?.activated_at);if(active[1]?.activated_at!==activated||activated>e.at)fail(); + const originals=verifyMailboxSlotOwnerInputs(entries,{data:resources.data.offer,metadata:resources.metadata.offer},{expectedSlot:e.slot,expectedOwner:e.owner, + expectedTarget:e.target,targetStorageEpoch:e.epoch,limitPolicy:e.limits,at:activated,policy,budget}); + const slot=originals.slot.payload,activation=originals.activation.payload; + if(!same(slot.sender,dual(e.sender,budget))||!same(slot.recipient,dual(e.owner,budget)))fail(); + for(const name of ['slot','read','maintenance','bootstrap'] as const){const p=originals[name].payload; + if(!(p.issued_at<=e.at&&e.at=Math.min(p.windows.admit_until,p.windows.retain_until)))fail('repair_resource_expired');} + const checked:Obj={}; + for(const [name,purpose] of [['data','mailbox_data'],['metadata','feed_metadata']]){ + checked[name]=verifyMailboxResourceInputs({...resources[name],activation:entries.activation},{expectedRoot:e.slot.root_key,expectedOwner:e.owner,expectedTarget:e.target, + targetStorageEpoch:e.epoch,expectedPurpose:purpose,expectedScope:activation.scope,expectedAuthorityRefs:activation.authority_refs,expectedOfferRefs:activation.resource_offer_refs,at:e.at,policy,budget}); + if(!same(checked[name].active.payload.resource,slot[name+'_resource_ref']))fail(); + } + const contact=verifyContactOriginals(Object.fromEntries(Object.entries({node:'source.descriptor',request:'contact.request',policy:'contact.policy',decision:'contact.decision',grant:'contact.store_grant',knock_lease:'contact.knock_lease',delivery_lease:'contact.delivery_lease'}).map(([n,r])=>[n,get(r).raw])), + {senderKeyId:e.sender.signing_key.key_id,senderEncryptionKeyId:e.sender.encryption_key.key_id,recipientKeyId:e.owner.signing_key.key_id, + recipientEncryptionKeyId:e.owner.encryption_key.key_id,nodeKeyId:e.target.signing_key.key_id,storageEpoch:e.epoch,at:e.at,policy,budget}); + return {originals,resources:Object.freeze(checked),contact,roles:Object.freeze(roles),accepted_at:e.at}; +} +/** Each observation must cover its duty, and every scope it contains belongs to + * that signer. Reuse is bounded to exact bytes plus context in this invocation. */ +function historicalStatuses(roles:Obj,duties:Obj[],root:unknown,at:number,policy:RepairPolicy,budget:RepairBudget):readonly AuthenticatedStatusOriginal[]{ + const statuses:AuthenticatedStatusOriginal[]=[],cache=new Map(),revisions=new Map(),floors=new Map(); + for(const duty of duties){ + const permitted=duties.filter(v=>same(v.signer,duty.signer)),entry=roles['historical.status.'+duty.role];if(!entry)fail('repair_original_missing'); + const raw=entry.raw,preview=fields(parseNewWire(raw,policy,budget).value,['payload','proof']).payload; + if(!preview||!Array.isArray(preview.entries))fail('repair_invalid_status'); + const present=new Set(preview.entries.map((v:Obj)=>{if(!v||typeof v.scope_kind!=='string'||typeof v.scope_id!=='string')fail('repair_invalid_status');return v.scope_kind+':'+v.scope_id;})); + const required=permitted.filter(v=>present.has(v.scope_kind+':'+v.scope_id)).map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id,document_revision:v.document_revision,operation_mask:v.operation_mask})); + if(!required.some(v=>v.scope_kind===duty.scope_kind&&v.scope_id===duty.scope_id))fail('repair_status_missing'); + const context={expectedRoot:root,expectedSigningKey:duty.signer,at,allowedScopes:permitted.map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),required}; + const contextRaw=buildNewWire(context,policy,budget).raw,ref=rawRef(entry.ref); + if(ref.namespace!=='meta'||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + const identity=budget.hash(buildNewWire({ref,context},policy,budget).raw);let observed=cache.get(identity); + if(!observed){observed=verifyStatusOriginal({raw,ref},{...context,policy,budget});budget.retain(raw.length+contextRaw.length);cache.set(identity,observed);} + const rev=duty.signer.key_id+':'+observed.payload.revision,prior=revisions.get(rev); + if(prior!==undefined&&prior!==observed.canonical_sha256)fail('repair_status_conflict');revisions.set(rev,observed.canonical_sha256);statuses.push(observed); + for(const row of observed.payload.entries){const key=duty.signer.key_id+':'+row.scope_kind+':'+row.scope_id,held=floors.get(key)??[]; + held.push([observed.payload.revision,row.minimum_document_revision]);floors.set(key,held);} + } + for(const values of floors.values()){let minimum=0;for(const [,value] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(value({raw:roles[n].raw,ref:roles[n].ref}); + const checked=verifyAckOfferBootstrapOriginal(entry('bootstrap.ack_offer'),{root:entry('ack.root_authority'),write:entry('ack.write_grant')}, + {expectedAckSlot:write.ack_slot,expectedOwner:e.sender,expectedReceiptWriter:e.owner,expectedMessageId:m.message_id,expectedEnvelopeRef:m.envelope_ref,at:e.at,limitPolicy:INDEX_LIMITS,policy,budget}); + const root=checked.originals.root.payload.ack_slot.root_key,duties=[]; + for(const [name,role,mask] of [['root','ack_root',3],['write','ack_write',1],['bootstrap','ack_offer_bootstrap',3]] as const){const item=checked.originals[name]; + duties.push({role,scope_kind:'authority',scope_id:statusScope(root,'authority',{authority_kind:item.payload.kind,authority_sha256:item.ref.raw_sha256},policy,budget),document_revision:item.payload.revision,operation_mask:mask,signer:e.sender.signing_key});} + return Object.freeze({originals:checked.originals,statuses:historicalStatuses(roles,duties,root,e.at,policy,budget)}); +} +function member(resolved:DraftHistoryInputs,e:Obj,policy:RepairPolicy,budget:RepairBudget):Obj{ + const graph=setup(resolved,e,policy,budget),{roles,originals}=graph,m=resolved.manifest.value as Obj,root=m.root_key,key=m.slot_key,at=e.at; + if(!roleSet(roles,BASE)&&!roleSet(roles,[...BASE,...ACK]))fail(); + const control=(role:string,names:string,signer:Obj):Obj=>{const signed=fields(parseNewWire(roles[role].raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key',...names.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==role||u53(p.issued_at)>=u53(p.expires_at))fail(); + if(!(p.issued_at<=at&&atu53(dp.budget[n])>slot.budget[n])||WINDOWS.some(n=>u53(dp.windows[n])>slot.windows[n]||dp.windows[n]>dp.windows.retain_until)||at>=Math.min(dp.windows.admit_until,dp.windows.retain_until))fail(); + for(const [n,r] of [['contact_request_ref','contact.request'],['contact_policy_ref','contact.policy'],['contact_knock_lease_ref','contact.knock_lease'],['contact_decision_ref','contact.decision'],['store_grant_ref','contact.store_grant']])if(!same(dp[n],roles[r].ref))fail(); + u53(cp.revision,1);if(u53(cp.operation_mask)>127||rawRef(cp.envelope_ref).namespace!=='object')fail(); + if(!same(cp.root_key,root)||!same(cp.maintenance_root_ref,roles['mailbox.maintenance_root'].ref)||cp.issued_at>ap.issued_at||(cp.operation_mask&65)!==65|| + !(atMath.min(slot.windows.retain_until,maintenance.windows.retain_until))fail(); + const required=['contact.request','delivery.attempt','message.disclosure','authority.status.disclosure'],allowed=cp.allowed_roles; + if(!Array.isArray(allowed)||allowed.some((v:any,i:number)=>typeof v!=='string'||(i>0&&v<=allowed[i-1])||![...required,...ACK].includes(v))||required.some(n=>!allowed.includes(n)))fail('repair_status_disclosure'); + const returned=fields(cp.bootstrap_return,['subject','consumer','roles','until']); + if(!same(returned.subject,slot.recipient)||returned.consumer!=='mailbox_feed'||!same(returned.roles,['authority.status.disclosure','message.disclosure'])||!(at!allowed.includes(n)))fail('repair_status_disclosure');if(!same(ap.ack_grant_ref,roles['ack.write_grant'].ref))fail();ack=ackConfiguration(roles,e,m,policy,budget);} + const duties:Obj[]=[],add=(role:string,kind:string,subject:unknown,revision:number,mask:number,signer:Obj)=>duties.push({role,scope_kind:kind,scope_id:statusScope(root,kind,subject,policy,budget),document_revision:revision,operation_mask:mask,signer}); + add('slot','mailbox_slot',key,slot.revision,65,e.owner.signing_key); + for(const [name,role,p,mask,signer] of [['read','mailbox.read_grant',originals.read.payload,2,e.owner.signing_key],['maintenance','mailbox.maintenance_root',maintenance,65,e.owner.signing_key], + ['bootstrap','bootstrap.mailbox_feed',originals.bootstrap.payload,10,e.owner.signing_key],['destination','delivery.destination',dp,1,e.owner.signing_key],['disclosure','message.disclosure',cp,65,e.sender.signing_key]] as const){ + add(name,'authority',{authority_kind:p.kind,authority_sha256:roles[role].ref.raw_sha256},p.revision??1,mask,signer);} + for(const n of ['data','metadata']){const p=graph.resources[n].active.payload;add(n+'_resource','resource',p.resource,p.reservation_generation,65,e.target.signing_key);} + return Object.freeze({...graph,destination:dp,attempt:ap,disclosure:cp,statuses:historicalStatuses(roles,duties,root,at,policy,budget),obligations:Object.freeze(duties.map(Object.freeze)),ack_configuration:ack}); +} +export function verifyMailboxMemberInputs(manifest:unknown,resolver:LocalRawResolver,options:MailboxMemberOptions):Readonly{ + const a=fields(options,['expectedSlot','expectedOwner','expectedSender','expectedTarget','targetStorageEpoch','acceptedAt','limitPolicy','policy','budget']),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,owner:a.expectedOwner,sender:a.expectedSender,target:a.expectedTarget,epoch:a.targetStorageEpoch,at:a.acceptedAt,limits:a.limitPolicy},policy,budget).value as Obj; + u53(e.at);for(const k of [e.owner,e.sender,e.target])dual(k,budget); + const entry=fields(manifest,['raw','ref']),ref=rawRef(entry.ref),raw=parseNewWire(entry.raw,policy,budget).raw; + if(ref.namespace!=='meta'||ref.size!==raw.length||ref.raw_sha256!==budget.hash(raw))fail('repair_ref_mismatch'); + return member(resolveHistoricalInputs(raw,resolver,policy,budget),e,policy,budget); +} diff --git a/tests/test_open_repair_mailbox_member_typescript.py b/tests/test_open_repair_mailbox_member_typescript.py new file mode 100644 index 0000000..e0fc334 --- /dev/null +++ b/tests/test_open_repair_mailbox_member_typescript.py @@ -0,0 +1,105 @@ +"""Native admission graph consumes originals from a real HTTP mailbox commit.""" +import copy +import json +import subprocess +import unittest +from dataclasses import asdict +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_delivery_http as http_fixture +from tests.test_open_repair_probe_typescript import encoded +from tests.open_repair_ack_fixtures import reference +import memory_vault_open_repair_wire as wire +import memory_vault_open_repair_history as history +from memory_vault_open_repair_mailbox_snapshot import MailboxSnapshotSource +from memory_vault_open_repair_mailbox_activation import verify_mailbox_member_inputs + +DRIVER=runtime.DRIVER.split('const {Agent}',1)[0]+r''' +const {RepairBudget,LocalRawResolver}=await import('./open-repair-wire.ts'); +const {verifyMailboxMemberInputs}=await import('./open-repair-mailbox-member.ts'); +const {verifyMailboxFeedSourceEvent}=await import('./open-repair-mailbox-feed.ts'); +const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk); +const calls=JSON.parse(Buffer.concat(chunks).toString()),decode=e=>({raw:Buffer.from(e.raw,'base64'),ref:e.ref}),out=[]; +for(const v of calls){let budget;try{ + budget=new RepairBudget(v.policy);const resolver=new LocalRawResolver(v.policy,budget); + for(const p of v.packs){const e=decode(p);resolver.put(e.ref.namespace,e.ref.key,e.raw);} + if(v.operation==='feed'){const result=verifyMailboxFeedSourceEvent(decode(v.manifest),resolver,decode(v.custody),{...v.options,policy:v.policy,budget});out.push({ok:true,count:result.graph.head.count,statuses:result.graph.statuses.map(v=>v.canonical_sha256),retain_until:result.retain_until,metrics:budget.snapshot()});continue;} + const graph=verifyMailboxMemberInputs(decode(v.manifest),resolver,{...v.options,policy:v.policy,budget}); + out.push({ok:true,accepted_at:graph.accepted_at,refs:Object.fromEntries(Object.entries(graph.roles).map(([k,e])=>[k,e.ref])), + statuses:graph.statuses.map(v=>v.canonical_sha256),ack:graph.ack_configuration!==null,metrics:budget.snapshot()}); +}catch(e){out.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack),metrics:budget?.snapshot()});}} +process.stdout.write(JSON.stringify({out,subprocessCalls})); +''' + +class NativeMailboxMemberTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def native(self,calls): + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(calls).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=40) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout) + self.assertEqual(out['subprocessCalls'],0);return out['out'] + + def test_committed_http_member_originals_and_denied_cross_bindings(self): + h=http_fixture.MailboxStagingHTTPTests('test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources') + h.setUp();self.addCleanup(h.doCleanups);visited=[] + def inspect(staging,key,head): + visited.append(True);s=staging.source + rid=staging.db.execute('SELECT resource_id FROM open_repair_mailbox_roots').fetchone()[0] + held=MailboxSnapshotSource(staging).load(rid,key,head,max_bytes=2_000_000,max_items=256);part=held.parts[2] + def entry(ref):return dict(raw=held.read(ref.as_dict()),ref=ref.as_dict()) + packs=[entry(ref) for role,ref in part.transfer if role=='history.raw_pack'];manifest=entry(part.manifest) + core=next(ref for role,ref in part.transfer if role=='member.core');accepted=json.loads(held.read(core.as_dict()))['payload']['accepted_at'] + owner=json.loads(staging.db.execute('SELECT owner_keys FROM open_repair_mailbox_resources LIMIT 1').fetchone()[0]) + envelope=json.loads(held.read(held.envelopes[0].as_dict())) + sender=dict(signing_key=h.ai.public_descriptor(),encryption_key=envelope['payload']['context']['sender_encryption_key']) + options=dict(expectedSlot=key,expectedOwner=owner,expectedSender=sender,expectedTarget=s.target, + targetStorageEpoch=s.node['payload']['storage_epoch'],acceptedAt=accepted,limitPolicy=s.limits) + budget=wire.RepairBudget(s.policy);resolver=wire.LocalRawResolver(s.policy,budget) + for p in packs:resolver.put(p['ref']['namespace'],p['ref']['key'],p['raw']) + resolved=history.resolve_historical_inputs(manifest['raw'],resolver,s.policy,budget) + expected=verify_mailbox_member_inputs(resolved,expected_slot=key,expected_owner=owner,expected_sender=sender,expected_target=s.target, + target_storage_epoch=options['targetStorageEpoch'],accepted_at=accepted,limit_policy=s.limits,policy=s.policy,budget=budget) + call=dict(manifest=encoded(manifest),packs=[encoded(p) for p in packs],options=options,policy=asdict(s.policy)) + call=json.loads(json.dumps(call)) + calls=[call];labels=['exact'] + for name,value in [('expectedSender',owner),('targetStorageEpoch','different_epoch'),('acceptedAt',accepted+10000)]: + wrong=copy.deepcopy(call);wrong['options'][name]=value;calls.append(wrong);labels.append(name) + wrong=copy.deepcopy(call);wrong['manifest']['ref']['raw_sha256']='0'*64;calls.append(wrong);labels.append('manifest_ref') + wrong=copy.deepcopy(call);wrong['policy']['max_signature_checks']=3;calls.append(wrong);labels.append('signature_budget') + # Repack exact original documents under a wrong role. Every pack hash + # remains valid; typed admission checks must reject the substitution. + roles={row.role:dict(raw=row.original.raw,ref=row.original.ref.as_dict()) for row in resolved.roles} + for target,other in [('historical.status.disclosure','historical.status.slot'),('contact.decision','contact.policy'),('delivery.destination','delivery.attempt')]: + altered=dict(roles);altered[target]=roles[other];meter=wire.RepairBudget(s.policy) + pack=wire.build_raw_pack([v['raw'] for v in altered.values()],s.policy,meter) + indices={(v.raw_sha256,v.size):i for i,v in enumerate(pack.entries)} + value=dict(json.loads(manifest['raw']),roles=[dict(role=role,document_ref=v['ref'],pack_ref=pack.ref.as_dict(),entry_index=indices[(v['ref']['raw_sha256'],v['ref']['size'])]) for role,v in sorted(altered.items())]) + changed=history.build_historical_manifest(value,s.policy,meter) + wrong=copy.deepcopy(call);wrong['manifest']=encoded(dict(raw=changed.raw,ref=reference(changed.raw,'native_mutated_member'))) + wrong['packs']=[encoded(dict(raw=pack.raw,ref=pack.ref.as_dict()))];calls.append(wrong);labels.append(target) + results=self.native(calls);good=results[0];self.assertTrue(good['ok'],good) + self.assertEqual(good['accepted_at'],accepted);self.assertEqual(good['refs'],{k:v['ref'] for k,v in expected['roles'].items()}) + self.assertEqual(good['statuses'],[v.canonical_sha256 for v in expected['statuses']]);self.assertEqual(good['ack'],expected['ack_configuration'] is not None) + from memory_vault_open_repair_mailbox_activation import verify_mailbox_feed_source_event + feedpart=held.parts[1];feedpacks=[entry(ref) for role,ref in feedpart.transfer if role=='history.raw_pack'] + feedoptions={k:v for k,v in options.items() if k not in ('acceptedAt','targetStorageEpoch')} + feedcall=json.loads(json.dumps(dict(operation='feed',manifest=encoded(entry(feedpart.manifest)),custody=encoded(entry(feedpart.custody)),packs=[encoded(p) for p in feedpacks],options=feedoptions,policy=asdict(s.policy)))) + meter=wire.RepairBudget(s.policy);local=wire.LocalRawResolver(s.policy,meter) + for pack in feedpacks:local.put(pack['ref']['namespace'],pack['ref']['key'],pack['raw']) + feedexpected=verify_mailbox_feed_source_event(entry(feedpart.manifest),local,entry(feedpart.custody),expected_slot=key,expected_owner=owner,expected_sender=sender,expected_target=s.target,limit_policy=s.limits,policy=s.policy,budget=meter) + feedcalls=[feedcall] + wrong=copy.deepcopy(feedcall);wrong['options']['expectedSender']=owner;feedcalls.append(wrong) + wrong=copy.deepcopy(feedcall);wrong['custody']['ref']['raw_sha256']='0'*64;feedcalls.append(wrong) + feedresults=self.native(feedcalls);self.assertTrue(feedresults[0]['ok'],feedresults[0]) + self.assertEqual(feedresults[0]['count'],1);self.assertEqual(feedresults[0]['retain_until'],feedexpected['retain_until']) + self.assertEqual(feedresults[0]['statuses'],[v.canonical_sha256 for v in feedexpected['graph']['statuses']]) + for result in feedresults[1:]:self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error',result) + for label,result in zip(labels[1:],results[1:]): + with self.subTest(label=label):self.assertFalse(result['ok'],result);self.assertNotEqual(result['code'],'untyped_error',result) + h.inspect_committed_mailbox=inspect + h.test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources();self.assertEqual(visited,[True]) + +if __name__=='__main__':unittest.main() From d86d62083566a30804f3cae5ad6643da73e28f9a Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:45:55 +0800 Subject: [PATCH 09/16] Keep exact public SDK inventory aligned with native receipt return --- tests/test_network_packaging.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 427a99d..c207d07 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -49,7 +49,7 @@ "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-repair-probe.ts", - "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts")} + "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} TS_ENDPOINT_TESTS = {"tests/test_network_typescript_" + name + ".py" for name in ("nodes", "records", "vault", "peer", "peer_race", "transport", "setup", "retrieval_text", "retrieval", "agent", "agent_network", "topics")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 57) + self.assertEqual(len(TS_NETWORK), 59) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -125,6 +125,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", + "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", "tests/test_open_repair_proof.py", "tests/test_open_repair_proof_typescript.py", "tests/test_open_repair_access.py", @@ -192,7 +193,8 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", - "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts"}) + "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", + "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts"}) for name in repair_sources: self.assertTrue((ROOT / name).is_file()) for name in ("docs/NATIVE_OPEN_PROVIDER.md", "docs/CONTINUATION_TRIAL.md", "docs/OPEN_NETWORK_QUICKSTART.md"): From 08626c6bade2320a342b3a4ac48b299ffeb8ba74 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:58:49 +0800 Subject: [PATCH 10/16] Resume retained mailbox memory imports and saved receipts in native Agent --- clients/typescript/network/README.md | 20 +++ .../network/open-delivery-client.ts | 33 +++- .../network/open-repair-mailbox-inbox.ts | 38 +++++ .../network/open-repair-mailbox-read.ts | 151 ++++++++++++++++++ .../typescript/network/open-repair-receipt.ts | 2 +- clients/typescript/network/package.json | 7 +- scripts/build_client_plugin.py | 5 + scripts/build_release.py | 6 + scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 14 +- ...t_open_repair_mailbox_member_typescript.py | 98 +++++++++++- 11 files changed, 356 insertions(+), 22 deletions(-) create mode 100644 clients/typescript/network/open-repair-mailbox-inbox.ts create mode 100644 clients/typescript/network/open-repair-mailbox-read.ts diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index 31a286e..a378e96 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -277,3 +277,23 @@ in `open-repair-admin.ts`: `recover-ack`, `recover-empty`, and verified status observations across restarts, including failed recovery. See [request formats and commands](../../../docs/OPEN_ACK_RECOVERY.md#native-recovery-commands-development-after-alpha019). This command entry is development after the frozen alpha.0.19 archives. + +### Retained mailbox inbox recovery (development) + +A native `Agent.handle({op: "receive", limit: 1})` can resume an ordinary mailbox transfer +already durably staged by the Python client in the same protected inbox. It +rechecks the original owner/contact approvals, activated storage resources, +complete feed prefix, recipient-encrypted directory and admission core, and +READ observations recorded at receipt time. A retained revocation or missing +sender permission blocks the object read. The original memory share then +uses the existing Vault import and its idempotent transfer receipt. The saved +receipt remains available to an explicit `connect/return_receipt` request; +mailbox access does not grant an independent receipt return. + +`readMailboxIndex` and `readMailboxAdmission` accept a caller-authorized exact +reference reader. They support asynchronous reads without delegating to Python. +`verifyMailboxInboxEvidence` reopens the retained ordinary-mailbox format without +network access. This is historical local recovery, not a claim about current +source availability. Native discovery/download of a new remote mailbox and the +replica inbox format remain separate unfinished work; use the Python client for +those paths. These additions are development source, not yet a public release. diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index d69c7b8..5c28e98 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -2,6 +2,7 @@ * Original share bytes and author trust remain separate from network identity. */ import {randomBytes} from 'node:crypto'; +import {verifyMailboxInboxEvidence} from './open-repair-mailbox-inbox.ts'; import {performance} from 'node:perf_hooks'; import type {DatabaseSync} from 'node:sqlite'; import {canonicalBytes,document,objectFields,sha256,encodeBase64url,decodeBase64url,opaqueId,safeInteger, @@ -118,7 +119,8 @@ export class OpenDeliveryClient{ })); } private static keys(session:Obj):Obj{ - if(session.intent)session=session.intent.payload.authority; + if(Object.hasOwn(session,'mailbox'))session=session.authority; + else if(session.intent)session=session.intent.payload.authority; const request=session.request.payload,policy=session.policy.payload; return {sender_signing_key:request.signing_key,sender_encryption_key:request.encryption_key, recipient_signing_key:policy.signing_key,recipient_encryption_key:policy.encryption_key}; @@ -324,7 +326,7 @@ export class OpenDeliveryClient{ private stageInbox(messageId:string,sender:string,envelope:Uint8Array,body:Uint8Array,session:Obj):Obj{ originalDocument(envelope,{maximum:MAX_ENVELOPE_BYTES});const digest=sha256(envelope),content=validateContent(body); if(!['message','memory_transfer'].includes(content.kind))fail('open_delivery_invalid_content_kind'); - const proof=canonicalBytes(document(session,MAX_INBOX_SESSION_BYTES)); + const proof=canonicalBytes(OpenDeliveryClient.inboxSession(session)); return this.db(db=>transaction(db,()=>{const old=db.prepare('SELECT * FROM open_delivery_inbox WHERE message_id=?').get(messageId) as Obj|undefined; if(old){if(old.envelope_sha256!==digest||old.sender!==sender||!raw(old.body).equals(raw(body)))fail('network_inbox_identity_conflict');return old;} const size=this.size(db,'open_delivery_inbox');if(size.count>=4096||size.bytes+envelope.length+body.length+proof.length+2*MAX_RESULT_BYTES>MAX_LOCAL_BYTES)fail('network_inbox_capacity'); @@ -337,10 +339,23 @@ export class OpenDeliveryClient{ if(!row)fail('network_message_not_found');if(['saved','rejected'].includes(row.phase))return document(row.result,MAX_RESULT_BYTES); const result={message_id:messageId,state:'rejected',code};db.prepare("UPDATE open_delivery_inbox SET phase='rejected',result=? WHERE message_id=?").run(canonicalBytes(result),messageId);return result; }));} + private static inboxSession(value:DocumentInput):Obj{ + const session=document(value,8*1024*1024) as Obj; + if(!Object.hasOwn(session,'mailbox'))return document(session,MAX_INBOX_SESSION_BYTES) as Obj; + return objectFields(session,['mailbox','authority','source_node']); + } + private async verifyMailboxInbox(session:Obj,row:Obj):Promise{ + const verified=await verifyMailboxInboxEvidence(session.mailbox,row.envelope,{owner:{signing_key:validateSigningIdentity(this.participant.identity), + encryption_key:validateEncryptionIdentity(this.encryption)},encryptionIdentity:this.encryption,stagedAt:row.created_at}); + if(verified.core.message_id!==row.message_id||session.mailbox.sender.signing_key.key_id!==row.sender)fail('network_inbox_identity_conflict'); + const expected=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([name,role])=>[name,document(verified.setup.roles[role].raw)])); + if(!same(session.authority,expected))fail('network_inbox_identity_conflict'); + } private async finishInbox(messageId:string):Promise{ const row=this.inbox(messageId);if(!row)fail('network_message_not_found');if(['saved','rejected'].includes(row.phase))return document(row.result,MAX_RESULT_BYTES); - const session=document(row.session,MAX_INBOX_SESSION_BYTES) as Obj;if(sha256(row.envelope)!==row.envelope_sha256)fail('network_inbox_identity_conflict'); - verifyStorageReceipt(session.storage_receipt,{node:session.source_node,intent:session.intent});const keys=OpenDeliveryClient.keys(session); + const session=OpenDeliveryClient.inboxSession(row.session);if(sha256(row.envelope)!==row.envelope_sha256)fail('network_inbox_identity_conflict'); + if(Object.hasOwn(session,'mailbox'))await this.verifyMailboxInbox(session,row); + else verifyStorageReceipt(session.storage_receipt,{node:session.source_node,intent:session.intent});const keys=OpenDeliveryClient.keys(session); if(!same(keys.recipient_signing_key,validateSigningIdentity(this.participant.identity)))fail('open_delivery_key_binding_mismatch'); const reopened=await decryptEnvelope(row.envelope,{...keys,encryption_identity:this.encryption} as any); if(!raw(reopened).equals(raw(row.body)))fail('network_inbox_identity_conflict');const content=validateContent(row.body);let imported:Obj|null=null; @@ -373,10 +388,11 @@ export class OpenDeliveryClient{ } /** Read the actually saved inbox receipt for an explicitly requested return. * The caller cannot substitute receipt bytes or a saved-state flag. */ - savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):{raw:Uint8Array;ref:Obj}{ + async savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):Promise<{raw:Uint8Array;ref:Obj}>{ if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); - const keys=OpenDeliveryClient.keys(document(row.session,MAX_INBOX_SESSION_BYTES) as Obj); + const session=OpenDeliveryClient.inboxSession(row.session);if(Object.hasOwn(session,'mailbox'))await this.verifyMailboxInbox(session,row); + const keys=OpenDeliveryClient.keys(session); if(sha256(row.envelope)!==row.envelope_sha256||row.sender!==owner.signing_key.key_id||!same(envelopeRef(row.envelope),expectedEnvelope)|| !same(keys,{sender_signing_key:owner.signing_key,sender_encryption_key:owner.encryption_key, recipient_signing_key:validateSigningIdentity(this.participant.identity),recipient_encryption_key:validateEncryptionIdentity(this.encryption)}))fail('repair_saved_tuple_mismatch'); @@ -384,7 +400,7 @@ export class OpenDeliveryClient{ if(result.state!=='validated_saved'||result.message_id!==messageId||result.sender_key_id!==row.sender|| !['message','memory_transfer'].includes(result.content_kind))fail('repair_receipt_not_saved'); this.savedReceipt(messageId);const saved=this.inbox(messageId); - if(!saved||saved.phase!=='saved'||saved.sender!==row.sender||!raw(saved.envelope).equals(raw(row.envelope))||!raw(saved.body).equals(raw(row.body)))fail('repair_saved_tuple_mismatch'); + if(!saved||saved.phase!=='saved'||saved.sender!==row.sender||!raw(saved.envelope).equals(raw(row.envelope))||!raw(saved.body).equals(raw(row.body))||!raw(saved.session).equals(raw(row.session))||saved.created_at!==row.created_at)fail('repair_saved_tuple_mismatch'); const bytes=Buffer.from(saved.receipt); verifyRecipientReceipt(document(bytes,4096),{recipient_signing_key:validateSigningIdentity(this.participant.identity), sender_key_id:row.sender,message_id:messageId,envelope_ref:expectedEnvelope} as any); @@ -392,7 +408,8 @@ export class OpenDeliveryClient{ } private async sendReceipt(messageId:string,budget:DeliveryBudget,node?:SignedNode){ const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('open_delivery_not_saved');if(row.receipt_sent)return; - const receipt=this.savedReceipt(messageId);verifyRecipientReceipt(receipt,{recipient_signing_key:validateSigningIdentity(this.participant.identity), + const receipt=this.savedReceipt(messageId);if(Object.hasOwn(OpenDeliveryClient.inboxSession(row.session),'mailbox'))return; + verifyRecipientReceipt(receipt,{recipient_signing_key:validateSigningIdentity(this.participant.identity), sender_key_id:row.sender,message_id:messageId,envelope_ref:envelopeRef(row.envelope)}); if(!node){const session=document(row.session,MAX_INBOX_SESSION_BYTES) as Obj,child=budget.routing(); try{node=await this.sessionNode({node:session.source_node},child);}finally{budget.merge(child);}} diff --git a/clients/typescript/network/open-repair-mailbox-inbox.ts b/clients/typescript/network/open-repair-mailbox-inbox.ts new file mode 100644 index 0000000..c2d1c63 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-inbox.ts @@ -0,0 +1,38 @@ +/** Reopen an actually retained mailbox transfer at its recorded receipt time. + * No network operation or new permission is implied by local historical bytes. */ +import {buildNewWire,objectFields,rawRef,u53,RepairError,RepairBudget,LocalRawResolver} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import {decodeBase64url} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import {verifyMailboxFeedSourceEvent} from './open-repair-mailbox-feed.ts'; +import {readMailboxIndex,readMailboxAdmission} from './open-repair-mailbox-read.ts'; +type Obj=Record; +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +const key=(value:unknown)=>{const r=rawRef(value);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +function fail(code:string):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +export async function verifyMailboxInboxEvidence(evidence:unknown,envelope:Uint8Array,options:{owner:unknown;encryptionIdentity:EncryptionIdentityDocument;stagedAt:number}):Promise>{ + const a=fields(options,['owner','encryptionIdentity','stagedAt']); + const value=fields(evidence,['schema_version','received_at','slot','sender','target','limits','member','manifest_ref','custody_ref','originals','status_refs']); + if(value.schema_version!=='memory-vault-mailbox-inbox/v1')fail('repair_invalid_context'); + const at=u53(value.received_at);if(at>u53(a.stagedAt))fail('repair_invalid_context'); + const policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}),budget=new RepairBudget(policy); + const selected=buildNewWire({slot:value.slot,sender:value.sender,target:value.target,owner:a.owner,member:value.member,limits:value.limits,manifest_ref:value.manifest_ref,custody_ref:value.custody_ref,status_refs:value.status_refs},policy,budget).value as Obj; + const limits=fields(selected.limits,Object.keys(DEFAULT_REPAIR_CLIENT_LIMITS));for(const n of Object.keys(limits))u53(limits[n],1); + if(!Array.isArray(value.originals)||value.originals.length<1||value.originals.length>limits.max_proof_items||!Array.isArray(selected.status_refs)||selected.status_refs.length<1||selected.status_refs.length>16)fail('repair_over_budget'); + const originals=new Map;raw:Uint8Array}>(),resolver=new LocalRawResolver(policy,budget);let total=0; + for(const item of value.originals){const v=fields(item,['ref','raw_base64url']),ref=rawRef(v.ref),id=key(ref); + if(ref.namespace!=='meta'||originals.has(id)||ref.size>policy.max_document_bytes)fail('repair_ref_mismatch'); + const raw=decodeBase64url(v.raw_base64url,policy.max_document_bytes);total+=raw.length;if(total>limits.max_proof_bytes)fail('repair_over_budget'); + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + if(!same(resolver.put(ref.namespace,ref.key,raw).ref,ref))fail('repair_ref_mismatch');originals.set(id,{raw,ref}); + } + const entry=(ref:unknown)=>{const v=originals.get(key(ref));if(!v)fail('repair_original_missing');return v;}; + const source=verifyMailboxFeedSourceEvent(entry(selected.manifest_ref),resolver,entry(selected.custody_ref),{expectedSlot:selected.slot,expectedOwner:selected.owner,expectedSender:selected.sender,expectedTarget:selected.target,limitPolicy:limits,policy,budget}); + if(!(source.custody.payload.stored_at<=at&&at[v.role,{ref:v.original.ref,raw:v.original.raw}])); + const members=await readMailboxIndex(roles.get('feed.head'),roles.get('feed.checkpoint'),{expectedSlot:selected.slot,expectedSigningKey:selected.target.signing_key,encryptionIdentity:a.encryptionIdentity,readOriginal:ref=>entry(ref).raw,at,maxMessages:65536,policy,budget}); + if(!members.some(v=>same(v,selected.member)))fail('repair_mailbox_member_mismatch'); + return readMailboxAdmission(selected.member,{expectedSlot:selected.slot,expectedSigningKey:selected.target.signing_key,expectedOwner:selected.owner,expectedSender:selected.sender,expectedTarget:selected.target, + encryptionIdentity:a.encryptionIdentity,readOriginal:ref=>ref.namespace==='object'?envelope:entry(ref).raw,at,currentStatuses:selected.status_refs.map(entry),knownStatuses:[],statusObligations:source.graph.obligations,limitPolicy:limits,policy,budget}); +} diff --git a/clients/typescript/network/open-repair-mailbox-read.ts b/clients/typescript/network/open-repair-mailbox-read.ts new file mode 100644 index 0000000..8f7c534 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-read.ts @@ -0,0 +1,151 @@ +/** Read and decrypt a complete mailbox directory through an authorized reader. + * Enumeration alone grants no message access; each admission still needs its + * original authority, source history and current status before object reads. */ +import {isUint8Array} from 'node:util/types'; +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,RawRef} from './open-repair-wire.ts'; +import {verifyBoundedControlSignature} from './open-repair-original.ts'; +import {emptyMailboxState,appendMailboxState} from './open-repair-mailbox-range.ts'; +import {decryptBytes,validateEncryptionIdentity} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +type Obj=Record; +export interface MailboxIndexReadOptions{ + readonly expectedSlot:unknown;readonly expectedSigningKey:unknown;readonly encryptionIdentity:EncryptionIdentityDocument; + readonly readOriginal:(reference:RawRef)=>Uint8Array|Promise; + readonly at:number;readonly maxMessages:number;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +function fail(code='repair_mailbox_range_mismatch'):never{throw new RepairError(code);} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +export async function readMailboxIndex(headEntry:unknown,checkpointEntry:unknown,options:MailboxIndexReadOptions):Promise{ + const a=fields(options,['expectedSlot','expectedSigningKey','encryptionIdentity','readOriginal','at','maxMessages','policy','budget']),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,at:a.at,max:a.maxMessages,identity:a.encryptionIdentity},policy,budget).value as Obj; + const reader=a.readOriginal;u53(e.at);u53(e.max,1);if(e.max>65536||typeof reader!=='function')fail('repair_invalid_context'); + validateEncryptionIdentity(e.identity);const identity=e.identity as EncryptionIdentityDocument; + const checked=(raw:unknown,reference:unknown):Obj=>{const ref=rawRef(reference),doc=parseNewWire(raw as Uint8Array,policy,budget),held=doc.raw; + if(ref.namespace!=='meta'||held.length!==ref.size||budget.hash(held)!==ref.raw_sha256)fail('repair_ref_mismatch');return doc.value as Obj;}; + const event=(entry:unknown,kind:string,names:string):Obj=>{const v=fields(entry,['raw','ref']),signed=fields(checked(v.raw,v.ref),['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key','slot_key','committed_at','retain_until',...names.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==kind||!same(p.slot_key,e.slot))fail(); + verifyBoundedControlSignature(p,signed.proof,e.key,budget);if(!(u53(p.committed_at)<=e.at&&e.ate.max||count!==cp.count||!same(head.checkpoint_ref,fields(checkpointEntry,['raw','ref']).ref)||head.committed_at!==cp.committed_at||head.retain_until!==cp.retain_until)fail(); + const state=Object.fromEntries(['slot_binding','count','leaf_root','frontier'].map(n=>[n,cp[n]])); + let accumulated=emptyMailboxState(e.slot,policy,budget);const entries:Obj[]=[],seen=new Set(); + const load=async(reference:unknown):Promise=>{const ref=rawRef(reference);if(ref.namespace!=='meta')fail('repair_ref_mismatch');if(seen.has(ref.key))fail();seen.add(ref.key);return checked(await reader(ref),ref);}; + const walk=async(reference:unknown,level:number,start:number,end:number):Promise=>{ + const value=fields(await load(reference),['schema_version','kind','slot_key','start','end',...(level>=0?['level','children']:['sealed_page_ref','entries'])]); + if(value.schema_version!=='memory-vault-open-repair/v1'||!same(value.slot_key,e.slot)||u53(value.start)!==start||u53(value.end)!==end)fail(); + if(level>=0){ + if(value.kind!=='range.index'||u53(value.level)!==level||!Array.isArray(value.children)||value.children.length<1||value.children.length>16)fail();let cursor=start;const span=16**(level+1); + for(const child of value.children){fields(child,['start','end','ref']);const stop=Math.min(cursor+span,end);if(u53(child.start)!==cursor||u53(child.end)!==stop||stop<=cursor)fail();await walk(child.ref,level-1,cursor,stop);cursor=stop;} + if(cursor!==end)fail();return; + } + if(value.kind!=='range.repair_page'||!Array.isArray(value.entries)||value.entries.length!==end-start||end-start<1||end-start>16)fail(); + for(let i=0;iv!==privateRaw[i]))fail(); + for(const row of value.entries){accumulated=appendMailboxState(accumulated,row.sealed_core_ref.raw_sha256,e.slot,policy,budget);entries.push(row);} + }; + if(count){let level=0;while(count>16**(level+2))level++;await walk(head.range_root_ref,level,0,count);} + else if(head.range_root_ref!==null)fail(); + if(!same(accumulated,state))fail();return Object.freeze(entries); +} + +import {LocalRawResolver} from './open-repair-wire.ts'; +import {parseHistoricalManifest} from './open-repair-history.ts'; +import {verifyMailboxMemberInputs} from './open-repair-mailbox-member.ts'; +import {verifyMailboxInclusion} from './open-repair-mailbox-range.ts'; +import {authenticateStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +export interface MailboxAdmissionReadOptions extends Omit{ + readonly expectedOwner:unknown;readonly expectedSender:unknown;readonly expectedTarget:unknown;readonly limitPolicy:unknown; + readonly currentStatuses:readonly unknown[];readonly knownStatuses:readonly unknown[];readonly statusObligations:readonly unknown[]; + readonly onStatusAuthenticated?: (item:AuthenticatedStatusOriginal)=>void; +} +function currentRead(setup:Obj,entries:readonly unknown[],known:readonly unknown[],additional:readonly unknown[],at:number, + policy:RepairPolicy,budget:RepairBudget,onAuthenticated?: (item:AuthenticatedStatusOriginal)=>void):readonly AuthenticatedStatusOriginal[]{ + if(!Array.isArray(entries)||entries.length<1||entries.length>16||!Array.isArray(known)||known.length>32||!Array.isArray(additional)||additional.length>80|| + (onAuthenticated!==undefined&&typeof onAuthenticated!=='function'))fail('repair_invalid_status'); + const root=setup.originals.slot.payload.slot_key.root_key; + const obligations=setup.obligations.map((v:Obj)=>({...v,operation_mask:v.role==='destination'?0:v.role==='bootstrap'?10:2})); + for(const n of ['read','maintenance'])if((setup.originals[n].payload.operation_mask&2)!==2)fail('repair_authority_revoked'); + if((setup.disclosure.operation_mask&2)!==2)fail('repair_authority_revoked'); + const required=new Map(obligations.map((v:Obj)=>[[v.signer.key_id,v.scope_kind,v.scope_id].join(':'),v])); + const added=buildNewWire(additional,policy,budget).value as Obj[]; + for(const duty of added){fields(duty,['role','scope_kind','scope_id','document_revision','operation_mask','signer']);fields(duty.signer,['schema_version','algorithm','key_id','public_key']);} + const disclosed=[...obligations,...added]; + const authenticate=(entry:unknown,current:boolean):AuthenticatedStatusOriginal=>{const v=fields(entry,['raw','ref']),preview=fields(parseNewWire(v.raw,policy,budget).value,['payload','proof']).payload; + if(!preview||!preview.scope_key)fail('repair_invalid_status');const issuer=preview.scope_key.issuer_key_id,permitted=disclosed.filter(v=>v.signer.key_id===issuer); + if(!permitted.length||u53(preview.issued_at)>at)fail('repair_status_mismatch'); + const scopes=new Map();for(const duty of permitted)scopes.set(duty.scope_kind+':'+duty.scope_id,{scope_kind:duty.scope_kind,scope_id:duty.scope_id}); + return authenticateStatusOriginal(v,{expectedRoot:root,expectedSigningKey:permitted[0].signer,at:current?at:preview.issued_at, + allowedScopes:[...scopes].sort(([a],[b])=>ab?1:0).map(([,v])=>v),policy,budget},onAuthenticated);}; + const retained=[...setup.statuses,...known.map(v=>authenticate(v,false))],current=entries.map(v=>authenticate(v,true)); + const seen=new Map(),floors=new Map(),covered=new Set(); + for(const observed of [...retained,...current]){const p=observed.payload,issuer=p.scope_key.issuer_key_id,revision=p.revision,id=issuer+':'+revision; + if(seen.has(id)&&seen.get(id)!==observed.canonical_sha256)fail('repair_status_conflict');seen.set(id,observed.canonical_sha256); + for(const row of p.entries){const key=[issuer,row.scope_kind,row.scope_id].join(':'),wanted=required.get(key);if(!wanted)continue; + const mask=wanted.operation_mask;if(row.status==='revoked'&&(row.operation_mask&mask))fail('repair_authority_revoked'); + if(mask&&row.minimum_document_revision>wanted.document_revision)fail('repair_status_revision'); + const held=floors.get(key)??[];held.push([revision,row.minimum_document_revision]);floors.set(key,held);} + } + for(const values of floors.values()){let minimum=0;for(const [,floor] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(floorv[0]>p.revision))fail('repair_status_rollback'); + const mask=wanted.operation_mask;if(mask&&row.status==='active'&&(row.operation_mask&mask)===mask)covered.add(key);}} + const expected=[...required].filter(([,v])=>v.operation_mask).map(([k])=>k);if(covered.size!==expected.length||expected.some(k=>!covered.has(k)))fail('repair_status_missing'); + return Object.freeze(current); +} +/** Authenticate/decrypt metadata and check present authority before fetching + * the envelope. The enclosing verified feed determines which member to select. */ +export async function readMailboxAdmission(member:unknown,options:MailboxAdmissionReadOptions):Promise>{ + const names=['expectedSlot','expectedSigningKey','encryptionIdentity','readOriginal','at','policy','budget','expectedOwner','expectedSender','expectedTarget','limitPolicy','currentStatuses','knownStatuses','statusObligations']; + const a=fields(options,Object.hasOwn(options,'onStatusAuthenticated')?[...names,'onStatusAuthenticated']:names),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,owner:a.expectedOwner,sender:a.expectedSender,target:a.expectedTarget,limits:a.limitPolicy,at:a.at,identity:a.encryptionIdentity,member},policy,budget).value as Obj; + if(typeof a.readOriginal!=='function')fail('repair_invalid_context');u53(e.at);validateEncryptionIdentity(e.identity); + if(!same(e.key,e.target.signing_key))fail('repair_mailbox_member_mismatch'); + const item=fields(e.member,['sequence','admission_link_ref','sealed_core_ref']),sequence=u53(item.sequence);if(sequence>=65536)fail('repair_invalid_context'); + const originals=new Map(); + const load=async(reference:unknown,namespace='meta'):Promise=>{const ref=rawRef(reference);if(ref.namespace!==namespace||(namespace==='object'&&ref.size>6291456))fail('repair_ref_mismatch'); + const input=await a.readOriginal(ref);let size:number;try{size=Object.getOwnPropertyDescriptor(Object.getPrototypeOf(Uint8Array.prototype),'byteLength')!.get!.call(input);}catch{fail('repair_invalid_bytes');} + if(size!==ref.size)fail('repair_ref_mismatch');budget.input(size);budget.retain(size);if(!isUint8Array(input))fail('repair_invalid_bytes'); + const proto=Object.getPrototypeOf(Uint8Array.prototype),buffer=Object.getOwnPropertyDescriptor(proto,'buffer')!.get!.call(input),offset=Object.getOwnPropertyDescriptor(proto,'byteOffset')!.get!.call(input); + const raw=Uint8Array.from(Buffer.from(buffer,offset,size)); + if(budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');const key=ref.namespace+':'+ref.key,prior=originals.get(key);if(prior&&!same(prior.ref,ref))fail('repair_ref_mismatch');originals.set(key,{ref,raw});return raw;}; + const event=async(reference:unknown,kind:string,names:string):Promise<[Uint8Array,Obj]>=>{const raw=await load(reference),signed=fields(parseNewWire(raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key','slot_key',...names.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==kind||!same(p.slot_key,e.slot))fail('repair_mailbox_member_mismatch');verifyBoundedControlSignature(p,signed.proof,e.key,budget);return [raw,p];}; + const [,link]=await event(item.admission_link_ref,'admission.link','sequence message_id envelope_ref core_ref sealed_core_ref historical_manifest_ref checkpoint_ref inclusion_path'); + if(u53(link.sequence)!==sequence||!same(link.sealed_core_ref,item.sealed_core_ref))fail('repair_mailbox_member_mismatch'); + const [coreRaw,core]=await event(link.core_ref,'admission.core','core_id sequence message_id envelope_ref attempt_ref historical_manifest_ref data_resource_ref metadata_resource_ref accepted_at object_until enum_until'); + for(const id of [core.core_id,core.message_id])if(typeof id!=='string'||/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.exec(id)?.[0]!==id)fail('repair_mailbox_member_mismatch'); + if(u53(core.sequence)!==sequence||['message_id','envelope_ref','historical_manifest_ref'].some(n=>!same(core[n],link[n]))||!(u53(core.accepted_at)<=e.at&&e.atv!==coreRaw[i]))fail('repair_mailbox_member_mismatch'); + const [,checkpoint]=await event(link.checkpoint_ref,'mailbox.checkpoint','slot_binding count leaf_root frontier committed_at retain_until'); + if(u53(checkpoint.count)!==sequence+1||!(core.accepted_at<=u53(checkpoint.committed_at)&&checkpoint.committed_at<=e.at)||u53(checkpoint.retain_until)[n,checkpoint[n]])),sequence,item.sealed_core_ref.raw_sha256,link.inclusion_path,e.slot,policy,budget); + const manifestRaw=await load(core.historical_manifest_ref),manifest=parseHistoricalManifest(manifestRaw,policy,budget),m=manifest.value as Obj; + if(m.variant!=='mailbox_member'||!same(m.root_key,e.slot.root_key)||!same(m.slot_key,e.slot)||['message_id','envelope_ref','attempt_ref'].some(n=>!same(m[n],core[n])))fail('repair_mailbox_member_mismatch'); + const resolver=new LocalRawResolver(policy,budget),packs=new Map(); + for(const row of m.roles){const ref=rawRef(row.pack_ref),prior=packs.get(ref.key);if(prior&&!same(prior,ref))fail('repair_ref_mismatch');if(!prior){resolver.put('meta',ref.key,await load(ref));packs.set(ref.key,ref);}} + const setup=verifyMailboxMemberInputs({raw:manifestRaw,ref:core.historical_manifest_ref},resolver,{expectedSlot:e.slot,expectedOwner:e.owner,expectedSender:e.sender,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,acceptedAt:core.accepted_at,limitPolicy:e.limits,policy,budget}); + for(const n of ['data','metadata'])if(!same(core[n+'_resource_ref'],setup.resources[n].active.payload.resource))fail('repair_mailbox_member_mismatch'); + const deadlines=[setup.disclosure.consent_until,setup.disclosure.bootstrap_return.until,setup.destination.windows.retain_until]; + for(const n of ['slot','read','maintenance','bootstrap']){const p=setup.originals[n].payload;deadlines.push(p.expires_at);if(p.windows)deadlines.push(p.windows.read_until,p.windows.retain_until);if(n==='bootstrap')deadlines.push(p.probe_until,p.proof_until,p.upload_until);} + for(const n of ['data','metadata']){const p=setup.resources[n].active.payload;deadlines.push(p.windows.read_until,p.windows.retain_until);} + if(core.enum_until>Math.min(...deadlines))fail('repair_mailbox_member_mismatch'); + const statuses=currentRead(setup,a.currentStatuses,a.knownStatuses,a.statusObligations,e.at,policy,budget,a.onStatusAuthenticated); + const envelope=await load(core.envelope_ref,'object'); + const retained=Object.freeze([...originals.values()].map(v=>Object.freeze({ref:v.ref,get raw(){budget.output(v.raw.length);return Uint8Array.from(v.raw);}}))); + return Object.freeze({core,link,checkpoint,setup,history:manifest,current_statuses:statuses,originals:retained,get envelope(){budget.output(envelope.length);return Uint8Array.from(envelope);}}); +} diff --git a/clients/typescript/network/open-repair-receipt.ts b/clients/typescript/network/open-repair-receipt.ts index af24445..e635810 100644 --- a/clients/typescript/network/open-repair-receipt.ts +++ b/clients/typescript/network/open-repair-receipt.ts @@ -94,7 +94,7 @@ export class SavedAckReceiptPublisher{ async publishSaved(baseUrl:string,value:unknown,timeout=30):Promise{ if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); const client=this.#client,policy=this.#policy,budget=new RepairBudget(policy),r=this.#request(value,budget); - const receipt=this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; + const receipt=await this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; const receiptPayload=(parseOriginalControl(receipt.raw,policy,budget).value as Obj).payload; const authorities=verifyAckOfferBootstrapOriginal(r.bootstrap_entry,{root:r.root_entry,write:r.write_entry}, {expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedReceiptWriter:client.subject,expectedMessageId:r.message_id, diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index b2d36c5..b8a5d5b 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -41,7 +41,12 @@ "./open-repair-wire": "./open-repair-wire.ts", "./open-repair-bound": "./open-repair-bound.ts", "./open-repair-empty": "./open-repair-empty.ts", - "./open-repair-occupied": "./open-repair-occupied.ts" + "./open-repair-occupied": "./open-repair-occupied.ts", + "./open-repair-mailbox-authority": "./open-repair-mailbox-authority.ts", + "./open-repair-mailbox-member": "./open-repair-mailbox-member.ts", + "./open-repair-mailbox-feed": "./open-repair-mailbox-feed.ts", + "./open-repair-mailbox-read": "./open-repair-mailbox-read.ts", + "./open-repair-mailbox-inbox": "./open-repair-mailbox-inbox.ts" }, "types": "./crypto.ts", "engines": { diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 268b2c4..31ae18e 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -142,6 +142,11 @@ "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", "clients/typescript/network/open-repair-mailbox-range.ts", + "clients/typescript/network/open-repair-mailbox-authority.ts", + "clients/typescript/network/open-repair-mailbox-member.ts", + "clients/typescript/network/open-repair-mailbox-feed.ts", + "clients/typescript/network/open-repair-mailbox-read.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 298113c..a0c2ab0 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -123,6 +123,7 @@ "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", "tests/test_open_repair_receipt_typescript.py", + "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", @@ -243,6 +244,11 @@ "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", "clients/typescript/network/open-repair-mailbox-range.ts", + "clients/typescript/network/open-repair-mailbox-authority.ts", + "clients/typescript/network/open-repair-mailbox-member.ts", + "clients/typescript/network/open-repair-mailbox-feed.ts", + "clients/typescript/network/open-repair-mailbox-read.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index a2c633b..e084862 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -54,7 +54,7 @@ "tests.test_open_repair_index_prepare", "tests.test_open_repair_index_prepare_admin", "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", - "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +244,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 427a99d..427923d 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -46,10 +46,10 @@ "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", - "open-repair-status.ts", "open-repair-mailbox-range.ts", + "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-repair-ack.ts", "open-repair-probe.ts", - "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts")} + "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} TS_ENDPOINT_TESTS = {"tests/test_network_typescript_" + name + ".py" for name in ("nodes", "records", "vault", "peer", "peer_race", "transport", "setup", "retrieval_text", "retrieval", "agent", "agent_network", "topics")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 57) + self.assertEqual(len(TS_NETWORK), 64) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -122,9 +122,10 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_status.py", "tests/test_open_repair_status_typescript.py", "tests/open_repair_ack_fixtures.py", "tests/test_open_repair_ack.py", "tests/test_open_repair_ack_typescript.py", "tests/test_open_repair_state.py", "tests/test_open_repair_mailbox_resources.py", "tests/test_open_repair_copy_resources.py", - "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", + "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", + "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", "tests/test_open_repair_proof.py", "tests/test_open_repair_proof_typescript.py", "tests/test_open_repair_access.py", @@ -187,12 +188,13 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-wire.ts", "clients/typescript/network/open-repair-history.ts", "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", - "clients/typescript/network/open-repair-mailbox-range.ts", + "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", - "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts"}) + "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", + "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts"}) for name in repair_sources: self.assertTrue((ROOT / name).is_file()) for name in ("docs/NATIVE_OPEN_PROVIDER.md", "docs/CONTINUATION_TRIAL.md", "docs/OPEN_NETWORK_QUICKSTART.md"): diff --git a/tests/test_open_repair_mailbox_member_typescript.py b/tests/test_open_repair_mailbox_member_typescript.py index e0fc334..4b67273 100644 --- a/tests/test_open_repair_mailbox_member_typescript.py +++ b/tests/test_open_repair_mailbox_member_typescript.py @@ -17,16 +17,26 @@ const {RepairBudget,LocalRawResolver}=await import('./open-repair-wire.ts'); const {verifyMailboxMemberInputs}=await import('./open-repair-mailbox-member.ts'); const {verifyMailboxFeedSourceEvent}=await import('./open-repair-mailbox-feed.ts'); +const {readMailboxIndex,readMailboxAdmission}=await import('./open-repair-mailbox-read.ts'); +const {createHash}=await import('node:crypto'); +const {verifyMailboxInboxEvidence}=await import('./open-repair-mailbox-inbox.ts'); const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk); const calls=JSON.parse(Buffer.concat(chunks).toString()),decode=e=>({raw:Buffer.from(e.raw,'base64'),ref:e.ref}),out=[]; -for(const v of calls){let budget;try{ +for(const v of calls){let budget,objectReads=0,observed=0;try{ budget=new RepairBudget(v.policy);const resolver=new LocalRawResolver(v.policy,budget); for(const p of v.packs){const e=decode(p);resolver.put(e.ref.namespace,e.ref.key,e.raw);} + if(v.operation==='inbox'){const result=await verifyMailboxInboxEvidence(v.evidence,Buffer.from(v.envelope,'base64'),v.options);out.push({ok:true,envelopeHash:createHash('sha256').update(result.envelope).digest('hex'),message:result.core.message_id});continue;} + if(v.operation==='admission'){const originals=new Map(v.originals.map(e=>{const d=decode(e);return [JSON.stringify(d.ref),d.raw];})); + const result=await readMailboxAdmission(v.member,{...v.options,currentStatuses:v.current.map(decode),knownStatuses:v.known.map(decode),statusObligations:[],onStatusAuthenticated:()=>{observed++;},policy:v.policy,budget,readOriginal:async ref=>{if(ref.namespace==='object')objectReads++;const raw=originals.get(JSON.stringify(ref));if(!raw)throw Error('missing exact original');return raw;}}); + out.push({ok:true,envelopeHash:createHash('sha256').update(result.envelope).digest('hex'),objectReads,observed,metrics:budget.snapshot()});continue;} + if(v.operation==='index'){let reads=0;const originals=new Map(v.originals.map(e=>{const d=decode(e);return [JSON.stringify(d.ref),d.raw];})); + const entries=await readMailboxIndex(decode(v.head),decode(v.checkpoint),{...v.options,policy:v.policy,budget,readOriginal:async ref=>{reads++;if(ref.namespace!=='meta')throw Error('premature object read');const bytes=originals.get(JSON.stringify(ref));if(!bytes)throw Error('missing exact ref');return bytes;}}); + out.push({ok:true,entries,reads,metrics:budget.snapshot()});continue;} if(v.operation==='feed'){const result=verifyMailboxFeedSourceEvent(decode(v.manifest),resolver,decode(v.custody),{...v.options,policy:v.policy,budget});out.push({ok:true,count:result.graph.head.count,statuses:result.graph.statuses.map(v=>v.canonical_sha256),retain_until:result.retain_until,metrics:budget.snapshot()});continue;} const graph=verifyMailboxMemberInputs(decode(v.manifest),resolver,{...v.options,policy:v.policy,budget}); out.push({ok:true,accepted_at:graph.accepted_at,refs:Object.fromEntries(Object.entries(graph.roles).map(([k,e])=>[k,e.ref])), statuses:graph.statuses.map(v=>v.canonical_sha256),ack:graph.ack_configuration!==null,metrics:budget.snapshot()}); -}catch(e){out.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack),metrics:budget?.snapshot()});}} +}catch(e){out.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack),objectReads,observed,metrics:budget?.snapshot()});}} process.stdout.write(JSON.stringify({out,subprocessCalls})); ''' @@ -35,6 +45,18 @@ class NativeMailboxMemberTests(unittest.TestCase): def setUpClass(cls): runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) (cls.fixture/'driver.mjs').write_text(DRIVER) + from tests.test_open_ack_typescript_agent import DRIVER as agent_driver + agent_driver=agent_driver.replace('process.stdout.write(JSON.stringify({results,calls,subprocessCalls}));',r''' +let savedReceipt; +if(input.saved_receipt){ + const {OpenNetworkClient}=await import('./open-client.ts'),{OpenDeliveryClient}=await import('./open-delivery-client.ts'),{loadClient}=await import('./client-config.ts'); + const network=new OpenNetworkClient(input.network_config),delivery=new OpenDeliveryClient(network.participant,network.encryption,loadClient(input.client_config)); + try{const item=await delivery.savedReceiptForAck(input.saved_receipt.message_id,input.saved_receipt.owner,input.saved_receipt.envelope_ref);savedReceipt={raw:Buffer.from(item.raw).toString('base64'),ref:item.ref};} + finally{delivery.close();network.close();} +} +process.stdout.write(JSON.stringify({results,calls,subprocessCalls,savedReceipt})); +''') + (cls.fixture/'agent-driver.mjs').write_text(agent_driver) def native(self,calls): result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, @@ -43,8 +65,26 @@ def native(self,calls): self.assertEqual(out['subprocessCalls'],0);return out['out'] def test_committed_http_member_originals_and_denied_cross_bindings(self): - h=http_fixture.MailboxStagingHTTPTests('test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources') + self.check_committed(False) + + def test_committed_http_member_with_independent_ack_configuration(self): + self.check_committed(True) + + def check_committed(self,with_ack): + method='test_ack_configuration_survives_mailbox_custody' if with_ack else 'test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources' + h=http_fixture.MailboxStagingHTTPTests(method) h.setUp();self.addCleanup(h.doCleanups);visited=[] + from memory_vault_trust import TrustStore + from memory_vault_client import ClientConfig + TrustStore(ClientConfig.load(h.b.client_config).trust_path).add(h.ai.public_descriptor()) + call=h.call + def share(agent,**request): + if agent is h.a and (request['op']=='send' or (request.get('invitation',{}).get('action')=='prepare' and request.get('invitation',{}).get('schema_version')=='memory-vault-open-ack-connect/v1')) and not (request.get('memory_ids') or request.get('invitation',{}).get('memory_ids')): + remembered=call(h.a,op='remember',request_id='req_native_cold_memory',kind='observation',text='Synthetic native cold recovery memory.') + if request['op']=='send':request=dict(request,memory_ids=[remembered['memory_id']]) + else:request=dict(request,invitation=dict(request['invitation'],memory_ids=[remembered['memory_id']])) + return call(agent,**request) + h.call=share def inspect(staging,key,head): visited.append(True);s=staging.source rid=staging.db.execute('SELECT resource_id FROM open_repair_mailbox_roots').fetchone()[0] @@ -97,9 +137,59 @@ def entry(ref):return dict(raw=held.read(ref.as_dict()),ref=ref.as_dict()) self.assertEqual(feedresults[0]['count'],1);self.assertEqual(feedresults[0]['retain_until'],feedexpected['retain_until']) self.assertEqual(feedresults[0]['statuses'],[v.canonical_sha256 for v in feedexpected['graph']['statuses']]) for result in feedresults[1:]:self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error',result) + from memory_vault_open_repair_client import read_mailbox_index + with h.b._network() as network:encryption=network.encryption + headref=wire.raw_ref(json.loads(held.read(feedpart.manifest.as_dict()))['feed_head_ref']) + cpref=wire.raw_ref(feedexpected['graph']['head']['checkpoint_ref']) + indexoptions=dict(expectedSlot=key,expectedSigningKey=s.target['signing_key'],encryptionIdentity=encryption.private_document(),at=feedexpected['custody'].payload['stored_at'],maxMessages=10) + indexcall=json.loads(json.dumps(dict(operation='index',head=encoded(entry(headref)),checkpoint=encoded(entry(cpref)),originals=[encoded(entry(ref)) for ref in held.originals],packs=[],options=indexoptions,policy=asdict(s.policy)))) + expectedindex=read_mailbox_index(entry(headref),entry(cpref),expected_slot=key,expected_signing_key=s.target['signing_key'],encryption_identity=encryption,read_original=held.read,at=indexoptions['at'],max_messages=10,policy=s.policy) + from memory_vault_network_crypto import EncryptionIdentity + wrong=copy.deepcopy(indexcall);wrong['options']['encryptionIdentity']=EncryptionIdentity.generate().private_document() + indexresults=self.native([indexcall,wrong]);self.assertTrue(indexresults[0]['ok'],indexresults[0]);self.assertEqual(indexresults[0]['entries'],list(expectedindex));self.assertGreater(indexresults[0]['reads'],0) + self.assertFalse(indexresults[1]['ok']);self.assertEqual(indexresults[1]['code'],'repair_mailbox_range_mismatch') + from memory_vault_open_repair_client import read_mailbox_admission + statuses=list({row.original.ref.raw_sha256:dict(raw=row.original.raw,ref=row.original.ref.as_dict()) for row in resolved.roles if row.role.startswith('historical.status.') and not row.role.startswith('historical.status.ack_')}.values()) + admissionoptions=dict(indexoptions,expectedOwner=owner,expectedSender=sender,expectedTarget=s.target,limitPolicy=s.limits);admissionoptions.pop('maxMessages') + admissioncall=json.loads(json.dumps(dict(operation='admission',member=expectedindex[0],originals=indexcall['originals'],packs=[],options=admissionoptions,current=[encoded(v) for v in statuses],known=[],policy=asdict(s.policy)))) + checked=read_mailbox_admission(expectedindex[0],expected_slot=key,expected_signing_key=s.target['signing_key'],expected_owner=owner,expected_sender=sender,expected_target=s.target,limit_policy=s.limits,current_statuses=statuses,encryption_identity=encryption,read_original=held.read,at=indexoptions['at'],policy=s.policy) + from memory_vault_open_provider import issue_status + from tests.test_open_repair_status import status_entry + owner_status=next(json.loads(v['raw'])['payload'] for v in statuses if json.loads(v['raw'])['payload']['signing_key']['key_id']==h.bi.key_id) + denied=status_entry(issue_status(h.bi,root=key['root_key'],revision=99,entries=[dict(row,status='revoked') for row in owner_status['entries']],issued_at=indexoptions['at']-10,valid_until=indexoptions['at']-1)) + wrong=copy.deepcopy(admissioncall);wrong['known']=[encoded(denied)] + missing=copy.deepcopy(admissioncall);missing['current']=[encoded(v) for v in statuses if json.loads(v['raw'])['payload']['signing_key']['key_id']!=h.ai.key_id] + admissionresults=self.native([admissioncall,wrong,missing]);self.assertTrue(admissionresults[0]['ok'],admissionresults[0]) + import hashlib + self.assertEqual(admissionresults[0]['envelopeHash'],hashlib.sha256(checked['envelope']).hexdigest());self.assertEqual(admissionresults[0]['objectReads'],1) + for result,code in zip(admissionresults[1:],['repair_authority_revoked','repair_status_missing']): + self.assertFalse(result['ok'],result);self.assertEqual(result['code'],code,result);self.assertEqual(result['objectReads'],0);self.assertGreater(result['observed'],0) + from memory_vault_open_repair_client import verify_mailbox_inbox_evidence + from memory_vault_network_crypto import b64url + evidence=dict(schema_version='memory-vault-mailbox-inbox/v1',received_at=indexoptions['at'],slot=key,sender=sender,target=s.target,limits=s.limits,member=expectedindex[0],manifest_ref=feedpart.manifest.as_dict(),custody_ref=feedpart.custody.as_dict(),originals=[dict(ref=ref.as_dict(),raw_base64url=b64url(held.read(ref.as_dict()))) for ref in held.originals if ref.namespace=='meta'],status_refs=[v['ref'] for v in statuses]) + verify_mailbox_inbox_evidence(evidence,checked['envelope'],owner=owner,encryption_identity=encryption,staged_at=indexoptions['at']) + import base64 + inboxcall=json.loads(json.dumps(dict(operation='inbox',packs=[],policy=asdict(s.policy),evidence=evidence,envelope=base64.b64encode(checked['envelope']).decode(),options=dict(owner=owner,encryptionIdentity=encryption.private_document(),stagedAt=indexoptions['at'])))) + wrong=copy.deepcopy(inboxcall);wrong['evidence']['received_at']+=1 + inboxresults=self.native([inboxcall,wrong]);self.assertTrue(inboxresults[0]['ok'],inboxresults[0]);self.assertEqual(inboxresults[0]['message'],checked['core']['message_id']);self.assertFalse(inboxresults[1]['ok']);self.assertEqual(inboxresults[1]['code'],'repair_invalid_context') + # The Python recipient stops after durable staging. A fresh native + # Agent resumes that inbox and imports the original memory bytes. + from memory_vault_open_delivery import decrypt_envelope + from memory_vault_network_crypto import document + plaintext=decrypt_envelope(checked['envelope'],encryption_identity=encryption,sender_signing_key=sender['signing_key'],sender_encryption_key=sender['encryption_key'],recipient_signing_key=owner['signing_key'],recipient_encryption_key=owner['encryption_key']) + session=dict(mailbox=evidence,authority={name:document(roles[role]['raw']) for name,role in [('request','contact.request'),('policy','contact.policy')]},source_node=s.node) + with h.b._network() as network: + row=network._delivery()._stage_inbox(message_id=checked['core']['message_id'],sender_key_id=h.ai.key_id,envelope=checked['envelope'],body=plaintext,session=session) + self.assertEqual(row['phase'],'staged') + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'agent-driver.mjs')],cwd=self.fixture,input=json.dumps(dict(client_config=str(h.b.client_config),network_config=str(h.b.network_config),requests=[dict(op='receive',limit=1),dict(op='receive',message_id=checked['core']['message_id'])],no_network=True,saved_receipt=dict(message_id=checked['core']['message_id'],owner=sender,envelope_ref=checked['core']['envelope_ref']))).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=40) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);native=json.loads(result.stdout);self.assertEqual(native['subprocessCalls'],0);self.assertEqual(native['calls'],[]) + for result in native['results']:self.assertTrue(result['ok'],result) + messages=native['results'][0]['result']['messages'];self.assertEqual(len(messages),1,messages);self.assertEqual(messages[0]['state'],'validated_saved');self.assertEqual(messages[0]['content_kind'],'memory_transfer');self.assertEqual(messages[0]['share']['records_added'],1) + with h.b._network() as network: + row=network._delivery()._inbox(checked['core']['message_id']);self.assertEqual(row['phase'],'saved');self.assertIsNotNone(row['receipt']);self.assertFalse(row['receipt_sent']);self.assertEqual(base64.b64decode(native['savedReceipt']['raw']),bytes(row['receipt'])) for label,result in zip(labels[1:],results[1:]): with self.subTest(label=label):self.assertFalse(result['ok'],result);self.assertNotEqual(result['code'],'untyped_error',result) h.inspect_committed_mailbox=inspect - h.test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources();self.assertEqual(visited,[True]) + getattr(h,method)();self.assertEqual(visited,[True]) if __name__=='__main__':unittest.main() From c052c5a784959d4001bfdd9b4c602bd0b7369ddf Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:14:40 +0800 Subject: [PATCH 11/16] Receive registered remote mailbox memories in native Agent --- clients/typescript/network/README.md | 49 +++--- clients/typescript/network/open-client.ts | 5 +- .../network/open-delivery-client.ts | 36 +++- .../typescript/network/open-mailbox-client.ts | 160 ++++++++++++++++++ .../network/open-mailbox-journal.ts | 41 +++++ .../network/open-mailbox-receivers.ts | 67 ++++++++ clients/typescript/network/package.json | 5 +- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 4 +- scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 8 +- tests/test_open_mailbox_client_typescript.py | 132 +++++++++++++++ 12 files changed, 478 insertions(+), 35 deletions(-) create mode 100644 clients/typescript/network/open-mailbox-client.ts create mode 100644 clients/typescript/network/open-mailbox-journal.ts create mode 100644 clients/typescript/network/open-mailbox-receivers.ts create mode 100644 tests/test_open_mailbox_client_typescript.py diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index a378e96..1266908 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -278,22 +278,33 @@ verified status observations across restarts, including failed recovery. See [request formats and commands](../../../docs/OPEN_ACK_RECOVERY.md#native-recovery-commands-development-after-alpha019). This command entry is development after the frozen alpha.0.19 archives. -### Retained mailbox inbox recovery (development) - -A native `Agent.handle({op: "receive", limit: 1})` can resume an ordinary mailbox transfer -already durably staged by the Python client in the same protected inbox. It -rechecks the original owner/contact approvals, activated storage resources, -complete feed prefix, recipient-encrypted directory and admission core, and -READ observations recorded at receipt time. A retained revocation or missing -sender permission blocks the object read. The original memory share then -uses the existing Vault import and its idempotent transfer receipt. The saved -receipt remains available to an explicit `connect/return_receipt` request; -mailbox access does not grant an independent receipt return. - -`readMailboxIndex` and `readMailboxAdmission` accept a caller-authorized exact -reference reader. They support asynchronous reads without delegating to Python. -`verifyMailboxInboxEvidence` reopens the retained ordinary-mailbox format without -network access. This is historical local recovery, not a claim about current -source availability. Native discovery/download of a new remote mailbox and the -replica inbox format remain separate unfinished work; use the Python client for -those paths. These additions are development source, not yet a public release. +### Registered mailbox recovery (development) + +A native `Agent.handle({op: "connect", invitation})` accepts the existing +`memory-vault-open-mailbox-connect/v1` `register`, `list`, and `remove` actions. +Registration retains an explicitly supplied owner slot, READ and maintenance +grants, feed bootstrap, expected sender/target keys, limits and signed source +endpoint. Registration itself performs no network requests. Python and native +clients share the same bounded receiver registry and retained status journal. + +`Agent.handle({op: "receive", limit: 1})` checks registered ordinary mailboxes, +downloads and authenticates the complete feed prefix, opens the recipient-encrypted +directory and admission core, then downloads the exact authorized body. It checks +current READ permissions before fetching the body and preserves authenticated +revocations across process restarts. Existing approvals and storage grants remain +required. A missing ordinary delivery lease does not prevent authorized recovery +from the mailbox's retained copy. + +Downloaded messages use the existing durable inbox and Vault import. A new +process can also resume an ordinary mailbox transfer staged by either client, +without network access, rechecking its original approvals and received-time +permissions. Memory transfers retain their idempotent import receipt. The saved +recipient receipt is available to explicit `connect/return_receipt` with separate +return authority; mailbox registration does not supply that authority. + +The native SDK exposes `MailboxFeedRecoveryClient` and `MailboxSetupJournal` for +bounded feed/body recovery, and `readMailboxIndex`, `readMailboxAdmission`, and +`verifyMailboxInboxEvidence` for exact-reference/local recovery. These paths do +not delegate to a Python process. Native mailbox provisioning and replica inbox +recovery remain unfinished; use the Python client for those paths. These additions +are development source, not yet a public release. diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index 082ab86..21176ca 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -6,6 +6,7 @@ import {absolutePath,readPrivate,NetworkError} from './io.ts'; import {loadClient} from './client-config.ts'; import type {Client} from './client-config.ts'; import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {RegisteredMailboxReceivers,MAILBOX_CONNECT_SCHEMA} from './open-mailbox-receivers.ts'; import {OpenParticipant} from './open-participant.ts'; import {OpenContactClient,CONNECT_SCHEMA} from './open-contact-client.ts'; import type {SignedNode} from './open-control.ts'; @@ -36,6 +37,8 @@ export class OpenNetworkClient{ private result(value:Record):Record{if(canonicalBytes(value).length>8192)fail('network_response_too_large');return value;} async connect(invitation?:unknown,requestId?:string):Promise>{ if(invitation!=null){ + if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA) + return this.result(new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).connect(invitation)); if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA) return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); if(typeof invitation!=='object'||Array.isArray(invitation)||(invitation as any).schema_version!==CONNECT_SCHEMA)fail('open_private_invitation_unsupported'); @@ -54,7 +57,7 @@ export class OpenNetworkClient{ return this.result(await delivery.send(requestId,recipients,text,memoryIds,control, (row,deadline)=>recoverPrepared(this.participant,this.encryption,delivery,row,deadline))); } - async receive(limit=4):Promise>{return this.result(await this.delivery().receive(limit));} + async receive(limit=4):Promise>{return this.result(await new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).receive(limit));} readMessage(messageId:string,offset=0):Record{return this.result(this.delivery().readMessage(messageId,offset));} respondTo(..._args:any[]):never{return fail('open_hint_exchange_unsupported');} readReceivedBatch(..._args:any[]):never{return fail('open_hint_exchange_unsupported');} diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 5c28e98..5aacd71 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -3,6 +3,8 @@ */ import {randomBytes} from 'node:crypto'; import {verifyMailboxInboxEvidence} from './open-repair-mailbox-inbox.ts'; +import {MailboxFeedRecoveryClient} from './open-mailbox-client.ts'; +import type {MailboxRecoverOptions} from './open-mailbox-client.ts'; import {performance} from 'node:perf_hooks'; import type {DatabaseSync} from 'node:sqlite'; import {canonicalBytes,document,objectFields,sha256,encodeBase64url,decodeBase64url,opaqueId,safeInteger, @@ -42,7 +44,8 @@ function preview(text:string):string{ const MAX_LOCAL_BYTES=256*1024*1024,MAX_SESSION_BYTES=32768,MAX_INBOX_SESSION_BYTES=65536, MAX_INTENT_BYTES=49152,MAX_RESULT_BYTES=16384; class DeliveryBudget{ - readonly deadline=performance.now()/1000+60; + readonly deadline:number; + constructor(deadline?:number){this.deadline=Math.min(performance.now()/1000+60,deadline??Infinity);if(!Number.isFinite(this.deadline))fail('open_delivery_budget_exhausted',true);} requests=0;requestBytes=0;responseBytes=0; check(){if(performance.now()/1000>=this.deadline||this.requests>256||this.requestBytes+this.responseBytes>64*1024*1024)fail('open_delivery_budget_exhausted',true);} request(bytes:number){this.check();if(this.requests>=256||this.requestBytes+this.responseBytes+bytes>64*1024*1024)fail('open_delivery_budget_exhausted',true);this.requests++;this.requestBytes+=bytes;} @@ -446,6 +449,28 @@ export class OpenDeliveryClient{ this.stageInbox(entry.message_id,session.request.payload.signing_key.key_id,envelope,body, {intent:original,storage_receipt:handle.storage_receipt,source_node:handle.source_node});return [await this.finishInbox(entry.message_id),true]; } + async receiveMailbox(reader:MailboxFeedRecoveryClient,baseUrl:string,options:MailboxRecoverOptions,limit=4):Promise{ + if(!(reader instanceof MailboxFeedRecoveryClient)||!same(reader.subject,{signing_key:validateSigningIdentity(this.participant.identity),encryption_key:validateEncryptionIdentity(this.encryption)}))fail('open_delivery_key_binding_mismatch'); + if(!Number.isSafeInteger(limit)||limit<1||limit>4)fail('network_invalid_receive_limit'); + return this.serial(async()=>{ + const deadline=performance.now()/1000+(options.timeout??60),feed=await reader.recover(baseUrl,options),messages:Obj[]=[]; + for(const member of feed.entries){if(messages.length>=limit)break; + const candidates=(feed.source.manifest.manifest.value as Obj).members.filter((v:Obj)=>same(v.admission_link_ref,member.admission_link_ref));if(candidates.length!==1)fail('repair_proof_mismatch'); + const selected=candidates[0],prior=this.inbox(selected.message_id);let result:Obj,fresh:boolean; + if(prior){if(prior.envelope_sha256!==selected.envelope_ref.raw_sha256)fail('network_inbox_identity_conflict');fresh=prior.phase==='staged';result=await this.finishInbox(selected.message_id);} + else{ + const remaining=Math.min(30,deadline-performance.now()/1000);if(remaining<=0)fail('open_delivery_budget_exhausted',true); + const recovered=await reader.readMember(baseUrl,feed,member,{knownStatuses:options.knownStatuses,timeout:remaining}),roles=recovered.setup.roles; + const authority=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([n,r])=>[n,document(roles[r].raw)])); + const session={authority,source_node:document((options.targetNodeEntry as Obj).raw),mailbox:reader.inboxEvidence(feed,member)},envelope=recovered.envelope; + const body=await decryptEnvelope(envelope,{...OpenDeliveryClient.keys(session),encryption_identity:this.encryption} as any); + this.stageInbox(selected.message_id,authority.request.payload.signing_key.key_id,envelope,body,session);result=await this.finishInbox(selected.message_id);fresh=true; + } + if(fresh)messages.push(result);if(result.state==='validated_saved')this.savedReceipt(selected.message_id); + } + return {messages,errors:[],state:'observed',body_transport:'mailbox_retained_copy',receipt_state:'retained_for_independent_return'}; + }); + } private pendingReceiptBatch():Obj[]{return this.db(db=>{ const select="SELECT message_id,phase,created_at FROM open_delivery_inbox WHERE (phase='staged' OR (phase='saved' AND receipt_sent=0))", order=' ORDER BY created_at,message_id LIMIT ?', @@ -460,13 +485,14 @@ export class OpenDeliveryClient{ const created=safeInteger(row.created_at),id=opaqueId(row.message_id); this.db(db=>db.prepare('INSERT OR REPLACE INTO open_delivery_retry_cursor VALUES(1,?,?)').run(created,id)); } - async receive(limit=4):Promise{if(!Number.isSafeInteger(limit)||limit<1||limit>4)fail('network_invalid_receive_limit');return this.serial(()=>this.receiveInternal(limit));} - private async receiveInternal(limit:number):Promise{ - const budget=new DeliveryBudget(),messages:Obj[]=[],errors:Obj[]=[]; - const pending=this.pendingReceiptBatch(); + async receive(limit=4,options:{pendingOnly?:boolean;skipPending?:boolean;deadline?:number}={}):Promise{if(!Number.isSafeInteger(limit)||limit<1||limit>4)fail('network_invalid_receive_limit');return this.serial(()=>this.receiveInternal(limit,options));} + private async receiveInternal(limit:number,options:{pendingOnly?:boolean;skipPending?:boolean;deadline?:number}):Promise{ + const budget=new DeliveryBudget(options.deadline),messages:Obj[]=[],errors:Obj[]=[]; + const pending=options.skipPending?[]:this.pendingReceiptBatch(); for(const row of pending){if(messages.length>=limit)break;try{const result=await this.finishInbox(row.message_id);if(row.phase==='staged')messages.push(result); if(result.state==='validated_saved')await this.sendReceipt(row.message_id,budget);}catch(error){errors.push({message_id:row.message_id,...errorData(error)});} finally{this.advancePendingReceipt(row);}} + if(options.pendingOnly)return {messages,errors:errors.slice(0,4),network_accessed:budget.requests>0}; for(const candidate of this.contactSessions(false)){if(messages.length>=limit)break; try{const session=await this.checkedSession(candidate,budget),node=session.node,leaseId=session.decision.payload.grant.payload.resource_lease.payload.lease_id; let [key,after]=this.cursor(session);const intent=readerIntent('list',{lease_id:leaseId,caller_key_id:this.participant.keyId}); diff --git a/clients/typescript/network/open-mailbox-client.ts b/clients/typescript/network/open-mailbox-client.ts new file mode 100644 index 0000000..f02cf05 --- /dev/null +++ b/clients/typescript/network/open-mailbox-client.ts @@ -0,0 +1,160 @@ +/** Bounded native HTTP recovery of an original mailbox feed and message body. */ +import {performance} from 'node:perf_hooks'; +import {OpenHTTPTransport,endpoint} from './open-transport.ts'; +import {RepairBudget,RepairError,buildNewWire,parseNewWire,objectFields,rawRef,u53,LocalRawResolver} from './open-repair-wire.ts'; +import type {RawOriginal,RawRef,RepairPolicy} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import {validateSigningIdentity,validateEncryptionIdentity,encodeBase64url} from './crypto.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {verifyMailboxFeedSourceEvent} from './open-repair-mailbox-feed.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {verifyBootstrapProofResponse,makeBootstrapChildRequest,makeMailboxBodyRequest} from './open-repair-proof.ts'; +import {authenticateStatusOriginal,verifyStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +import {readMailboxIndex,readMailboxAdmission} from './open-repair-mailbox-read.ts'; +import {MailboxSetupJournal} from './open-mailbox-journal.ts'; +type Obj=Record; +const monotonic=()=>performance.now()/1000; +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function fail(code:string):never{throw new RepairError(code);} +function options(v:unknown,required:string[],optional:string[]):Obj{ + if(!v||typeof v!=='object')fail('repair_invalid_context');return fields(v,[...required,...optional.filter(n=>Object.hasOwn(v,n))]); +} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +const id=(v:unknown)=>{const r=rawRef(v);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +function snapshot(v:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{const e=fields(v,['raw','ref']),ref=rawRef(e.ref),raw=parseNewWire(e.raw,policy,budget).raw;if(ref.namespace!=='meta'||ref.size!==raw.length||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {ref,raw};} +function statusPreview(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{ + const p=fields(fields(parseNewWire(raw,policy,budget).value,['payload','proof']).payload,['schema_version','kind','signing_key','scope_key','revision','issued_at','valid_until','entries']); + if(!Array.isArray(p.entries)||p.entries.length<1||p.entries.length>16)fail('repair_invalid_status');for(const row of p.entries)fields(row,['scope_kind','scope_id','status','minimum_document_revision','operation_mask']);return p; +} +function floors(retained:readonly Obj[],current:readonly Obj[],obligations:readonly Obj[]):void{ + const required=new Map(obligations.map(v=>[[v.signer.key_id,v.kind,v.scope_id].join(':'),v])),seen=new Map(),values=new Map(); + for(const observed of [...retained,...current]){const p=observed.payload,issuer=p.scope_key.issuer_key_id,revision=p.revision,key=issuer+':'+revision; + if(seen.has(key)&&seen.get(key)!==observed.canonical_sha256)fail('repair_status_conflict');seen.set(key,observed.canonical_sha256); + for(const row of p.entries){const k=[issuer,row.scope_kind,row.scope_id].join(':'),wanted=required.get(k); + if(wanted&&row.status==='revoked'&&(row.operation_mask&wanted.mask))fail('repair_authority_revoked'); + if(wanted&&row.minimum_document_revision>wanted.revision)fail('repair_status_revision'); + const held=values.get(k)??[];held.push([revision,row.minimum_document_revision]);values.set(k,held);} + } + for(const held of values.values()){let minimum=0;for(const [,v] of held.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(vv[0]>p.revision))fail('repair_status_rollback');} +} +export interface MailboxClientOptions{readonly policy?:RepairPolicy;readonly limitPolicy?:unknown;readonly allowLoopback?:boolean;readonly transport?:OpenHTTPTransport;readonly clock?:()=>number;readonly statusObserver?:(item:AuthenticatedStatusOriginal)=>void;readonly networkObserver?:()=>void;} +export interface MailboxRecoverOptions{readonly targetNodeEntry:unknown;readonly expectedTarget:unknown;readonly expectedSender:unknown;readonly expectedSlot:unknown;readonly slotEntries:unknown;readonly journal:MailboxSetupJournal;readonly knownStatuses?:readonly unknown[];readonly archiveStatuses?:readonly unknown[];readonly timeout?:number;} +export interface RecoveredMailboxFeed{readonly source:Readonly;readonly proof:Obj;readonly current_statuses:readonly AuthenticatedStatusOriginal[];readonly originals:readonly RawOriginal[];readonly entries:readonly Obj[];readonly metrics:Readonly;readonly base_url:string;} +export class MailboxFeedRecoveryClient{ + readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #limits:Obj; + readonly #transport:OpenHTTPTransport;readonly #ownTransport:boolean;readonly #loopback:boolean;readonly #clock:()=>number;readonly #observer?: (v:AuthenticatedStatusOriginal)=>void;readonly #networkObserver?:()=>void; + readonly #feeds=new WeakMap(); + constructor(identity:unknown,encryptionIdentity:unknown,value:MailboxClientOptions={}){ + const a=options(value,[],['policy','limitPolicy','allowLoopback','transport','clock','statusObserver','networkObserver']);const budget=new RepairBudget(a.policy??{...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512});this.#policy=budget.policy; + const held=buildNewWire({identity,encryption:encryptionIdentity,limits:a.limitPolicy??DEFAULT_REPAIR_CLIENT_LIMITS},this.#policy,budget).value as Obj; + this.#identity=held.identity;this.#encryption=held.encryption;this.#subject=buildNewWire({signing_key:validateSigningIdentity(held.identity),encryption_key:validateEncryptionIdentity(held.encryption)},this.#policy,budget).value as Obj; + this.#limits=fields(held.limits,Object.keys(DEFAULT_REPAIR_CLIENT_LIMITS));for(const n of Object.keys(this.#limits))u53(this.#limits[n],1); + if((a.allowLoopback!==undefined&&typeof a.allowLoopback!=='boolean')||(a.clock!==undefined&&typeof a.clock!=='function')||(a.statusObserver!==undefined&&typeof a.statusObserver!=='function')||(a.networkObserver!==undefined&&typeof a.networkObserver!=='function'))fail('repair_invalid_policy'); + this.#loopback=a.allowLoopback??false;this.#clock=a.clock??(()=>Date.now()/1000);this.#observer=a.statusObserver;this.#networkObserver=a.networkObserver;this.#transport=a.transport??new OpenHTTPTransport({allow_loopback:this.#loopback});this.#ownTransport=a.transport===undefined;Object.freeze(this); + } + get subject():Readonly{return this.#subject;} + get limits():Readonly{return this.#limits;} + inboxEvidence(feed:RecoveredMailboxFeed,member:unknown):Readonly{ + const c=this.#feeds.get(feed);if(!c||!feed.entries.some(v=>same(v,member)))fail('repair_invalid_context'); + const children=(c.held.manifest.value as Obj).children,reference=(role:string)=>{const found=children.filter((v:Obj)=>v.role===role);if(found.length!==1)fail('repair_proof_mismatch');return found[0].ref;}; + return Object.freeze({schema_version:'memory-vault-mailbox-inbox/v1',received_at:this.#now(),slot:c.e.slot,sender:c.e.sender,target:c.e.target,limits:this.#limits,member, + manifest_ref:reference('history.mailbox_feed'),custody_ref:reference('feed.custody'),originals:feed.originals.map(v=>({ref:v.ref,raw_base64url:encodeBase64url(v.raw)})),status_refs:feed.current_statuses.map(v=>v.ref)}); + } + close():void{if(this.#ownTransport)this.#transport.close();} + #now():number{return u53(Math.floor(this.#clock()));} + #timeout(v:unknown):number{if(typeof v!=='number'||!Number.isFinite(v)||v<=0||v>60)fail('repair_invalid_deadline');return v;} + #observe(journal:MailboxSetupJournal,key:string,item:AuthenticatedStatusOriginal):void{journal.observe(key,item);this.#observer?.(item);} + #known(entries:readonly unknown[],e:Obj,at:number,budget:RepairBudget,journal:MailboxSetupJournal,key:string):AuthenticatedStatusOriginal[]{ + if(!Array.isArray(entries)||entries.length>32)fail('repair_status_history_capacity');const result=[]; + for(const item of entries){const entry=snapshot(item,this.#policy,budget),p=statusPreview(entry.raw,this.#policy,budget),signer=p.signing_key,permitted=new Set(); + if(same(signer,this.#subject.signing_key)){permitted.add('mailbox_slot');permitted.add('authority');} + if(same(signer,e.sender.signing_key))permitted.add('authority');if(same(signer,e.target.signing_key))permitted.add('resource'); + if(!permitted.size||p.entries.some((v:Obj)=>!permitted.has(v.scope_kind))||u53(p.issued_at)>at)fail('repair_status_disclosure'); + const observed=authenticateStatusOriginal(entry,{expectedRoot:e.slot.root_key,expectedSigningKey:signer,at:p.issued_at,allowedScopes:p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),policy:this.#policy,budget},v=>this.#observe(journal,key,v)); + result.push(observed);if(p.entries.some((v:Obj)=>v.status==='revoked'&&(v.operation_mask&10)))fail('repair_authority_revoked'); + }return result; + } + async recover(baseUrl:string,value:MailboxRecoverOptions):Promise{ + const a=options(value,['targetNodeEntry','expectedTarget','expectedSender','expectedSlot','slotEntries','journal'],['knownStatuses','archiveStatuses','timeout']); + if(!(a.journal instanceof MailboxSetupJournal))fail('repair_invalid_context');const journal=a.journal as MailboxSetupJournal; + const timeout=this.#timeout(a.timeout??60),policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),deadline=monotonic()+timeout; + const e=buildNewWire({slot:a.expectedSlot,target:a.expectedTarget,sender:a.expectedSender},policy,budget).value as Obj; + const setup=verifyMailboxFeedBootstrap(a.slotEntries,{expectedSlot:e.slot,expectedOwner:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,limitPolicy:this.#limits,at:started,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:e.sender.signing_key.key_id,encryption_key_id:e.sender.encryption_key.key_id}))fail('repair_proof_mismatch'); + const nodeEntry=snapshot(a.targetNodeEntry,policy,budget),node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:e.target.signing_key,expectedStorageEpoch:e.slot.writer_storage_epoch,at:started,policy,budget}); + if(!same(endpoint(baseUrl,this.#loopback),endpoint(node.payload.base_url,this.#loopback)))fail('repair_proof_mismatch'); + const journalKey=journal.start({kind:'mailbox.feed_recovery',slot_key:e.slot,owner:this.#subject,sender:e.sender,target:e.target,entries:Object.fromEntries(Object.entries(setup).map(([n,v])=>[n,v.ref]))}); + const retained=new Map();for(const group of [a.knownStatuses??[],a.archiveStatuses??[],journal.statuses(journalKey)]){ + if(!Array.isArray(group)||group.length>32)fail('repair_status_history_capacity');for(const item of group){const entry=snapshot(item,policy,budget);retained.set(entry.ref.raw_sha256+':'+entry.ref.size,entry);}} + const known=this.#known([...retained.values()],e,started,budget,journal,journalKey),grant=setup.bootstrap.payload; + const expiry=Math.min(started+Math.min(60,Math.max(1,Math.floor(timeout))),node.payload.expires_at as number,grant.probe_until,grant.proof_until,...Object.values(setup).map(v=>v.payload.expires_at as number));if(expiry<=started)fail('repair_access_expired'); + const counts={requests:0,wire_bytes:0,proof_bytes:0}; + const request=async(raw:Uint8Array,child=false):Promise=>{if(counts.requests>=grant.limits.max_requests||monotonic()>=deadline)fail('repair_over_budget');counts.requests++; + this.#networkObserver?.();const reply=await this.#transport.requestRepair(baseUrl,raw,deadline,{child});counts.wire_bytes+=raw.length+reply.length;return reply;}; + const binding={expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,bootstrapGrantSha256:setup.bootstrap.ref.raw_sha256,selector:grant.selector,consumer:'mailbox_feed' as const,policy,budget}; + const outgoing=await makeBootstrapProbe(this.#identity,{...binding,at:started,expiresAt:expiry}),outgoingRaw=outgoing.original.raw;if(outgoingRaw.length>grant.limits.max_probe_bytes)fail('repair_over_budget'); + const challengeRaw=await request(outgoingRaw),digest=budget.hash(challengeRaw),challenge={raw:challengeRaw,ref:rawRef({namespace:'meta',key:digest,raw_sha256:digest,size:challengeRaw.length})}; + const preview=fields(parseNewWire(challengeRaw,policy,budget).value,['payload','proof']); + const answer=await solveBootstrapChallenge({raw:outgoingRaw,ref:outgoing.original.ref},challenge,{...binding,signer:this.#identity,encryptionIdentity:this.#encryption,targetNonce:outgoing.nonce,at:this.#now(),expiresAt:Math.min(expiry,u53(preview.payload?.expires_at))}); + const response=await request(answer.raw),held=verifyBootstrapProofResponse(response,{expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,selector:grant.selector,bootstrapGrantRef:setup.bootstrap.ref, + probeRef:outgoing.original.ref,challengeRef:challenge.ref,answerRef:answer.ref,at:this.#now(),maxProofItems:grant.limits.max_proof_items,maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'feed',consumer:'mailbox_feed',policy,budget}); + counts.proof_bytes=response.length+held.handle.raw.length+held.manifest.raw.length; + const originals=new Map(),roles=new Map(),available=new Map(); + for(const entry of [...Object.values(setup),...known,nodeEntry])available.set(id(entry.ref),entry.raw); + const children=(held.manifest.value as Obj).children; + for(const item of children){const reference=rawRef(item.ref);roles.set(item.role,[...(roles.get(item.role)??[]),reference]);} + const download=async(item:Obj):Promise=>{const ref=rawRef(item.ref),key=id(ref);if(originals.has(key))return; + if(ref.size>policy.max_document_bytes||counts.proof_bytes+ref.size>grant.limits.max_proof_bytes)fail('repair_over_budget');let raw=available.get(key); + if(raw)budget.input(raw.length);else{const chunks:Uint8Array[]=[];let offset=0;while(offset{for(const row of tree.roles){const key=id(row.original.ref),raw=row.original.raw,prior=available.get(key);if(prior&&!Buffer.from(prior).equals(Buffer.from(raw)))fail('repair_ref_conflict');available.set(key,raw);}for(const child of tree.predecessors)reuse(child);}; + for(const ref of roles.get('history.mailbox_feed')??[]){const tree=resolveHistoricalInputs(originals.get(id(ref))!.raw,packed,policy,budget);if((tree.manifest.value as Obj).variant!=='mailbox_feed')fail('repair_proof_mismatch');reuse(tree);} + for(const item of children)await download(item); + const entry=(ref:unknown)=>{const item=originals.get(id(ref));if(!item)fail('repair_original_missing');return item;}; + const one=(role:string)=>{const refs=roles.get(role);if(!refs||refs.length!==1)fail('repair_proof_mismatch');return entry(refs[0]);}; + const source=verifyMailboxFeedSourceEvent(one('history.mailbox_feed'),packed,one('feed.custody'),{expectedSlot:e.slot,expectedOwner:this.#subject,expectedSender:e.sender,expectedTarget:e.target,limitPolicy:this.#limits,policy,budget}); + for(const member of source.graph.members)for(const [name,parent] of Object.entries(setup)){const actual=member.originals[name];if(!same(actual.ref,parent.ref)||!Buffer.from(actual.raw).equals(Buffer.from(parent.raw)))fail('repair_proof_mismatch');} + const actual=new Set(),wanted=new Set();for(const [role,refs] of roles)if(!role.startsWith('current.status.'))for(const ref of refs)actual.add(role+'|'+id(ref)); + for(const tree of [source.manifest,...source.manifest.predecessors]){for(const row of tree.roles)wanted.add(row.role+'|'+id(row.original.ref));for(const row of tree.manifest.value.roles)wanted.add('history.raw_pack|'+id(row.pack_ref));} + for(const role of ['history.mailbox_feed','feed.custody'])wanted.add(role+'|'+id(one(role).ref));if(actual.size!==wanted.size||[...actual].some(k=>!wanted.has(k)))fail('repair_proof_mismatch'); + const duties=source.graph.obligations.map((v:Obj)=>({role:'current.status.'+v.role,kind:v.scope_kind,scope_id:v.scope_id,revision:v.document_revision,signer:v.signer,mask:['slot','maintenance','bootstrap'].includes(v.role)?10:2})); + const allowed=new Map();for(const member of source.graph.members)for(const v of member.obligations)allowed.set([v.signer.key_id,v.scope_kind,v.scope_id].join(':'),v); + const current:AuthenticatedStatusOriginal[]=[],covered=new Set(); + for(const [role,refs] of roles){if(!role.startsWith('current.status.'))continue;for(const ref of refs){const item=entry(ref),p=statusPreview(item.raw,policy,budget),present=new Set(p.entries.map((v:Obj)=>v.scope_kind+':'+v.scope_id)),permitted=duties.filter((v:Obj)=>same(v.signer,p.signing_key)),matched=permitted.filter((v:Obj)=>v.role===role&&present.has(v.kind+':'+v.scope_id)); + if(!matched.length)fail('repair_status_disclosure');const observed=verifyStatusOriginal(item,{expectedRoot:e.slot.root_key,expectedSigningKey:matched[0].signer,at:this.#now(),allowedScopes:[...allowed.values()].filter(v=>v.signer.key_id===p.signing_key.key_id).map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id})), + required:permitted.filter((v:Obj)=>present.has(v.kind+':'+v.scope_id)).map((v:Obj)=>({scope_kind:v.kind,scope_id:v.scope_id,document_revision:v.revision,operation_mask:v.mask})),policy,budget},v=>this.#observe(journal,journalKey,v)); + current.push(observed);for(const v of matched)covered.add([v.role,v.kind,v.scope_id].join(':'));}} + const expectedCoverage=new Set(duties.map((v:Obj)=>[v.role,v.kind,v.scope_id].join(':')));if(covered.size!==expectedCoverage.size||[...expectedCoverage].some(k=>!covered.has(k)))fail('repair_status_missing'); + const historic=[...source.graph.statuses,...source.graph.members.flatMap((v:Obj)=>v.statuses)];floors([...known,...historic],current,duties); + const members=await readMailboxIndex(one('feed.head'),one('feed.checkpoint'),{expectedSlot:e.slot,expectedSigningKey:e.target.signing_key,encryptionIdentity:this.#encryption as any,readOriginal:ref=>entry(ref).raw,at:this.#now(),maxMessages:setup.slot.payload.max_appends,policy,budget}); + const until=Math.min(expiry,held.handle.payload.expires_at as number,source.read_until,source.retain_until,...current.map(v=>v.payload.valid_until as number));if(this.#now()>=until||monotonic()>=deadline)fail('repair_access_expired'); + verifyMailboxFeedBootstrap(Object.fromEntries(Object.entries(setup).map(([n,v])=>[n,{ref:v.ref,raw:v.raw}])),{expectedSlot:e.slot,expectedOwner:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,limitPolicy:this.#limits,at:this.#now(),policy,budget}); + const result=Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:Object.freeze([...originals.values()].map(v=>Object.freeze({ref:v.ref,get raw(){return Uint8Array.from(v.raw);}}))),entries:members,metrics:Object.freeze({...counts,...budget.snapshot()}),base_url:baseUrl}); + this.#feeds.set(result,{e,setup,source,held,originals,journal,journalKey,current,counts,until,budget});return result; + } + async readMember(baseUrl:string,feed:RecoveredMailboxFeed,member:unknown,value:{knownStatuses?:readonly unknown[];timeout?:number}={}):Promise>{ + const c=this.#feeds.get(feed),a=options(value,[],['knownStatuses','timeout']);if(!c||!same(endpoint(baseUrl,this.#loopback),endpoint(feed.base_url,this.#loopback)))fail('repair_proof_mismatch'); + const {e,source,held,originals,journal,journalKey,current,counts,budget}=c,policy=this.#policy,timeout=this.#timeout(a.timeout??30),deadline=monotonic()+timeout; + const selected=buildNewWire(member,policy,budget).value;if(!feed.entries.some(v=>same(v,selected)))fail('repair_invalid_context'); + const guard=()=>{if(this.#now()>=c.until||monotonic()>=deadline)fail('repair_access_expired');};guard(); + const known=a.knownStatuses??[];if(!Array.isArray(known)||known.length>32)fail('repair_status_history_capacity'); + const retained=new Map();for(const item of [...known,...journal.statuses(journalKey)]){const entry=snapshot(item,policy,budget);retained.set(entry.ref.raw_sha256+':'+entry.ref.size,entry);}if(retained.size>32)fail('repair_status_history_capacity'); + const item=selected as Obj;const link=originals.get(id(item.admission_link_ref));if(!link)fail('repair_original_missing');const linkValue=fields(parseNewWire(link.raw,policy,budget).value,['payload','proof']).payload; + const children=(held.manifest.value as Obj).children.filter((v:Obj)=>v.role==='member.core'&&same(v.ref,linkValue.core_ref));if(children.length!==1)fail('repair_proof_mismatch'); + const readOriginal=async(ref:RawRef):Promise=>{if(ref.namespace==='meta'){const original=originals.get(id(ref));if(!original)fail('repair_original_missing');return original.raw;} + guard();if(counts.proof_bytes+ref.size>c.setup.bootstrap.payload.limits.max_proof_bytes)fail('repair_over_budget');const chunks:Uint8Array[]=[];let offset=0; + while(offset=c.setup.bootstrap.payload.limits.max_requests)fail('repair_over_budget');const count=Math.min(65536,ref.size-offset),packet=makeMailboxBodyRequest(this.#identity,held,ref,{subject:this.#subject,target:e.target,at:this.#now(),expiresAt:held.handle.payload.expires_at as number,childIndex:children[0].index,offset,requestedBytes:count,policy,budget}); + counts.requests++;this.#networkObserver?.();const raw=await this.#transport.requestRepair(baseUrl,packet.raw,deadline,{child:true});counts.wire_bytes+=packet.raw.length+raw.length;if(raw.length!==count)fail('repair_ref_mismatch');budget.input(raw.length);chunks.push(raw);counts.proof_bytes+=raw.length;offset+=raw.length;} + guard();budget.output(ref.size);return Buffer.concat(chunks);}; + const result=await readMailboxAdmission(selected,{expectedSlot:e.slot,expectedSigningKey:e.target.signing_key,expectedOwner:this.#subject,expectedSender:e.sender,expectedTarget:e.target,encryptionIdentity:this.#encryption as any,readOriginal,at:this.#now(), + currentStatuses:current.map((v:AuthenticatedStatusOriginal)=>({ref:v.ref,raw:v.raw})),knownStatuses:[...retained.values()],statusObligations:source.graph.obligations,onStatusAuthenticated:v=>this.#observe(journal,journalKey,v),limitPolicy:this.#limits,policy,budget});guard();return result; + } +} diff --git a/clients/typescript/network/open-mailbox-journal.ts b/clients/typescript/network/open-mailbox-journal.ts new file mode 100644 index 0000000..f8d3a65 --- /dev/null +++ b/clients/typescript/network/open-mailbox-journal.ts @@ -0,0 +1,41 @@ +/** Bounded durable mailbox setup/status journal shared with the Python client. */ +import type {OpenParticipant} from './open-participant.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +import {canonicalBytes,sha256} from './crypto.ts'; +import {RepairError} from './open-repair-wire.ts'; +import type {RawOriginal} from './open-repair-wire.ts'; +import {transaction} from './io.ts'; +type Obj=Record; +function fail(code:string):never{throw new RepairError(code);} +export class MailboxSetupJournal{ + readonly #participant:OpenParticipant; + constructor(participant:OpenParticipant){this.#participant=participant;participant.providerStorage(db=>db.exec(` + CREATE TABLE IF NOT EXISTS open_mailbox_setup_jobs(job_key TEXT PRIMARY KEY,plan BLOB NOT NULL,blocked INTEGER NOT NULL DEFAULT 0); + CREATE TABLE IF NOT EXISTS open_mailbox_setup_steps(job_key TEXT NOT NULL,stage TEXT NOT NULL,request BLOB NOT NULL,response BLOB,PRIMARY KEY(job_key,stage)); + CREATE TABLE IF NOT EXISTS open_mailbox_setup_statuses(job_key TEXT NOT NULL,digest TEXT NOT NULL,raw BLOB NOT NULL,PRIMARY KEY(job_key,digest));`));} + start(plan:unknown):string{ + const raw=canonicalBytes(plan,65536),key=sha256(raw); + this.#participant.providerStorage(db=>transaction(db,()=>{ + const old=db.prepare('SELECT plan,blocked FROM open_mailbox_setup_jobs WHERE job_key=?').get(key) as Obj|undefined; + if(old){if(!Buffer.from(old.plan).equals(Buffer.from(raw)))fail('repair_setup_journal_conflict');if(old.blocked)fail('repair_setup_journal_capacity');return;} + const size=db.prepare('SELECT count(*) AS n FROM open_mailbox_setup_jobs').get() as Obj;if(size.n>=16)fail('repair_setup_journal_capacity'); + db.prepare('INSERT INTO open_mailbox_setup_jobs(job_key,plan) VALUES(?,?)').run(key,raw); + }));return key; + } + #check(db:any,key:string):void{if(!/^[0-9a-f]{64}$/.test(key))fail('repair_invalid_context');const row=db.prepare('SELECT blocked FROM open_mailbox_setup_jobs WHERE job_key=?').get(key);if(!row||row.blocked)fail('repair_setup_journal_unavailable');} + statuses(key:string):readonly RawOriginal[]{return this.#participant.providerStorage(db=>{ + this.#check(db,key);const rows=db.prepare('SELECT digest,raw FROM open_mailbox_setup_statuses WHERE job_key=? ORDER BY digest LIMIT 33').all(key) as Obj[]; + if(rows.length>32||rows.reduce((n,v)=>n+v.raw.length,0)>262144)fail('repair_setup_journal_capacity'); + return rows.map(v=>{const raw=Uint8Array.from(v.raw);if(sha256(raw)!==v.digest)fail('repair_storage_corrupt');return {raw,ref:{namespace:'meta' as const,key:v.digest,raw_sha256:v.digest,size:raw.length}};}); + });} + observe(key:string,item:AuthenticatedStatusOriginal):void{ + const raw=item.raw,digest=item.raw_sha256;if(raw.length>16384||sha256(raw)!==digest)fail('repair_invalid_context'); + const saved=this.#participant.providerStorage(db=>transaction(db,()=>{ + this.#check(db,key);const old=db.prepare('SELECT raw FROM open_mailbox_setup_statuses WHERE job_key=? AND digest=?').get(key,digest) as Obj|undefined; + if(old){if(!Buffer.from(old.raw).equals(Buffer.from(raw)))fail('repair_storage_corrupt');return true;} + const size=db.prepare('SELECT count(*) AS n,coalesce(sum(length(raw)),0) AS size FROM open_mailbox_setup_statuses WHERE job_key=?').get(key) as Obj; + if(size.n>=32||size.size+raw.length>262144){db.prepare('UPDATE open_mailbox_setup_jobs SET blocked=1 WHERE job_key=?').run(key);return false;} + db.prepare('INSERT INTO open_mailbox_setup_statuses VALUES(?,?,?)').run(key,digest,raw);return true; + }));if(!saved)fail('repair_setup_journal_capacity'); + } +} diff --git a/clients/typescript/network/open-mailbox-receivers.ts b/clients/typescript/network/open-mailbox-receivers.ts new file mode 100644 index 0000000..f99cb55 --- /dev/null +++ b/clients/typescript/network/open-mailbox-receivers.ts @@ -0,0 +1,67 @@ +/** Existing explicit mailbox registration and ordinary Agent receive polling. */ +import {performance} from 'node:perf_hooks'; +import {canonicalBytes,document,objectFields,sha256,opaqueId,validateSigningIdentity,validateEncryptionIdentity} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {MailboxFeedRecoveryClient} from './open-mailbox-client.ts'; +import {MailboxSetupJournal} from './open-mailbox-journal.ts'; +import {RepairBudget,RepairError,parseNewWire,rawRef} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {endpoint} from './open-transport.ts'; +import {transaction} from './io.ts'; +type Obj=Record; +export const MAILBOX_CONNECT_SCHEMA='memory-vault-open-mailbox-connect/v1'; +function fail(code:string):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +export class RegisteredMailboxReceivers{ + readonly #participant:OpenParticipant;readonly #encryption:EncryptionIdentityDocument;readonly #delivery:OpenDeliveryClient; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient){this.#participant=participant;this.#encryption=encryption;this.#delivery=delivery; + participant.providerStorage(db=>db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_receivers(receiver_id TEXT PRIMARY KEY,body BLOB NOT NULL,last_attempt INTEGER NOT NULL DEFAULT 0)'));} + #decode(config:Obj):Obj{ + const entry=(v:unknown)=>{const e=objectFields(v,['raw','ref']);if(typeof e.raw!=='string')fail('open_invalid_mailbox_receiver');return {raw:Buffer.from(e.raw,'utf8'),ref:e.ref};}; + const slots=objectFields(config.slot_entries,['slot','read','maintenance','bootstrap']); + return {expectedSlot:config.expected_slot,expectedSender:config.expected_sender,expectedTarget:config.expected_target,targetNodeEntry:entry(config.target_node_entry),slotEntries:Object.fromEntries(Object.entries(slots).map(([n,v])=>[n,entry(v)]))}; + } + #validate(value:Obj):Obj{ + objectFields(value,['schema_version','action','base_url','limit_policy','expected_slot','expected_sender','expected_target','target_node_entry','slot_entries']);if(value.schema_version!==MAILBOX_CONNECT_SCHEMA||value.action!=='register')fail('open_invalid_mailbox_receiver'); + const options=this.#decode(value),policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512},budget=new RepairBudget(policy),at=Math.floor(Date.now()/1000),owner={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)}; + const setup=verifyMailboxFeedBootstrap(options.slotEntries,{expectedSlot:value.expected_slot,expectedOwner:owner,expectedTarget:value.expected_target,targetStorageEpoch:value.expected_slot.writer_storage_epoch,limitPolicy:value.limit_policy,at,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:value.expected_sender.signing_key.key_id,encryption_key_id:value.expected_sender.encryption_key.key_id}))fail('open_invalid_mailbox_receiver'); + const ref=rawRef(options.targetNodeEntry.ref),raw=parseNewWire(options.targetNodeEntry.raw,policy,budget).raw; + if(ref.namespace!=='meta'||ref.size!==raw.length||budget.hash(raw)!==ref.raw_sha256)fail('open_invalid_mailbox_receiver'); + const node=verifySourceNodeOriginal(raw,{expectedSigningKey:value.expected_target.signing_key,expectedStorageEpoch:value.expected_slot.writer_storage_epoch,at,policy,budget}); + if(!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url,this.#participant.transport.allow_loopback)))fail('open_invalid_mailbox_receiver');return options; + } + connect(invitation:unknown):Obj{ + const value=document(invitation as any,65536) as Obj;if(value.schema_version!==MAILBOX_CONNECT_SCHEMA)fail('open_invalid_mailbox_receiver'); + if(value.action==='list'){objectFields(value,['schema_version','action']);const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id FROM open_mailbox_receivers ORDER BY receiver_id LIMIT 17').all()) as Obj[];if(rows.length>16)fail('open_mailbox_receiver_capacity');return {state:'configured',mailboxes:rows.map(v=>v.receiver_id),network_accessed:false};} + if(value.action==='remove'){objectFields(value,['schema_version','action','receiver_id']);opaqueId(value.receiver_id);this.#participant.providerStorage(db=>db.prepare('DELETE FROM open_mailbox_receivers WHERE receiver_id=?').run(value.receiver_id));return {state:'removed',receiver_id:value.receiver_id,network_accessed:false};} + this.#validate(value);const receiver='mailbox_'+sha256(canonicalBytes(value.expected_slot)),raw=canonicalBytes(value); + this.#participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?').get(receiver) as Obj|undefined; + if(old){if(!Buffer.from(old.body).equals(Buffer.from(raw)))fail('open_mailbox_receiver_conflict');return;} + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_receivers').get() as Obj).n>=16)fail('open_mailbox_receiver_capacity');db.prepare('INSERT INTO open_mailbox_receivers(receiver_id,body) VALUES(?,?)').run(receiver,raw); + }));return {state:'registered',receiver_id:receiver,network_accessed:false,receipt_return:'separate_authority_required'}; + } + async receive(limit:number):Promise{ + const deadline=performance.now()/1000+60,result=await this.#delivery.receive(limit,{pendingOnly:true,deadline}); + const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id,body FROM open_mailbox_receivers ORDER BY last_attempt,receiver_id LIMIT 17').all()) as Obj[]; + if(rows.length>16)fail('open_mailbox_receiver_capacity'); + for(const row of rows){let remaining=deadline-performance.now()/1000;if(result.messages.length>=limit||remaining<=0)break; + this.#participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receivers SET last_attempt=? WHERE receiver_id=?').run(BigInt(Date.now())*1000000n,row.receiver_id)); + let reader:MailboxFeedRecoveryClient|undefined; + try{ + const config=document(row.body,65536) as Obj;if(row.receiver_id!=='mailbox_'+sha256(canonicalBytes(config.expected_slot)))fail('open_invalid_mailbox_receiver'); + const options=this.#validate(config),journal=new MailboxSetupJournal(this.#participant); + reader=new MailboxFeedRecoveryClient(this.#participant.identity,this.#encryption,{limitPolicy:config.limit_policy,allowLoopback:this.#participant.transport.allow_loopback,transport:this.#participant.transport,networkObserver:()=>{result.network_accessed=true;}}); + remaining=deadline-performance.now()/1000;if(remaining<=0)fail('repair_access_expired');const received=await this.#delivery.receiveMailbox(reader,config.base_url,{...options,journal,timeout:Math.min(60,remaining)} as any,limit-result.messages.length); + result.messages.push(...received.messages);result.errors.push(...received.errors); + }catch(error){result.errors.push({receiver_id:row.receiver_id,code:(error as any)?.code??'network_storage_unavailable',retryable:(error as any)?.retryable===true});} + finally{reader?.close();} + } + if(result.messages.length({raw:Buffer.from(e.raw,'base64'),ref:e.ref}); +const calls=[],original=OpenHTTPTransport.prototype.requestRepair;let denyHttp=false; +OpenHTTPTransport.prototype.requestRepair=async function(...args){const p=JSON.parse(Buffer.from(args[1]).toString());calls.push(p.kind??p.payload?.kind);if(denyHttp)throw Error('network after retained revocation');return original.apply(this,args);}; +let network,client;const results=[]; +try{ + network=new OpenNetworkClient(v.network_config);const journal=new MailboxSetupJournal(network.participant); + const make=()=>new MailboxFeedRecoveryClient(network.identity,network.encryption,{policy:v.policy,limitPolicy:v.limits,allowLoopback:true});client=make(); + const opts={...v.options,targetNodeEntry:decode(v.node),slotEntries:Object.fromEntries(Object.entries(v.slots).map(([n,e])=>[n,decode(e)])),journal}; + const start=performance.now(),feed=await client.recover(v.base,opts),body=await client.readMember(v.base,feed,feed.entries[0]); + const plain=await decryptEnvelope(body.envelope,{sender_signing_key:v.options.expectedSender.signing_key,sender_encryption_key:v.options.expectedSender.encryption_key,recipient_signing_key:validateSigningIdentity(network.identity),recipient_encryption_key:validateEncryptionIdentity(network.encryption),encryption_identity:network.encryption}); + results.push({ok:true,count:feed.entries.length,envelopeHash:createHash('sha256').update(body.envelope).digest('hex'),plaintext:Buffer.from(plain).toString(),requests:feed.metrics.requests,seconds:(performance.now()-start)/1000}); + let code;const before=calls.length;try{await client.readMember(v.base,{...feed},feed.entries[0]);}catch(e){code=e.code;}results.push({forged:code,calls:calls.length-before}); + denyHttp=true;const count=calls.length;try{await client.recover(v.base,{...opts,knownStatuses:[decode(v.revoked)]});results.push({revoked:'accepted'});}catch(e){results.push({revoked:e.code,calls:calls.length-count});} + client.close();client=make();const prior=calls.length;try{await client.recover(v.base,opts);results.push({restart:'accepted'});}catch(e){results.push({restart:e.code,calls:calls.length-prior});} +}catch(e){results.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack)});} +finally{client?.close();network?.close();} +process.stdout.write(JSON.stringify({results,calls,subprocessCalls})); +''' + +class NativeMailboxClientTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + from tests.test_open_ack_typescript_agent import DRIVER as agent_driver + (cls.fixture/'agent-driver.mjs').write_text(agent_driver) + + def test_native_http_feed_body_and_durable_denial(self): + self.check_http(False) + + def test_registered_agent_downloads_imports_and_retains_receipt(self): + self.check_http(True) + + def test_registered_agent_recovers_mailbox_with_independent_ack(self): + self.check_http(True,True) + + def check_http(self,agent_mode,with_ack=False): + method='test_ack_configuration_survives_mailbox_custody' if with_ack else 'test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources' + h=fixture.MailboxStagingHTTPTests(method);h.setUp();self.addCleanup(h.doCleanups);visited=[] + if agent_mode: + from memory_vault_trust import TrustStore + from memory_vault_client import ClientConfig + TrustStore(ClientConfig.load(h.b.client_config).trust_path).add(h.ai.public_descriptor()) + original_call=h.call + def share(agent,**request): + if agent is h.a and (request['op']=='send' or (request.get('invitation',{}).get('action')=='prepare' and request.get('invitation',{}).get('schema_version')=='memory-vault-open-ack-connect/v1')): + remembered=original_call(h.a,op='remember',request_id='req_native_http_memory',kind='observation',text='Synthetic remotely recovered memory.') + if request['op']=='send':request=dict(request,memory_ids=[remembered['memory_id']]) + else:request=dict(request,invitation=dict(request['invitation'],memory_ids=[remembered['memory_id']])) + return original_call(agent,**request) + h.call=share + def inspect(staging,key,head): + visited.append(True);s=staging.source;rid=staging.db.execute('SELECT resource_id FROM open_repair_mailbox_roots').fetchone()[0] + snapshot=MailboxSnapshotSource(staging).load(rid,key,head,max_bytes=2_000_000,max_items=256) + part=snapshot.parts[2];budget=wire.RepairBudget(s.policy);resolver=wire.LocalRawResolver(s.policy,budget) + for role,ref in part.transfer: + if role=='history.raw_pack':resolver.put(ref.namespace,ref.key,snapshot.read(ref.as_dict())) + tree=history.resolve_historical_inputs(snapshot.read(part.manifest.as_dict()),resolver,s.policy,budget) + roles={v.role:dict(raw=v.original.raw,ref=v.original.ref.as_dict()) for v in tree.roles} + envelope=snapshot.read(snapshot.envelopes[0].as_dict());context=json.loads(envelope)['payload']['context'] + sender=dict(signing_key=h.ai.public_descriptor(),encryption_key=context['sender_encryption_key']) + from memory_vault_open_node import OpenParticipant,OpenHTTPServer + participant=OpenParticipant(s.identity,h.root/'node_0/transport',seeds=[],descriptor=s.node,encryption_identity=s.encryption_identity,allow_loopback=True,repair_policy=dict(enabled=True,limit_policy=s.limits),contact_policy=dict(enabled=True),delivery_policy=dict(enabled=True)) + server=OpenHTTPServer(('127.0.0.1',0),participant);thread=threading.Thread(target=server.serve_forever,kwargs=dict(poll_interval=.02),daemon=True);thread.start() + def close():server.shutdown();server.server_close();thread.join(timeout=3);participant.close() + self.addCleanup(close) + base='http://127.0.0.1:'+str(server.server_port) + node=signed_entry(dict(s.node['payload'],base_url=base,revision=s.node['payload']['revision']+1),s.identity,'synthetic_native_feed_node') + # The ordinary delivery transport cannot supply this message anymore. + staging.db.execute('DELETE FROM open_delivery_messages');staging.db.execute('DELETE FROM open_contact_resource_leases');staging.db.commit() + from memory_vault_open_provider import issue_status + from tests.test_open_repair_status import status_entry + p=json.loads(roles['historical.status.slot']['raw'])['payload'];now=int(time.time()) + revoked=status_entry(issue_status(h.bi,root=key['root_key'],revision=99,entries=[dict(v,status='revoked') for v in p['entries']],issued_at=now-10,valid_until=now-1)) + call=dict(network_config=str(h.b.network_config),base=base,node=encoded(node),slots={n:encoded(roles[r]) for n,r in [('slot','mailbox.slot'),('read','mailbox.read_grant'),('maintenance','mailbox.maintenance_root'),('bootstrap','bootstrap.mailbox_feed')]},options=dict(expectedSlot=key,expectedSender=sender,expectedTarget=s.target),policy=asdict(replace(s.policy,max_signature_checks=512)),limits=s.limits,revoked=encoded(revoked)) + if agent_mode: + invitation=dict(schema_version='memory-vault-open-mailbox-connect/v1',action='register',base_url=base,limit_policy=s.limits,expected_slot=key,expected_sender=sender,expected_target=s.target,target_node_entry=dict(raw=node['raw'].decode(),ref=node['ref']),slot_entries={n:dict(raw=roles[r]['raw'].decode(),ref=roles[r]['ref']) for n,r in [('slot','mailbox.slot'),('read','mailbox.read_grant'),('maintenance','mailbox.maintenance_root'),('bootstrap','bootstrap.mailbox_feed')]}) + message=json.loads(roles['delivery.attempt']['raw'])['payload']['message_id'] + def native(requests,no_network=False,allow_failure=False): + v=dict(client_config=str(h.b.client_config),network_config=str(h.b.network_config),requests=requests,no_network=no_network) + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'agent-driver.mjs')],cwd=self.fixture,input=json.dumps(v).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=75) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2000:]);out=json.loads(run.stdout);self.assertEqual(out['subprocessCalls'],0) + if no_network:self.assertEqual(out['calls'],[]) + if not allow_failure: + for row in out['results']:self.assertTrue(row['ok'],row) + return out + invalid=native([dict(op='connect',invitation=dict(invitation,base_url=base+'/wrong')),dict(op='connect',invitation=dict(invitation,expected_sender=s.target)),dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='list'))],True,True)['results'] + self.assertFalse(invalid[0]['ok']);self.assertFalse(invalid[1]['ok']);self.assertEqual(invalid[2]['result']['mailboxes'],[]) + registered=native([dict(op='connect',invitation=invitation)],True)['results'][0]['result'];self.assertEqual(registered['state'],'registered') + self.assertEqual(native([dict(op='connect',invitation=invitation)],True)['results'][0]['result'],registered) + received=native([dict(op='receive',limit=1)]) + result=received['results'][0]['result'];self.assertTrue(result['network_accessed']);self.assertEqual(result['errors'],[]);self.assertEqual(len(result['messages']),1,result) + saved=result['messages'][0];self.assertEqual(saved['state'],'validated_saved');self.assertEqual(saved['content_kind'],'memory_transfer');self.assertEqual(saved['share']['records_added'],1) + reopened=native([dict(op='receive',message_id=message),dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='list'))],True) + self.assertEqual(reopened['results'][1]['result']['mailboxes'],[registered['receiver_id']]) + with h.b._network() as network: + inbox=network._delivery()._inbox(message);self.assertEqual(inbox['phase'],'saved');self.assertEqual(inbox['receipt_sent'],0);self.assertIsNotNone(inbox['receipt']) + # Python reopens the exact native evidence and authenticates its saved receipt. + network._delivery()._verify_mailbox_inbox(network._delivery()._inbox_session(inbox['session']),inbox) + self.assertEqual(network._delivery()._saved_receipt(message),json.loads(inbox['receipt'])) + self.assertEqual(network._mailbox_saved_receiver(registered['receiver_id']),invitation) + removed=native([dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='remove',receiver_id=registered['receiver_id'])),dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='list'))],True) + self.assertEqual(removed['results'][1]['result']['mailboxes'],[]) + return + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture,input=json.dumps(call).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=75) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout);self.assertEqual(out['subprocessCalls'],0) + self.assertEqual(len(out['results']),4,out);good,forged,denied,restarted=out['results'];self.assertTrue(good['ok'],out);self.assertEqual(good['envelopeHash'],hashlib.sha256(envelope).hexdigest());self.assertIn('Synthetic mailbox staging message',good['plaintext']);self.assertEqual(good['count'],1) + self.assertIn('mailbox.body_read',out['calls']);self.assertEqual(forged,dict(forged='repair_proof_mismatch',calls=0));self.assertEqual(denied,dict(revoked='repair_authority_revoked',calls=0));self.assertEqual(restarted,dict(restart='repair_authority_revoked',calls=0)) + h.inspect_committed_mailbox=inspect;getattr(h,method)();self.assertEqual(visited,[True]) + +if __name__=='__main__':unittest.main() From 0614aa4b92bea70ddab308a14375f4ba750fba67 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:19:30 +0800 Subject: [PATCH 12/16] Prepare alpha36 native remote mailbox recovery candidate --- clients/typescript/network/README.md | 4 ++-- memory_vault.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index 1266908..a4314db 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -278,7 +278,7 @@ verified status observations across restarts, including failed recovery. See [request formats and commands](../../../docs/OPEN_ACK_RECOVERY.md#native-recovery-commands-development-after-alpha019). This command entry is development after the frozen alpha.0.19 archives. -### Registered mailbox recovery (development) +### Registered mailbox recovery (alpha.0.36 candidate) A native `Agent.handle({op: "connect", invitation})` accepts the existing `memory-vault-open-mailbox-connect/v1` `register`, `list`, and `remove` actions. @@ -307,4 +307,4 @@ bounded feed/body recovery, and `readMailboxIndex`, `readMailboxAdmission`, and `verifyMailboxInboxEvidence` for exact-reference/local recovery. These paths do not delegate to a Python process. Native mailbox provisioning and replica inbox recovery remain unfinished; use the Python client for those paths. These additions -are development source, not yet a public release. +are in the alpha.0.36 candidate; public release acceptance remains pending. diff --git a/memory_vault.py b/memory_vault.py index 0c1e1f5..0b5d092 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.35" +VERSION = "0.28.0-alpha.0.36" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" From 5945c9ae86355867ca1a23f09a76f0cf45d4ffd0 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:22:03 +0800 Subject: [PATCH 13/16] Bound the expanded native mailbox release inventory at 640 files --- scripts/release_source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/release_source.py b/scripts/release_source.py index e1f1f93..1c8c0fa 100644 --- a/scripts/release_source.py +++ b/scripts/release_source.py @@ -20,9 +20,10 @@ MAX_SOURCE_BYTES = 2 * 1024 * 1024 MAX_TREE_BYTES = 2 * 1024 * 1024 MAX_TREE_ENTRIES = 8192 -# The explicit native ACK runtime/review inventory now exceeds 608 files. +# The explicit native mailbox runtime/review inventory adds eleven files +# to the alpha35 inventory and now exceeds 624 selected source files. # Keep a finite source-count ceiling independent of the unchanged byte caps. -MAX_SELECTED_FILES = 624 +MAX_SELECTED_FILES = 640 MAX_SELECTED_BYTES = 32 * 1024 * 1024 GIT_TIMEOUT_SECONDS = 15 _SHA = re.compile(r"[0-9a-f]{40}") From bd3ca97ee56ca59a85c87eb1395e738e1a8c56ca Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:23:35 +0800 Subject: [PATCH 14/16] Align alpha36 plugin and installation guides with mailbox recovery --- packaging/CLIENT_README.md | 16 ++++++++++++++-- packaging/PROTOCOL_README.md | 16 ++++++++++++++-- packaging/RELEASE_NOTES.md | 18 +++++++++++++++--- packaging/REVIEW_README.md | 14 +++++++++++++- .../.codex-plugin/plugin.json | 2 +- 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 1bc2989..9704411 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 — authorized full client +# Memory Vault v0.28.0-alpha.0.36 — authorized full client + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected @@ -136,7 +148,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.35 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.36 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 15e70c4..0446a56 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 — independent protocol +# Memory Vault v0.28.0-alpha.0.36 — independent protocol + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected @@ -153,7 +165,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.35; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.36; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index f35a776..7af802a 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 — native saved receipt return and restart recovery +# Memory Vault v0.28.0-alpha.0.36 — native remote mailbox memory recovery + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected @@ -162,7 +174,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 6648812..e49b3c6 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 independent review kit +# Memory Vault v0.28.0-alpha.0.36 independent review kit + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index 6f856be..a74930e 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.35", + "version": "0.28.0-alpha.0.36", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" From b82d66bb607207f8328eae08bd85e5aa5e7e8a7a Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:50:38 +0800 Subject: [PATCH 15/16] Return retained cold mailbox receipts during native Agent receive --- clients/typescript/network/README.md | 24 ++++ clients/typescript/network/open-ack-client.ts | 1 + clients/typescript/network/open-client.ts | 5 +- .../network/open-delivery-client.ts | 15 ++- .../network/open-mailbox-receipts.ts | 113 +++++++++++++++++ .../network/open-mailbox-receivers.ts | 4 + .../typescript/network/open-repair-receipt.ts | 5 +- clients/typescript/network/package.json | 3 +- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 4 +- scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 8 +- tests/test_open_delivery_http.py | 3 + .../test_open_mailbox_receipts_typescript.py | 115 ++++++++++++++++++ 14 files changed, 289 insertions(+), 17 deletions(-) create mode 100644 clients/typescript/network/open-mailbox-receipts.ts create mode 100644 tests/test_open_mailbox_receipts_typescript.py diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index a4314db..b8ed1df 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -308,3 +308,27 @@ bounded feed/body recovery, and `readMailboxIndex`, `readMailboxAdmission`, and not delegate to a Python process. Native mailbox provisioning and replica inbox recovery remain unfinished; use the Python client for those paths. These additions are in the alpha.0.36 candidate; public release acceptance remains pending. + +### Retained receipt destinations (development after alpha.0.36) + +The native Agent accepts `memory-vault-open-ack-connect/v1` actions +`register_mailbox_receipt_return`, `list_mailbox_receipt_returns`, +`inspect_mailbox_receipt_return` and `remove_mailbox_receipt_return`. +Registration retains an exact `message_id`, `source_url`, `source_key_id` and +`repair_profile` (`receipt` or `receipt-index`) locally. It can happen before +reception; it grants no upload or disclosure permission. + +Ordinary `receive` attempts up to two ready, saved-message returns within its +shared deadline. It reopens the mailbox evidence and original independent ACK +configuration, proves the selected source, and persists the exact preparation +before returning the actual saved receipt. A lost reply or process restart +resumes the same original request. Authenticated original-root status history +remains durable; retained denials stop a pending upload before retransmission. +Expired uncertain returns become `reconciliation_required` and stop polling. + +The bounded selection and return journals use the same protected SQLite tables +as Python. Removing a selection preserves the saved message and original return +history. Re-registering a completed selection can reuse authenticated local +history without another upload. Direct `return_mailbox_receipt` supports the same +message and source selection. Native replica inbox support remains unfinished. +These additions are development source after the frozen alpha.0.36 candidate. diff --git a/clients/typescript/network/open-ack-client.ts b/clients/typescript/network/open-ack-client.ts index e5f1222..e25e7b9 100644 --- a/clients/typescript/network/open-ack-client.ts +++ b/clients/typescript/network/open-ack-client.ts @@ -21,6 +21,7 @@ type Obj=Record; export const ACK_CONNECT_SCHEMA='memory-vault-open-ack-connect/v1'; const PROFILES={receipt:{...DEFAULT_REPAIR_CLIENT_LIMITS,max_signature_checks:2048,max_proof_bytes:1048576}, 'receipt-index':{...DEFAULT_REPAIR_CLIENT_LIMITS,max_signature_checks:4096,max_proof_bytes:4194304,max_requests:256,max_replay_records:256}}; +export {PROFILES as ACK_REPAIR_PROFILES}; function fail(code:string):never{throw new NetworkError(code);} function decode(value:unknown):Obj{ const entry=objectFields(value,['raw','ref']); diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index 21176ca..a3abe78 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -7,6 +7,7 @@ import {loadClient} from './client-config.ts'; import type {Client} from './client-config.ts'; import {OpenDeliveryClient} from './open-delivery-client.ts'; import {RegisteredMailboxReceivers,MAILBOX_CONNECT_SCHEMA} from './open-mailbox-receivers.ts'; +import {MailboxReceiptJobs,MAILBOX_RECEIPT_ACTIONS,returnMailboxReceipt} from './open-mailbox-receipts.ts'; import {OpenParticipant} from './open-participant.ts'; import {OpenContactClient,CONNECT_SCHEMA} from './open-contact-client.ts'; import type {SignedNode} from './open-control.ts'; @@ -39,8 +40,10 @@ export class OpenNetworkClient{ if(invitation!=null){ if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA) return this.result(new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).connect(invitation)); - if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA) + if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA){ + if(MAILBOX_RECEIPT_ACTIONS.has((invitation as any).action))return this.result((invitation as any).action==='return_mailbox_receipt'?await returnMailboxReceipt(this.participant,this.encryption,this.delivery(),invitation):new MailboxReceiptJobs(this.participant,this.encryption,this.delivery()).connect(invitation)); return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); + } if(typeof invitation!=='object'||Array.isArray(invitation)||(invitation as any).schema_version!==CONNECT_SCHEMA)fail('open_private_invitation_unsupported'); const result=await new OpenContactClient(this.participant,this.encryption).dispatch(invitation,requestId); return this.result({...result,profile:'open-routing-v1',network_accessed:true,open_messaging_supported:true}); diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 5aacd71..8ddb1f3 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -347,12 +347,12 @@ export class OpenDeliveryClient{ if(!Object.hasOwn(session,'mailbox'))return document(session,MAX_INBOX_SESSION_BYTES) as Obj; return objectFields(session,['mailbox','authority','source_node']); } - private async verifyMailboxInbox(session:Obj,row:Obj):Promise{ + private async verifyMailboxInbox(session:Obj,row:Obj):Promise{ const verified=await verifyMailboxInboxEvidence(session.mailbox,row.envelope,{owner:{signing_key:validateSigningIdentity(this.participant.identity), encryption_key:validateEncryptionIdentity(this.encryption)},encryptionIdentity:this.encryption,stagedAt:row.created_at}); if(verified.core.message_id!==row.message_id||session.mailbox.sender.signing_key.key_id!==row.sender)fail('network_inbox_identity_conflict'); const expected=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([name,role])=>[name,document(verified.setup.roles[role].raw)])); - if(!same(session.authority,expected))fail('network_inbox_identity_conflict'); + if(!same(session.authority,expected))fail('network_inbox_identity_conflict');return verified; } private async finishInbox(messageId:string):Promise{ const row=this.inbox(messageId);if(!row)fail('network_message_not_found');if(['saved','rejected'].includes(row.phase))return document(row.result,MAX_RESULT_BYTES); @@ -389,8 +389,15 @@ export class OpenDeliveryClient{ db.prepare('UPDATE open_delivery_inbox SET receipt=? WHERE message_id=?').run(encoded,messageId);return receipt; })); } - /** Read the actually saved inbox receipt for an explicitly requested return. - * The caller cannot substitute receipt bytes or a saved-state flag. */ + /** Reauthenticate the saved mailbox evidence before deriving return authority. */ + async savedMailboxForAck(messageId:string):Promise{ + if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); + const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); + const session=OpenDeliveryClient.inboxSession(row.session);if(!Object.hasOwn(session,'mailbox'))fail('open_ack_mailbox_configuration_missing'); + const verified=await this.verifyMailboxInbox(session,row);if(!verified.setup.ack_configuration)fail('open_ack_mailbox_configuration_missing'); + return {owner:session.mailbox.sender,roles:Object.fromEntries(Object.entries(verified.setup.roles).map(([n,e]:[string,any])=>[n,{ref:e.ref,raw:e.raw}]))}; + } + /** Read the actual saved receipt; never accept substituted receipt bytes. */ async savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):Promise<{raw:Uint8Array;ref:Obj}>{ if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); diff --git a/clients/typescript/network/open-mailbox-receipts.ts b/clients/typescript/network/open-mailbox-receipts.ts new file mode 100644 index 0000000..9be05aa --- /dev/null +++ b/clients/typescript/network/open-mailbox-receipts.ts @@ -0,0 +1,113 @@ +/** Explicit retained receipt destinations and original-authority cold returns. */ +import {performance} from 'node:perf_hooks'; +import {canonicalBytes,document,objectFields,sha256,encodeBase64url,decodeBase64url} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import {NetworkError,transaction} from './io.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import type {OpenDeliveryClient} from './open-delivery-client.ts'; +import {endpoint} from './open-transport.ts'; +import type {OpenHTTPTransport} from './open-transport.ts'; +import {coordinate,verifyNode} from './open-control.ts'; +import {LookupBudget} from './open-routing.ts'; +import {OpenProviderClient} from './open-provider-client.ts'; +import {AckReceiptClient} from './open-repair-offer-client.ts'; +import {SavedAckReceiptPublisher,SAVED_ACK_REQUEST_FIELDS} from './open-repair-receipt.ts'; +import {ACK_CONNECT_SCHEMA,ACK_REPAIR_PROFILES} from './open-ack-client.ts'; +import {OriginalAckStatusJournal} from './open-ack-status.ts'; +type Obj=Record; +const monotonic=()=>performance.now()/1000; +const FIELDS=['schema_version','action','message_id','source_url','source_key_id','repair_profile']; +const SCHEMA='memory-vault-open-saved-ack-request/v1',MAX_BUNDLE=1048576; +export const MAILBOX_RECEIPT_ACTIONS=new Set(['return_mailbox_receipt','register_mailbox_receipt_return','list_mailbox_receipt_returns','inspect_mailbox_receipt_return','remove_mailbox_receipt_return']); +function fail(code:string,retryable=false):never{throw new NetworkError(code,retryable);} +function message(v:unknown):string{if(typeof v!=='string'||!/^msg_[0-9a-f]{64}$/.test(v))fail('open_invalid_ack_request');return v;} +const equal=(a:unknown,b:unknown)=>Buffer.from(canonicalBytes(a)).equals(Buffer.from(canonicalBytes(b))); +function selection(v:unknown,participant:OpenParticipant,action:string):Obj{ + const r=objectFields(v,FIELDS);if(r.schema_version!==ACK_CONNECT_SCHEMA||r.action!==action)fail('open_invalid_ack_request'); + message(r.message_id);coordinate(r.source_key_id); + if(typeof r.repair_profile!=='string'||!Object.hasOwn(ACK_REPAIR_PROFILES,r.repair_profile))fail('open_invalid_repair_policy'); + endpoint(r.source_url,participant.transport.allow_loopback);return {...r,source_url:r.source_url.replace(/\/$/,'')}; +} +function decodedBundle(raw:Uint8Array,base:string,profile:string):Obj{ + const bundle=objectFields(document(raw,MAX_BUNDLE),['schema_version','base_url','repair_profile','request']); + if(bundle.schema_version!==SCHEMA||bundle.base_url!==base||bundle.repair_profile!==profile)fail('open_ack_mailbox_return_corrupt'); + const r=objectFields(bundle.request,SAVED_ACK_REQUEST_FIELDS),decode=(v:unknown)=>{const e=objectFields(v,['ref','raw_base64url']);return {ref:e.ref,raw:decodeBase64url(e.raw_base64url,MAX_BUNDLE)};}; + if(!Array.isArray(r.current_statuses)||r.current_statuses.length>7)fail('open_ack_mailbox_return_corrupt'); + return Object.fromEntries(Object.entries(r).map(([n,v])=>[n,n==='current_statuses'?v.map(decode):n.endsWith('_entry')?decode(v):v])); +} +export async function returnMailboxReceipt(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient,value:unknown, + options:{deadline?:number;networkObserver?:()=>void}={}):Promise{ + objectFields(options,[...(Object.hasOwn(options,'deadline')?['deadline']:[]),...(Object.hasOwn(options,'networkObserver')?['networkObserver']:[])]); + if((options.deadline!==undefined&&!Number.isFinite(options.deadline))||(options.networkObserver!==undefined&&typeof options.networkObserver!=='function'))fail('repair_invalid_deadline'); + const deadline=Math.min(monotonic()+60,options.deadline??Infinity),remaining=()=>{const n=deadline-monotonic();if(n<=0)fail('open_delivery_budget_exhausted',true);return Math.min(60,n);}; + remaining(); + const v=selection(document(value as any,4096),participant,'return_mailbox_receipt'),base=v.source_url; + const saved=await delivery.savedMailboxForAck(v.message_id),roles=saved.roles,owner=saved.owner; + const [root,write,bootstrap]=['ack.root_authority','ack.write_grant','bootstrap.ack_offer'].map(n=>roles[n]); + const wp=document(write.raw,65536).payload as Obj,binding=sha256(canonicalBytes({message_id:v.message_id,envelope_ref:wp.envelope_ref,source_url:base,source_key_id:v.source_key_id,repair_profile:v.repair_profile,write_ref:write.ref})); + participant.providerStorage(db=>db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_ack_returns(message_id TEXT PRIMARY KEY,binding TEXT NOT NULL,request BLOB NOT NULL,request_sha256 TEXT NOT NULL)')); + const held=participant.providerStorage(db=>db.prepare('SELECT * FROM open_mailbox_ack_returns WHERE message_id=?').get(v.message_id)) as Obj|undefined; + const decodeHeld=(row:Obj)=>{if(row.binding!==binding)fail('open_ack_mailbox_return_conflict');if(row.request.length>MAX_BUNDLE||sha256(row.request)!==row.request_sha256)fail('open_ack_mailbox_return_corrupt');return decodedBundle(row.request,base,v.repair_profile);}; + const statuses=new OriginalAckStatusJournal(participant,wp.ack_slot.root_key),authorityHistory=()=>statuses.load(new Set([owner.signing_key.key_id,v.source_key_id])); + let accessed=false;const observe=()=>{accessed=true;options.networkObserver?.();}; + const transport={requestRepair:async(url:string,raw:Uint8Array,until:number,o:unknown)=>{remaining();observe();return participant.transport.requestRepair(url,raw,Math.min(until,deadline),o as any);}} as OpenHTTPTransport; + const reader=new AckReceiptClient(participant.identity,encryption,{transport,statusObserver:item=>statuses.observe(item),allowLoopback:participant.transport.allow_loopback,limitPolicy:ACK_REPAIR_PROFILES[v.repair_profile as keyof typeof ACK_REPAIR_PROFILES]}); + try{ + let request:Obj; + if(held)request=decodeHeld(held); + else{ + remaining();observe();const node=(await participant.transport.requestNode(base,Math.min(deadline,monotonic()+15))).response,p=verifyNode(node); + if(p.status!=='active'||p.signing_key.key_id!==v.source_key_id||!equal(endpoint(p.base_url,participant.transport.allow_loopback),endpoint(base,participant.transport.allow_loopback)))fail('open_ack_mailbox_return_conflict'); + participant.acceptProviderControl(node);const targetRecord=await new OpenProviderClient(participant,encryption).proveTarget(node,new LookupBudget({maximum_seconds:Math.min(10,remaining())})); + const target={signing_key:targetRecord.payload.signing_key,encryption_key:targetRecord.payload.targetEncryptionKey},raw=canonicalBytes(node),digest=sha256(raw),nodeEntry={raw,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}}; + const statuses=new Map();for(const role of ['ack_root','ack_write','ack_offer_bootstrap']){const e=roles['historical.status.'+role];statuses.set(e.ref.raw_sha256,e);} + remaining();const prepared=await reader.prepareReturn(base,{targetNodeEntry:nodeEntry,expectedTarget:target,expectedAckSlot:wp.ack_slot,expectedOwner:owner,expectedMessageId:v.message_id,expectedEnvelopeRef:wp.envelope_ref,rootEntry:root,writeEntry:write,bootstrapEntry:bootstrap,knownStatuses:[...statuses.values()],archiveStatuses:authorityHistory(),timeout:30}); + request={message_id:v.message_id,envelope_ref:wp.envelope_ref,ack_slot:wp.ack_slot,owner,target,target_node_entry:nodeEntry,root_entry:root,write_entry:write,bootstrap_entry:bootstrap,binding_entry:{ref:prepared.source.binding.ref,raw:prepared.source.binding.raw},current_statuses:prepared.current_statuses.map(e=>({ref:e.ref,raw:e.raw})),read_until:prepared.source.read_until,retain_until:prepared.source.retain_until}; + const encode=(e:Obj)=>({ref:e.ref,raw_base64url:encodeBase64url(e.raw)}),bundle=canonicalBytes({schema_version:SCHEMA,base_url:base,repair_profile:v.repair_profile,request:Object.fromEntries(Object.entries(request).map(([n,e])=>[n,n==='current_statuses'?e.map(encode):n.endsWith('_entry')?encode(e):e]))},MAX_BUNDLE); + request=participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT * FROM open_mailbox_ack_returns WHERE message_id=?').get(v.message_id) as Obj|undefined;if(old)return decodeHeld(old); + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_ack_returns').get() as Obj).n>=16)fail('open_ack_mailbox_return_capacity');db.prepare('INSERT INTO open_mailbox_ack_returns VALUES(?,?,?,?)').run(v.message_id,binding,bundle,sha256(bundle));return request; + })); + } + remaining();const result=await new SavedAckReceiptPublisher(delivery,reader).publishSaved(base,request,30,{archiveStatuses:authorityHistory(),knownDisclosureStatuses:statuses.load(new Set([participant.identity.key_id]))}); + return {state:'retained_at_ack_source',message_id:v.message_id,receipt_ref:result.source.inputs.receipt.ref,commit_ref:result.source.commit.ref,from_local_history:result.from_local_history,network_accessed:accessed}; + }finally{reader.close();} +} +export class MailboxReceiptJobs{ + readonly participant:OpenParticipant;readonly encryption:EncryptionIdentityDocument;readonly delivery:OpenDeliveryClient; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient){ + this.participant=participant;this.encryption=encryption;this.delivery=delivery; + participant.providerStorage(db=>db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_receipt_jobs(message_id TEXT PRIMARY KEY,body BLOB NOT NULL,body_sha256 TEXT NOT NULL,phase TEXT NOT NULL,result BLOB,result_sha256 TEXT,last_attempt INTEGER NOT NULL DEFAULT 0)')); + } + #body(row:Obj,name:string):Obj{const raw=row[name];if(!raw||sha256(raw)!==row[name+'_sha256'])fail('open_ack_mailbox_return_corrupt');return document(raw,8192) as Obj;} + connect(value:unknown):Obj{ + const v=document(value as any,4096) as Obj;if(v.schema_version!==ACK_CONNECT_SCHEMA)fail('open_invalid_ack_request'); + if(v.action==='register_mailbox_receipt_return'){ + const body=selection(v,this.participant,v.action),raw=canonicalBytes(body,4096); + this.participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT message_id,body,body_sha256,phase,result,result_sha256 FROM open_mailbox_receipt_jobs WHERE message_id=?').get(v.message_id) as Obj|undefined; + if(old){if(!equal(this.#body(old,'body'),body))fail('open_ack_mailbox_return_conflict');return;} + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_receipt_jobs').get() as Obj).n>=16)fail('open_ack_mailbox_return_capacity');db.prepare("INSERT INTO open_mailbox_receipt_jobs(message_id,body,body_sha256,phase) VALUES(?,?,?,'pending')").run(v.message_id,raw,sha256(raw)); + }));return {state:'registered',message_id:v.message_id,network_accessed:false,receipt_return:'ordinary_receive_after_saved_original_authority_check'}; + } + if(v.action==='list_mailbox_receipt_returns'){objectFields(v,['schema_version','action']);const rows=this.participant.providerStorage(db=>db.prepare('SELECT message_id,phase FROM open_mailbox_receipt_jobs ORDER BY message_id LIMIT 17').all()) as Obj[]; + if(rows.length>16)fail('open_ack_mailbox_return_capacity');return {state:'configured',returns:rows.map(r=>({message_id:r.message_id,state:r.phase})),network_accessed:false};} + objectFields(v,['schema_version','action','message_id']);message(v.message_id); + if(v.action==='remove_mailbox_receipt_return'){this.participant.providerStorage(db=>db.prepare('DELETE FROM open_mailbox_receipt_jobs WHERE message_id=?').run(v.message_id));return {state:'removed',message_id:v.message_id,network_accessed:false};} + if(v.action!=='inspect_mailbox_receipt_return')fail('open_invalid_ack_request'); + const row=this.participant.providerStorage(db=>db.prepare('SELECT message_id,body,body_sha256,phase,result,result_sha256 FROM open_mailbox_receipt_jobs WHERE message_id=?').get(v.message_id)) as Obj|undefined;if(!row)fail('open_ack_mailbox_return_missing'); + return {state:row.phase,selection:this.#body(row,'body'),result:row.result===null?null:this.#body(row,'result'),network_accessed:false,source_rechecked:false}; + } + async poll(result:Obj,deadline:number,attempted:Set):Promise{ + if(attempted.size>=2||monotonic()>=deadline)return; + // Keep nanosecond scheduling timestamps in SQLite; they exceed JS safe integers. + const rows=this.participant.providerStorage(db=>db.prepare("SELECT j.message_id,j.body,j.body_sha256,j.phase,j.result,j.result_sha256 FROM open_mailbox_receipt_jobs AS j JOIN open_delivery_inbox AS i ON i.message_id=j.message_id WHERE j.phase='pending' AND i.phase='saved' ORDER BY j.last_attempt,j.message_id LIMIT 3").all()) as Obj[]; + const row=rows.find(r=>!attempted.has(r.message_id));if(!row)return;const mid=row.message_id;attempted.add(mid); + this.participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receipt_jobs SET last_attempt=? WHERE message_id=?').run(BigInt(Date.now())*1000000n,mid)); + const store=(phase:string,v:Obj)=>{const raw=canonicalBytes(v,8192);this.participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receipt_jobs SET phase=?,result=?,result_sha256=? WHERE message_id=? AND body_sha256=?').run(phase,raw,sha256(raw),mid,row.body_sha256));}; + try{ + const selected=selection(this.#body(row,'body'),this.participant,'register_mailbox_receipt_return');if(selected.message_id!==mid)fail('open_ack_mailbox_return_corrupt'); + const returned=await returnMailboxReceipt(this.participant,this.encryption,this.delivery,{...selected,action:'return_mailbox_receipt'},{deadline,networkObserver:()=>{result.network_accessed=true;}}); + store('complete',returned);(result.receipt_returns??=[]).push({message_id:mid,state:returned.state,from_local_history:returned.from_local_history});result.network_accessed ||=returned.network_accessed; + }catch(error){const e=error as any,code=e?.code??'network_storage_unavailable',retryable=e?.retryable===true,phase=['repair_reconciliation_required','repair_saved_reconciliation_required'].includes(code)?'reconciliation_required':'pending'; + store(phase,{state:phase,code,retryable});result.errors.push({message_id:mid,operation:'return_mailbox_receipt',code,retryable});} + } +} diff --git a/clients/typescript/network/open-mailbox-receivers.ts b/clients/typescript/network/open-mailbox-receivers.ts index f99cb55..faf0b7b 100644 --- a/clients/typescript/network/open-mailbox-receivers.ts +++ b/clients/typescript/network/open-mailbox-receivers.ts @@ -12,6 +12,7 @@ import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; import {verifySourceNodeOriginal} from './open-repair-original.ts'; import {endpoint} from './open-transport.ts'; import {transaction} from './io.ts'; +import {MailboxReceiptJobs} from './open-mailbox-receipts.ts'; type Obj=Record; export const MAILBOX_CONNECT_SCHEMA='memory-vault-open-mailbox-connect/v1'; function fail(code:string):never{throw new RepairError(code);} @@ -47,6 +48,8 @@ export class RegisteredMailboxReceivers{ } async receive(limit:number):Promise{ const deadline=performance.now()/1000+60,result=await this.#delivery.receive(limit,{pendingOnly:true,deadline}); + const jobs=new MailboxReceiptJobs(this.#participant,this.#encryption,this.#delivery),attempted=new Set(); + await jobs.poll(result,Math.min(deadline,performance.now()/1000+30),attempted); const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id,body FROM open_mailbox_receivers ORDER BY last_attempt,receiver_id LIMIT 17').all()) as Obj[]; if(rows.length>16)fail('open_mailbox_receiver_capacity'); for(const row of rows){let remaining=deadline-performance.now()/1000;if(result.messages.length>=limit||remaining<=0)break; @@ -62,6 +65,7 @@ export class RegisteredMailboxReceivers{ finally{reader?.close();} } if(result.messages.length!sameEntry(source.inputs[name as keyof typeof source.inputs],entries[name])))fail('repair_saved_corrupt'); return Object.freeze({source,originals:Object.freeze([...originals.values()].map(e=>Object.freeze({ref:e.ref,get raw(){return Uint8Array.from(e.raw);}}))),from_local_history:true}); } - async publishSaved(baseUrl:string,value:unknown,timeout=30):Promise{ + async publishSaved(baseUrl:string,value:unknown,timeout=30,history:{archiveStatuses?:readonly unknown[];knownDisclosureStatuses?:readonly unknown[]}={}):Promise{ + objectFields(history,[...(Object.hasOwn(history,'archiveStatuses')?['archiveStatuses']:[]),...(Object.hasOwn(history,'knownDisclosureStatuses')?['knownDisclosureStatuses']:[])]); if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); const client=this.#client,policy=this.#policy,budget=new RepairBudget(policy),r=this.#request(value,budget); const receipt=await this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; @@ -157,7 +158,7 @@ export class SavedAckReceiptPublisher{ db.prepare('UPDATE open_repair_saved_acks SET '+column+'=?,attempted=1 WHERE slot_digest=?').run(raw,slotDigest); })); }; - const options={currentStatuses:[...r.current_statuses,entries.status],readUntil:r.read_until,retainUntil:r.retain_until, + const options={currentStatuses:[...r.current_statuses,entries.status],archiveStatuses:history.archiveStatuses??[],knownDisclosureStatuses:history.knownDisclosureStatuses??[],readUntil:r.read_until,retainUntil:r.retain_until, targetNodeEntry:r.target_node_entry,expectedTarget:r.target,expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedMessageId:r.message_id, expectedEnvelopeRef:r.envelope_ref,rootEntry:r.root_entry,writeEntry:r.write_entry,bootstrapEntry:r.bootstrap_entry,timeout,journal}; const result=resume===null?await client.put(baseUrl,entries.receipt,entries.disclosure,entries.put,options): diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index 709a521..41aaa86 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -49,7 +49,8 @@ "./open-repair-mailbox-inbox": "./open-repair-mailbox-inbox.ts", "./open-mailbox-client": "./open-mailbox-client.ts", "./open-mailbox-journal": "./open-mailbox-journal.ts", - "./open-mailbox-receivers": "./open-mailbox-receivers.ts" + "./open-mailbox-receivers": "./open-mailbox-receivers.ts", + "./open-mailbox-receipts": "./open-mailbox-receipts.ts" }, "types": "./crypto.ts", "engines": { diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 834b99f..184be1f 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -146,7 +146,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 11634fa..18887cf 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -123,7 +123,7 @@ "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", "tests/test_open_repair_receipt_typescript.py", - "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", + "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", @@ -248,7 +248,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index ce01db6..5ef5617 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -54,7 +54,7 @@ "tests.test_open_repair_index_prepare", "tests.test_open_repair_index_prepare_admin", "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", - "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +244,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 7740c28..6a19020 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -46,7 +46,7 @@ "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", - "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", + "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-repair-ack.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 67) + self.assertEqual(len(TS_NETWORK), 68) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -122,7 +122,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_status.py", "tests/test_open_repair_status_typescript.py", "tests/open_repair_ack_fixtures.py", "tests/test_open_repair_ack.py", "tests/test_open_repair_ack_typescript.py", "tests/test_open_repair_state.py", "tests/test_open_repair_mailbox_resources.py", "tests/test_open_repair_copy_resources.py", - "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", + "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", @@ -188,7 +188,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-wire.ts", "clients/typescript/network/open-repair-history.ts", "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", - "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", + "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", diff --git a/tests/test_open_delivery_http.py b/tests/test_open_delivery_http.py index 0e53e37..d0acd5a 100644 --- a/tests/test_open_delivery_http.py +++ b/tests/test_open_delivery_http.py @@ -822,6 +822,9 @@ def corrupt_envelope(reference): self.assertGreater(status_count,0) self.assertEqual(recipient_db.execute('SELECT count(*) FROM open_mailbox_setup_statuses').fetchone()[0],status_count) self.assertEqual(delivery._inbox(sent['message_id'])['phase'],'saved') + # This recovery session is finished before a different runtime + # reopens the same recipient state for independent receipt return. + recipient_context.close() def _remote_message_admission(self, source, owner_status, slot, slot_entries, owner, sender_encryption, sent, now, limits, envelope): import hashlib diff --git a/tests/test_open_mailbox_receipts_typescript.py b/tests/test_open_mailbox_receipts_typescript.py new file mode 100644 index 0000000..8f010ee --- /dev/null +++ b/tests/test_open_mailbox_receipts_typescript.py @@ -0,0 +1,115 @@ +"""Actual cold saved memory to native ordinary-receive receipt return.""" +import base64 +import json +import subprocess +import unittest +import sqlite3 +import time +from pathlib import Path +from unittest.mock import patch +from tests import test_open_delivery_http as fixture +from tests import test_open_ack_typescript_agent as runtime +from memory_vault_open_client import ACK_CONNECT_SCHEMA + +DRIVER=runtime.DRIVER.replace("const agent=new Agent", "if(input.clock_offset){const clock=Date.now;Date.now=()=>clock()+input.clock_offset*1000;}\nconst agent=new Agent").replace("if(name==='requestNode'", """if(name==='requestRepair'&&input.lose_reply&&JSON.parse(Buffer.from(args[1]).toString()).kind==='ack.put_request'&&JSON.parse(Buffer.from(result).toString()).kind==='ack.put_response'){ + const error=Error('synthetic lost ACK reply');error.code='synthetic_lost_ack_reply';throw error; + } + if(name==='requestNode'""") + +class NativeMailboxReceiptTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.NativeAckAgentTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def native(self,agent,requests,*,lose_reply=False,no_network=False,clock_offset=0): + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config),requests=requests,lose_reply=lose_reply,no_network=no_network,clock_offset=clock_offset)).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=75) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2500:]);out=json.loads(run.stdout);self.assertEqual(out['subprocessCalls'],0) + state=Path(json.loads(Path(agent.network_config).read_bytes())['state_directory'])/'network.sqlite3' + db=sqlite3.connect(state) + try:self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(),[('ok',)]) + finally:db.close() + for row in out['results']:self.assertTrue(row['ok'],(row,run.stderr.decode(errors='replace')[-2000:])) + self.assertNotIn('network_storage_unavailable',json.dumps(out),(out,run.stderr.decode(errors='replace')[-2000:])) + if no_network:self.assertEqual(out['calls'],[]) + return out + + def test_native_ordinary_receive_returns_cold_memory_receipt_after_lost_reply(self): + self.check_return('resume') + + def test_expired_retained_revocation_stops_pending_native_return(self): + self.check_return('revoke') + + def test_expired_uncertain_return_stops_ordinary_polling(self): + self.check_return('expired') + + def check_return(self,mode): + h=fixture.MailboxStagingHTTPTests('test_cold_mailbox_returns_independent_receipt');h.setUp();self.addCleanup(h.doCleanups) + from memory_vault_trust import TrustStore + from memory_vault_client import ClientConfig + TrustStore(ClientConfig.load(h.b.client_config).trust_path).add(h.ai.public_descriptor()) + call=h.call + def share(agent,**request): + invitation=request.get('invitation',{}) + if agent is h.a and ((request['op']=='send' and not request.get('memory_ids')) or (invitation.get('action')=='prepare' and invitation.get('schema_version')==ACK_CONNECT_SCHEMA and not invitation.get('memory_ids'))): + m=call(h.a,op='remember',request_id='req_native_retained_return_memory',kind='observation',text='Synthetic cold mailbox return memory.') + if request['op']=='send':request=dict(request,memory_ids=[m['memory_id']]) + else:request=dict(request,invitation=dict(invitation,memory_ids=[m['memory_id']])) + return call(agent,**request) + h.call=share;visited=[] + def returned(host): + visited.append(True);write=json.loads(h.ack_configuration['ack.write_grant']['raw'])['payload'];mid=write['message_id'] + selected=dict(schema_version=ACK_CONNECT_SCHEMA,action='register_mailbox_receipt_return',message_id=mid,source_url=host.nodes[0]['payload']['base_url'],source_key_id=host.identities[0].key_id,repair_profile='receipt') + query=dict(schema_version=ACK_CONNECT_SCHEMA,action='inspect_mailbox_receipt_return',message_id=mid) + self.native(h.b,[dict(op='connect',invitation=selected),dict(op='connect',invitation=selected)],no_network=True) + first=self.native(h.b,[dict(op='receive',limit=1)],lose_reply=True)['results'][0]['result'] + self.assertTrue(any(e['code']=='synthetic_lost_ack_reply' for e in first['errors']),first);self.assertTrue(first['network_accessed']);self.assertFalse(first.get('receipt_returns')) + pending=self.native(h.b,[dict(op='connect',invitation=query)],no_network=True)['results'][0]['result'];self.assertEqual(pending['state'],'pending');self.assertEqual(pending['result']['code'],'synthetic_lost_ack_reply') + if mode=='expired': + expired=self.native(h.b,[dict(op='receive',limit=1)],clock_offset=120) + errors=expired['results'][0]['result']['errors'];self.assertTrue(any(e['code'] in {'repair_reconciliation_required','repair_saved_reconciliation_required'} for e in errors),errors) + self.assertFalse(any(c['kind']=='requestRepair' for c in expired['calls'])) + held=self.native(h.b,[dict(op='connect',invitation=query)],no_network=True)['results'][0]['result'];self.assertEqual(held['state'],'reconciliation_required') + stopped=self.native(h.b,[dict(op='receive',limit=1)],clock_offset=120);self.assertFalse(any(c['kind']=='requestRepair' for c in stopped['calls']));self.assertFalse(stopped['results'][0]['result'].get('receipt_returns')) + return + if mode=='revoke': + from memory_vault_open_provider import issue_status + from tests.test_open_repair_status import status_entry + from memory_vault_open_repair_status import authenticate_status_original + from memory_vault_open_repair_admin import _ReplicaStatusJournal + from memory_vault_open_repair_state import DEFAULT_POLICY + from memory_vault_open_repair_wire import RepairBudget + old=json.loads(h.ack_configuration['historical.status.ack_root']['raw'])['payload'];now=int(time.time()) + denied=status_entry(issue_status(h.ai,root=write['ack_slot']['root_key'],revision=99,entries=[dict(e,status='revoked') for e in old['entries']],issued_at=now-10,valid_until=now-1)) + checked=authenticate_status_original(denied,expected_root=write['ack_slot']['root_key'],expected_signing_key=h.ai.public_descriptor(),at=now-10,allowed_scopes=[dict(scope_kind=e['scope_kind'],scope_id=e['scope_id']) for e in old['entries']],policy=DEFAULT_POLICY,budget=RepairBudget(DEFAULT_POLICY)) + with h.b._network() as network: + with network.participant.state.db() as db:_ReplicaStatusJournal(db,write['ack_slot']['root_key'],original_source=True).observe(checked) + for _ in range(2): + refused=self.native(h.b,[dict(op='receive',limit=1)]) + result=refused['results'][0]['result'];self.assertFalse(result.get('receipt_returns'));self.assertTrue(any(e['code']=='repair_authority_revoked' for e in result['errors']),result) + self.assertFalse(any(c['kind']=='requestRepair' for c in refused['calls']),refused['calls']) + return + resumed=self.native(h.b,[dict(op='receive',limit=1)])['results'][0]['result'];self.assertEqual(resumed['receipt_returns'],[dict(message_id=mid,state='retained_at_ack_source',from_local_history=False)]) + completed=self.native(h.b,[dict(op='connect',invitation=query)],no_network=True)['results'][0]['result'];self.assertEqual(completed['state'],'complete') + again=self.native(h.b,[dict(op='receive',limit=1)]);self.assertFalse(again['results'][0]['result'].get('receipt_returns'));self.assertFalse(any(c['kind']=='requestRepair' for c in again['calls'])) + with h.b._network() as network: + inbox=network._delivery()._inbox(mid);saved=json.loads(inbox['result']);self.assertEqual(saved['content_kind'],'memory_transfer');self.assertEqual(saved['share']['records_added'],1) + inspected=network.connect(invitation=query);self.assertEqual(inspected,completed) + with patch.object(network.participant.transport,'request_repair',side_effect=AssertionError('completed native request repeated HTTP')): + py=network.connect(invitation=dict(selected,action='return_mailbox_receipt')) + self.assertTrue(py['from_local_history']);self.assertEqual(py['commit_ref'],completed['result']['commit_ref']) + chunks=[];cursor=None + while True: + q=dict(schema_version=ACK_CONNECT_SCHEMA,action='export_preparation',request_id=h.ack_preparation_request_id,part='owner_invitation') + if cursor is not None:q['cursor']=cursor + page=h.call(h.a,op='connect',invitation=q);chunks.append(base64.b64decode(page['bundle_chunk']));cursor=page['next_cursor'] + if cursor is None:break + recovered=h.call(h.a,op='connect',invitation=json.loads(b''.join(chunks)));self.assertEqual(recovered['message_id'],mid) + with h.a._network() as network: + with network.participant.state.db() as db:self.assertIsNotNone(db.execute('SELECT acknowledgement FROM open_delivery_outbox WHERE message_id=?',(mid,)).fetchone()[0]) + self.native(h.b,[dict(op='connect',invitation=dict(query,action='remove_mailbox_receipt_return')),dict(op='connect',invitation=selected)],no_network=True) + replay=self.native(h.b,[dict(op='receive',limit=1)]);self.assertTrue(replay['results'][0]['result']['receipt_returns'][0]['from_local_history']);self.assertFalse(any(c['kind']=='requestRepair' for c in replay['calls'])) + h._return_cold_ack=returned;h.test_cold_mailbox_returns_independent_receipt();self.assertEqual(visited,[True]) + +if __name__=='__main__':unittest.main() From 0b6b1c4f03c6496f1e9670a12c6c8acd31e67c20 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:59:09 +0800 Subject: [PATCH 16/16] Prepare alpha37 automatic native mailbox receipt release --- memory_vault.py | 2 +- packaging/CLIENT_README.md | 13 +++++++++++-- packaging/PROTOCOL_README.md | 13 +++++++++++-- packaging/RELEASE_NOTES.md | 15 ++++++++++++--- packaging/REVIEW_README.md | 11 ++++++++++- .../memory-vault-client/.codex-plugin/plugin.json | 2 +- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/memory_vault.py b/memory_vault.py index 0b5d092..d7b0e14 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.36" +VERSION = "0.28.0-alpha.0.37" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 9704411..0fa2a7f 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 — authorized full client +# Memory Vault v0.28.0-alpha.0.37 — authorized full client + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, @@ -148,7 +157,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.36 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.37 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 0446a56..4c230e7 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 — independent protocol +# Memory Vault v0.28.0-alpha.0.37 — independent protocol + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, @@ -165,7 +174,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.36; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.37; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index 7af802a..4b49e7a 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 — native remote mailbox memory recovery +# Memory Vault v0.28.0-alpha.0.37 — automatic native mailbox receipt returns + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, @@ -174,7 +183,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index e49b3c6..0374631 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 independent review kit +# Memory Vault v0.28.0-alpha.0.37 independent review kit + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index a74930e..fd07619 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.36", + "version": "0.28.0-alpha.0.37", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors"