From 36d006a1ee059ab83c70f3cddbc93221244ae943 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:19:27 +0800 Subject: [PATCH 01/24] Read independent ACK sources with the native receipt writer --- .../network/open-repair-offer-client.ts | 281 ++++++++++++++++++ clients/typescript/network/package.json | 1 + docs/OPEN_ACK_RECOVERY.md | 26 ++ scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 2 +- scripts/run_open_network_ci.py | 4 +- ...est_open_repair_offer_client_typescript.py | 119 ++++++++ 7 files changed, 431 insertions(+), 4 deletions(-) create mode 100644 clients/typescript/network/open-repair-offer-client.ts create mode 100644 tests/test_open_repair_offer_client_typescript.py diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts new file mode 100644 index 0000000..3be1f55 --- /dev/null +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -0,0 +1,281 @@ +/** Native receipt-writer preflight over independently authorized empty sources. + * This reads original source evidence only; it does not authorize or upload a receipt. + */ +import {performance} from 'node:perf_hooks'; +import {isProxy} from 'node:util/types'; +import {OpenHTTPTransport,endpoint} from './open-transport.ts'; +import {RepairBudget,RepairError,buildNewWire,objectFields,rawRef,u53,LocalRawResolver,parseNewWire} from './open-repair-wire.ts'; +import type {RepairPolicy,RawRef,RawOriginal} from './open-repair-wire.ts'; +import {verifyAckOfferBootstrapOriginal} from './open-repair-bound.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import type {AckOwnerRecoveryOptions,RecoveredAckOwnerEmptyProof} from './open-repair-client.ts'; +import {verifyAckEmptySourceEvent} from './open-repair-empty.ts'; +import type {AuthenticatedAckEmptySourceEvent} from './open-repair-empty.ts'; +import {parseOriginalControl,verifySourceNodeOriginal,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {verifyBootstrapProofResponse,makeBootstrapChildRequest} from './open-repair-proof.ts'; +import {statusScope,verifyStatusOriginal,authenticateStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; + +type Obj=Record; +const clock=()=>performance.now()/1000; +export interface AckOfferOptions { + readonly targetNodeEntry:unknown;readonly expectedTarget:unknown;readonly expectedAckSlot:unknown; + readonly expectedOwner:unknown;readonly expectedMessageId:string;readonly expectedEnvelopeRef:unknown; + readonly rootEntry:unknown;readonly writeEntry:unknown;readonly bootstrapEntry:unknown; + readonly knownStatuses?:readonly unknown[];readonly archiveStatuses?:readonly unknown[];readonly timeout?:number; +} +export type PreparedAckOffer=RecoveredAckOwnerEmptyProof; +interface Obligation{role:string|null;kind:string;subject:unknown;revision:number;signer:Obj;scope_id:string;mask?:number;} +function fail(code:string):never{throw new RepairError(code);} +function fields(value:unknown,names:readonly string[]):Obj{return objectFields(value,names);} +function options(value:unknown,required:readonly string[],optional:readonly string[]):Obj{ + if(value===null||typeof value!=='object'||isProxy(value))fail('repair_unknown_fields'); + const names=Reflect.ownKeys(value);if(names.some(name=>typeof name!=='string'||![...required,...optional].includes(name)))fail('repair_unknown_fields'); + return fields(value,[...required,...optional.filter(name=>Object.hasOwn(value,name))]); +} +function same(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(a===null||b===null||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(name=>Object.hasOwn(b,name)&&same((a as Obj)[name],(b as Obj)[name])); +} +function refKey(ref:RawRef):string{return `${ref.namespace}:${ref.key}:${ref.raw_sha256}:${ref.size}`;} +function scopeKey(item:Obj):string{return item.scope_kind+':'+item.scope_id;} +function checkedStatus(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{ + const signed=fields(parseOriginalControl(raw,policy,budget).value,['payload','proof']); + const payload=fields(signed.payload,['schema_version','kind','signing_key','scope_key','revision','issued_at','valid_until','entries']); + if(!Array.isArray(payload.entries)||payload.entries.length<1||payload.entries.length>16)fail('repair_invalid_status'); + for(const item of payload.entries)fields(item,['scope_kind','scope_id','minimum_document_revision','status','operation_mask']); + return payload; +} +function snapshotEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget,legacy=false):RawOriginal{ + const held=fields(value,['raw','ref']),ref=rawRef(held.ref),document=legacy?parseOriginalControl(held.raw,policy,budget):parseNewWire(held.raw,policy,budget); + const raw=document.raw;if(ref.namespace!=='meta'||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + return {raw,ref}; +} +export class AckOfferClient{ + readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #limits:Obj; + readonly #statusObserver?: (item:AuthenticatedStatusOriginal)=>void; + readonly #transport:OpenHTTPTransport;readonly #allowLoopback:boolean;readonly #clock:()=>number;readonly #ownTransport:boolean; + constructor(identity:unknown,encryptionIdentity:unknown,value:AckOwnerRecoveryOptions={}){ + const args=options(value,[],['policy','limitPolicy','allowLoopback','transport','clock','statusObserver']); + const meter=new RepairBudget(args.policy??DEFAULT_REPAIR_CLIENT_POLICY);this.#policy=meter.policy; + const held=buildNewWire({identity,encryption:encryptionIdentity,limits:args.limitPolicy??DEFAULT_REPAIR_CLIENT_LIMITS},this.#policy,meter).value as Obj; + this.#identity=fields(held.identity,['schema_version','algorithm','key_id','public_key','private_key']); + this.#encryption=fields(held.encryption,['schema_version','algorithm','key_id','public_key','private_key']); + this.#limits=fields(held.limits,Object.keys(DEFAULT_REPAIR_CLIENT_LIMITS));for(const amount of Object.values(this.#limits))u53(amount,1); + this.#subject=buildNewWire({signing_key:{schema_version:'universal-memory-public-key/v1',algorithm:this.#identity.algorithm, + key_id:this.#identity.key_id,public_key:this.#identity.public_key},encryption_key:{schema_version:'memory-vault-network-encryption-key/v1', + algorithm:this.#encryption.algorithm,key_id:this.#encryption.key_id,public_key:this.#encryption.public_key}},this.#policy,meter).value as Obj; + if(args.allowLoopback!==undefined&&typeof args.allowLoopback!=='boolean')fail('repair_invalid_policy'); + if(args.clock!==undefined&&typeof args.clock!=='function')fail('repair_invalid_policy'); + if(args.statusObserver!==undefined&&typeof args.statusObserver!=='function')fail('repair_invalid_policy'); + this.#allowLoopback=args.allowLoopback??false;this.#clock=args.clock??(()=>Date.now()/1000); + this.#transport=args.transport??new OpenHTTPTransport({allow_loopback:this.#allowLoopback});this.#ownTransport=args.transport===undefined; + this.#statusObserver=args.statusObserver; + Object.freeze(this); + } + close():void{if(this.#ownTransport)this.#transport.close();} + #now():number{return u53(Math.floor(this.#clock()));} + async preflight(baseUrl:string,value:AckOfferOptions):Promise{ + const args=options(value,['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId', + 'expectedEnvelopeRef','rootEntry','writeEntry','bootstrapEntry'],['knownStatuses','archiveStatuses','timeout']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + const policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),deadline=clock()+timeout; + const expected=buildNewWire({target:args.expectedTarget,slot:args.expectedAckSlot,owner:args.expectedOwner, + message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const history=checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget); + const setup=verifyAckOfferBootstrapOriginal(args.bootstrapEntry,{root:args.rootEntry,write:args.writeEntry}, + {expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject, + expectedMessageId:expected.message,expectedEnvelopeRef:expected.envelope,at:started,limitPolicy:this.#limits,policy,budget}); + const nodeEntry=snapshotEntry(args.targetNodeEntry,policy,budget,true),nodePayload=checkedNodePayload(nodeEntry.raw,policy,budget); + const node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:expected.target.signing_key, + expectedStorageEpoch:nodePayload.storage_epoch,at:started,policy,budget}); + if(!same(endpoint(baseUrl,this.#allowLoopback),endpoint(node.payload.base_url,this.#allowLoopback)))fail('repair_proof_mismatch'); + const grant=setup.originals.bootstrap.payload as Obj; + const known=checks.known(history,checks.obligations(setup.originals,expected.slot,budget),expected.slot.root_key,expected.target,budget,true); + const expiry=Math.min(started+Math.min(60,Math.max(1,Math.floor(timeout))),grant.probe_until,grant.proof_until,grant.expires_at, + node.payload.expires_at as number,setup.originals.root.payload.expires_at as number,setup.originals.write.payload.expires_at as number); + if(expiry<=started)fail('repair_access_expired'); + const binding={expectedSubject:this.#subject,expectedTarget:expected.target,targetStorageEpoch:node.payload.storage_epoch as string, + bootstrapGrantSha256:setup.originals.bootstrap.ref.raw_sha256,selector:grant.selector,policy,budget,consumer:'ack_offer' as const}; + let requests=0,wireBytes=0,proofBytes=0; + const request=async(body:Uint8Array,child=false):Promise=>{ + if(requests>=grant.limits.max_requests||clock()>=deadline)fail('repair_over_budget');requests++; + const reply=await this.#transport.requestRepair(baseUrl,body,deadline,{child});wireBytes+=body.length+reply.length;return reply; + }; + const outgoing=await makeBootstrapProbe(this.#identity,{...binding,at:started,expiresAt:expiry}); + if(outgoing.original.raw.length>grant.limits.max_probe_bytes)fail('repair_over_budget'); + const challengeRaw=await request(outgoing.original.raw),challengeDigest=budget.hash(challengeRaw); + const challenge={raw:challengeRaw,ref:rawRef({namespace:'meta',key:challengeDigest,raw_sha256:challengeDigest,size:challengeRaw.length})}; + const challengeValue=fields(parseNewWire(challengeRaw,policy,budget).value,['payload','proof']); + const answer=await solveBootstrapChallenge({raw:outgoing.original.raw,ref:outgoing.original.ref},challenge, + {...binding,signer:this.#identity,encryptionIdentity:this.#encryption,targetNonce:outgoing.nonce,at:this.#now(), + expiresAt:Math.min(expiry,u53((challengeValue.payload as Obj)?.expires_at))}); + const response=await request(answer.raw),held=verifyBootstrapProofResponse(response,{expectedSubject:this.#subject,expectedTarget:expected.target, + targetStorageEpoch:node.payload.storage_epoch as string,selector:grant.selector,bootstrapGrantRef:setup.originals.bootstrap.ref, + probeRef:outgoing.original.ref,challengeRef:challenge.ref,answerRef:answer.ref,at:this.#now(),maxProofItems:grant.limits.max_proof_items, + maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'empty',consumer:'ack_offer',policy,budget}); + proofBytes=response.length+held.handle.raw.length+held.manifest.raw.length; + // Reuse only exact already-held originals. Verification below still checks + // the complete independently authenticated source and both history events. + const available=new Map(); + for(const item of [...Object.values(setup.originals),...history,nodeEntry] as RawOriginal[])available.set(refKey(item.ref),item.raw); + const originals=new Map(),roles=new Map(); + for(const item of (held.manifest.value as Obj).children){ + const reference=rawRef(item.ref),key=refKey(reference);roles.set(item.role,[...(roles.get(item.role)??[]),reference]);if(originals.has(key))continue; + if(reference.size>policy.max_document_bytes||proofBytes+reference.size>grant.limits.max_proof_bytes)fail('repair_over_budget'); + let assembled:Uint8Array; + if(available.has(key)){assembled=available.get(key)!;budget.input(assembled.length);} + else{ + const chunks:Uint8Array[]=[];let offset=0; + while(offset=(held.handle.payload.expires_at as number)||clock()>=deadline)fail('repair_access_expired'); + const entry=(role:string):RawOriginal=>originals.get(refKey(roles.get(role)![0]))!; + const resolver=new LocalRawResolver(policy,budget); + for(const reference of roles.get('history.raw_pack')!){const item=originals.get(refKey(reference))!; + if(!same(resolver.put(reference.namespace,reference.key,item.raw).ref,reference))fail('repair_ref_mismatch');} + const source=verifyAckEmptySourceEvent(entry('history.ack_empty'),resolver,entry('ack.empty_custody'),{ + expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedTarget:expected.target, + targetStorageEpoch:node.payload.storage_epoch as string,expectedReceiptWriter:this.#subject, + expectedMessageId:expected.message,expectedEnvelopeRef:expected.envelope,limitPolicy:this.#limits,policy,budget}); + this.#emptyHead(entry('ack.head'),source,expected.target,budget); + const prior=source.predecessor; + for(const item of source.manifest.roles){const actual=originals.get(refKey(item.original.ref)); + if(!same(roles.get(item.role),[item.original.ref])||!actual||!Buffer.from(actual.raw).equals(Buffer.from(item.original.raw)))fail('repair_proof_mismatch');} + if(!same(prior.resources.originals.root.ref,setup.originals.root.ref)||!same(source.authorities.originals.write.ref,setup.originals.write.ref)|| + !same(source.authorities.originals.bootstrap.ref,setup.originals.bootstrap.ref))fail('repair_proof_mismatch'); + const usedPacks=new Set();for(const manifest of [source.manifest,prior.manifest]) + for(const row of (manifest.manifest.value as Obj).roles)usedPacks.add(refKey(rawRef(row.pack_ref))); + const suppliedPacks=roles.get('history.raw_pack')!.map(refKey); + if(suppliedPacks.length!==usedPacks.size||suppliedPacks.some(key=>!usedPacks.has(key)))fail('repair_unused_pack'); + const current=checks.current(source,roles,originals,expected.target,budget,known),now=this.#now(); + if(now>=Math.min(expiry,held.handle.payload.expires_at as number,source.read_until,...current.map(item=>item.payload.valid_until as number))||clock()>=deadline)fail('repair_access_expired'); + const saved=Object.freeze([...originals.values()].map(item=>Object.freeze({ref:item.ref,get raw(){return Uint8Array.from(item.raw);}}))); + return Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:saved, + metrics:Object.freeze({requests,wire_bytes:wireBytes,proof_bytes:proofBytes,...budget.snapshot()})}); + } + #emptyHead(entry:RawOriginal,source:AuthenticatedAckEmptySourceEvent,target:Obj,budget:RepairBudget):void{ + const input=snapshotEntry(entry,this.#policy,budget),signed=fields(parseNewWire(input.raw,this.#policy,budget).value,['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key','ack_slot','generation','observed_at','retain_until','state','root_authority_ref','grant_ref','binding_ref']); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!=='ack.head'||p.state!=='empty'||u53(p.generation,1)!==1|| + !same(p.ack_slot,source.custody.payload.ack_slot)||p.observed_at!==source.stored_at||p.retain_until!==source.retain_until|| + ['root_authority_ref','grant_ref','binding_ref'].some(name=>!same(p[name],source.custody.payload[name])))fail('repair_ack_empty_mismatch'); + verifyBoundedControlSignature(p,signed.proof,target.signing_key,budget); + } +} +class OfferStatusChecks{ + readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #clock:()=>number; + readonly #statusObserver?: (item:AuthenticatedStatusOriginal)=>void; + constructor(owner:Obj,policy:RepairPolicy,now:()=>number,observer?: (item:AuthenticatedStatusOriginal)=>void){ + this.#subject=owner;this.#policy=policy;this.#clock=now;this.#statusObserver=observer; + } + #now():number{return u53(Math.floor(this.#clock()));} + obligations(originals:Obj,slot:Obj,budget:RepairBudget):Obligation[]{ + const result:Obligation[]=[]; + for(const [name,role,kind,mask] of [['root','current.status.ack_root','ack.root_authority',3], + ['write','current.status.ack_write','ack.write_grant',1],['bootstrap','current.status.ack_offer_bootstrap','bootstrap.grant',3]] as const){ + const item=originals[name],subject={authority_kind:kind,authority_sha256:item.ref.raw_sha256}; + result.push({role,kind:'authority',subject,revision:item.payload.revision,signer:this.#subject.signing_key,mask, + scope_id:statusScope(slot.root_key,'authority',subject,this.#policy,budget)}); + } + result.push({role:'current.status.ack_slot',kind:'ack_slot',subject:slot,revision:originals.root.payload.revision, + signer:this.#subject.signing_key,mask:3,scope_id:statusScope(slot.root_key,'ack_slot',slot,this.#policy,budget)});return result; + } + history(known:unknown,archive:unknown,budget:RepairBudget):RawOriginal[]{ + // Both sets remain signed evidence. Archive entries carry old conflict + // witnesses as well as revocation/floor observations; no summary is trusted. + const merged=new Map(); + for(const [values,maximum] of [[known,16],[archive,32]] as const){ + if(!Array.isArray(values)||isProxy(values))fail('repair_invalid_status'); + if(values.length>maximum)fail(maximum===16?'repair_invalid_status':'repair_status_history_capacity'); + const descriptors=Object.getOwnPropertyDescriptors(values); + for(let index=0;index=32)fail('repair_status_history_capacity');merged.set(key,item); + } + } + return [...merged.values()]; + } + known(values:unknown,obligations:Obligation[],root:Obj,target:Obj,budget:RepairBudget,deferredAuthorities=false):AuthenticatedStatusOriginal[]{ + // Copy raw entries before the first asynchronous operation. + if(!Array.isArray(values)||isProxy(values)||values.length>32)fail('repair_invalid_status'); + const descriptors=Object.getOwnPropertyDescriptors(values);for(let index=0;indexthis.#now()+30)fail('repair_status_mismatch');let signer:Obj,allowed:Obj[]; + if(issuer===this.#subject.signing_key.key_id){signer=this.#subject.signing_key;allowed=obligations.map(v=>({scope_kind:v.kind,scope_id:v.scope_id})); + if(deferredAuthorities){const scopes=new Map(allowed.map(item=>[scopeKey(item),item])); + for(const item of payload.entries)if(item.scope_kind==='authority')scopes.set(scopeKey(item),{scope_kind:item.scope_kind,scope_id:item.scope_id});allowed=[...scopes.values()];} + if(issuer===target.signing_key.key_id)allowed.push(...payload.entries.filter((v:Obj)=>v.scope_kind==='resource').map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})));} + else if(issuer===target.signing_key.key_id){signer=target.signing_key; + if(payload.entries.some((v:Obj)=>v.scope_kind!=='resource'))fail('repair_status_disclosure'); + allowed=payload.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id}));} + else fail('repair_status_mismatch'); + checked.push(authenticateStatusOriginal(item,{expectedRoot:root,expectedSigningKey:signer,at:issued,allowedScopes:allowed,policy:this.#policy,budget},this.#statusObserver)); + } + // Activation's historical owner READ grant is checked against the exact + // recovered history below; it cannot revoke the writer's independent ADMIT. + this.#floors(checked,[],obligations,true,deferredAuthorities);return checked; + } + #floors(known:readonly AuthenticatedStatusOriginal[],current:readonly AuthenticatedStatusOriginal[],obligations:Obligation[],probePhase=false,deferredAuthorities=false):void{ + const seen=new Map(),floors=new Map(),required=new Map(obligations.map(v=>[v.signer.key_id+':'+v.kind+':'+v.scope_id,v])); + for(const observed of [...known,...current]){ + const payload=observed.payload as Obj,issuer=payload.scope_key.issuer_key_id,revision=payload.revision,id=issuer+':'+revision; + if(seen.has(id)&&seen.get(id)!==observed.canonical_sha256)fail('repair_status_conflict');seen.set(id,observed.canonical_sha256); + for(const entry of payload.entries){const key=issuer+':'+scopeKey(entry),wanted=required.get(key); + let mask=wanted?.mask??2; + if(probePhase&&wanted&&['current.status.ack_root','current.status.ack_offer_bootstrap'].includes(wanted.role??''))mask|=8; + if(deferredAuthorities&&!wanted&&issuer===this.#subject.signing_key.key_id&&entry.scope_kind==='authority')mask=0; + if(entry.status==='revoked'&&(entry.operation_mask&mask))fail('repair_authority_revoked'); + if(wanted&&mask&&entry.minimum_document_revision>wanted.revision)fail('repair_status_revision'); + floors.set(key,[...(floors.get(key)??[]),[revision,entry.minimum_document_revision]]); + } + } + for(const values of floors.values()){let minimum=0;for(const [,floor] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(floorv[0])))fail('repair_status_rollback');}} + } + current(source:AuthenticatedAckEmptySourceEvent,roles:Map,originals:Map,target:Obj, + budget:RepairBudget,known:AuthenticatedStatusOriginal[]):AuthenticatedStatusOriginal[]{ + const prior=source.predecessor,slot=source.custody.payload.ack_slot as Obj,root=slot.root_key,policy=this.#policy; + const obligations=this.obligations(source.authorities.originals,slot,budget),active=prior.resources.originals.active.payload as Obj; + obligations.push({role:'current.status.ack_resource',kind:'resource',subject:active.resource,revision:active.reservation_generation, + signer:target.signing_key,mask:3,scope_id:statusScope(root,'resource',active.resource,policy,budget)}); + for(const [item,kind] of [[prior.resources.originals.read,'ack.read_grant'],[prior.bootstrap.originals.bootstrap,'bootstrap.grant']] as const){ + const subject={authority_kind:kind,authority_sha256:item.ref.raw_sha256}; + obligations.push({role:null,kind:'authority',subject,revision:item.payload.revision as number,signer:this.#subject.signing_key,mask:0, + scope_id:statusScope(root,'authority',subject,policy,budget)}); + } + const permitted=new Set(obligations.map(v=>v.signer.key_id+':'+v.kind+':'+v.scope_id)); + for(const observed of known){const p=observed.payload as Obj; + if(p.entries.some((v:Obj)=>!permitted.has(p.scope_key.issuer_key_id+':'+scopeKey(v))))fail('repair_status_disclosure');} + const groups=new Map(); + for(const item of obligations){if(item.role===null)continue;const ref=roles.get(item.role)![0],key=refKey(ref),group=groups.get(key)??{ref,signer:item.signer,required:[]}; + if(!same(group.signer,item.signer))fail('repair_proof_mismatch'); + group.required.push({scope_kind:item.kind,scope_id:item.scope_id,document_revision:item.revision,operation_mask:item.mask});groups.set(key,group);} + const checked:AuthenticatedStatusOriginal[]=[]; + for(const [key,group] of groups){ + const allowed=obligations.filter(v=>same(v.signer,group.signer)),entry=originals.get(key)!; + checked.push(verifyStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:group.signer,at:this.#now(), + allowedScopes:allowed.map(v=>({scope_kind:v.kind,scope_id:v.scope_id})),required:group.required,policy,budget},this.#statusObserver)); + } + this.#floors([...prior.statuses,...source.statuses,...known],checked,obligations); + if(this.#now()>=Math.min(...['admit_until','read_until','retain_until'].map(n=>active.windows[n])))fail('repair_access_expired'); + return checked; + } +} +function checkedNodePayload(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{return fields(parseOriginalControl(raw,policy,budget).value,['payload','proof']).payload;} diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index 08c8aed..e456d82 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -36,6 +36,7 @@ "./open-delivery-client": "./open-delivery-client.ts", "./open-repair-admin": "./open-repair-admin.ts", "./open-repair-client": "./open-repair-client.ts", + "./open-repair-offer-client": "./open-repair-offer-client.ts", "./open-repair-wire": "./open-repair-wire.ts", "./open-repair-bound": "./open-repair-bound.ts", "./open-repair-empty": "./open-repair-empty.ts", diff --git a/docs/OPEN_ACK_RECOVERY.md b/docs/OPEN_ACK_RECOVERY.md index df2fd31..b782b41 100644 --- a/docs/OPEN_ACK_RECOVERY.md +++ b/docs/OPEN_ACK_RECOVERY.md @@ -1029,3 +1029,29 @@ same queue. Completed jobs stay local on subsequent runs. Use `list` to inspect the queue or `remove --job-id selected-message` to stop retaining a job. Removing it preserves the copy preparation journal and remote storage. Keep the private request and exported result outside public repositories. + + +## Native writer preflight (development after alpha.0.34) + +The native TypeScript `AckOfferClient` in `open-repair-offer-client.ts` can read +and authenticate a message-bound empty ACK source directly over HTTP. It checks +the source's two keys, both storage history events, the caller's exact original +owner root, write grant and offer bootstrap, and current ADMIT/READ/DISCOVER +permissions. Retained status originals keep revocations and revision floors in +force. A historical owner READ revocation does not remove the recipient's +separately granted ADMIT permission. + +Construct the client with the receipt writer's signing and encryption identities, +using the same bounded policy and transport options as `AckOwnerRecoveryClient`. +Call `preflight(baseUrl, options)` with `targetNodeEntry`, `expectedTarget`, +`expectedAckSlot`, `expectedOwner`, `expectedMessageId`, `expectedEnvelopeRef`, +`rootEntry`, `writeEntry`, and `bootstrapEntry`. Raw entries are `{raw: Uint8Array, +ref: RawRef}` containing independently held signed originals. Optional +`knownStatuses` and `archiveStatuses` retain original status evidence; `timeout` +is a positive deadline of at most sixty seconds. + +The result contains the authenticated empty source, bounded proof, current +statuses, original bytes and actual request/work counts. This preflight uploads +no receipt and grants no upload authority. The complete native saved-receipt +upload and restart journal remain subsequent integration work; Python's existing +receipt-return workflow remains the available complete path. diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 89c5409..93f2dca 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -145,7 +145,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 40312f5..874dfa7 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -243,7 +243,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 1ec261b..580f751 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -47,7 +47,7 @@ "repair_http", "repair_client", "repair_client_typescript", "repair_admin", "repair_admin_typescript", "repair_runtime_review", "repair_bound", "repair_bound_typescript", "repair_empty", "repair_empty_typescript", "repair_empty_review", "repair_empty_access", "repair_empty_http", "repair_empty_client", "repair_empty_client_typescript", - "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( + "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( "tests.test_open_repair_stage", "tests.test_open_repair_index", "tests.test_open_repair_index_access", "tests.test_open_repair_index_journal", "tests.test_open_repair_index_state", "tests.test_open_repair_index_http", "tests.test_open_repair_index_client", "tests.test_open_repair_index_recovery", "tests.test_open_repair_index_admin", @@ -245,7 +245,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-capacity.ts", - "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", + "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", "agent.ts", "peer.ts", "io.ts", "crypto.ts", "nodes.ts")] diff --git a/tests/test_open_repair_offer_client_typescript.py b/tests/test_open_repair_offer_client_typescript.py new file mode 100644 index 0000000..2c34fe3 --- /dev/null +++ b/tests/test_open_repair_offer_client_typescript.py @@ -0,0 +1,119 @@ +"""Native writer reads the independently authorized source over actual HTTP.""" +import hashlib +import json +import subprocess +import unittest + +from memory_vault import canonical_bytes +from memory_vault_open_repair_offer_access import RepairAckOfferAccess +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_repair_probe_typescript as probe_ts +from tests import test_open_repair_proof_typescript as proof_ts +from tests.open_repair_ack_fixtures import signed_entry +from tests.test_open_repair_empty_http import EmptyHTTPFixture + +DRIVER=r""" +import child from 'node:child_process'; +import crypto from 'node:crypto'; +import {syncBuiltinESMExports} from 'node:module'; +let subprocessCalls=0,nativeChecks=0; +const deny=()=>{subprocessCalls++;throw Error('native offer must not delegate');}; +for(const name of ['spawn','spawnSync','exec','execSync','execFile','execFileSync','fork'])child[name]=deny; +const originalVerify=crypto.verify;crypto.verify=(...args)=>{nativeChecks++;return originalVerify(...args);};syncBuiltinESMExports(); +const {AckOfferClient}=await import('./open-repair-offer-client.ts'); +const {OpenHTTPTransport}=await import('./open-transport.ts'); +const chunks=[];let size=0;for await(const chunk of process.stdin){size+=chunk.length;if(size>1048576)throw Error('fixture limit');chunks.push(chunk);} +const input=JSON.parse(Buffer.concat(chunks).toString('utf8')),decode=item=>({raw:Buffer.from(item.raw,'base64'),ref:item.ref}); +const transport=new OpenHTTPTransport({allow_loopback:true}),calls=[],request=transport.requestRepair.bind(transport); +transport.requestRepair=async(base,raw,...rest)=>{calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);return request(base,raw,...rest);}; +const client=new AckOfferClient(input.signing,input.encryption,{policy:input.policy,limitPolicy:input.limits,transport, + allowLoopback:true,clock:()=>input.now}); +const options={...input.options}; +for(const name of ['targetNodeEntry','rootEntry','writeEntry','bootstrapEntry'])options[name]=decode(options[name]); +for(const name of ['knownStatuses','archiveStatuses'])if(options[name])options[name]=options[name].map(decode); +try{ + const result=await client.preflight(input.base,options); + const original=result.originals[0],before=Buffer.from(original.raw).toString('hex');original.raw.fill(0); + process.stdout.write(JSON.stringify({ok:true,metrics:result.metrics,consumer:result.proof.handle.payload.consumer, + write:result.source.authorities.originals.write.ref,bootstrap:result.source.authorities.originals.bootstrap.ref, + custody:Buffer.from(result.source.custody.raw).toString('base64'),originals:result.originals.length, + immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls})); +}catch(error){process.stdout.write(JSON.stringify({ok:false,code:error.code??'untyped_error',detail:error.code?undefined:String(error.stack),calls,nativeChecks,subprocessCalls}));} +finally{client.close();transport.close();} +""" + + +class NativeOfferClientTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.host=EmptyHTTPFixture(self) + self.f=self.host.f + + def call(self,**extra): + f=self.f;e=f['expected'];h=self.host + return dict(base=h.http.base,signing=probe_ts.private_signing(f['signers']['writer']), + encryption=f['encryption']['writer'].private_document(),now=2_000_000_007, + policy=proof_ts.POLICY,limits=e['limit_policy'],options=dict( + targetNodeEntry=probe_ts.encoded(f['entries']['descriptor']),expectedTarget=e['expected_target'], + expectedAckSlot=e['expected_ack_slot'],expectedOwner=e['expected_owner'], + expectedMessageId=h.expected['expected_message_id'],expectedEnvelopeRef=h.expected['expected_envelope_ref'], + rootEntry=probe_ts.encoded(f['entries']['root']),writeEntry=probe_ts.encoded(h.write),bootstrapEntry=probe_ts.encoded(h.offer))|extra) + + def native(self,value): + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(value).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=40) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]) + value=json.loads(result.stdout);self.assertEqual(value['subprocessCalls'],0);return value + + def revoke(self,kind,original,mask): + root=self.f['expected']['expected_ack_slot']['root_key'] + scope=hashlib.sha256(canonical_bytes(dict(kind='authority',root_key=root, + authority_kind=kind,authority_sha256=original['ref']['raw_sha256']))).hexdigest() + payload=json.loads(self.host.expected['current_statuses'][0]['raw'])['payload'];payload['revision']=3 + for item in payload['entries']: + if item['scope_id']==scope:item.update(status='revoked',operation_mask=mask) + return signed_entry(payload,self.f['signers']['owner'],'synthetic-native-offer-revocation') + + def test_native_complete_source_read_after_restart(self): + self.host.http.restart();result=self.native(self.call()) + self.assertTrue(result['ok'],result);self.assertTrue(result['immutable']) + self.assertEqual(result['consumer'],'ack_offer') + self.assertEqual(result['write'],self.host.write['ref']);self.assertEqual(result['bootstrap'],self.host.offer['ref']) + self.assertEqual(result['metrics']['requests'],12);self.assertEqual(result['originals'],12) + self.assertEqual(result['nativeChecks'],result['metrics']['signature_checks']) + self.assertEqual(result['calls'][:2],['bootstrap.probe','bootstrap.answer']) + self.assertTrue(all(kind=='bootstrap.proof_child_request' for kind in result['calls'][2:])) + + def test_writer_admit_revocation_refuses_before_any_private_probe(self): + revoked=self.revoke('ack.write_grant',self.host.write,1) + result=self.native(self.call(knownStatuses=[probe_ts.encoded(revoked)])) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_authority_revoked');self.assertEqual(result['calls'],[]) + + def test_historical_owner_read_revocation_does_not_revoke_writer_admit(self): + revoked=self.revoke('ack.read_grant',self.f['entries']['read'],2) + gate=RepairAckOfferAccess(self.host.source.state);gate.initialize() + prepared=gate.prepare(self.host.source.resource_id,action='proof',current_statuses=[revoked,self.host.expected['current_statuses'][1]]) + with self.host.source.state._transaction():self.assertTrue(gate.check_locked(prepared).allowed) + result=self.native(self.call(knownStatuses=[probe_ts.encoded(revoked)])) + self.assertTrue(result['ok'],result);self.assertEqual(result['metrics']['requests'],12) + + def test_root_discover_revocation_and_archived_write_revocation_refuse_before_probe(self): + cases=[('ack.root_authority',self.f['entries']['root'],8,'knownStatuses'), + ('ack.write_grant',self.host.write,1,'archiveStatuses')] + for kind,original,mask,field in cases: + with self.subTest(kind=kind): + revoked=self.revoke(kind,original,mask) + result=self.native(self.call(**{field:[probe_ts.encoded(revoked)]})) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_authority_revoked');self.assertEqual(result['calls'],[]) + + def test_message_binding_mismatch_refuses_before_network(self): + result=self.native(self.call(expectedMessageId='msg_'+'f'*64)) + self.assertFalse(result['ok']);self.assertEqual(result['calls'],[]) + self.assertNotEqual(result['code'],'untyped_error') + + +if __name__=='__main__':unittest.main() From 6d36f3f19e50ce1af2e2b3cdd4a30ca8c78ca7f7 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:33:05 +0800 Subject: [PATCH 02/24] Upload native saved receipts and retain interoperable recovery journals --- .../network/open-repair-occupied.ts | 9 +- .../network/open-repair-offer-client.ts | 168 +++++++++++++++++- docs/OPEN_ACK_RECOVERY.md | 19 +- scripts/run_open_network_ci.py | 2 +- .../test_open_repair_put_client_typescript.py | 107 +++++++++++ 5 files changed, 294 insertions(+), 11 deletions(-) create mode 100644 tests/test_open_repair_put_client_typescript.py diff --git a/clients/typescript/network/open-repair-occupied.ts b/clients/typescript/network/open-repair-occupied.ts index 40a8e6d..e40d601 100644 --- a/clients/typescript/network/open-repair-occupied.ts +++ b/clients/typescript/network/open-repair-occupied.ts @@ -123,14 +123,14 @@ function floors(previous:readonly AuthenticatedStatusOriginal[],current:readonly for(const observed of previous){const p=observed.payload as Obj;for(const item of p.entries){const scope=p.scope_key.issuer_key_id+':'+scopeKey(item),held=prior.get(scope)??[0,0];prior.set(scope,[Math.max(held[0],p.revision),Math.max(held[1],item.minimum_document_revision)]);}} for(const observed of current){const p=observed.payload as Obj;for(const item of p.entries){const held=prior.get(p.scope_key.issuer_key_id+':'+scopeKey(item));if(held&&(p.revision!same(p[name],c[name])))mismatch();return head; } + +// The native writer uses the same receipt, consent, window and historical +// checks as the independently verified occupied source response. +export {inputs as verifyAckReceiptInputs, obligations as ackReceiptObligations, + windows as ackReceiptWindows, floors as checkAckReceiptHistoryFloors}; diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 3be1f55..1577c22 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -2,6 +2,7 @@ * This reads original source evidence only; it does not authorize or upload a receipt. */ import {performance} from 'node:perf_hooks'; +import {randomBytes} from 'node:crypto'; import {isProxy} from 'node:util/types'; import {OpenHTTPTransport,endpoint} from './open-transport.ts'; import {RepairBudget,RepairError,buildNewWire,objectFields,rawRef,u53,LocalRawResolver,parseNewWire} from './open-repair-wire.ts'; @@ -12,11 +13,14 @@ import type {AckOwnerRecoveryOptions,RecoveredAckOwnerEmptyProof} from './open-r import {verifyAckEmptySourceEvent} from './open-repair-empty.ts'; import type {AuthenticatedAckEmptySourceEvent} from './open-repair-empty.ts'; import {parseOriginalControl,verifySourceNodeOriginal,verifyBoundedControlSignature} from './open-repair-original.ts'; -import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge,signBoundedBootstrapOriginal} from './open-repair-probe.ts'; import {verifyBootstrapProofResponse,makeBootstrapChildRequest} from './open-repair-proof.ts'; import {statusScope,verifyStatusOriginal,authenticateStatusOriginal} from './open-repair-status.ts'; import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +import {verifyAckReceiptInputs,ackReceiptObligations,ackReceiptWindows,checkAckReceiptHistoryFloors, + verifyAckOccupiedSourceEvent,verifyAckOccupiedHead} from './open-repair-occupied.ts'; +import type {AuthenticatedAckOccupiedSourceEvent} from './open-repair-occupied.ts'; type Obj=Record; const clock=()=>performance.now()/1000; export interface AckOfferOptions { @@ -77,10 +81,13 @@ export class AckOfferClient{ close():void{if(this.#ownTransport)this.#transport.close();} #now():number{return u53(Math.floor(this.#clock()));} async preflight(baseUrl:string,value:AckOfferOptions):Promise{ + return this.#preflight(baseUrl,value,new RepairBudget(this.#policy)); + } + async #preflight(baseUrl:string,value:AckOfferOptions,budget:RepairBudget):Promise{ const args=options(value,['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId', 'expectedEnvelopeRef','rootEntry','writeEntry','bootstrapEntry'],['knownStatuses','archiveStatuses','timeout']); const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); - const policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),deadline=clock()+timeout; + const policy=this.#policy,started=this.#now(),deadline=clock()+timeout; const expected=buildNewWire({target:args.expectedTarget,slot:args.expectedAckSlot,owner:args.expectedOwner, message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); @@ -164,6 +171,114 @@ export class AckOfferClient{ return Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:saved, metrics:Object.freeze({requests,wire_bytes:wireBytes,proof_bytes:proofBytes,...budget.snapshot()})}); } + async put(baseUrl:string,receiptEntry:unknown,disclosureEntry:unknown,putEntry:unknown,value:AckReceiptPutOptions):Promise{ + const preflightNames=['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', + 'rootEntry','writeEntry','bootstrapEntry']; + const args=options(value,[...preflightNames,'currentStatuses','readUntil','retainUntil'], + ['knownStatuses','archiveStatuses','knownDisclosureStatuses','timeout','journal']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + if(args.journal!==undefined&&typeof args.journal!=='function')fail('repair_invalid_context'); + const started=clock(),policy=this.#policy,budget=new RepairBudget(policy); + const expected=buildNewWire({owner:args.expectedOwner,target:args.expectedTarget,message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; + const readUntil=u53(args.readUntil),retainUntil=u53(args.retainUntil); + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const retained=checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget),disclosureHistory=checks.history(args.knownDisclosureStatuses??[],[],budget); + if(!Array.isArray(args.currentStatuses)||isProxy(args.currentStatuses)||args.currentStatuses.length<1||args.currentStatuses.length>8)fail('repair_invalid_status'); + const currentEntries=checks.history(args.currentStatuses,[],budget); + // Snapshot private upload originals before the first asynchronous read. + const receipt=snapshotReceiptEntry(receiptEntry,policy,budget),disclosure=snapshotEntry(disclosureEntry,policy,budget,true),put=snapshotEntry(putEntry,policy,budget,true); + const preflight=Object.fromEntries(preflightNames.map(name=>[name,args[name]])); + preflight.archiveStatuses=retained;preflight.timeout=timeout; + const prepared=await this.#preflight(baseUrl,preflight as unknown as AckOfferOptions,budget),source=prepared.source; + const inputs=verifyAckReceiptInputs(source,receipt,disclosure,put,expected.owner,this.#subject,this.#now(),policy,budget); + if(!same((inputs.disclosure.payload as Obj).bootstrap_return.roles,['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt']))fail('repair_disclosure_permission'); + const windows=ackReceiptWindows(source,inputs,this.#now()); + if(!(this.#now()=expiry||clock()>=started+timeout)fail('repair_access_expired'); + if(budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); + budget.output(16);const useId='use_'+randomBytes(16).toString('hex'); + const use=signBoundedBootstrapOriginal({schema_version:'memory-vault-open-repair/v1',kind:'bootstrap.use',signing_key:this.#subject.signing_key, + issued_at:this.#now(),expires_at:expiry,use_id:useId,subject:handle.subject,target:handle.target,target_storage_epoch:handle.target_storage_epoch, + consumer:'ack_offer',operation:'ack.put',bootstrap_probe_ref:handle.probe_ref,bootstrap_manifest_ref:prepared.proof.manifest_ref, + request_ref:inputs.put.ref},this.#identity,policy,budget); + const packet=buildNewWire({schema_version:'memory-vault-open-repair/v1',kind:'ack.put_request',use:encodePutEntry(use,budget), + receipt:encodePutEntry(inputs.receipt,budget),disclosure:encodePutEntry(inputs.disclosure,budget),put:encodePutEntry(inputs.put,budget), + current_statuses:checked.map(item=>encodePutEntry(item,budget)),read_until:readUntil,retain_until:retainUntil},policy,budget); + const grant=source.authorities.originals.bootstrap.payload as Obj; + const uploadBytes=packet.raw.length+use.raw.length+[...Object.values(inputs),...checked].reduce((sum,item)=>sum+item.raw.length,0); + if(packet.raw.length>65536||prepared.metrics.requests+1>grant.limits.max_requests||prepared.metrics.proof_bytes+uploadBytes>grant.limits.max_proof_bytes)fail('repair_over_budget'); + if(args.journal){ + const digest=budget.hash(packet.raw),journal=buildNewWire({schema_version:'memory-vault-ack-put-journal/v1', + packet:encodePutEntry({raw:packet.raw,ref:rawRef({namespace:'meta',key:digest,raw_sha256:digest,size:packet.raw.length})},budget), + handle:encodePutEntry(prepared.proof.handle,budget),manifest:encodePutEntry({raw:prepared.proof.manifest.raw,ref:prepared.proof.manifest_ref},budget), + originals:prepared.originals.map(item=>encodePutEntry(item,budget)),retained:retained.map(item=>encodePutEntry(item,budget)), + disclosure_statuses:disclosureHistory.map(item=>encodePutEntry(item,budget))},policy,budget); + await journalBeforeDeadline(args.journal,'request',journal.raw,started+timeout); + } + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); + const response=await this.#transport.requestRepair(baseUrl,packet.raw,started+timeout); + if(response.length===0||response.length>65536)fail('repair_invalid_response'); + const body=fields(parseNewWire(response,policy,budget).value,['schema_version','kind','use_ref','manifest','commit','head','packs']); + if(body.schema_version!=='memory-vault-open-repair/v1'||body.kind!=='ack.put_response'||!same(rawRef(body.use_ref),use.ref))fail('repair_put_mismatch'); + const manifest=decodePutEntry(body.manifest,policy,budget),commit=decodePutEntry(body.commit,policy,budget),head=decodePutEntry(body.head,policy,budget); + if(!Array.isArray(body.packs)||body.packs.length<1||body.packs.length>policy.max_entries)fail('repair_invalid_pack'); + const packs:RawOriginal[]=body.packs.map((item:unknown)=>decodePutEntry(item,policy,budget)); + const transferred=prepared.metrics.proof_bytes+uploadBytes+response.length+[manifest,commit,head,...packs].reduce((sum,item)=>sum+item.raw.length,0); + if(transferred>grant.limits.max_proof_bytes)fail('repair_over_budget'); + const originals=new Map(prepared.originals.map(item=>[refKey(item.ref),item])),resolver=new LocalRawResolver(policy,budget); + for(const item of (prepared.proof.manifest.value as Obj).children)if(item.role==='history.raw_pack'){ + const reference=rawRef(item.ref),raw=originals.get(refKey(reference))!.raw; + if(!same(resolver.put(reference.namespace,reference.key,raw).ref,reference))fail('repair_ref_mismatch'); + } + const wanted=new Set((parseNewWire(manifest.raw,policy,budget).value as Obj).roles.map((item:Obj)=>refKey(rawRef(item.pack_ref)))); + if(packs.length!==wanted.size||new Set(packs.map(item=>refKey(item.ref))).size!==wanted.size||packs.some(item=>!wanted.has(refKey(item.ref))))fail('repair_unused_pack'); + for(const item of packs)if(!same(resolver.put(item.ref.namespace,item.ref.key,item.raw).ref,item.ref))fail('repair_ref_mismatch'); + const complete=verifyAckOccupiedSourceEvent(manifest,resolver,commit,{expectedAckSlot:slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject, + expectedMessageId:expected.message,expectedEnvelopeRef:expected.envelope,expectedTarget:expected.target, + targetStorageEpoch:handle.target_storage_epoch,limitPolicy:this.#limits,policy,budget}); + verifyAckOccupiedHead(head,complete,{expectedTarget:expected.target,policy,budget}); + if(!same(complete.predecessor.custody.ref,source.custody.ref)||complete.read_until!==readUntil||complete.retain_until!==retainUntil|| + Object.entries(inputs).some(([name,item])=>!same(complete.inputs[name as keyof typeof inputs].ref,item.ref)|| + !Buffer.from(complete.inputs[name as keyof typeof inputs].raw).equals(Buffer.from(item.raw))))fail('repair_put_mismatch'); + checkAckReceiptHistoryFloors(checked,complete.statuses); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); + for(const item of [manifest,commit,head,...packs,...complete.manifest.roles.map(item=>item.original)])originals.set(refKey(item.ref),item); + if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); + return Object.freeze({source:complete,originals:Object.freeze([...originals.values()].map(item=>Object.freeze({ref:item.ref,get raw(){return Uint8Array.from(item.raw);}}))), + metrics:Object.freeze({requests:prepared.metrics.requests+1,transfer_bytes:transferred,...budget.snapshot()})}); + } + #putStatuses(entries:unknown,retained:RawOriginal[],duties:Obj[],source:AuthenticatedAckEmptySourceEvent,owner:Obj,budget:RepairBudget, + retainedAuthorities:RawOriginal[],currentAuthorities:readonly AuthenticatedStatusOriginal[]):AuthenticatedStatusOriginal[]{ + if(!Array.isArray(entries)||isProxy(entries)||entries.length<1||entries.length>8)fail('repair_invalid_status'); + if(retained.length>16)fail('repair_status_history_capacity'); + const authenticated:AuthenticatedStatusOriginal[]=[],old:AuthenticatedStatusOriginal[]=[],root=(source.custody.payload.ack_slot as Obj).root_key; + for(const [index,value] of [...entries,...retained,...retainedAuthorities].entries()){ + const item=snapshotEntry(value,this.#policy,budget,true),payload=checkedStatus(item.raw,this.#policy,budget),issuer=payload.scope_key.issuer_key_id; + const allowed=duties.filter(duty=>duty.signer.key_id===issuer); + if(!allowed.length||(index>=entries.length&&indexthis.#now()+30)fail('repair_status_mismatch'); + const checked=authenticateStatusOriginal(item,{expectedRoot:root,expectedSigningKey:allowed[0].signer,at, + allowedScopes:allowed.map(duty=>({scope_kind:duty.scope_kind,scope_id:duty.scope_id})),policy:this.#policy,budget},this.#statusObserver); + (index(item.payload as Obj).scope_key.issuer_key_id===duty.signer.key_id? + (item.payload as Obj).entries.filter((entry:Obj)=>entry.scope_kind===duty.scope_kind&&entry.scope_id===duty.scope_id).map((entry:Obj)=>({revision:item.payload.revision,entry})):[]); + if(!observations.length)fail('repair_status_missing'); + observations.sort((a,b)=>b.revision-a.revision);const entry=observations[0].entry; + if(entry.status!=='active'||(entry.operation_mask&duty.mask)!==duty.mask)fail('repair_authority_revoked'); + if(entry.minimum_document_revision>duty.revision)fail('repair_status_revision'); + } + const obligations=duties.map(duty=>({kind:duty.scope_kind,scope_id:duty.scope_id,signer:duty.signer,mask:duty.mask,revision:duty.revision,role:duty.role,subject:null})); + new OfferStatusChecks(owner,this.#policy,this.#clock,this.#statusObserver).floors( + [...source.predecessor.statuses,...source.statuses,...currentAuthorities,...old],authenticated,obligations); + return authenticated; + } #emptyHead(entry:RawOriginal,source:AuthenticatedAckEmptySourceEvent,target:Obj,budget:RepairBudget):void{ const input=snapshotEntry(entry,this.#policy,budget),signed=fields(parseNewWire(input.raw,this.#policy,budget).value,['payload','proof']); const p=fields(signed.payload,['schema_version','kind','signing_key','ack_slot','generation','observed_at','retain_until','state','root_authority_ref','grant_ref','binding_ref']); @@ -229,9 +344,9 @@ class OfferStatusChecks{ } // Activation's historical owner READ grant is checked against the exact // recovered history below; it cannot revoke the writer's independent ADMIT. - this.#floors(checked,[],obligations,true,deferredAuthorities);return checked; + this.floors(checked,[],obligations,true,deferredAuthorities);return checked; } - #floors(known:readonly AuthenticatedStatusOriginal[],current:readonly AuthenticatedStatusOriginal[],obligations:Obligation[],probePhase=false,deferredAuthorities=false):void{ + floors(known:readonly AuthenticatedStatusOriginal[],current:readonly AuthenticatedStatusOriginal[],obligations:Obligation[],probePhase=false,deferredAuthorities=false):void{ const seen=new Map(),floors=new Map(),required=new Map(obligations.map(v=>[v.signer.key_id+':'+v.kind+':'+v.scope_id,v])); for(const observed of [...known,...current]){ const payload=observed.payload as Obj,issuer=payload.scope_key.issuer_key_id,revision=payload.revision,id=issuer+':'+revision; @@ -273,9 +388,52 @@ class OfferStatusChecks{ checked.push(verifyStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:group.signer,at:this.#now(), allowedScopes:allowed.map(v=>({scope_kind:v.kind,scope_id:v.scope_id})),required:group.required,policy,budget},this.#statusObserver)); } - this.#floors([...prior.statuses,...source.statuses,...known],checked,obligations); + this.floors([...prior.statuses,...source.statuses,...known],checked,obligations); if(this.#now()>=Math.min(...['admit_until','read_until','retain_until'].map(n=>active.windows[n])))fail('repair_access_expired'); return checked; } } function checkedNodePayload(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{return fields(parseOriginalControl(raw,policy,budget).value,['payload','proof']).payload;} + +export interface AckReceiptPutOptions extends AckOfferOptions { + readonly currentStatuses:readonly unknown[];readonly readUntil:number;readonly retainUntil:number; + readonly knownDisclosureStatuses?:readonly unknown[]; + readonly journal?:(kind:'request'|'response',raw:Uint8Array)=>void|Promise; +} +export interface CommittedAckReceipt { + readonly source:AuthenticatedAckOccupiedSourceEvent;readonly originals:readonly RawOriginal[]; + readonly metrics:Readonly; +} +export class AckReceiptClient extends AckOfferClient { + constructor(identity:unknown,encryptionIdentity:unknown,value:AckOwnerRecoveryOptions={}){ + const args=options(value,[],['policy','limitPolicy','allowLoopback','transport','clock','statusObserver']); + super(identity,encryptionIdentity,{...args,policy:args.policy??{...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:96}}); + } +} +function snapshotReceiptEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const held=fields(value,['raw','ref']),ref=rawRef(held.ref); + if(ref.size>4096)fail('repair_put_too_large'); + const parsed=parseOriginalControl(held.raw,policy,budget),raw=parsed.raw; + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; +} +function encodePutEntry(item:RawOriginal,budget:RepairBudget):Obj{ + const ref=rawRef(item.ref),raw=item.raw;budget.input(raw.length); + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + budget.output(Math.ceil(raw.length*4/3));return {ref,raw_base64url:Buffer.from(raw).toString('base64url')}; +} +function decodePutEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const item=fields(value,['raw_base64url','ref']),ref=rawRef(item.ref),encoded=item.raw_base64url; + if(ref.namespace!=='meta'||ref.size>policy.max_document_bytes||typeof encoded!=='string'||encoded.length>Math.ceil(ref.size*4/3)||!/^[A-Za-z0-9_-]*$/.test(encoded))fail('repair_put_too_large'); + budget.output(ref.size);const raw=Buffer.from(encoded,'base64url'); + if(raw.length!==ref.size||raw.toString('base64url')!==encoded||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; +} + +async function journalBeforeDeadline(callback:NonNullable,kind:'request'|'response',raw:Uint8Array,deadline:number):Promise{ + const remaining=deadline-clock();if(remaining<=0)fail('repair_access_expired'); + let timer:ReturnType|undefined; + try{ + await Promise.race([Promise.resolve().then(()=>callback(kind,raw)),new Promise((_,reject)=>{ + timer=setTimeout(()=>reject(new RepairError('repair_access_expired')),Math.ceil(remaining*1000)); + })]); + }finally{if(timer!==undefined)clearTimeout(timer);} +} diff --git a/docs/OPEN_ACK_RECOVERY.md b/docs/OPEN_ACK_RECOVERY.md index b782b41..b0fa7b9 100644 --- a/docs/OPEN_ACK_RECOVERY.md +++ b/docs/OPEN_ACK_RECOVERY.md @@ -1052,6 +1052,19 @@ is a positive deadline of at most sixty seconds. The result contains the authenticated empty source, bounded proof, current statuses, original bytes and actual request/work counts. This preflight uploads -no receipt and grants no upload authority. The complete native saved-receipt -upload and restart journal remain subsequent integration work; Python's existing -receipt-return workflow remains the available complete path. +no receipt and grants no upload authority. `AckReceiptClient` from the same module additionally uploads the existing saved +receipt under its independent disclosure and put consent, then authenticates the +returned occupied storage history. It uses one finite work budget for preflight, +upload and response verification, with a default limit of 96 signature checks. + +Call `put(baseUrl, receiptEntry, disclosureEntry, putEntry, options)` with the +preflight options plus `currentStatuses`, `readUntil` and `retainUntil`. Optional +`knownDisclosureStatuses` retain prior recipient consent observations. A +`journal(kind, raw)` callback may persist the exact `request` journal before any +private receipt upload and the verified `response` afterward. The callback is +awaited; failure to store the request stops the upload. Journal data contains +private original records and belongs in the participant's private durable store. +The journal format is compatible with the existing Python receipt recovery +client. Native prepared-return reuse, native journal replay and automatic Agent +integration remain subsequent work; Python remains the complete automated +receipt-return path. diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 580f751..30bca3f 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -47,7 +47,7 @@ "repair_http", "repair_client", "repair_client_typescript", "repair_admin", "repair_admin_typescript", "repair_runtime_review", "repair_bound", "repair_bound_typescript", "repair_empty", "repair_empty_typescript", "repair_empty_review", "repair_empty_access", "repair_empty_http", "repair_empty_client", "repair_empty_client_typescript", - "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( + "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_client_typescript", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( "tests.test_open_repair_stage", "tests.test_open_repair_index", "tests.test_open_repair_index_access", "tests.test_open_repair_index_journal", "tests.test_open_repair_index_state", "tests.test_open_repair_index_http", "tests.test_open_repair_index_client", "tests.test_open_repair_index_recovery", "tests.test_open_repair_index_admin", diff --git a/tests/test_open_repair_put_client_typescript.py b/tests/test_open_repair_put_client_typescript.py new file mode 100644 index 0000000..feacf16 --- /dev/null +++ b/tests/test_open_repair_put_client_typescript.py @@ -0,0 +1,107 @@ +"""Native recipient uploads its actual receipt and checks the returned storage event.""" +import base64 +import json +import unittest +from unittest.mock import patch + +from tests import test_open_repair_offer_client_typescript as offer_ts +from tests import test_open_repair_put_client as put_py +from tests import test_open_repair_probe_typescript as probe_ts +from tests.test_open_repair_occupied import receipt_inputs + +DRIVER=offer_ts.DRIVER.replace("const {AckOfferClient}=", "const {AckReceiptClient}=").replace('new AckOfferClient(', 'new AckReceiptClient(') +DRIVER=DRIVER.replace("calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);","const packet=JSON.parse(Buffer.from(raw).toString());calls.push(packet.payload?.kind??packet.kind);") +DRIVER=DRIVER.replace("const input=JSON.parse", "const journal=[];\nconst input=JSON.parse") +DRIVER=DRIVER.replace("return request(base,raw,...rest);", """const response=await request(base,raw,...rest); + if(input.corruptReply&&calls.at(-1)==='ack.put_request'){ + const body=JSON.parse(Buffer.from(response).toString());body.commit.ref.raw_sha256='0'.repeat(64); + const {canonicalBytes}=await import('./crypto.ts');return canonicalBytes(body); + }return response;""") +DRIVER=DRIVER.replace("['knownStatuses','archiveStatuses']", "['knownStatuses','archiveStatuses','knownDisclosureStatuses','currentStatuses']") +DRIVER=DRIVER.replace('const result=await client.preflight(input.base,options);', """options.journal=async(kind,raw)=>{ + journal.push({kind,raw:Buffer.from(raw).toString('base64'),calls:[...calls]}); + if(input.failJournal&&kind==='request')throw Error('synthetic journal unavailable'); + if(input.blockJournal&&kind==='request')await new Promise(()=>{}); + }; + const result=await client.put(input.base,decode(input.receipt),decode(input.disclosure),decode(input.put),options);""") +start=DRIVER.index(' process.stdout.write(JSON.stringify({ok:true') +end=DRIVER.index('\n}catch(error)',start) +DRIVER=DRIVER[:start]+''' process.stdout.write(JSON.stringify({ok:true,metrics:result.metrics, + receipt:Buffer.from(result.source.inputs.receipt.raw).toString('base64'),commit:result.source.commit.ref, + immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls,journal}));'''+DRIVER[end:] +DRIVER=DRIVER.replace('calls,nativeChecks,subprocessCalls}));}', 'calls,nativeChecks,subprocessCalls,journal}));}') + + +class NativePutClientTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + offer_ts.NativeOfferClientTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.case=put_py.RepairPutClientTests();self.addCleanup(self.case.doCleanups) + if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage': + # One original finite grant funds both the upload and exact replay. + with patch.object(put_py.offer_fixture.RepairOfferClientTests,'fixture_limits',dict(proof_limit=524288),create=True):self.case.setUp() + else:self.case.setUp() + self.host=self.case.host;self.f=self.host.f + + def call(self,inputs=None,**extra): + value=offer_ts.NativeOfferClientTests.call(self) + receipt,disclosure,put,options=inputs or self.case.inputs + value.update(receipt=probe_ts.encoded(receipt),disclosure=probe_ts.encoded(disclosure),put=probe_ts.encoded(put),now=2_000_000_008,**extra) + value['policy']=dict(value['policy'],max_signature_checks=96) + value['options'].update(currentStatuses=[probe_ts.encoded(item) for item in options['current_statuses']],readUntil=options['read_until'],retainUntil=options['retain_until']) + return value + + native=offer_ts.NativeOfferClientTests.native + + def test_native_upload_saves_exact_receipt_and_records_request_before_network(self): + self.host.http.restart();result=self.native(self.call()) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'});self.assertTrue(result['immutable']) + self.assertEqual(result['receipt'],probe_ts.encoded(self.case.inputs[0])['raw']) + self.assertEqual(result['metrics']['requests'],13) + self.assertEqual(result['nativeChecks'],result['metrics']['signature_checks']) + self.assertEqual([item['kind'] for item in result['journal']],['request','response']) + self.assertNotIn('ack.put_request',result['journal'][0]['calls']);self.assertEqual(result['calls'][-1],'ack.put_request') + self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'occupied') + + def test_legacy_disclosure_does_not_send_receipt(self): + from memory_vault_open_repair_occupied import RETURN_ROLES_LEGACY + inputs=receipt_inputs(self.case.fixture,bootstrap_roles=RETURN_ROLES_LEGACY) + result=self.native(self.call(inputs));self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_disclosure_permission') + self.assertNotIn('ack.put_request',result['calls']) + + def test_original_signature_budget_covers_preflight_and_upload_together(self): + value=self.call();value['policy']['max_signature_checks']=64 + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_over_budget') + self.assertNotIn('ack.put_request',result['calls']) + + def test_journal_failure_stops_before_private_upload(self): + result=self.native(self.call(failJournal=True));self.assertFalse(result['ok']) + self.assertNotIn('ack.put_request',result['calls']) + self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'empty') + + def test_unfinished_journal_obeys_original_deadline_without_upload(self): + value=self.call(blockJournal=True);value['options']['timeout']=8 + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + self.assertNotIn('ack.put_request',result['calls']) + + def test_bad_returned_commit_cannot_claim_verified_storage(self): + result=self.native(self.call(corruptReply=True));self.assertFalse(result['ok']) + self.assertEqual(result['code'],'repair_ref_mismatch') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'occupied') + # The exact native journal is interoperable with the existing durable + # Python recovery path; no second probe or replacement put is created. + receipt,disclosure,put,options=self.case.inputs + recovered=self.case.client().resume(self.host.http.base,base64.b64decode(result['journal'][0]['raw']), + receipt,disclosure,put,**options,**self.case.case.args) + self.assertEqual(recovered.source.inputs['receipt'].raw,receipt['raw']) + self.assertEqual(recovered.metrics['requests'],1) + self.assertEqual(self.host.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + + +if __name__=='__main__':unittest.main() From 83011b902727e965b02d38d127caa4738194083e Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:44:16 +0800 Subject: [PATCH 03/24] Stop ACK upload when current permission expires during journal persistence --- .../network/open-repair-offer-client.ts | 2 +- memory_vault_open_repair_put_client.py | 5 ++- .../test_open_repair_put_client_typescript.py | 38 ++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 1577c22..6a2a96f 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -197,7 +197,7 @@ export class AckOfferClient{ const slot=source.custody.payload.ack_slot as Obj,duties=ackReceiptObligations(source,inputs,expected.owner,this.#subject,expected.target,slot,policy,budget); const checked=this.#putStatuses(currentEntries,disclosureHistory,duties,source,expected.owner,budget,retained,prepared.current_statuses); const handle=prepared.proof.handle.payload as Obj; - const expiry=Math.min(handle.expires_at,windows.admitUntil,this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); + const expiry=Math.min(handle.expires_at,windows.admitUntil,...checked.map(item=>item.payload.valid_until as number),this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); if(budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); budget.output(16);const useId='use_'+randomBytes(16).toString('hex'); diff --git a/memory_vault_open_repair_put_client.py b/memory_vault_open_repair_put_client.py index 7ea2354..2d36f15 100644 --- a/memory_vault_open_repair_put_client.py +++ b/memory_vault_open_repair_put_client.py @@ -94,7 +94,8 @@ def put(self, base_url, receipt_entry, disclosure_entry, put_entry, *, current_s checked = self._put_statuses(current_statuses,known_disclosure_statuses,duties,source,owner,budget, retained_authorities=retained,current_authorities=prepared.current_statuses) handle = prepared.proof.handle.payload - expiry = min(handle["expires_at"],admit_until,self._now()+max(1,int(timeout-(time.monotonic()-started)))) + expiry = min(handle["expires_at"],admit_until,*(item.payload["valid_until"] for item in checked), + self._now()+max(1,int(timeout-(time.monotonic()-started)))) if self._now() >= expiry or time.monotonic()-started >= timeout: _fail("repair_access_expired") # Reserve enough of this same local meter for the full occupied @@ -118,6 +119,8 @@ def put(self, base_url, receipt_entry, disclosure_entry, put_entry, *, current_s if _journal is not None: journal = self._journal_record(packet.raw,prepared,retained,known_disclosure_statuses,budget) _journal("request",journal) + if self._now() >= expiry or time.monotonic()-started >= timeout: + _fail("repair_access_expired") response = self.transport.request_repair(base_url,packet.raw,deadline=started+timeout) result=self._finish(response,prepared,inputs,checked,owner,target,use,read_until,retain_until, preflight_options,started,timeout,expiry,upload_bytes,budget) diff --git a/tests/test_open_repair_put_client_typescript.py b/tests/test_open_repair_put_client_typescript.py index feacf16..1f04b4a 100644 --- a/tests/test_open_repair_put_client_typescript.py +++ b/tests/test_open_repair_put_client_typescript.py @@ -8,6 +8,7 @@ from tests import test_open_repair_put_client as put_py from tests import test_open_repair_probe_typescript as probe_ts from tests.test_open_repair_occupied import receipt_inputs +from tests.open_repair_ack_fixtures import signed_entry DRIVER=offer_ts.DRIVER.replace("const {AckOfferClient}=", "const {AckReceiptClient}=").replace('new AckOfferClient(', 'new AckReceiptClient(') DRIVER=DRIVER.replace("calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);","const packet=JSON.parse(Buffer.from(raw).toString());calls.push(packet.payload?.kind??packet.kind);") @@ -20,6 +21,7 @@ DRIVER=DRIVER.replace("['knownStatuses','archiveStatuses']", "['knownStatuses','archiveStatuses','knownDisclosureStatuses','currentStatuses']") DRIVER=DRIVER.replace('const result=await client.preflight(input.base,options);', """options.journal=async(kind,raw)=>{ journal.push({kind,raw:Buffer.from(raw).toString('base64'),calls:[...calls]}); + if(input.advanceOnJournal&&kind==='request')input.now=input.advanceOnJournal; if(input.failJournal&&kind==='request')throw Error('synthetic journal unavailable'); if(input.blockJournal&&kind==='request')await new Promise(()=>{}); }; @@ -41,7 +43,7 @@ def setUpClass(cls): def setUp(self): self.case=put_py.RepairPutClientTests();self.addCleanup(self.case.doCleanups) if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage': - # One original finite grant funds both the upload and exact replay. + # One original finite grant funds the two actual client exchanges. with patch.object(put_py.offer_fixture.RepairOfferClientTests,'fixture_limits',dict(proof_limit=524288),create=True):self.case.setUp() else:self.case.setUp() self.host=self.case.host;self.f=self.host.f @@ -82,6 +84,40 @@ def test_journal_failure_stops_before_private_upload(self): self.assertNotIn('ack.put_request',result['calls']) self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'empty') + def short_statuses(self,case=None): + case=case or self.case + signers={signer.key_id:signer for signer in case.host.f['signers'].values()} + result=[] + for index,item in enumerate(case.inputs[3]['current_statuses']): + payload=json.loads(item['raw'])['payload'];payload['revision']+=1;payload['valid_until']=2_000_000_009 + result.append(signed_entry(payload,signers[payload['signing_key']['key_id']],f'synthetic-short-status-{index}')) + return result + + def test_status_expiry_during_journal_write_stops_native_and_python_private_upload(self): + # The receiving service still has a live clock; the clients must refuse + # before transmitting even if that remote service would accept the put. + statuses=self.short_statuses();value=self.call(advanceOnJournal=2_000_000_009) + value['options']['currentStatuses']=[probe_ts.encoded(item) for item in statuses] + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + self.assertNotIn('ack.put_request',result['calls']) + from memory_vault_open_repair_wire import RepairWireError + # Each implementation uses its own original finite source; the first + # preflight must not consume the second implementation's test budget. + other=put_py.RepairPutClientTests();self.addCleanup(other.doCleanups);other.setUp() + receipt,disclosure,put,options=other.inputs;client=other.client() + def journal(kind,raw): + if kind=='request':client.clock=lambda:2_000_000_009 + calls=[];request=client.transport.request_repair + def observed(base,raw,**options): + body=json.loads(raw);calls.append(body.get('kind',body.get('payload',{}).get('kind'))) + return request(base,raw,**options) + with patch.object(client.transport,'request_repair',side_effect=observed),self.assertRaisesRegex(RepairWireError,'repair_access_expired'): + client.put(other.host.http.base,receipt,disclosure,put,**(options|dict(current_statuses=self.short_statuses(other))),**other.case.args,_journal=journal) + self.assertNotIn('ack.put_request',calls) + for host in (self.host,other.host): + self.assertEqual(host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'empty') + def test_unfinished_journal_obeys_original_deadline_without_upload(self): value=self.call(blockJournal=True);value['options']['timeout']=8 result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') From 5c2594d323730bb656c6245f4ee681a74be513bc Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:02:34 +0800 Subject: [PATCH 04/24] Replay native receipt journals across restarts without renewing authorization --- .../network/open-repair-offer-client.ts | 200 ++++++++++++++++-- .../test_open_repair_put_client_typescript.py | 103 ++++++++- 2 files changed, 281 insertions(+), 22 deletions(-) diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 6a2a96f..9be5f38 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -1,5 +1,5 @@ -/** Native receipt-writer preflight over independently authorized empty sources. - * This reads original source evidence only; it does not authorize or upload a receipt. +/** Native receipt-writer preflight, consent-bound upload and exact journal replay. + * Preflight alone reads evidence; upload requires the independent receipt consent. */ import {performance} from 'node:perf_hooks'; import {randomBytes} from 'node:crypto'; @@ -30,6 +30,10 @@ export interface AckOfferOptions { readonly knownStatuses?:readonly unknown[];readonly archiveStatuses?:readonly unknown[];readonly timeout?:number; } export type PreparedAckOffer=RecoveredAckOwnerEmptyProof; +const OFFER_FIELDS=['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', + 'rootEntry','writeEntry','bootstrapEntry'] as const; +interface CachedAckReturn {context:string;prepared:PreparedAckOffer;budget:RepairBudget;started:number;timeout:number;retained:RawOriginal[];} + interface Obligation{role:string|null;kind:string;subject:unknown;revision:number;signer:Obj;scope_id:string;mask?:number;} function fail(code:string):never{throw new RepairError(code);} function fields(value:unknown,names:readonly string[]):Obj{return objectFields(value,names);} @@ -57,6 +61,7 @@ function snapshotEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget,leg return {raw,ref}; } export class AckOfferClient{ + #preparedReturn:CachedAckReturn|undefined;#preparedGeneration=0; readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #limits:Obj; readonly #statusObserver?: (item:AuthenticatedStatusOriginal)=>void; readonly #transport:OpenHTTPTransport;readonly #allowLoopback:boolean;readonly #clock:()=>number;readonly #ownTransport:boolean; @@ -78,14 +83,32 @@ export class AckOfferClient{ this.#statusObserver=args.statusObserver; Object.freeze(this); } - close():void{if(this.#ownTransport)this.#transport.close();} + close():void{this.#preparedReturn=undefined;this.#preparedGeneration++;if(this.#ownTransport)this.#transport.close();} #now():number{return u53(Math.floor(this.#clock()));} async preflight(baseUrl:string,value:AckOfferOptions):Promise{ return this.#preflight(baseUrl,value,new RepairBudget(this.#policy)); } + async prepareReturn(baseUrl:string,value:AckOfferOptions):Promise{ + this.#preparedReturn=undefined;const generation=++this.#preparedGeneration; + const args=options(value,OFFER_FIELDS,['knownStatuses','archiveStatuses','timeout']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + const started=clock(),budget=new RepairBudget(this.#policy),context=this.#offerContext(baseUrl,args,budget); + const owner=buildNewWire(args.expectedOwner,this.#policy,budget).value as Obj; + const retained=new OfferStatusChecks(owner,this.#policy,this.#clock,this.#statusObserver).history(args.knownStatuses??[],args.archiveStatuses??[],budget); + const selected={...args,knownStatuses:[],archiveStatuses:retained} as unknown as AckOfferOptions; + const prepared=await this.#preflight(baseUrl,selected,budget); + if(generation!==this.#preparedGeneration)fail('repair_invalid_context'); + if(clock()>=started+timeout)fail('repair_access_expired'); + this.#preparedReturn={context,prepared,budget,started,timeout,retained};return prepared; + } + #offerContext(baseUrl:string,args:Obj,budget:RepairBudget):string{ + const selected:Obj={base_url:baseUrl}; + for(const name of OFFER_FIELDS)selected[name]=['targetNodeEntry','rootEntry','writeEntry','bootstrapEntry'].includes(name)? + snapshotEntry(args[name],this.#policy,budget,true).ref:args[name]; + return budget.hash(buildNewWire(selected,this.#policy,budget).raw); + } async #preflight(baseUrl:string,value:AckOfferOptions,budget:RepairBudget):Promise{ - const args=options(value,['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId', - 'expectedEnvelopeRef','rootEntry','writeEntry','bootstrapEntry'],['knownStatuses','archiveStatuses','timeout']); + const args=options(value,OFFER_FIELDS,['knownStatuses','archiveStatuses','timeout']); const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); const policy=this.#policy,started=this.#now(),deadline=clock()+timeout; const expected=buildNewWire({target:args.expectedTarget,slot:args.expectedAckSlot,owner:args.expectedOwner, @@ -172,24 +195,36 @@ export class AckOfferClient{ metrics:Object.freeze({requests,wire_bytes:wireBytes,proof_bytes:proofBytes,...budget.snapshot()})}); } async put(baseUrl:string,receiptEntry:unknown,disclosureEntry:unknown,putEntry:unknown,value:AckReceiptPutOptions):Promise{ - const preflightNames=['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', - 'rootEntry','writeEntry','bootstrapEntry']; + const preflightNames=OFFER_FIELDS; const args=options(value,[...preflightNames,'currentStatuses','readUntil','retainUntil'], ['knownStatuses','archiveStatuses','knownDisclosureStatuses','timeout','journal']); - const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + let timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); if(args.journal!==undefined&&typeof args.journal!=='function')fail('repair_invalid_context'); - const started=clock(),policy=this.#policy,budget=new RepairBudget(policy); + const cached=this.#preparedReturn;this.#preparedReturn=undefined;this.#preparedGeneration++; + const started=cached?.started??clock(),policy=this.#policy,budget=cached?.budget??new RepairBudget(policy); + if(cached){ + timeout=Math.min(timeout,cached.timeout); + if(clock()>=started+timeout)fail('repair_access_expired'); + if(cached.context!==this.#offerContext(baseUrl,args,budget))fail('repair_proof_mismatch'); + } const expected=buildNewWire({owner:args.expectedOwner,target:args.expectedTarget,message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; const readUntil=u53(args.readUntil),retainUntil=u53(args.retainUntil); const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); - const retained=checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget),disclosureHistory=checks.history(args.knownDisclosureStatuses??[],[],budget); + const retained=mergeRetained(cached?.retained??[],checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget)), + disclosureHistory=checks.history(args.knownDisclosureStatuses??[],[],budget); if(!Array.isArray(args.currentStatuses)||isProxy(args.currentStatuses)||args.currentStatuses.length<1||args.currentStatuses.length>8)fail('repair_invalid_status'); const currentEntries=checks.history(args.currentStatuses,[],budget); // Snapshot private upload originals before the first asynchronous read. const receipt=snapshotReceiptEntry(receiptEntry,policy,budget),disclosure=snapshotEntry(disclosureEntry,policy,budget,true),put=snapshotEntry(putEntry,policy,budget,true); const preflight=Object.fromEntries(preflightNames.map(name=>[name,args[name]])); preflight.archiveStatuses=retained;preflight.timeout=timeout; - const prepared=await this.#preflight(baseUrl,preflight as unknown as AckOfferOptions,budget),source=prepared.source; + const prepared=cached?.prepared??await this.#preflight(baseUrl,preflight as unknown as AckOfferOptions,budget),source=prepared.source; + // The cached proof does not renew its node descriptor or permit a changed + // source. Recheck the same exact original before private upload. + const nodeEntry=snapshotEntry(args.targetNodeEntry,policy,budget,true); + const sourceNode=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:expected.target.signing_key, + expectedStorageEpoch:prepared.proof.handle.payload.target_storage_epoch,at:this.#now(),policy,budget}); + if(!same(endpoint(baseUrl,this.#allowLoopback),endpoint(sourceNode.payload.base_url,this.#allowLoopback)))fail('repair_proof_mismatch'); const inputs=verifyAckReceiptInputs(source,receipt,disclosure,put,expected.owner,this.#subject,this.#now(),policy,budget); if(!same((inputs.disclosure.payload as Obj).bootstrap_return.roles,['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt']))fail('repair_disclosure_permission'); const windows=ackReceiptWindows(source,inputs,this.#now()); @@ -197,7 +232,7 @@ export class AckOfferClient{ const slot=source.custody.payload.ack_slot as Obj,duties=ackReceiptObligations(source,inputs,expected.owner,this.#subject,expected.target,slot,policy,budget); const checked=this.#putStatuses(currentEntries,disclosureHistory,duties,source,expected.owner,budget,retained,prepared.current_statuses); const handle=prepared.proof.handle.payload as Obj; - const expiry=Math.min(handle.expires_at,windows.admitUntil,...checked.map(item=>item.payload.valid_until as number),this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); + const expiry=Math.min(handle.expires_at,sourceNode.payload.expires_at as number,windows.admitUntil,...checked.map(item=>item.payload.valid_until as number),this.#now()+Math.max(1,Math.floor(timeout-(clock()-started)))); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); if(budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); budget.output(16);const useId='use_'+randomBytes(16).toString('hex'); @@ -221,6 +256,14 @@ export class AckOfferClient{ } if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); const response=await this.#transport.requestRepair(baseUrl,packet.raw,started+timeout); + const result=this.#finish(response,{prepared,inputs,checked,expected,use,readUntil,retainUntil,started,timeout,expiry,uploadBytes,budget}); + if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired');return result; + } + #finish(response:Uint8Array,context:PutFinishContext):CommittedAckReceipt{ + const {prepared,inputs,checked,expected,use,readUntil,retainUntil,started,timeout,expiry,uploadBytes,budget}=context; + const source=prepared.source,policy=this.#policy,slot=source.custody.payload.ack_slot as Obj, + handle=prepared.proof.handle.payload as Obj,grant=source.authorities.originals.bootstrap.payload as Obj; if(response.length===0||response.length>65536)fail('repair_invalid_response'); const body=fields(parseNewWire(response,policy,budget).value,['schema_version','kind','use_ref','manifest','commit','head','packs']); if(body.schema_version!=='memory-vault-open-repair/v1'||body.kind!=='ack.put_response'||!same(rawRef(body.use_ref),use.ref))fail('repair_put_mismatch'); @@ -247,10 +290,119 @@ export class AckOfferClient{ checkAckReceiptHistoryFloors(checked,complete.statuses); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); for(const item of [manifest,commit,head,...packs,...complete.manifest.roles.map(item=>item.original)])originals.set(refKey(item.ref),item); - if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired'); return Object.freeze({source:complete,originals:Object.freeze([...originals.values()].map(item=>Object.freeze({ref:item.ref,get raw(){return Uint8Array.from(item.raw);}}))), - metrics:Object.freeze({requests:prepared.metrics.requests+1,transfer_bytes:transferred,...budget.snapshot()})}); + metrics:Object.freeze({requests:context.replay?1:prepared.metrics.requests+1,transfer_bytes:transferred,...budget.snapshot()})}); + } + async resume(baseUrl:string,journalRaw:Uint8Array,receiptEntry:unknown,disclosureEntry:unknown,putEntry:unknown,value:AckReceiptPutOptions):Promise{ + const args=options(value,[...OFFER_FIELDS,'currentStatuses','readUntil','retainUntil'], + ['knownStatuses','archiveStatuses','knownDisclosureStatuses','timeout','journal']); + const timeout=args.timeout??30;if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + if(args.journal!==undefined&&typeof args.journal!=='function')fail('repair_invalid_context'); + this.#preparedReturn=undefined;this.#preparedGeneration++; + const started=clock(),policy=this.#policy,budget=new RepairBudget(policy); + const journal=fields(parseNewWire(journalRaw,policy,budget).value,['schema_version','packet','handle','manifest','originals','retained','disclosure_statuses']); + if(journal.schema_version!=='memory-vault-ack-put-journal/v1')fail('repair_invalid_context'); + for(const [name,maximum] of [['originals',policy.max_entries],['retained',32],['disclosure_statuses',16]] as const) + if(!Array.isArray(journal[name])||journal[name].length>maximum)fail('repair_over_budget'); + const packet=decodePutEntry(journal.packet,policy,budget);if(packet.raw.length>65536)fail('repair_put_too_large'); + const body=fields(parseNewWire(packet.raw,policy,budget).value,['schema_version','kind','use','receipt','disclosure','put','current_statuses','read_until','retain_until']); + if(body.schema_version!=='memory-vault-open-repair/v1'||body.kind!=='ack.put_request')fail('repair_put_mismatch'); + const useEntry=decodePutEntry(body.use,policy,budget),signed=fields(parseNewWire(useEntry.raw,policy,budget).value,['payload','proof']); + const use=fields(signed.payload,['schema_version','kind','signing_key','issued_at','expires_at','use_id','subject','target','target_storage_epoch', + 'consumer','operation','bootstrap_probe_ref','bootstrap_manifest_ref','request_ref']); + if(this.#now()>=u53(use.expires_at))fail('repair_reconciliation_required'); + verifyBoundedControlSignature(use,signed.proof,this.#subject.signing_key,budget); + const expected=buildNewWire({owner:args.expectedOwner,target:args.expectedTarget,slot:args.expectedAckSlot, + message:args.expectedMessageId,envelope:args.expectedEnvelopeRef},policy,budget).value as Obj; + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const retained=mergeRetained(journal.retained.map((item:unknown)=>decodePutEntry(item,policy,budget)), + checks.history(args.knownStatuses??[],args.archiveStatuses??[],budget)); + const prepared=this.#restore(baseUrl,journal,use,args,expected,retained,budget),source=prepared.source,handle=prepared.proof.handle.payload as Obj; + const dual=(keys:Obj)=>({signing_key_id:keys.signing_key.key_id,encryption_key_id:keys.encryption_key.key_id}); + if(use.schema_version!=='memory-vault-open-repair/v1'||use.kind!=='bootstrap.use'||!same(use.signing_key,this.#subject.signing_key)|| + use.consumer!=='ack_offer'||use.operation!=='ack.put'||!same(use.subject,dual(this.#subject))||!same(use.target,dual(expected.target))|| + use.target_storage_epoch!==handle.target_storage_epoch||!same(rawRef(use.bootstrap_probe_ref),rawRef(handle.probe_ref))|| + !same(rawRef(use.bootstrap_manifest_ref),prepared.proof.manifest_ref))fail('repair_put_mismatch'); + if(!(u53(use.issued_at)<=this.#now()&&use.expires_at-use.issued_at>=1&&use.expires_at-use.issued_at<=60)|| + typeof use.use_id!=='string'||!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(use.use_id))fail('repair_invalid_probe'); + const decoded={receipt:decodePutEntry(body.receipt,policy,budget,true),disclosure:decodePutEntry(body.disclosure,policy,budget),put:decodePutEntry(body.put,policy,budget)}; + for(const [name,value] of [['receipt',receiptEntry],['disclosure',disclosureEntry],['put',putEntry]] as const){ + const item=name==='receipt'?snapshotReceiptEntry(value,policy,budget):snapshotEntry(value,policy,budget,true); + if(!same(item.ref,decoded[name].ref)||!Buffer.from(item.raw).equals(Buffer.from(decoded[name].raw)))fail('repair_put_mismatch'); + } + const readUntil=u53(args.readUntil),retainUntil=u53(args.retainUntil); + if(!same(rawRef(use.request_ref),decoded.put.ref)||body.read_until!==readUntil||body.retain_until!==retainUntil)fail('repair_put_mismatch'); + const inputs=verifyAckReceiptInputs(source,decoded.receipt,decoded.disclosure,decoded.put,expected.owner,this.#subject,this.#now(),policy,budget); + if(!same((inputs.disclosure.payload as Obj).bootstrap_return.roles,['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt']))fail('repair_disclosure_permission'); + const windows=ackReceiptWindows(source,inputs,this.#now()),grant=source.authorities.originals.bootstrap.payload as Obj; + if(!(this.#now()Math.min(handle.expires_at,windows.admitUntil,grant.upload_until))fail('repair_put_mismatch'); + if(!Array.isArray(body.current_statuses)||body.current_statuses.length<1||body.current_statuses.length>8)fail('repair_invalid_status'); + const packetStatuses=body.current_statuses.map((item:unknown)=>decodePutEntry(item,policy,budget)); + const disclosureHistory=mergeRetained(journal.disclosure_statuses.map((item:unknown)=>decodePutEntry(item,policy,budget)), + checks.history(args.knownDisclosureStatuses??[],[],budget)); + if(disclosureHistory.length>16)fail('repair_status_history_capacity'); + const duties=ackReceiptObligations(source,inputs,expected.owner,this.#subject,expected.target,source.custody.payload.ack_slot as Obj,policy,budget); + const current=this.#putStatuses(args.currentStatuses,disclosureHistory,duties,source,expected.owner,budget,retained,prepared.current_statuses); + const checked=this.#putStatuses(packetStatuses,[],duties,source,expected.owner,budget,[],[...prepared.current_statuses,...current]); + const uploadBytes=packet.raw.length+useEntry.raw.length+[...Object.values(inputs),...checked].reduce((sum,item)=>sum+item.raw.length,0); + if(prepared.metrics.proof_bytes+uploadBytes>grant.limits.max_proof_bytes||budget.snapshot().signature_checks+32>(policy.max_signature_checks??0))fail('repair_over_budget'); + const node=snapshotEntry(args.targetNodeEntry,policy,budget,true),nodePayload=checkedNodePayload(node.raw,policy,budget); + const expiry=Math.min(use.expires_at,nodePayload.expires_at,...checked.map(item=>item.payload.valid_until as number),...current.map(item=>item.payload.valid_until as number)); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_reconciliation_required'); + // Every byte, nonce and consent is the original retained carrier. A fresh + // local read budget cannot renew the source's original use or resource. + const response=await this.#transport.requestRepair(baseUrl,packet.raw,started+timeout); + const result=this.#finish(response,{prepared,inputs,checked,expected,use:useEntry,readUntil,retainUntil,started,timeout,expiry,uploadBytes,budget,replay:true}); + if(args.journal)await journalBeforeDeadline(args.journal,'response',Uint8Array.from(response),started+timeout); + if(this.#now()>=expiry||clock()>=started+timeout)fail('repair_access_expired');return result; + } + #restore(baseUrl:string,journal:Obj,use:Obj,args:Obj,expected:Obj,retained:RawOriginal[],budget:RepairBudget):PreparedAckOffer{ + const policy=this.#policy,setup=verifyAckOfferBootstrapOriginal(args.bootstrapEntry,{root:args.rootEntry,write:args.writeEntry}, + {expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject,expectedMessageId:expected.message, + expectedEnvelopeRef:expected.envelope,at:this.#now(),limitPolicy:this.#limits,policy,budget}); + const nodeEntry=snapshotEntry(args.targetNodeEntry,policy,budget,true),nodePayload=checkedNodePayload(nodeEntry.raw,policy,budget); + const node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:expected.target.signing_key,expectedStorageEpoch:nodePayload.storage_epoch,at:this.#now(),policy,budget}); + if(!same(endpoint(baseUrl,this.#allowLoopback),endpoint(node.payload.base_url,this.#allowLoopback)))fail('repair_proof_mismatch'); + const handle=decodePutEntry(journal.handle,policy,budget),manifest=decodePutEntry(journal.manifest,policy,budget); + const hp=fields(parseNewWire(handle.raw,policy,budget).value,['payload','proof']).payload as Obj,grant=setup.originals.bootstrap.payload as Obj; + const handleEncoded=encodePutEntry(handle,budget),manifestEncoded=encodePutEntry(manifest,budget); + const response=buildNewWire({schema_version:'memory-vault-open-repair/v1',kind:'bootstrap.proof_response', + handle_raw_base64url:handleEncoded.raw_base64url,manifest_raw_base64url:manifestEncoded.raw_base64url},policy,budget); + const held=verifyBootstrapProofResponse(response.raw,{expectedSubject:this.#subject,expectedTarget:expected.target, + targetStorageEpoch:node.payload.storage_epoch as string,selector:grant.selector,bootstrapGrantRef:setup.originals.bootstrap.ref, + probeRef:rawRef(use.bootstrap_probe_ref),challengeRef:rawRef(hp.challenge_ref),answerRef:rawRef(hp.answer_ref),at:this.#now(), + maxProofItems:grant.limits.max_proof_items,maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'empty',consumer:'ack_offer',policy,budget}); + if(!same(held.handle.ref,handle.ref)||!same(held.manifest_ref,manifest.ref))fail('repair_ref_mismatch'); + const originals=new Map(),roles=new Map(); + for(const encoded of journal.originals){const item=decodePutEntry(encoded,policy,budget),key=refKey(item.ref);if(originals.has(key))fail('repair_invalid_context');originals.set(key,item);} + for(const item of (held.manifest.value as Obj).children){const reference=rawRef(item.ref);roles.set(item.role,[...(roles.get(item.role)??[]),reference]);} + const required=new Set([...roles.values()].flat().map(refKey)); + if(originals.size!==required.size||[...originals.keys()].some(key=>!required.has(key)))fail('repair_proof_mismatch'); + const resolver=new LocalRawResolver(policy,budget); + for(const reference of roles.get('history.raw_pack')!){const item=originals.get(refKey(reference))!; + if(!same(resolver.put(reference.namespace,reference.key,item.raw).ref,reference))fail('repair_ref_mismatch');} + const entry=(role:string):RawOriginal=>originals.get(refKey(roles.get(role)![0]))!; + const source=verifyAckEmptySourceEvent(entry('history.ack_empty'),resolver,entry('ack.empty_custody'),{ + expectedAckSlot:expected.slot,expectedOwner:expected.owner,expectedReceiptWriter:this.#subject,expectedMessageId:expected.message, + expectedEnvelopeRef:expected.envelope,expectedTarget:expected.target,targetStorageEpoch:node.payload.storage_epoch as string,limitPolicy:this.#limits,policy,budget}); + this.#emptyHead(entry('ack.head'),source,expected.target,budget); + for(const item of source.manifest.roles){const actual=originals.get(refKey(item.original.ref)); + if(!same(roles.get(item.role),[item.original.ref])||!actual||!Buffer.from(actual.raw).equals(Buffer.from(item.original.raw)))fail('repair_proof_mismatch');} + for(const [name,item] of [['root',source.predecessor.resources.originals.root],['write',source.authorities.originals.write],['bootstrap',source.authorities.originals.bootstrap]] as const) + if(!same(item.ref,setup.originals[name].ref)||!Buffer.from(item.raw).equals(Buffer.from(setup.originals[name].raw)))fail('repair_proof_mismatch'); + const packs=new Set();for(const historical of [source.manifest,source.predecessor.manifest]) + for(const item of (historical.manifest.value as Obj).roles)packs.add(refKey(rawRef(item.pack_ref))); + const supplied=roles.get('history.raw_pack')!.map(refKey); + if(packs.size!==supplied.length||supplied.some(key=>!packs.has(key)))fail('repair_unused_pack'); + const checks=new OfferStatusChecks(expected.owner,policy,this.#clock,this.#statusObserver); + const known=checks.known(retained,checks.obligations(setup.originals,expected.slot,budget),expected.slot.root_key,expected.target,budget,true); + const current=checks.current(source,roles,originals,expected.target,budget,known); + const transferred=response.raw.length+handle.raw.length+manifest.raw.length+[...originals.values()].reduce((sum,item)=>sum+item.raw.length,0); + if(transferred>grant.limits.max_proof_bytes)fail('repair_over_budget'); + return Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:Object.freeze([...originals.values()]), + metrics:Object.freeze({requests:0,wire_bytes:0,proof_bytes:transferred,...budget.snapshot()})}); } #putStatuses(entries:unknown,retained:RawOriginal[],duties:Obj[],source:AuthenticatedAckEmptySourceEvent,owner:Obj,budget:RepairBudget, retainedAuthorities:RawOriginal[],currentAuthorities:readonly AuthenticatedStatusOriginal[]):AuthenticatedStatusOriginal[]{ @@ -421,9 +573,9 @@ function encodePutEntry(item:RawOriginal,budget:RepairBudget):Obj{ if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); budget.output(Math.ceil(raw.length*4/3));return {ref,raw_base64url:Buffer.from(raw).toString('base64url')}; } -function decodePutEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ +function decodePutEntry(value:unknown,policy:RepairPolicy,budget:RepairBudget,receipt=false):RawOriginal{ const item=fields(value,['raw_base64url','ref']),ref=rawRef(item.ref),encoded=item.raw_base64url; - if(ref.namespace!=='meta'||ref.size>policy.max_document_bytes||typeof encoded!=='string'||encoded.length>Math.ceil(ref.size*4/3)||!/^[A-Za-z0-9_-]*$/.test(encoded))fail('repair_put_too_large'); + if((!receipt&&ref.namespace!=='meta')||ref.size>(receipt?4096:policy.max_document_bytes)||typeof encoded!=='string'||encoded.length>Math.ceil(ref.size*4/3)||!/^[A-Za-z0-9_-]*$/.test(encoded))fail('repair_put_too_large'); budget.output(ref.size);const raw=Buffer.from(encoded,'base64url'); if(raw.length!==ref.size||raw.toString('base64url')!==encoded||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; } @@ -437,3 +589,19 @@ async function journalBeforeDeadline(callback:NonNullable(); + for(const item of [...first,...second]){ + const key=refKey(item.ref),old=merged.get(key); + if(old&&!Buffer.from(old.raw).equals(Buffer.from(item.raw)))fail('repair_ref_conflict'); + merged.set(key,item);if(merged.size>32)fail('repair_status_history_capacity'); + } + return [...merged.values()]; +} + +interface PutFinishContext { + prepared:PreparedAckOffer;inputs:AuthenticatedAckOccupiedSourceEvent['inputs'];checked:AuthenticatedStatusOriginal[]; + expected:Obj;use:RawOriginal;readUntil:number;retainUntil:number;started:number;timeout:number;expiry:number; + uploadBytes:number;budget:RepairBudget;replay?:boolean; +} diff --git a/tests/test_open_repair_put_client_typescript.py b/tests/test_open_repair_put_client_typescript.py index 1f04b4a..e3ec36f 100644 --- a/tests/test_open_repair_put_client_typescript.py +++ b/tests/test_open_repair_put_client_typescript.py @@ -12,8 +12,9 @@ DRIVER=offer_ts.DRIVER.replace("const {AckOfferClient}=", "const {AckReceiptClient}=").replace('new AckOfferClient(', 'new AckReceiptClient(') DRIVER=DRIVER.replace("calls.push(JSON.parse(Buffer.from(raw).toString()).payload.kind);","const packet=JSON.parse(Buffer.from(raw).toString());calls.push(packet.payload?.kind??packet.kind);") -DRIVER=DRIVER.replace("const input=JSON.parse", "const journal=[];\nconst input=JSON.parse") -DRIVER=DRIVER.replace("return request(base,raw,...rest);", """const response=await request(base,raw,...rest); +DRIVER=DRIVER.replace("const input=JSON.parse", "const journal=[],sentPackets=[];\nconst input=JSON.parse") +DRIVER=DRIVER.replace("return request(base,raw,...rest);", """if(calls.at(-1)==='ack.put_request')sentPackets.push(Buffer.from(raw).toString('base64')); + const response=await request(base,raw,...rest); if(input.corruptReply&&calls.at(-1)==='ack.put_request'){ const body=JSON.parse(Buffer.from(response).toString());body.commit.ref.raw_sha256='0'.repeat(64); const {canonicalBytes}=await import('./crypto.ts');return canonicalBytes(body); @@ -25,13 +26,22 @@ if(input.failJournal&&kind==='request')throw Error('synthetic journal unavailable'); if(input.blockJournal&&kind==='request')await new Promise(()=>{}); }; - const result=await client.put(input.base,decode(input.receipt),decode(input.disclosure),decode(input.put),options);""") + if(input.prepare){ + const selected=Object.fromEntries(['targetNodeEntry','expectedTarget','expectedAckSlot','expectedOwner','expectedMessageId','expectedEnvelopeRef', + 'rootEntry','writeEntry','bootstrapEntry','knownStatuses','archiveStatuses','timeout'].filter(name=>Object.hasOwn(options,name)).map(name=>[name,options[name]])); + if(input.prepareTimeout)selected.timeout=input.prepareTimeout; + await client.prepareReturn(input.base,selected); + if(input.changedBase)input.base=input.changedBase; + if(input.newKnownAfterPrepare)options.knownStatuses=input.newKnownAfterPrepare.map(decode); + if(input.waitAfterPrepare)await new Promise(resolve=>setTimeout(resolve,input.waitAfterPrepare)); + } + const result=input.resumeJournal?await client.resume(input.base,Buffer.from(input.resumeJournal,'base64'),decode(input.receipt),decode(input.disclosure),decode(input.put),options):await client.put(input.base,decode(input.receipt),decode(input.disclosure),decode(input.put),options);""") start=DRIVER.index(' process.stdout.write(JSON.stringify({ok:true') end=DRIVER.index('\n}catch(error)',start) DRIVER=DRIVER[:start]+''' process.stdout.write(JSON.stringify({ok:true,metrics:result.metrics, receipt:Buffer.from(result.source.inputs.receipt.raw).toString('base64'),commit:result.source.commit.ref, - immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls,journal}));'''+DRIVER[end:] -DRIVER=DRIVER.replace('calls,nativeChecks,subprocessCalls}));}', 'calls,nativeChecks,subprocessCalls,journal}));}') + immutable:Buffer.from(original.raw).toString('hex')===before,calls,nativeChecks,subprocessCalls,journal,sentPackets}));'''+DRIVER[end:] +DRIVER=DRIVER.replace('calls,nativeChecks,subprocessCalls}));}', 'calls,nativeChecks,subprocessCalls,journal,sentPackets}));}') class NativePutClientTests(unittest.TestCase): @@ -42,7 +52,7 @@ def setUpClass(cls): def setUp(self): self.case=put_py.RepairPutClientTests();self.addCleanup(self.case.doCleanups) - if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage': + if self._testMethodName=='test_bad_returned_commit_cannot_claim_verified_storage' or 'resume' in self._testMethodName: # One original finite grant funds the two actual client exchanges. with patch.object(put_py.offer_fixture.RepairOfferClientTests,'fixture_limits',dict(proof_limit=524288),create=True):self.case.setUp() else:self.case.setUp() @@ -68,6 +78,35 @@ def test_native_upload_saves_exact_receipt_and_records_request_before_network(se self.assertNotIn('ack.put_request',result['journal'][0]['calls']);self.assertEqual(result['calls'][-1],'ack.put_request') self.assertEqual(self.host.source.db.execute('SELECT status FROM open_repair_ack_resources').fetchone()[0],'occupied') + def test_prepared_return_reuses_actual_preflight_and_original_work_meter(self): + result=self.native(self.call(prepare=True)) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'}) + self.assertEqual(result['calls'].count('bootstrap.probe'),1) + self.assertEqual(result['metrics']['requests'],13) + self.assertEqual(result['metrics']['signature_checks'],result['nativeChecks']) + self.assertGreater(result['metrics']['signature_checks'],64) + + def test_prepared_return_rejects_changed_source_before_network(self): + result=self.native(self.call(prepare=True,changedBase='http://127.0.0.1:1')) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_proof_mismatch') + self.assertEqual(len(result['calls']),12);self.assertNotIn('ack.put_request',result['calls']) + + def test_prepared_return_does_not_reset_local_signature_allowance(self): + value=self.call(prepare=True);value['policy']['max_signature_checks']=64 + result=self.native(value);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_over_budget') + self.assertEqual(result['calls'].count('bootstrap.probe'),1);self.assertNotIn('ack.put_request',result['calls']) + + def test_prepared_return_keeps_original_deadline(self): + result=self.native(self.call(prepare=True,prepareTimeout=8,waitAfterPrepare=8000)) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_access_expired') + self.assertEqual(len(result['calls']),12);self.assertNotIn('ack.put_request',result['calls']) + + def test_new_revocation_after_preparation_stops_private_upload(self): + revoked=self.case.case.revoke('ack.write_grant',self.host.write) + result=self.native(self.call(prepare=True,newKnownAfterPrepare=[probe_ts.encoded(revoked)])) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_authority_revoked') + self.assertEqual(len(result['calls']),12);self.assertNotIn('ack.put_request',result['calls']) + def test_legacy_disclosure_does_not_send_receipt(self): from memory_vault_open_repair_occupied import RETURN_ROLES_LEGACY inputs=receipt_inputs(self.case.fixture,bootstrap_roles=RETURN_ROLES_LEGACY) @@ -124,6 +163,58 @@ def test_unfinished_journal_obeys_original_deadline_without_upload(self): self.assertEqual([item['kind'] for item in result['journal']],['request']) self.assertNotIn('ack.put_request',result['calls']) + def pending_native(self): + result=self.native(self.call(corruptReply=True)) + self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_ref_mismatch') + self.assertEqual([item['kind'] for item in result['journal']],['request']) + return result['journal'][0]['raw'] + + def test_resume_exact_native_journal_after_client_and_service_restart(self): + journal=self.pending_native();self.host.http.restart() + result=self.native(self.call(resumeJournal=journal)) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'}) + self.assertEqual(result['calls'],['ack.put_request']) + packet=json.loads(base64.b64decode(journal))['packet']['raw_base64url'] + self.assertEqual(base64.b64decode(result['sentPackets'][0]),base64.urlsafe_b64decode(packet+'='*((-len(packet))%4))) + self.assertEqual(result['metrics']['requests'],1) + self.assertEqual(result['nativeChecks'],result['metrics']['signature_checks']) + self.assertEqual(result['receipt'],probe_ts.encoded(self.case.inputs[0])['raw']) + self.assertEqual([item['kind'] for item in result['journal']],['response']) + self.assertEqual(self.host.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + def test_resume_rejects_changed_source_message_and_raw_journal_without_network(self): + journal=self.pending_native() + values=[] + value=self.call(resumeJournal=journal);value['base']='http://127.0.0.1:1';values.append(value) + value=self.call(resumeJournal=journal);value['options']['expectedMessageId']='msg_'+'f'*64;values.append(value) + value=self.call(resumeJournal=journal);body=json.loads(base64.b64decode(journal)) + body['packet']['ref']['raw_sha256']='0'*64 + value['resumeJournal']=base64.b64encode(json.dumps(body,sort_keys=True,separators=(',',':')).encode()).decode();values.append(value) + for value in values: + with self.subTest(): + result=self.native(value);self.assertFalse(result['ok']) + self.assertNotEqual(result['code'],'untyped_error');self.assertEqual(result['calls'],[]) + + def test_resume_python_journal_in_native_client(self): + journal=[];receipt,disclosure,put,options=self.case.inputs + self.case.client().put(self.host.http.base,receipt,disclosure,put,**options,**self.case.case.args,_journal=lambda kind,raw:journal.append((kind,raw))) + self.host.http.restart() + result=self.native(self.call(resumeJournal=base64.b64encode(journal[0][1]).decode())) + self.assertTrue(result['ok'],{k:v for k,v in result.items() if k!='journal'}) + self.assertEqual(result['calls'],['ack.put_request']) + self.assertEqual(self.host.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + def test_resume_expired_original_use_requires_reconciliation_without_network(self): + journal=self.pending_native();value=self.call(resumeJournal=journal);value['now']+=61 + result=self.native(value);self.assertFalse(result['ok']) + self.assertEqual(result['code'],'repair_reconciliation_required');self.assertEqual(result['calls'],[]) + + def test_resume_refuses_new_revocation_before_network(self): + journal=self.pending_native();revoked=self.case.case.revoke('ack.write_grant',self.host.write) + value=self.call(resumeJournal=journal);value['options']['knownStatuses']=[probe_ts.encoded(revoked)] + result=self.native(value);self.assertFalse(result['ok']) + self.assertEqual(result['code'],'repair_authority_revoked');self.assertEqual(result['calls'],[]) + def test_bad_returned_commit_cannot_claim_verified_storage(self): result=self.native(self.call(corruptReply=True));self.assertFalse(result['ok']) self.assertEqual(result['code'],'repair_ref_mismatch') From e383f65122cb6248874b1e3ba59b8ed738b13ebf Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:10:59 +0800 Subject: [PATCH 05/24] Return actual saved receipts through the native Agent and shared durable journal --- clients/typescript/network/open-ack-client.ts | 18 ++ clients/typescript/network/open-client.ts | 4 +- .../network/open-delivery-client.ts | 19 ++ .../typescript/network/open-repair-empty.ts | 4 + .../network/open-repair-offer-client.ts | 4 + .../typescript/network/open-repair-receipt.ts | 175 ++++++++++++++++++ clients/typescript/network/package.json | 1 + docs/OPEN_ACK_PROVISIONING.md | 30 ++- docs/OPEN_ACK_RECOVERY.md | 14 +- memory_vault.py | 2 +- packaging/CLIENT_README.md | 17 +- packaging/PROTOCOL_README.md | 17 +- packaging/RELEASE_NOTES.md | 19 +- packaging/REVIEW_README.md | 15 +- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 2 +- scripts/run_open_network_ci.py | 4 +- tests/test_open_repair_receipt_typescript.py | 93 ++++++++++ 18 files changed, 416 insertions(+), 24 deletions(-) create mode 100644 clients/typescript/network/open-repair-receipt.ts create mode 100644 tests/test_open_repair_receipt_typescript.py diff --git a/clients/typescript/network/open-ack-client.ts b/clients/typescript/network/open-ack-client.ts index 6b3a853..e5f1222 100644 --- a/clients/typescript/network/open-ack-client.ts +++ b/clients/typescript/network/open-ack-client.ts @@ -5,6 +5,8 @@ import {endpoint} from './open-transport.ts'; import {envelopeRef} from './open-delivery-control.ts'; import {DiscoveredAckRecoveryClient} from './open-ack-discovery.ts'; import {OpenProviderClient} from './open-provider-client.ts'; +import {AckReceiptClient} from './open-repair-offer-client.ts'; +import {SavedAckReceiptPublisher,SAVED_ACK_REQUEST_FIELDS} from './open-repair-receipt.ts'; import {OriginalAckStatusJournal} from './open-ack-status.ts'; import type {DatabaseSync} from 'node:sqlite'; import {canonicalBytes,document,objectFields,sha256,validateSigningIdentity,decodeBase64url} from './crypto.ts'; @@ -73,6 +75,22 @@ class Journal{ if(!saved)fail('repair_setup_journal_capacity'); } } +export async function returnSavedReceipt(participant:OpenParticipant,encryption:EncryptionIdentityDocument, + delivery:OpenDeliveryClient,invitation:unknown):Promise{ + const v=objectFields(document(invitation as any,65536),['schema_version','action','repair_profile','base_url','request']); + if(v.schema_version!==ACK_CONNECT_SCHEMA||v.action!=='return_receipt'||typeof v.repair_profile!=='string'||!Object.hasOwn(PROFILES,v.repair_profile))fail('open_invalid_ack_request'); + const request=objectFields(v.request,SAVED_ACK_REQUEST_FIELDS) as Obj; + const decoded={...request};for(const name of SAVED_ACK_REQUEST_FIELDS.filter(n=>n.endsWith('_entry')))decoded[name]=decode(request[name]); + if(!Array.isArray(request.current_statuses)||request.current_statuses.length>7)fail('open_invalid_ack_request'); + decoded.current_statuses=request.current_statuses.map(decode); + const client=new AckReceiptClient(participant.identity,encryption,{limitPolicy:PROFILES[v.repair_profile as keyof typeof PROFILES], + transport:participant.transport,allowLoopback:participant.transport.allow_loopback}); + try{ + const result=await new SavedAckReceiptPublisher(delivery,client).publishSaved(v.base_url as string,decoded); + return {state:'retained_at_ack_source',message_id:request.message_id,receipt_ref:result.source.inputs.receipt.ref, + commit_ref:result.source.commit.ref,from_local_history:result.from_local_history,network_accessed:!result.from_local_history}; + }finally{client.close();} +} export async function recoverReceipt(participant:OpenParticipant,encryption:EncryptionIdentityDocument, delivery:OpenDeliveryClient,invitation:unknown,deadline?:number):Promise{ const parsed=document(invitation as any,65536),routed=parsed.action==='recover_routed_receipt',discovered=routed||parsed.action==='recover_discovered_receipt'; diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index d1738a0..082ab86 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -9,7 +9,7 @@ import {OpenDeliveryClient} from './open-delivery-client.ts'; import {OpenParticipant} from './open-participant.ts'; import {OpenContactClient,CONNECT_SCHEMA} from './open-contact-client.ts'; import type {SignedNode} from './open-control.ts'; -import {ACK_CONNECT_SCHEMA,recoverReceipt,recoverPrepared} from './open-ack-client.ts'; +import {ACK_CONNECT_SCHEMA,recoverReceipt,recoverPrepared,returnSavedReceipt} from './open-ack-client.ts'; export const OPEN_CLIENT_CONFIG='memory-vault-open-client-config/v1'; function fail(code:string):never{throw new NetworkError(code);} export class OpenNetworkClient{ @@ -37,7 +37,7 @@ export class OpenNetworkClient{ async connect(invitation?:unknown,requestId?:string):Promise>{ if(invitation!=null){ if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA) - return this.result(await recoverReceipt(this.participant,this.encryption,this.delivery(),invitation)); + return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); if(typeof invitation!=='object'||Array.isArray(invitation)||(invitation as any).schema_version!==CONNECT_SCHEMA)fail('open_private_invitation_unsupported'); const result=await new OpenContactClient(this.participant,this.encryption).dispatch(invitation,requestId); return this.result({...result,profile:'open-routing-v1',network_accessed:true,open_messaging_supported:true}); diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 5589efa..d69c7b8 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -371,6 +371,25 @@ export class OpenDeliveryClient{ db.prepare('UPDATE open_delivery_inbox SET receipt=? WHERE message_id=?').run(encoded,messageId);return receipt; })); } + /** Read the actually saved inbox receipt for an explicitly requested return. + * The caller cannot substitute receipt bytes or a saved-state flag. */ + savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):{raw:Uint8Array;ref:Obj}{ + if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); + const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); + const keys=OpenDeliveryClient.keys(document(row.session,MAX_INBOX_SESSION_BYTES) as Obj); + if(sha256(row.envelope)!==row.envelope_sha256||row.sender!==owner.signing_key.key_id||!same(envelopeRef(row.envelope),expectedEnvelope)|| + !same(keys,{sender_signing_key:owner.signing_key,sender_encryption_key:owner.encryption_key, + recipient_signing_key:validateSigningIdentity(this.participant.identity),recipient_encryption_key:validateEncryptionIdentity(this.encryption)}))fail('repair_saved_tuple_mismatch'); + const result=document(row.result,MAX_RESULT_BYTES) as Obj; + if(result.state!=='validated_saved'||result.message_id!==messageId||result.sender_key_id!==row.sender|| + !['message','memory_transfer'].includes(result.content_kind))fail('repair_receipt_not_saved'); + this.savedReceipt(messageId);const saved=this.inbox(messageId); + if(!saved||saved.phase!=='saved'||saved.sender!==row.sender||!raw(saved.envelope).equals(raw(row.envelope))||!raw(saved.body).equals(raw(row.body)))fail('repair_saved_tuple_mismatch'); + const bytes=Buffer.from(saved.receipt); + verifyRecipientReceipt(document(bytes,4096),{recipient_signing_key:validateSigningIdentity(this.participant.identity), + sender_key_id:row.sender,message_id:messageId,envelope_ref:expectedEnvelope} as any); + const digest=sha256(bytes);return {raw:bytes,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:bytes.length}}; + } private async sendReceipt(messageId:string,budget:DeliveryBudget,node?:SignedNode){ const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('open_delivery_not_saved');if(row.receipt_sent)return; const receipt=this.savedReceipt(messageId);verifyRecipientReceipt(receipt,{recipient_signing_key:validateSigningIdentity(this.participant.identity), diff --git a/clients/typescript/network/open-repair-empty.ts b/clients/typescript/network/open-repair-empty.ts index 9ffa874..83adc4d 100644 --- a/clients/typescript/network/open-repair-empty.ts +++ b/clients/typescript/network/open-repair-empty.ts @@ -140,3 +140,7 @@ export function verifyAckEmptySourceEvent(manifestEntry:unknown,resolver:LocalRa const observed=statuses(roles,obligations(prior,authorities,owner,target,ackSlot,policy,budget),ackSlot.root_key,at,policy,budget);historyFloors(prior.statuses,observed); return Object.freeze({manifest,custody,predecessor:prior,binding,authorities,statuses:Object.freeze(observed),stored_at:at,read_until:event.read_until,retain_until:event.retain_until}); } + +export function verifyAckBindingOriginal(value:unknown,target:Record,policy:RepairPolicy,budget:RepairBudget):AuthenticatedRepairOriginal{ + return signed(value,target,'ack.binding',policy,budget); +} diff --git a/clients/typescript/network/open-repair-offer-client.ts b/clients/typescript/network/open-repair-offer-client.ts index 9be5f38..c1e90f4 100644 --- a/clients/typescript/network/open-repair-offer-client.ts +++ b/clients/typescript/network/open-repair-offer-client.ts @@ -84,6 +84,10 @@ export class AckOfferClient{ Object.freeze(this); } close():void{this.#preparedReturn=undefined;this.#preparedGeneration++;if(this.#ownTransport)this.#transport.close();} + get subject():Readonly{return this.#subject;} + get policy():RepairPolicy{return this.#policy;} + get limits():Readonly{return this.#limits;} + get timestamp():number{return this.#now();} #now():number{return u53(Math.floor(this.#clock()));} async preflight(baseUrl:string,value:AckOfferOptions):Promise{ return this.#preflight(baseUrl,value,new RepairBudget(this.#policy)); diff --git a/clients/typescript/network/open-repair-receipt.ts b/clients/typescript/network/open-repair-receipt.ts new file mode 100644 index 0000000..af24445 --- /dev/null +++ b/clients/typescript/network/open-repair-receipt.ts @@ -0,0 +1,175 @@ +/** Explicitly return an actually saved inbox receipt. Shares the protected, + * finite Python journal; a local historical retry is never a fresh network claim. */ +import type {DatabaseSync} from 'node:sqlite'; +import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {AckReceiptClient} from './open-repair-offer-client.ts'; +import type {CommittedAckReceipt} from './open-repair-offer-client.ts'; +import {RepairBudget,RepairError,buildNewWire,parseNewWire,objectFields,rawRef,u53,LocalRawResolver} from './open-repair-wire.ts'; +import type {RawOriginal,RepairPolicy} from './open-repair-wire.ts'; +import {parseOriginalControl,verifyOriginalControl} from './open-repair-original.ts'; +import {verifyAckOfferBootstrapOriginal} from './open-repair-bound.ts'; +import {verifyAckBindingOriginal} from './open-repair-empty.ts'; +import {verifyAckOccupiedSourceEvent} from './open-repair-occupied.ts'; +import type {AuthenticatedAckOccupiedSourceEvent} from './open-repair-occupied.ts'; +import {signBoundedBootstrapOriginal} from './open-repair-probe.ts'; +import {statusScope} from './open-repair-status.ts'; +import {validateSigningIdentity,validateEncryptionIdentity} from './crypto.ts'; +import {transaction} from './io.ts'; +type Obj=Record; +export const SAVED_ACK_REQUEST_FIELDS=['message_id','envelope_ref','ack_slot','owner','target','target_node_entry', + 'root_entry','write_entry','bootstrap_entry','binding_entry','current_statuses','read_until','retain_until'] as const; +const MAX_RECORDS=16,MAX_BYTES=1048576; +const key=(r:Obj)=>`${r.namespace}:${r.key}:${r.raw_sha256}:${r.size}`; +function fail(code:string):never{throw new RepairError(code);} +function equal(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(n=>Object.hasOwn(b,n)&&equal((a as Obj)[n],(b as Obj)[n])); +} +function sameEntry(a:RawOriginal,b:RawOriginal):boolean{return equal(a.ref,b.ref)&&Buffer.from(a.raw).equals(Buffer.from(b.raw));} +export interface PublishedSavedAck{readonly source:AuthenticatedAckOccupiedSourceEvent;readonly originals:readonly RawOriginal[];readonly from_local_history:boolean;} +export class SavedAckReceiptPublisher{ + readonly #delivery:OpenDeliveryClient;readonly #client:AckReceiptClient;readonly #policy:RepairPolicy; + constructor(delivery:OpenDeliveryClient,client:AckReceiptClient){ + if(!(delivery instanceof OpenDeliveryClient))fail('repair_saved_client_required'); + if(!equal(client.subject,{signing_key:validateSigningIdentity(delivery.participant.identity),encryption_key:validateEncryptionIdentity(delivery.encryption)}))fail('repair_saved_identity_mismatch'); + this.#delivery=delivery;this.#client=client;this.#policy=client.policy; + this.#db(db=>transaction(db,()=>{ + db.exec(`CREATE TABLE IF NOT EXISTS open_repair_saved_ack_roots(root_digest TEXT PRIMARY KEY,revision INTEGER NOT NULL); + CREATE TABLE IF NOT EXISTS open_repair_saved_acks(slot_digest TEXT PRIMARY KEY,root_digest TEXT NOT NULL, + request_digest TEXT NOT NULL,issued_at INTEGER NOT NULL,status_revision INTEGER NOT NULL, + originals BLOB,result BLOB,attempted INTEGER NOT NULL DEFAULT 0);`); + const columns=new Set((db.prepare('PRAGMA table_info(open_repair_saved_acks)').all() as Obj[]).map(r=>r.name)); + if(!columns.has('attempted')){db.exec('ALTER TABLE open_repair_saved_acks ADD COLUMN attempted INTEGER NOT NULL DEFAULT 0'); + db.exec('UPDATE open_repair_saved_acks SET attempted=1 WHERE originals IS NOT NULL AND result IS NULL');} + for(const name of ['put_journal','put_response'])if(!columns.has(name))db.exec('ALTER TABLE open_repair_saved_acks ADD COLUMN '+name+' BLOB'); + })); + } + #db(operation:(db:DatabaseSync)=>T):T{return this.#delivery.participant.contactStorage(operation);} + #entry(value:unknown,budget:RepairBudget):RawOriginal{ + const e=objectFields(value,['raw','ref']),ref=rawRef(e.ref),raw=parseOriginalControl(e.raw,this.#policy,budget).raw; + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; + } + #request(value:unknown,budget:RepairBudget):Obj{ + const v=objectFields(value,SAVED_ACK_REQUEST_FIELDS),scalars=SAVED_ACK_REQUEST_FIELDS.filter(n=>!n.endsWith('_entry')&&n!=='current_statuses'); + const r={...buildNewWire(Object.fromEntries(scalars.map(n=>[n,v[n]])),this.#policy,budget).value as Obj}; + for(const n of SAVED_ACK_REQUEST_FIELDS.filter(n=>n.endsWith('_entry')))r[n]=this.#entry(v[n],budget); + if(!Array.isArray(v.current_statuses)||v.current_statuses.length<1||v.current_statuses.length>7)fail('repair_invalid_status'); + r.current_statuses=v.current_statuses.map((e:unknown)=>this.#entry(e,budget));return r; + } + #decodeEntries(raw:Uint8Array,budget:RepairBudget):Record{ + const v=objectFields(parseNewWire(raw,this.#policy,budget).value,['receipt','disclosure','put','status']); + return Object.fromEntries(Object.entries(v).map(([name,item])=>{ + const e=objectFields(item,['raw','ref']);if(typeof e.raw!=='string')fail('repair_saved_corrupt'); + return [name,this.#entry({raw:Buffer.from(e.raw,'utf8'),ref:e.ref},budget)]; + })); + } + #archive(result:CommittedAckReceipt,budget:RepairBudget):Uint8Array{ + const originals=[...result.originals].sort((a,b)=>(a.ref.namespaceb.ref.namespace?1:a.ref.keyb.ref.key?1:0)); + const raw=buildNewWire({manifest_ref:result.source.commit.payload.historical_manifest_ref,commit_ref:result.source.commit.ref, + originals:originals.map(e=>({ref:e.ref,raw_base64url:Buffer.from(e.raw).toString('base64url')}))},this.#policy,budget).raw; + if(raw.length>MAX_BYTES)fail('repair_saved_capacity');return raw; + } + #restore(raw:Uint8Array,r:Obj,entries:Record,budget:RepairBudget):PublishedSavedAck{ + const value=objectFields(parseNewWire(raw,this.#policy,budget).value,['manifest_ref','commit_ref','originals']); + if(!Array.isArray(value.originals)||value.originals.length<1||value.originals.length>128)fail('repair_saved_corrupt'); + const resolver=new LocalRawResolver(this.#policy,budget),originals=new Map(); + for(const item of value.originals){ + const encoded=objectFields(item,['ref','raw_base64url']),ref=rawRef(encoded.ref); + if(typeof encoded.raw_base64url!=='string'||!/^[A-Za-z0-9_-]*$/.test(encoded.raw_base64url))fail('repair_saved_corrupt'); + budget.output(ref.size);const body=Buffer.from(encoded.raw_base64url,'base64url'); + if(body.toString('base64url')!==encoded.raw_base64url||body.length!==ref.size||budget.hash(body)!==ref.raw_sha256||originals.has(key(ref)))fail('repair_saved_corrupt'); + originals.set(key(ref),{raw:body,ref});if(!equal(resolver.put(ref.namespace,ref.key,body).ref,ref))fail('repair_saved_corrupt'); + } + const manifest=originals.get(key(rawRef(value.manifest_ref))),commit=originals.get(key(rawRef(value.commit_ref))); + if(!manifest||!commit)fail('repair_saved_corrupt'); + const node=r.target_node_entry,nodeValue=parseOriginalControl(node.raw,this.#policy,budget).value as Obj; + const checked=verifyOriginalControl(node.raw,{expectedSigningKey:r.target.signing_key,expectedSchema:'memory-vault-open-control/v1',expectedKind:'node', + at:nodeValue.payload.issued_at,policy:this.#policy,budget}); + const source=verifyAckOccupiedSourceEvent(manifest,resolver,commit,{expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedReceiptWriter:this.#client.subject, + expectedMessageId:r.message_id,expectedEnvelopeRef:r.envelope_ref,expectedTarget:r.target,targetStorageEpoch:checked.payload.storage_epoch as string, + limitPolicy:this.#client.limits,policy:this.#policy,budget}); + if(['receipt','disclosure','put'].some(name=>!sameEntry(source.inputs[name as keyof typeof source.inputs],entries[name])))fail('repair_saved_corrupt'); + return Object.freeze({source,originals:Object.freeze([...originals.values()].map(e=>Object.freeze({ref:e.ref,get raw(){return Uint8Array.from(e.raw);}}))),from_local_history:true}); + } + async publishSaved(baseUrl:string,value:unknown,timeout=30):Promise{ + if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); + const client=this.#client,policy=this.#policy,budget=new RepairBudget(policy),r=this.#request(value,budget); + const receipt=this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; + const receiptPayload=(parseOriginalControl(receipt.raw,policy,budget).value as Obj).payload; + const authorities=verifyAckOfferBootstrapOriginal(r.bootstrap_entry,{root:r.root_entry,write:r.write_entry}, + {expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedReceiptWriter:client.subject,expectedMessageId:r.message_id, + expectedEnvelopeRef:r.envelope_ref,at:now,limitPolicy:client.limits,policy,budget}); + const binding=verifyAckBindingOriginal(r.binding_entry,r.target,policy,budget),{root,write,bootstrap:grant}=authorities.originals; + if(!equal(binding.payload.ack_slot,r.ack_slot)||!equal(binding.payload.root_authority_ref,root.ref)||!equal(binding.payload.grant_ref,write.ref)|| + binding.payload.bound_at>receiptPayload.saved_at||!(receiptPayload.saved_at<=now&&nowroot.payload.windows.read_until||r.retain_until>Math.min(binding.payload.retain_until as number,write.payload.windows.retain_until,root.payload.windows.retain_until))fail('repair_saved_tuple_mismatch'); + const hash=(v:unknown)=>budget.hash(buildNewWire(v,policy,budget).raw),rootDigest=hash(r.ack_slot.root_key),slotDigest=hash(r.ack_slot); + const requestDigest=hash({slot:r.ack_slot,receipt:receipt.ref,...Object.fromEntries(['owner','target','read_until','retain_until'].map(n=>[n,r[n]])), + ...Object.fromEntries(['root_entry','write_entry','bootstrap_entry','binding_entry'].map(n=>[n,r[n].ref]))}); + let saved=this.#db(db=>transaction(db,()=>{ + let row=db.prepare('SELECT * FROM open_repair_saved_acks WHERE slot_digest=?').get(slotDigest) as Obj|undefined; + if(!row){ + if((db.prepare('SELECT count(*) AS n FROM open_repair_saved_acks').get() as Obj).n>=MAX_RECORDS)fail('repair_saved_capacity'); + const prior=db.prepare('SELECT revision FROM open_repair_saved_ack_roots WHERE root_digest=?').get(rootDigest) as Obj|undefined; + const revision=u53((prior?.revision??0)+1,1); + db.prepare('INSERT INTO open_repair_saved_ack_roots VALUES(?,?) ON CONFLICT(root_digest) DO UPDATE SET revision=excluded.revision').run(rootDigest,revision); + db.prepare('INSERT INTO open_repair_saved_acks(slot_digest,root_digest,request_digest,issued_at,status_revision) VALUES(?,?,?,?,?)').run(slotDigest,rootDigest,requestDigest,now,revision); + row=db.prepare('SELECT * FROM open_repair_saved_acks WHERE slot_digest=?').get(slotDigest) as Obj; + } + if(row.root_digest!==rootDigest||row.request_digest!==requestDigest)fail('repair_saved_retry_mismatch');return row; + })); + if(saved.originals===null){ + const issued=u53(saved.issued_at),retain=r.retain_until,read=r.read_until,expiry=Math.min(retain,write.payload.expires_at,grant.payload.upload_until,issued+300); + if(expiry<=issued)fail('repair_access_expired'); + const sign=(payload:Obj)=>signBoundedBootstrapOriginal(payload,this.#delivery.participant.identity as unknown as Obj,policy,budget); + const common={schema_version:'memory-vault-open-repair/v1',signing_key:client.subject.signing_key}; + const disclosure=sign({...common,kind:'ack.disclosure',issued_at:issued,expires_at:retain,consent_id:'consent_'+slotDigest, + ack_slot:r.ack_slot,root_authority_ref:root.ref,grant_ref:write.ref,receipt_ref:receipt.ref,recipient:r.ack_slot.receipt_writer, + owner:r.ack_slot.root_key.owner,allowed_roles:['ack.disclosure','ack.put','historical.status.ack_disclosure','recipient.receipt'],operation_mask:67, + consent_until:retain,bootstrap_return:{subject:r.ack_slot.root_key.owner,consumer:'ack_owner', + roles:['ack.disclosure','ack.put','authority.status.disclosure','recipient.receipt'],until:read},revision:1}); + const put=sign({...common,kind:'ack.put',issued_at:issued,expires_at:expiry,put_id:'put_'+slotDigest,ack_slot:r.ack_slot, + grant_ref:write.ref,binding_ref:binding.ref,receipt_ref:receipt.ref,disclosure_ref:disclosure.ref,operation:'receipt.put'}); + const scope=statusScope(r.ack_slot.root_key,'authority',{authority_kind:'ack.disclosure',authority_sha256:disclosure.ref.raw_sha256},policy,budget); + const status=sign({schema_version:'memory-vault-open-authority/v1',kind:'authority.status',signing_key:client.subject.signing_key, + scope_key:{root_key:r.ack_slot.root_key,issuer_key_id:client.subject.signing_key.key_id},revision:saved.status_revision, + issued_at:issued,valid_until:Math.min(retain,issued+3600),entries:[{scope_kind:'authority',scope_id:scope,minimum_document_revision:1,status:'active',operation_mask:67}]}); + const encoded=buildNewWire(Object.fromEntries(Object.entries({receipt,disclosure,put,status}).map(([name,e])=>[name,{raw:Buffer.from(e.raw).toString('utf8'),ref:e.ref}])),policy,budget).raw; + if(encoded.length>65536)fail('repair_saved_capacity'); + saved=this.#db(db=>transaction(db,()=>{ + db.prepare('UPDATE open_repair_saved_acks SET originals=? WHERE slot_digest=? AND request_digest=? AND originals IS NULL').run(encoded,slotDigest,requestDigest); + return db.prepare('SELECT * FROM open_repair_saved_acks WHERE slot_digest=?').get(slotDigest) as Obj; + })); + } + const entries=this.#decodeEntries(saved.originals,budget);if(!sameEntry(entries.receipt,receipt))fail('repair_saved_corrupt'); + if(saved.result!==null)return this.#restore(saved.result,r,entries,budget); + const pending=this.#db(db=>db.prepare('SELECT attempted,put_journal FROM open_repair_saved_acks WHERE slot_digest=? AND request_digest=?').get(slotDigest,requestDigest)) as Obj|undefined; + if(!pending||(pending.attempted&&pending.put_journal===null))fail('repair_saved_reconciliation_required'); + const resume=pending.attempted?Buffer.from(pending.put_journal):null; + const journal=(phase:'request'|'response',raw:Uint8Array)=>{ + if(!raw.length||raw.length>MAX_BYTES)fail('repair_saved_capacity');const column=phase==='request'?'put_journal':'put_response'; + this.#db(db=>transaction(db,()=>{ + const held=db.prepare('SELECT originals,put_journal,put_response,result FROM open_repair_saved_acks WHERE slot_digest=? AND request_digest=?').get(slotDigest,requestDigest) as Obj|undefined; + if(!held||!Buffer.from(held.originals).equals(Buffer.from(saved.originals)))fail('repair_saved_corrupt'); + if(held[column]!==null&&!Buffer.from(held[column]).equals(Buffer.from(raw)))fail('repair_saved_retry_mismatch'); + if(phase==='response'&&held.put_journal===null)fail('repair_saved_corrupt'); + if(Object.entries(held).reduce((n,[name,v])=>n+(name!==column&&v!==null?(v as Uint8Array).length:0),0)+raw.length>MAX_BYTES)fail('repair_saved_capacity'); + db.prepare('UPDATE open_repair_saved_acks SET '+column+'=?,attempted=1 WHERE slot_digest=?').run(raw,slotDigest); + })); + }; + const options={currentStatuses:[...r.current_statuses,entries.status],readUntil:r.read_until,retainUntil:r.retain_until, + targetNodeEntry:r.target_node_entry,expectedTarget:r.target,expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedMessageId:r.message_id, + expectedEnvelopeRef:r.envelope_ref,rootEntry:r.root_entry,writeEntry:r.write_entry,bootstrapEntry:r.bootstrap_entry,timeout,journal}; + const result=resume===null?await client.put(baseUrl,entries.receipt,entries.disclosure,entries.put,options): + await client.resume(baseUrl,resume,entries.receipt,entries.disclosure,entries.put,options); + const archive=this.#archive(result,budget); + this.#db(db=>transaction(db,()=>{ + const current=db.prepare('SELECT originals,result,put_journal,put_response FROM open_repair_saved_acks WHERE slot_digest=? AND request_digest=?').get(slotDigest,requestDigest) as Obj|undefined; + if(!current||!Buffer.from(current.originals).equals(Buffer.from(saved.originals)))fail('repair_saved_corrupt'); + if(current.result!==null&&!Buffer.from(current.result).equals(Buffer.from(archive)))fail('repair_saved_retry_mismatch'); + if(['originals','put_journal','put_response'].reduce((n,k)=>n+(current[k]?.length??0),0)+archive.length>MAX_BYTES)fail('repair_saved_capacity'); + db.prepare('UPDATE open_repair_saved_acks SET result=? WHERE slot_digest=?').run(archive,slotDigest); + })); + return Object.freeze({source:result.source,originals:result.originals,from_local_history:false}); + } +} diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index e456d82..b2d36c5 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -37,6 +37,7 @@ "./open-repair-admin": "./open-repair-admin.ts", "./open-repair-client": "./open-repair-client.ts", "./open-repair-offer-client": "./open-repair-offer-client.ts", + "./open-repair-receipt": "./open-repair-receipt.ts", "./open-repair-wire": "./open-repair-wire.ts", "./open-repair-bound": "./open-repair-bound.ts", "./open-repair-empty": "./open-repair-empty.ts", diff --git a/docs/OPEN_ACK_PROVISIONING.md b/docs/OPEN_ACK_PROVISIONING.md index e92a5f9..2e01b37 100644 --- a/docs/OPEN_ACK_PROVISIONING.md +++ b/docs/OPEN_ACK_PROVISIONING.md @@ -516,6 +516,32 @@ or `ack_recovery_error` when that attempt is refused or unavailable; the ordinar send retains its existing storage/recipient receipt distinction. This adds owner receipt recovery from the selected original ACK source. ACK -source preparation, receipt return, cold-mailbox reception and ACK replica -recovery still require their Python workflows. It does not automatically select +source preparation, cold-mailbox reception and ACK replica recovery still require +their Python workflows; native receipt return is described below. It does not automatically select nodes, grant access, import memory or run a Python subprocess. + + +## Native saved receipt return (development after alpha.0.34) + +The native TypeScript Agent now accepts the existing `connect/return_receipt` +invitation shown above. The recipient uses the same private exported preparation +and `receipt` or `receipt-index` profile. It reads its protected delivery inbox, +requires an actually validated and saved message or memory transfer, and checks +the exact sender, recipient, envelope and source binding before returning the +original receipt. Receiving alone does not grant this disclosure permission. + +The recipient signs the existing bounded return consent and persists it before +private upload. The exact upload journal is durable before transmission. A +restart, or switching between Python and native clients, reuses those originals +and the same upload carrier while its original use remains valid. An expired +use needs independent reconciliation. A completed retry reauthenticates stored +history without contacting the network and sets `from_local_history: true` and +`network_accessed: false`; this is not a fresh source availability claim. + +Both clients share the existing SQLite journal, limited to sixteen saved ACK +records and one MiB per record. Conflicting retry inputs, changed inbox data, +failed journal persistence and insufficient original permissions stop the return. +The original delivery node may be offline: the sender can independently recover +this exact receipt from the authorized ACK source and match its original send. +This is the explicit original-source return path; automatic native mailbox and +replica orchestration remains unfinished. diff --git a/docs/OPEN_ACK_RECOVERY.md b/docs/OPEN_ACK_RECOVERY.md index b0fa7b9..ebc913c 100644 --- a/docs/OPEN_ACK_RECOVERY.md +++ b/docs/OPEN_ACK_RECOVERY.md @@ -1065,6 +1065,14 @@ private receipt upload and the verified `response` afterward. The callback is awaited; failure to store the request stops the upload. Journal data contains private original records and belongs in the participant's private durable store. The journal format is compatible with the existing Python receipt recovery -client. Native prepared-return reuse, native journal replay and automatic Agent -integration remain subsequent work; Python remains the complete automated -receipt-return path. +client. `prepareReturn(baseUrl, options)` saves one preflight for the next +matching `put` on that client. It retains the original work meter and deadline; +changed destinations, newer revocations and elapsed permissions stop upload. +`resume(baseUrl, journalRaw, receiptEntry, disclosureEntry, putEntry, options)` +reauthenticates the retained originals and replays the exact saved carrier. It +never creates another probe, consent or use. An expired use reports +`repair_reconciliation_required`; it does not silently renew authority. + +The native Agent now connects this writer to its actual saved inbox through +`connect/return_receipt`. See [saved receipt return](OPEN_ACK_PROVISIONING.md#native-saved-receipt-return-development-after-alpha034). +Preparation and complete mailbox/replica orchestration still use Python. diff --git a/memory_vault.py b/memory_vault.py index cc1b0bf..0c1e1f5 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.34" +VERSION = "0.28.0-alpha.0.35" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 33d7691..1bc2989 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 — authorized full client +# Memory Vault v0.28.0-alpha.0.35 — authorized full client + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them @@ -125,7 +136,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.34 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.35 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 0a81004..15e70c4 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 — independent protocol +# Memory Vault v0.28.0-alpha.0.35 — independent protocol + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them @@ -142,7 +153,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.34; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.35; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index 91bb699..f35a776 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 — directory republication and routed receipt recovery +# Memory Vault v0.28.0-alpha.0.35 — native saved receipt return and restart recovery + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them @@ -151,7 +162,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.34/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.34/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 5c3eff4..6648812 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,15 @@ -# Memory Vault v0.28.0-alpha.0.34 independent review kit +# Memory Vault v0.28.0-alpha.0.35 independent review kit + +Native TypeScript recipients can return an actually saved message or memory +receipt through the existing `connect/return_receipt` invitation. The protected +inbox supplies the exact original receipt. Independent return consent, source +binding and current permissions are checked before upload. The request is +persisted first; restart or switching from Python replays the same carrier while +its original use remains valid. Completed retries read authenticated local +history without claiming current source availability. The sender can separately +recover that receipt and confirm its original send while the delivery node is +offline. Preparation and complete native mailbox/replica orchestration remain +unfinished. See `docs/OPEN_ACK_PROVISIONING.md`. Original ACK sources can publish the same retained receipt location through another explicitly selected independent directory with new owner and recipient @@ -39,7 +50,7 @@ state. Fresh source identity, origin, epoch and revision floors remain checked. Confirmed repeats stay local, and signed refusal history is shared with Python across restart. Automatic recovery has twenty seconds within the existing sixty-second send deadline. No Python subprocess or new permission is created. -ACK source preparation, receipt return and replica workflows still use Python. +ACK source preparation and replica workflows still use Python. See `docs/OPEN_ACK_PROVISIONING.md`. Authorized maintainers can retain exact mailbox copy bundles and resume them diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 93f2dca..268b2c4 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -145,7 +145,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 874dfa7..089dde7 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -243,7 +243,7 @@ "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", - "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", + "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 30bca3f..a2c633b 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -47,7 +47,7 @@ "repair_http", "repair_client", "repair_client_typescript", "repair_admin", "repair_admin_typescript", "repair_runtime_review", "repair_bound", "repair_bound_typescript", "repair_empty", "repair_empty_typescript", "repair_empty_review", "repair_empty_access", "repair_empty_http", "repair_empty_client", "repair_empty_client_typescript", - "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_client_typescript", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_occupied_client_typescript")) + ( + "repair_bind_http", "repair_offer_access", "repair_offer_http", "repair_offer_client", "repair_offer_client_typescript", "repair_offer_http_typescript", "repair_occupied", "repair_occupied_typescript", "repair_occupied_access", "repair_occupied_client", "repair_put_http", "repair_put_client", "repair_put_client_typescript", "repair_put_recovery", "repair_roundtrip", "repair_receipt", "repair_receipt_typescript", "repair_occupied_client_typescript")) + ( "tests.test_open_repair_stage", "tests.test_open_repair_index", "tests.test_open_repair_index_access", "tests.test_open_repair_index_journal", "tests.test_open_repair_index_state", "tests.test_open_repair_index_http", "tests.test_open_repair_index_client", "tests.test_open_repair_index_recovery", "tests.test_open_repair_index_admin", @@ -245,7 +245,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-capacity.ts", - "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", + "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", "agent.ts", "peer.ts", "io.ts", "crypto.ts", "nodes.ts")] diff --git a/tests/test_open_repair_receipt_typescript.py b/tests/test_open_repair_receipt_typescript.py new file mode 100644 index 0000000..e8929fb --- /dev/null +++ b/tests/test_open_repair_receipt_typescript.py @@ -0,0 +1,93 @@ +"""Real saved inbox to native durable ACK return, with the delivery node offline.""" +import json +import subprocess +import unittest +from unittest.mock import patch +from tests import test_open_repair_receipt as saved_py +from tests import test_open_ack_typescript_agent as agent_ts + +DRIVER=agent_ts.DRIVER.replace("const agent=new Agent", "Date.now=()=>2_000_000_008_000;\nconst agent=new Agent") +DRIVER=DRIVER.replace("if(name==='requestNode'", """if(name==='requestRepair'&&input.corruptReply&&JSON.parse(Buffer.from(args[1]).toString()).kind==='ack.put_request'){ + const body=JSON.parse(Buffer.from(result).toString()); + if(body.kind==='ack.put_response'){body.commit.ref.raw_sha256='0'.repeat(64);return Buffer.from(JSON.stringify(body));} + } + if(name==='requestNode'""") + +class NativeSavedReceiptTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + agent_ts.NativeAckAgentTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.case=saved_py.SavedReceiptPublicationTests();self.addCleanup(self.case.doCleanups);self.case.setUp() + self.case.send(memory=True);self.case.save_without_old_receipt_upload() + def encode(e):return dict(raw=e['raw'].decode(),ref=e['ref']) + self.request={n:encode(v) if n.endswith('_entry') else [encode(e) for e in v] if n=='current_statuses' else v for n,v in self.case.request.items()} + self.invitation=dict(schema_version='memory-vault-open-ack-connect/v1',action='return_receipt',repair_profile='receipt',base_url=self.case.ack.http.base,request=self.request) + + def native(self,*,no_network=False,corrupt=False,agent=None,invitation=None): + agent=agent or self.case.delivery.b + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config), + requests=[dict(op='connect',invitation=invitation or self.invitation)],no_network=no_network,corruptReply=corrupt)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=45) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-3000:]) + output=json.loads(result.stdout);self.assertEqual(output['subprocessCalls'],0) + if no_network:self.assertEqual(output['calls'],[]) + return output + + def test_native_saved_memory_return_and_shared_completed_history(self): + result=self.native();returned=result['results'][0];self.assertTrue(returned['ok'],returned) + self.assertEqual(returned['result']['state'],'retained_at_ack_source');self.assertFalse(returned['result']['from_local_history']) + again=self.native(no_network=True)['results'][0];self.assertTrue(again['ok'],again) + self.assertTrue(again['result']['from_local_history']);self.assertEqual(again['result']['commit_ref'],returned['result']['commit_ref']) + publisher=self.case.publisher() + with patch.object(publisher.client,'put',side_effect=AssertionError('completed history accessed network')): + result=publisher.publish_saved(self.case.ack.http.base,self.case.request) + self.assertTrue(result.from_local_history);self.assertEqual(result.source.commit.ref.as_dict(),returned['result']['commit_ref']) + self.assertEqual(self.case.delivery.host.processes,{}) + self.case.ack.http.restart();f=self.case.ack.f + def encode(e):return dict(raw=e['raw'].decode(),ref=e['ref']) + request=dict(target_node_entry=encode(f['entries']['descriptor']),expected_target=f['expected']['expected_target'], + expected_ack_slot=f['expected']['expected_ack_slot'],root_entry=encode(f['entries']['root']),read_entry=encode(f['entries']['read']), + bootstrap_entry=encode(f['entries']['bootstrap']),expected_receipt_writer=self.case.ack.expected['expected_receipt_writer'], + expected_message_id=self.case.message_id,expected_envelope_ref=self.case.envelope_ref) + invitation=dict(schema_version=self.invitation['schema_version'],action='recover_receipt',repair_profile='receipt',base_url=self.case.ack.http.base,request=request) + recovered=self.native(agent=self.case.delivery.a,invitation=invitation)['results'][0] + self.assertTrue(recovered['ok'],recovered);self.assertTrue(recovered['result']['endpoint_validated']) + self.assertFalse(recovered['result']['acknowledgement_pending']) + self.assertEqual(recovered['result']['commit_ref'],returned['result']['commit_ref']) + + def test_native_resumes_pending_python_return(self): + publisher=self.case.publisher();exchange=publisher.client.transport.request_repair + def lose_response(base,raw,**options): + result=exchange(base,raw,**options) + if json.loads(raw).get('kind')=='ack.put_request':raise RuntimeError('synthetic lost committed response') + return result + with patch.object(publisher.client.transport,'request_repair',side_effect=lose_response),self.assertRaisesRegex(RuntimeError,'synthetic lost'): + publisher.publish_saved(self.case.ack.http.base,self.case.request) + self.case.ack.http.restart() + result=self.native();returned=result['results'][0];self.assertTrue(returned['ok'],returned) + self.assertEqual(len(result['calls']),1) + self.assertEqual(self.case.ack.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + + def test_unsaved_inbox_and_changed_envelope_refuse_before_network(self): + self.request['envelope_ref']=dict(self.request['envelope_ref'],raw_sha256='0'*64) + result=self.native(no_network=True)['results'][0];self.assertFalse(result['ok']) + self.assertEqual(result['error']['code'],'repair_saved_tuple_mismatch') + self.request['envelope_ref']=self.case.request['envelope_ref'] + with self.case.delivery.b._network() as network: + with network.participant.state.db() as db:db.execute("UPDATE open_delivery_inbox SET phase='staged'") + result=self.native(no_network=True)['results'][0];self.assertFalse(result['ok']) + self.assertEqual(result['error']['code'],'repair_receipt_not_saved') + + def test_native_restarts_and_replays_pending_put_without_new_consent(self): + result=self.native(corrupt=True)['results'][0];self.assertFalse(result['ok']);self.assertEqual(result['error']['code'],'repair_ref_mismatch') + self.case.ack.http.restart() + result=self.native();returned=result['results'][0];self.assertTrue(returned['ok'],returned) + self.assertEqual(len(result['calls']),1) + self.assertEqual(self.case.ack.source.db.execute('SELECT count(*) FROM open_repair_put_requests').fetchone()[0],1) + self.assertTrue(self.native(no_network=True)['results'][0]['result']['from_local_history']) + +if __name__=='__main__':unittest.main() From 2479cf16498752c6e3dc1b563c4beaa4e29cc9f8 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:13:44 +0800 Subject: [PATCH 06/24] Align alpha35 plugin version and include native writer review fixtures --- plugins/memory-vault-client/.codex-plugin/plugin.json | 2 +- scripts/build_release.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index 1006795..6f856be 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.34", + "version": "0.28.0-alpha.0.35", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" diff --git a/scripts/build_release.py b/scripts/build_release.py index 089dde7..298113c 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -110,6 +110,7 @@ "tests/test_open_repair_offer_access.py", "tests/test_open_repair_offer_http.py", "tests/test_open_repair_offer_client.py", + "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_offer_http_typescript.py", "tests/test_open_repair_occupied.py", "tests/test_open_repair_occupied_typescript.py", @@ -117,9 +118,11 @@ "tests/test_open_repair_occupied_client.py", "tests/test_open_repair_put_http.py", "tests/test_open_repair_put_client.py", + "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_put_recovery.py", "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", + "tests/test_open_repair_receipt_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", From 751978ab3d6461c71f9263387cbdb94babdfb7bb Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:29:29 +0800 Subject: [PATCH 07/24] Authenticate native mailbox owner controls and persisted resource activation --- .../network/open-repair-mailbox-authority.ts | 191 ++++++++++++++++++ ...pen_repair_mailbox_authority_typescript.py | 118 +++++++++++ 2 files changed, 309 insertions(+) create mode 100644 clients/typescript/network/open-repair-mailbox-authority.ts create mode 100644 tests/test_open_repair_mailbox_authority_typescript.py diff --git a/clients/typescript/network/open-repair-mailbox-authority.ts b/clients/typescript/network/open-repair-mailbox-authority.ts new file mode 100644 index 0000000..ac1e013 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-authority.ts @@ -0,0 +1,191 @@ +/** Authenticate retained mailbox owner controls and initial node history. + * Resource-offer signatures, live status and source custody are separate checks. + * No authority is created and no network or local storage is accessed here. */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,RawOriginal} from './open-repair-wire.ts'; +import type {AuthenticatedRepairOriginal} from './open-repair-resource.ts'; +import {originalPublicDescriptor,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {knownHistoricalRoles} from './open-repair-history.ts'; +import {emptyMailboxState} from './open-repair-mailbox-range.ts'; +type Obj=Record; +const COMMON=['schema_version','kind','signing_key']; +const FIELDS={ + maintenance:'issued_at expires_at root_key authority_id slot_key sender recipient maintainers operation_mask allowed_roles budget windows max_delegate_depth max_destinations_per_job max_concurrent_jobs revision', + read:'issued_at expires_at grant_id root_key slot_key reader serving_authority_id operation_mask budget windows revision', + slot:'issued_at expires_at revision slot_key feed_ref sender recipient data_resource_ref data_resource_offer_ref metadata_resource_ref metadata_resource_offer_ref read_grant_ref maintenance_root_ref max_appends max_live_items budget windows', + bootstrap:'issued_at expires_at grant_id revision owner subject root_key consumer selector parent_authority_ref caller_authority_ref probe_until proof_until upload_until probe_profile response_profile upload_roles limits', + activation:'issued_at expires_at activation_id subject target_node_key_id target_storage_epoch root_key scope resource_offer_refs authority_refs', +}; +const KINDS={maintenance:'mailbox.maintenance_root',read:'mailbox.read_grant',slot:'mailbox.slot',bootstrap:'bootstrap.grant',activation:'resource.activation'}; +const BUDGET=['max_live_bytes','max_meta_bytes','max_items','max_requests','max_pending','max_replay_records','max_jobs','max_job_bytes']; +const WINDOWS=['admit_until','read_until','copy_until','publish_until','retain_until']; +const PARENT_CAPS:Record={max_probe_bytes:['max_meta_bytes','max_job_bytes'],max_proof_bytes:['max_meta_bytes','max_job_bytes'], + max_proof_items:['max_items'],max_signature_checks:['max_requests'],max_requests:['max_requests'],max_pending:['max_pending'], + max_replay_records:['max_replay_records'],max_concurrent_handles:['max_pending','max_jobs'],max_candidate_attempts:['max_requests','max_jobs']}; +const ROLES=new Set([...knownHistoricalRoles,'bootstrap.grant','mailbox_root_service_v1','selected_slot_service_v1','ack_owner_service_v1','ack_offer_service_v1']); +const UPLOAD=new Set(['mailbox.slot','mailbox.read_grant','mailbox.maintenance_root','bootstrap.grant']); +const CONTROL_NAMES=['maintenance','read','slot','bootstrap'] as const; +export interface MailboxAuthorityOptions{ + readonly expectedSlot:unknown;readonly expectedOwner:unknown;readonly expectedTarget:unknown;readonly targetStorageEpoch:string; + readonly limitPolicy:unknown;readonly at:number;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +export type AuthenticatedMailboxControls=Readonly>; +function fail(code='repair_mailbox_activation_mismatch'):never{throw new RepairError(code);} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function same(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(n=>Object.hasOwn(b,n)&&same((a as Obj)[n],(b as Obj)[n])); +} +function pattern(value:unknown,re:RegExp):string{if(typeof value!=='string'||re.exec(value)?.[0]!==value)fail();return value;} +const opaque=(v:unknown)=>pattern(v,/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/); +function dual(value:unknown):Obj{const v=fields(value,['signing_key_id','encryption_key_id']);pattern(v.signing_key_id,/^ed25519_[0-9a-f]{64}$/);pattern(v.encryption_key_id,/^x25519_[0-9a-f]{64}$/);return v;} +function dualKey(value:unknown,budget:RepairBudget):Obj{const v=fields(value,['signing_key','encryption_key']);originalPublicDescriptor(v.signing_key,budget);originalPublicDescriptor(v.encryption_key,budget,true);return {signing_key_id:v.signing_key.key_id,encryption_key_id:v.encryption_key.key_id};} +function limits(value:unknown):Obj{const v=fields(value,Object.keys(PARENT_CAPS));for(const n of Object.keys(PARENT_CAPS))u53(v[n],1);return v;} +function ordered(value:unknown,get:(v:any)=>string,check:(v:any)=>unknown):void{ + if(!Array.isArray(value))fail();let prior:string|undefined;for(const item of value){check(item);const next=get(item);if(prior!==undefined&&next<=prior)fail();prior=next;} +} +function roles(value:unknown):void{ordered(value,v=>v,v=>{if(typeof v!=='string'||!ROLES.has(v))fail();});} +function original(value:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const v=fields(value,['raw','ref']),ref=rawRef(v.ref),raw=parseNewWire(v.raw,policy,budget).raw; + if(ref.namespace!=='meta'||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; +} +function authenticate(value:unknown,name:keyof typeof FIELDS,expected:Obj,policy:RepairPolicy,budget:RepairBudget):AuthenticatedRepairOriginal{ + const held=original(value,policy,budget),signed=fields(parseNewWire(held.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,...FIELDS[name].split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==KINDS[name]||u53(p.issued_at)>=u53(p.expires_at))fail(); + if(!(p.issued_at<=expected.at&&expected.at|undefined,policy:RepairPolicy,budget:RepairBudget):void{ + const key=e.slot,root=key.root_key,owner=dualKey(e.owner,budget),target=dualKey(e.target,budget),now=e.at; + const p=Object.fromEntries(Object.entries(held).map(([n,v])=>[n,(v as AuthenticatedRepairOriginal).payload])); + const {maintenance:m,read:r,slot:s,bootstrap:g,activation:a}=p; + if(!same(root.owner,owner)||!same(key.writer,target)||key.writer_storage_epoch!==e.epoch)fail(); + for(const name of ['maintenance','read','slot']){ + const v=p[name];emptyMailboxState(v.slot_key,policy,budget);u53(v.revision); + fields(v.budget,BUDGET);fields(v.windows,WINDOWS);for(const n of BUDGET)u53(v.budget[n]); + for(const n of WINDOWS)if(!(v.issued_at127)fail(); + for(const v of [m,s]){dual(v.sender);if(!same(dual(v.recipient),owner))fail();} + dual(r.reader);for(const v of [m.authority_id,r.grant_id,r.serving_authority_id])opaque(v); + if(!same(r.reader,owner)||!same(s.sender,m.sender)||r.serving_authority_id!==m.authority_id||!same(s.read_grant_ref,held.read.ref)||!same(s.maintenance_root_ref,held.maintenance.ref))fail(); + fields(s.feed_ref,['namespace','key']);if(s.feed_ref.namespace!=='feed')fail();pattern(s.feed_ref.key,/^[0-9a-f]{64}$/); + if(!(1<=u53(s.max_live_items)&&s.max_live_items<=u53(s.max_appends,1)&&s.max_appends<=65536&&s.max_live_items<=s.budget.max_items)||u53(m.max_delegate_depth)!==2)fail(); + u53(m.max_destinations_per_job);u53(m.max_concurrent_jobs);ordered(m.maintainers,v=>v.signing_key_id+':'+v.encryption_key_id,dual);roles(m.allowed_roles); + if((m.operation_mask&10)!==10||(r.operation_mask&2)!==2||(r.operation_mask&~m.operation_mask)!==0|| + !['bootstrap.grant','selected_slot_service_v1'].every(v=>m.allowed_roles.includes(v))||BUDGET.some(n=>r.budget[n]>m.budget[n])||WINDOWS.some(n=>r.windows[n]>m.windows[n])||r.expires_at>m.expires_at)fail(); + if(offers)for(const [name,purpose] of [['data','mailbox_data'],['metadata','feed_metadata']]){ + const offer=offers[name],v=(parseNewWire(offer.raw,policy,budget).value as Obj).payload,intent=v.intent; + if(!same(s[name+'_resource_ref'],v.resource)||!same(s[name+'_resource_offer_ref'],offer.ref)||intent.purpose!==purpose||!same(intent.root_key,root)|| + !same(intent.owner,e.owner)||!same(intent.target,e.target)||v.issued_at>Math.min(m.issued_at,r.issued_at,s.issued_at)||Math.min(...Object.values(intent.windows) as number[])<=now)fail(); + } + fields(g.selector,['root_key_sha256','slot_key_sha256','feed_ref','slot_sha256','read_grant_sha256','maintenance_root_sha256']);opaque(g.grant_id);u53(g.revision); + const selector={root_key_sha256:budget.hash(buildNewWire(root,policy,budget).raw),slot_key_sha256:budget.hash(buildNewWire(key,policy,budget).raw),feed_ref:s.feed_ref, + slot_sha256:held.slot.ref.raw_sha256,read_grant_sha256:held.read.ref.raw_sha256,maintenance_root_sha256:held.maintenance.ref.raw_sha256}; + if(!same(g.owner,owner)||!same(g.subject,owner)||!same(g.root_key,root)||g.consumer!=='mailbox_feed'||g.probe_profile!=='opaque_v1'||g.response_profile!=='selected_slot_service_v1'|| + !same(g.parent_authority_ref,held.maintenance.ref)||!same(g.caller_authority_ref,held.read.ref)||!same(g.selector,selector))fail(); + roles(g.upload_roles);if(g.upload_roles.some((v:string)=>!UPLOAD.has(v)||!m.allowed_roles.includes(v)))fail();limits(g.limits); + for(const [n,parents] of Object.entries(PARENT_CAPS))if(g.limits[n]>e.limits[n]||[m,r].some(v=>parents.some(k=>g.limits[n]>v.budget[k])))fail(); + const activated=offers?a.issued_at:now; + if(!(m.issued_at<=r.issued_at&&r.issued_at<=s.issued_at&&s.issued_at<=g.issued_at&&g.issued_at<=activated&&activated<=now&&g.expires_at<=Math.min(m.expires_at,r.expires_at)))fail(); + for(const n of ['probe_until','proof_until','upload_until'])if(!(now[m,r].some(v=>['read_until','retain_until'].some(w=>g[n]>v.windows[w]))))fail(); + if(!offers)return; + opaque(a.activation_id);const compare=(x:Obj,y:Obj)=>{for(const n of ['namespace','key','raw_sha256','size']){if(x[n]y[n])return 1;}return 0;}; + const refs=Object.values(offers).map(v=>v.ref).sort(compare),authorities=['maintenance','read','slot'].map(n=>({role:KINDS[n as keyof typeof KINDS],ref:held[n].ref})); + if(!same(a.subject,e.owner)||a.target_node_key_id!==e.target.signing_key.key_id||a.target_storage_epoch!==key.writer_storage_epoch||!same(a.root_key,root)|| + !same(a.scope,{kind:'mailbox_slot',slot_key:key,slot_ref:held.slot.ref})||!same(a.resource_offer_refs,refs)||!same(a.authority_refs,authorities))fail(); +} +export function verifyMailboxFeedBootstrap(entries:unknown,options:MailboxAuthorityOptions):AuthenticatedMailboxControls{ + const {expected:e,policy,budget}=context(options),v=fields(entries,CONTROL_NAMES),held:Obj={}; + for(const name of CONTROL_NAMES)held[name]=authenticate(v[name],name,e,policy,budget); + chain(held,e,undefined,policy,budget);return Object.freeze(held) as AuthenticatedMailboxControls; +} +export function verifyMailboxSlotOwnerInputs(entries:unknown,offers:unknown,options:MailboxAuthorityOptions):Readonly>{ + const {expected:e,policy,budget}=context(options),v=fields(entries,Object.keys(FIELDS)),held:Obj={},supplied=fields(offers,['data','metadata']); + for(const name of Object.keys(FIELDS) as (keyof typeof FIELDS)[])held[name]=authenticate(v[name],name,e,policy,budget); + const frozen={data:original(supplied.data,policy,budget),metadata:original(supplied.metadata,policy,budget)}; + chain(held,e,frozen,policy,budget);return Object.freeze(held) as Readonly>; +} +export function verifyMailboxGenesis(entries:unknown,options:{expectedSlot:unknown;expectedSigningKey:unknown;committedAt:number;at:number;policy:RepairPolicy;budget:RepairBudget}):Readonly>{ + const a=fields(options,['expectedSlot','expectedSigningKey','committedAt','at','policy','budget']),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,committed:a.committedAt,at:a.at},policy,budget).value as Obj; + u53(e.committed);u53(e.at);const initial=emptyMailboxState(e.slot,policy,budget),v=fields(entries,['checkpoint','head']),held:Obj={}; + for(const name of ['checkpoint','head']){ + const item=original(v[name],policy,budget),signed=fields(parseNewWire(item.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,'slot_key','committed_at','retain_until',...(name==='checkpoint'?Object.keys(initial):['checkpoint_ref','count','range_root_ref','catalog_generation'])]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==(name==='checkpoint'?'mailbox.checkpoint':'mailbox.feed_head')||!same(p.slot_key,e.slot)||p.committed_at!==e.committed||!(u53(p.committed_at)<=e.at&&e.at!same(p[k],value)))fail();} + else if(!same(p.checkpoint_ref,held.checkpoint.ref)||u53(p.count)!==0||p.range_root_ref!==null||u53(p.catalog_generation)!==0||p.retain_until!==held.checkpoint.payload.retain_until)fail(); + held[name]=Object.freeze({ref:item.ref,payload:p,get raw(){budget.output(item.raw.length);return Uint8Array.from(item.raw);}}); + } + return Object.freeze(held) as Readonly>; +} + +export interface MailboxResourceOptions{ + readonly expectedRoot:unknown;readonly expectedOwner:unknown;readonly expectedTarget:unknown;readonly targetStorageEpoch:string; + readonly expectedPurpose:string;readonly expectedScope:unknown;readonly expectedAuthorityRefs:unknown;readonly expectedOfferRefs:unknown; + readonly at:number;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +/** Scope and authority refs must come from the independently checked owner chain. */ +export function verifyMailboxResourceInputs(entries:unknown,options:MailboxResourceOptions):Readonly>{ + const a=fields(options,['expectedRoot','expectedOwner','expectedTarget','targetStorageEpoch','expectedPurpose','expectedScope','expectedAuthorityRefs','expectedOfferRefs','at','policy','budget']); + const {policy,budget}=a,e=buildNewWire({root:a.expectedRoot,owner:a.expectedOwner,target:a.expectedTarget,epoch:a.targetStorageEpoch,purpose:a.expectedPurpose, + scope:a.expectedScope,authorities:a.expectedAuthorityRefs,offers:a.expectedOfferRefs,at:a.at},policy,budget).value as Obj; + const mismatch=()=>fail('repair_resource_mismatch'); + const root=(v:unknown):Obj=>{const r=fields(v,['owner','root_kind','anchor_ref','owner_epoch','root_id']);dual(r.owner); + const anchor=fields(r.anchor_ref,['namespace','key']);if(r.root_kind!=='mailbox'||anchor.namespace!=='anchor')mismatch(); + pattern(anchor.key,/^[0-9a-f]{64}$/);opaque(r.owner_epoch);opaque(r.root_id);return r;}; + root(e.root);opaque(e.epoch);u53(e.at);const owner=dualKey(e.owner,budget),target=dualKey(e.target,budget); + if(!same(e.root.owner,owner)||!['anchor_catalog','feed_metadata','mailbox_data'].includes(e.purpose))mismatch(); + const shapes={allocate:'issued_at expires_at request_id target_node_key_id target_storage_epoch intent intent_sha256', + offer:'issued_at reservation_until offer_id allocation_request_ref intent intent_sha256 resource target_encryption_key reservation_generation budget windows', + activation:FIELDS.activation,active:'activated_at offer_ref activation_ref resource reservation_generation root_key purpose budget windows'}; + const v=fields(entries,Object.keys(shapes)),checked:Obj={}; + for(const name of Object.keys(shapes) as (keyof typeof shapes)[]){ + const item=original(v[name],policy,budget),signed=fields(parseNewWire(item.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,...shapes[name].split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!=='resource.'+name)fail('repair_invalid_resource'); + verifyBoundedControlSignature(p,signed.proof,(name==='offer'||name==='active'?e.target:e.owner).signing_key,budget); + checked[name]=Object.freeze({ref:item.ref,payload:p,get raw(){budget.output(item.raw.length);return Uint8Array.from(item.raw);}}); + } + const [allocate,offer,activation,active]=['allocate','offer','activation','active'].map(n=>checked[n].payload); + for(const p of [allocate,activation]){ + if(u53(p.issued_at)>=u53(p.expires_at))fail('repair_invalid_resource'); + if(p.target_node_key_id!==target.signing_key_id||p.target_storage_epoch!==e.epoch)mismatch(); + } + opaque(allocate.request_id);opaque(activation.activation_id);opaque(offer.offer_id); + const amounts=(v:unknown)=>{const p=fields(v,BUDGET);for(const n of BUDGET)u53(p[n]);}; + const windows=(v:unknown,issued?:number)=>{const p=fields(v,WINDOWS);for(const n of WINDOWS){u53(p[n]);if(p[n]>p.retain_until||(issued!==undefined&&p[n]<=issued))fail('repair_invalid_resource');}}; + for(const p of [allocate,offer]){ + const intent=fields(p.intent,['kind','allocation_id','root_key','owner','target','target_storage_epoch','purpose','budget','windows']); + if(intent.kind!=='resource.owner_intent'||intent.purpose!==e.purpose)fail('repair_invalid_resource'); + opaque(intent.allocation_id);opaque(intent.target_storage_epoch);root(intent.root_key);dualKey(intent.owner,budget);dualKey(intent.target,budget);amounts(intent.budget);windows(intent.windows); + if(p.intent_sha256!==budget.hash(buildNewWire(intent,policy,budget).raw))mismatch(); + } + const intent=allocate.intent; + if(!same(offer.intent,intent)||!same(intent.root_key,e.root)||!same(intent.owner,e.owner)||!same(intent.target,e.target)||intent.target_storage_epoch!==e.epoch||intent.purpose!==e.purpose)mismatch(); + windows(intent.windows,allocate.issued_at); + const resource=fields(offer.resource,['node_key_id','storage_epoch','lease_id','resource_id']);pattern(resource.node_key_id,/^ed25519_[0-9a-f]{64}$/); + for(const n of ['storage_epoch','lease_id','resource_id'])opaque(resource[n]); + if(resource.node_key_id!==target.signing_key_id||resource.storage_epoch!==e.epoch||!same(offer.target_encryption_key,e.target.encryption_key)|| + !same(rawRef(offer.allocation_request_ref),checked.allocate.ref)||!same(rawRef(active.offer_ref),checked.offer.ref)||!same(rawRef(active.activation_ref),checked.activation.ref))mismatch(); + if(!Array.isArray(e.offers)||!same(activation.subject,e.owner)||!same(activation.root_key,e.root)||!same(activation.scope,e.scope)|| + !same(activation.authority_refs,e.authorities)||!same(activation.resource_offer_refs,e.offers)||!e.offers.some((ref:unknown)=>same(ref,checked.offer.ref))||!same(active.root_key,e.root)||active.purpose!==e.purpose)mismatch(); + amounts(offer.budget);windows(offer.windows);u53(offer.reservation_generation,1); + if(['budget','windows'].some(n=>!same(offer[n],intent[n]))||['resource','reservation_generation','budget','windows'].some(n=>!same(active[n],offer[n])))mismatch(); + const issued=u53(offer.issued_at),reserved=u53(offer.reservation_until),activated=u53(active.activated_at); + if(!(allocate.issued_at<=issued&&issued>; +} diff --git a/tests/test_open_repair_mailbox_authority_typescript.py b/tests/test_open_repair_mailbox_authority_typescript.py new file mode 100644 index 0000000..1219836 --- /dev/null +++ b/tests/test_open_repair_mailbox_authority_typescript.py @@ -0,0 +1,118 @@ +"""Original Python mailbox owner controls consumed by the native reader.""" +import copy +import json +import subprocess +import unittest +from dataclasses import asdict +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_repair_mailbox_activation as activation +from tests.test_open_repair_probe_typescript import encoded +from tests.open_repair_ack_fixtures import signed_entry +from memory_vault_open_repair_mailbox_activation import verify_mailbox_feed_bootstrap,verify_mailbox_slot_owner_inputs,verify_mailbox_genesis +from memory_vault_open_repair_wire import RepairBudget,RepairWireError + +DRIVER=runtime.DRIVER.split('const {Agent}',1)[0]+r''' +const {RepairBudget}=await import('./open-repair-wire.ts'); +const api=await import('./open-repair-mailbox-authority.ts'); +const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk); +const v=JSON.parse(Buffer.concat(chunks).toString()),decode=e=>({raw:Buffer.from(e.raw,'base64'),ref:e.ref}), + entries=Object.fromEntries(Object.entries(v.entries).map(([k,e])=>[k,decode(e)])),budget=new RepairBudget(v.policy),options={...v.options,policy:v.policy,budget}; +try{ + const result=v.operation==='resource'?api.verifyMailboxResourceInputs(entries,options):v.operation==='genesis'?api.verifyMailboxGenesis(entries,options):v.operation==='bootstrap'?api.verifyMailboxFeedBootstrap(entries,options): + api.verifyMailboxSlotOwnerInputs(entries,Object.fromEntries(Object.entries(v.offers).map(([k,e])=>[k,decode(e)])),options); + const first=Object.values(result)[0],before=Buffer.from(first.raw).toString('hex');first.raw.fill(0); + process.stdout.write(JSON.stringify({ok:true,refs:Object.fromEntries(Object.entries(result).map(([k,e])=>[k,e.ref])), + immutable:before===Buffer.from(first.raw).toString('hex'),metrics:budget.snapshot(),subprocessCalls})); +}catch(e){process.stdout.write(JSON.stringify({ok:false,code:e.code??'untyped_error',detail:e.code?undefined:String(e.stack),metrics:budget.snapshot(),subprocessCalls}));} +''' + +class NativeMailboxAuthorityTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def setUp(self): + self.case=activation.MailboxActivationTests();self.case.setUp();self.addCleanup(self.case.doCleanups) + self.entries=self.case.setup_entries();self.policy=self.case.source.policy + self.options=dict(expectedSlot=self.case.slot_key,expectedOwner=self.case.owner,expectedTarget=self.case.source.target, + targetStorageEpoch=self.case.slot_key['writer_storage_epoch'],limitPolicy=self.case.source.limits,at=self.case.now) + self.offers={n:self.case.offers[p] for n,p in [('data','mailbox_data'),('metadata','feed_metadata')]} + + def native(self,entries=None,*,operation='owner',options=None,policy=None): + selected=entries or self.entries + if operation=='bootstrap':selected={k:v for k,v in selected.items() if k!='activation'} + value=dict(operation=operation,entries={k:encoded(v) for k,v in selected.items()},offers={k:encoded(v) for k,v in self.offers.items()}, + options=options or self.options,policy=policy or asdict(self.policy)) + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(value).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=25) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout) + self.assertEqual(out['subprocessCalls'],0);return out + + def python(self,entries,*,bootstrap=False,at=None): + c=self.case;args=dict(expected_slot=c.slot_key,expected_owner=c.owner,expected_target=c.source.target, + target_storage_epoch=c.slot_key['writer_storage_epoch'],limit_policy=c.source.limits,at=at or c.now, + policy=self.policy,budget=RepairBudget(self.policy)) + if bootstrap:return verify_mailbox_feed_bootstrap({k:v for k,v in entries.items() if k!='activation'},**args) + return verify_mailbox_slot_owner_inputs(entries,offers=self.offers,**args) + + def test_exact_owner_chain_and_bootstrap_keep_original_refs(self): + for operation in ['owner','bootstrap']: + native=self.native(operation=operation);self.assertTrue(native['ok'],native);self.assertTrue(native['immutable']) + checked=self.python(self.entries,bootstrap=operation=='bootstrap') + self.assertEqual(native['refs'],{k:v.ref.as_dict() for k,v in checked.items()}) + self.assertEqual(native['metrics']['signature_checks'],5 if operation=='owner' else 4) + + def test_owner_signed_wrong_reader_is_not_authorized(self): + entries=self.case.setup_entries(dict(read=dict(reader=self.case.dual(self.case.source.target)))) + for bootstrap in [False,True]: + with self.assertRaises(RepairWireError):self.python(entries,bootstrap=bootstrap) + result=self.native(entries,operation='bootstrap' if bootstrap else 'owner');self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + + def test_changed_scope_or_budget_never_repairs_signed_parent_chain(self): + for name,field,value in [('activation','target_storage_epoch','changed_epoch'),('slot','max_live_items',65537),('bootstrap','proof_until',self.case.now), + ('maintenance','allowed_roles',['bootstrap.grant','bootstrap.grant'])]: + with self.subTest(name=name): + entries=copy.deepcopy(self.entries);p=json.loads(entries[name]['raw'])['payload'];p[field]=value + entries[name]=signed_entry(p,self.case.f['signers']['owner'],'synthetic_mutated_owner') + with self.assertRaises(RepairWireError):self.python(entries) + result=self.native(entries);self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + + def test_invalid_reference_expiry_and_signature_budget_refuse(self): + entries=copy.deepcopy(self.entries);entries['slot']['ref']['raw_sha256']='0'*64 + result=self.native(entries);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_ref_mismatch') + result=self.native(options=dict(self.options,at=self.case.now+600));self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_resource_expired') + result=self.native(policy=dict(asdict(self.policy),max_signature_checks=4));self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_over_budget') + + def test_actual_durable_resource_chain_matches_owner_scope_and_original_limits(self): + from memory_vault_open_repair_resource import verify_mailbox_resource_inputs + c=self.case;activated=c.activate(self.entries);p=json.loads(self.entries['activation']['raw'])['payload'] + for name,purpose in [('data','mailbox_data'),('metadata','feed_metadata')]: + row=c.db.execute('SELECT allocation,allocation_ref FROM open_repair_mailbox_resources WHERE purpose=?',(purpose,)).fetchone() + entries=dict(allocate=dict(raw=bytes(row[0]),ref=json.loads(row[1])),offer=self.offers[name],activation=self.entries['activation'],active=activated[name]) + options=dict(expectedRoot=c.root,expectedOwner=c.owner,expectedTarget=c.source.target,targetStorageEpoch=c.slot_key['writer_storage_epoch'], + expectedPurpose=purpose,expectedScope=p['scope'],expectedAuthorityRefs=p['authority_refs'],expectedOfferRefs=p['resource_offer_refs'],at=c.now) + checked=verify_mailbox_resource_inputs(entries,expected_root=c.root,expected_owner=c.owner,expected_target=c.source.target, + target_storage_epoch=c.slot_key['writer_storage_epoch'],expected_purpose=purpose,expected_scope=p['scope'],expected_authority_refs=p['authority_refs'], + expected_offer_refs=p['resource_offer_refs'],at=c.now,policy=self.policy,budget=RepairBudget(self.policy)) + result=self.native(entries,operation='resource',options=options);self.assertTrue(result['ok'],result);self.assertTrue(result['immutable']) + self.assertEqual(result['refs'],{k:v.ref.as_dict() for k,v in checked.items()}) + result=self.native(entries,operation='resource',options=dict(options,expectedPurpose='anchor_catalog')) + self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + changed=json.loads(entries['active']['raw'])['payload'];changed['budget']['max_items']+=1 + wrong=dict(entries,active=signed_entry(changed,c.f['signers']['target'],'synthetic_inflated_resource')) + result=self.native(wrong,operation='resource',options=options);self.assertFalse(result['ok']);self.assertEqual(result['code'],'repair_resource_mismatch') + + def test_actual_node_genesis_requires_empty_count_and_exact_commit(self): + c=self.case;activated=c.activate(self.entries);entries={n:activated[n] for n in ('checkpoint','head')} + options=dict(expectedSlot=c.slot_key,expectedSigningKey=c.source.identity.public_descriptor(),committedAt=c.now,at=c.now) + result=self.native(entries,operation='genesis',options=options);self.assertTrue(result['ok'],result);self.assertTrue(result['immutable']) + self.assertEqual(result['refs'],{n:e['ref'] for n,e in entries.items()}) + for field,value in [('count',1),('catalog_generation',1),('range_root_ref',entries['checkpoint']['ref'])]: + p=json.loads(entries['head']['raw'])['payload'];p[field]=value + wrong=dict(entries,head=signed_entry(p,c.f['signers']['target'],'synthetic_nonempty_genesis')) + result=self.native(wrong,operation='genesis',options=options);self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error') + with self.assertRaises(RepairWireError):verify_mailbox_genesis(wrong,expected_slot=c.slot_key,expected_signing_key=c.source.identity.public_descriptor(), + committed_at=c.now,at=c.now,policy=self.policy,budget=RepairBudget(self.policy)) + +if __name__=='__main__':unittest.main() From 21d59cbdb41636bbb033b5f1b03b272145ef597f Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:44:47 +0800 Subject: [PATCH 08/24] Authenticate native mailbox admission and complete retained feed history --- .../network/open-repair-mailbox-feed.ts | 129 +++++++++++++++++ .../network/open-repair-mailbox-member.ts | 136 ++++++++++++++++++ ...t_open_repair_mailbox_member_typescript.py | 105 ++++++++++++++ 3 files changed, 370 insertions(+) create mode 100644 clients/typescript/network/open-repair-mailbox-feed.ts create mode 100644 clients/typescript/network/open-repair-mailbox-member.ts create mode 100644 tests/test_open_repair_mailbox_member_typescript.py diff --git a/clients/typescript/network/open-repair-mailbox-feed.ts b/clients/typescript/network/open-repair-mailbox-feed.ts new file mode 100644 index 0000000..c460f46 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-feed.ts @@ -0,0 +1,129 @@ +/** Original feed custody and complete retained prefix. This authenticates + * ciphertext layout; recipient decryption and current access remain separate. */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,LocalRawResolver,RawOriginal} from './open-repair-wire.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import {verifyBoundedControlSignature} from './open-repair-original.ts'; +import {verifyMailboxMemberInputs} from './open-repair-mailbox-member.ts'; +import {emptyMailboxState,appendMailboxState,verifyMailboxInclusion} from './open-repair-mailbox-range.ts'; +import {verifyStatusOriginal} from './open-repair-status.ts'; +import {validateJwe} from './crypto.ts'; +type Obj=Record; +const COMMON=['schema_version','kind','signing_key']; +const SLOT_INPUT='mailbox.slot mailbox.read_grant mailbox.maintenance_root bootstrap.mailbox_feed resource.data_allocate resource.metadata_allocate resource.data_offer resource.metadata_offer resource.slot_activation resource.data_active resource.metadata_active source.descriptor'.split(' '); +export interface MailboxFeedOptions{readonly expectedSlot:unknown;readonly expectedOwner:unknown;readonly expectedSender:unknown;readonly expectedTarget:unknown;readonly limitPolicy:unknown;readonly policy:RepairPolicy;readonly budget:RepairBudget;} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function fail(code='repair_mailbox_activation_mismatch'):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +const refKey=(v:unknown)=>{const r=rawRef(v);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +function parse(v:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{ + const entry=fields(v,['raw','ref']),ref=rawRef(entry.ref),raw=parseNewWire(entry.raw,policy,budget).raw; + if(ref.namespace!=='meta'||ref.size!==raw.length||ref.raw_sha256!==budget.hash(raw))fail('repair_ref_mismatch');return {raw,ref}; +} +export function verifyMailboxFeedSourceEvent(manifestEntry:unknown,resolver:LocalRawResolver,custodyEntry:unknown,options:MailboxFeedOptions):Readonly{ + const args=fields(options,['expectedSlot','expectedOwner','expectedSender','expectedTarget','limitPolicy','policy','budget']),{policy,budget}=args; + const e=buildNewWire({slot:args.expectedSlot,owner:args.expectedOwner,sender:args.expectedSender,target:args.expectedTarget,limits:args.limitPolicy},policy,budget).value as Obj,key=e.slot; + emptyMailboxState(key,policy,budget); + const custody=parse(custodyEntry,policy,budget),signed=fields(parseNewWire(custody.raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,[...COMMON,...'root_key slot_key slot_ref feed_head_ref covered_interval subtree historical_manifest_ref resource_refs stored_at read_until retain_until'.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!=='feed.custody'||!same(p.slot_key,key)||!same(p.root_key,key.root_key))fail(); + verifyBoundedControlSignature(p,signed.proof,e.target.signing_key,budget); + const at=u53(p.stored_at);if(!(at(),wanted=new Set(); + for(const row of resolved.roles){const id=row.role+'|'+refKey(row.original.ref);if(byRef.has(id))fail();byRef.set(id,row);} + const get=(role:string,ref:unknown):RawOriginal=>{const id=role+'|'+refKey(ref),row=byRef.get(id);if(!row)fail('repair_original_missing');wanted.add(id); + const raw=row.original.raw;if(raw.length!==row.original.ref.size||budget.hash(raw)!==row.original.ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref:row.original.ref};}; + const event=(role:string,ref:unknown,kind:string,names:string):Obj=>{const entry=get(role,ref),signed=fields(parseNewWire(entry.raw,policy,budget).value,['payload','proof']); + const value=fields(signed.payload,[...COMMON,'slot_key',...names.split(' ')]); + if(value.schema_version!=='memory-vault-open-repair/v1'||value.kind!==kind||!same(value.slot_key,key))fail();verifyBoundedControlSignature(value,signed.proof,e.target.signing_key,budget);return value;}; + const HEAD='checkpoint_ref count range_root_ref catalog_generation committed_at retain_until',CHECKPOINT='slot_binding count leaf_root frontier committed_at retain_until'; + const head=event('feed.head',m.feed_head_ref,'mailbox.feed_head',HEAD),checkpoint=event('feed.checkpoint',head.checkpoint_ref,'mailbox.checkpoint',CHECKPOINT),count=u53(head.count,1); + if(!same(m.covered_interval,{start:0,end:count})||!same(m.subtree,{root_ref:head.range_root_ref,parent_path_refs:[]})||checkpoint.count!==count||head.committed_at!==checkpoint.committed_at||head.retain_until!==checkpoint.retain_until||!(u53(head.committed_at)<=at&&at{ + const entry=get(role,ref),context={schema_version:'memory-vault-open-repair/v1',kind,slot_key:key,plaintext_sha256:budget.hash(raw),plaintext_size:raw.length,...extra}; + let value;try{value=validateJwe(entry.raw,{context});}catch{fail();} + if(value.recipients.length!==1||value.recipients[0].header?.kid!==e.owner.encryption_key.key_id)fail();}; + const predecessors=new Map(resolved.predecessors.map(v=>[budget.hash(v.manifest.raw),v])),members=new Map(),setups:Obj[]=[],deadlines=[head.retain_until]; + for(const member of m.members){ + const sequence=u53(member.sequence),manifest=get('history.member',member.historical_manifest_ref),child=predecessors.get(manifest.ref.raw_sha256); + if(!child)fail();const childValue=child.manifest.value as Obj; + const core=event('member.core',member.admission_core_ref,'admission.core','core_id sequence message_id envelope_ref attempt_ref historical_manifest_ref data_resource_ref metadata_resource_ref accepted_at object_until enum_until'); + if(core.sequence!==sequence||['message_id','envelope_ref','historical_manifest_ref'].some(n=>!same(core[n],member[n]))||['message_id','envelope_ref','attempt_ref'].some(n=>!same(core[n],childValue[n]))||!(u53(core.accepted_at)<=at&&at!same(link[n],core[n])))fail(); + sealed('member.sealed_core',link.sealed_core_ref,get('member.core',member.admission_core_ref).raw,'admission.sealed_core',{sequence}); + const cp=event('member.checkpoint',link.checkpoint_ref,'mailbox.checkpoint',CHECKPOINT); + if(cp.count!==sequence+1||!(core.accepted_at<=u53(cp.committed_at)&&cp.committed_at<=head.committed_at)||u53(cp.retain_until)[n,cp[n]])),sequence,link.sealed_core_ref.raw_sha256,link.inclusion_path,key,policy,budget); + const custody=event('member.custody',member.source_custody_ref,'message.custody','root_key message_id envelope_ref admission_link_ref checkpoint_ref feed_head_ref resource_refs stored_at object_until enum_until'); + if(!same(custody.root_key,key.root_key)||!same(custody.admission_link_ref,member.admission_link_ref)||!same(custody.checkpoint_ref,link.checkpoint_ref)||custody.stored_at!==cp.committed_at||!same(custody.resource_refs,{data:core.data_resource_ref,metadata:core.metadata_resource_ref})||['message_id','envelope_ref','object_until','enum_until'].some(n=>!same(custody[n],core[n])))fail(); + const oldHead=event('member.head',custody.feed_head_ref,'mailbox.feed_head',HEAD); + if(!same(oldHead.checkpoint_ref,link.checkpoint_ref)||oldHead.count!==sequence+1||oldHead.committed_at!==cp.committed_at||oldHead.retain_until!==cp.retain_until)fail(); + if(members.has(sequence))fail();members.set(sequence,{sequence,admission_link_ref:member.admission_link_ref,sealed_core_ref:link.sealed_core_ref}); + } + if(members.size!==count||predecessors.size!==count)fail(); + let state=emptyMailboxState(key,policy,budget); + const walk=(ref:unknown,level:number,start:number,end:number):void=>{ + const role=level>=0?'range.index':'range.repair_page',entry=get(role,ref),value=fields(parseNewWire(entry.raw,policy,budget).value,['schema_version','kind','slot_key','start','end',...(level>=0?['level','children']:['entries','sealed_page_ref'])]); + if(value.schema_version!=='memory-vault-open-repair/v1'||value.kind!==role||!same(value.slot_key,key)||u53(value.start)!==start||u53(value.end)!==end)fail(); + if(level>=0){ + if(u53(value.level)!==level||!Array.isArray(value.children)||value.children.length<1||value.children.length>16)fail();let cursor=start;const span=16**(level+1); + for(const row of value.children){fields(row,['start','end','ref']);const stop=Math.min(cursor+span,end);if(u53(row.start)!==cursor||u53(row.end)!==stop||cursor>=stop)fail();walk(row.ref,level-1,cursor,stop);cursor=stop;} + if(cursor!==end)fail(); + }else{ + if(end-start<1||end-start>16||!same(value.entries,Array.from({length:end-start},(_,i)=>members.get(start+i))))fail(); + const plain=buildNewWire({schema_version:'memory-vault-open-repair/v1',kind:'range.private_page',slot_key:key,start,end,entries:value.entries},policy,budget).raw; + sealed('range.sealed_page',value.sealed_page_ref,plain,'range.sealed_page',{start,end}); + for(const row of value.entries)state=appendMailboxState(state,row.sealed_core_ref.raw_sha256,key,policy,budget); + } + }; + let level=0;while(count>16**(level+2))level++;walk(head.range_root_ref,level,0,count); + if(Object.keys(state).some(n=>!same((state as unknown as Obj)[n],checkpoint[n])))fail(); + const obligations=new Map(),allowed=new Map(); + const dutyKey=(v:Obj)=>[v.signer.key_id,v.scope_kind,v.scope_id].join(':'); + for(const setup of setups){ + for(const duty of setup.obligations){const id=dutyKey(duty);allowed.set(id,duty);if(!['destination','data_resource'].includes(duty.role))obligations.set(id,{...duty,operation_mask:duty.role==='read'?2:duty.role==='bootstrap'?10:66});} + for(const n of ['slot','read','maintenance','bootstrap']){const v=setup.originals[n].payload;deadlines.push(v.expires_at);if(v.windows)deadlines.push(v.windows.read_until,v.windows.retain_until);} + const v=setup.resources.metadata.active.payload;deadlines.push(v.windows.read_until,v.windows.retain_until); + } + const covered=new Set(),statuses:Obj[]=[],revisions=new Map(),cache=new Map(); + for(const [id,{role,original:entry}] of byRef){ + if(!role.startsWith('historical.status.'))continue; + const raw=entry.raw,preview=fields(parseNewWire(raw,policy,budget).value,['payload','proof']).payload; + if(!preview||!preview.signing_key||!Array.isArray(preview.entries))fail('repair_invalid_status');const issuer=preview.signing_key.key_id; + const present=new Set(preview.entries.map((v:Obj)=>{if(!v||typeof v.scope_kind!=='string'||typeof v.scope_id!=='string')fail('repair_invalid_status');return v.scope_kind+':'+v.scope_id;})); + const included=([...obligations] as [string,Obj][]).filter(([,v])=>v.signer.key_id===issuer&&present.has(v.scope_kind+':'+v.scope_id)),matches=included.filter(([,v])=>role==='historical.status.'+v.role); + if(!matches.length)fail(); + const context={expectedRoot:key.root_key,expectedSigningKey:matches[0][1].signer,at,allowedScopes:[...allowed.values()].filter(v=>v.signer.key_id===issuer).map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id})), + required:included.map(([,v])=>({scope_kind:v.scope_kind,scope_id:v.scope_id,document_revision:v.document_revision,operation_mask:v.operation_mask}))}; + const contextRaw=buildNewWire(context,policy,budget).raw,cacheKey=refKey(entry.ref)+':'+budget.hash(contextRaw);let observed=cache.get(cacheKey); + if(!observed){observed=verifyStatusOriginal({raw,ref:entry.ref},{...context,policy,budget});budget.retain(raw.length+contextRaw.length);cache.set(cacheKey,observed);} + const revision=issuer+':'+observed.payload.revision;if(revisions.has(revision)&&revisions.get(revision)!==observed.canonical_sha256)fail('repair_status_conflict'); + revisions.set(revision,observed.canonical_sha256);for(const [identity] of matches)covered.add(identity);statuses.push(observed);wanted.add(id); + } + if(covered.size!==obligations.size||[...obligations.keys()].some(id=>!covered.has(id)))fail('repair_status_missing'); + const floors=new Map(); + for(const observed of [...statuses,...setups.flatMap(v=>v.statuses)]){ + const issuer=observed.payload.scope_key.issuer_key_id,revision=observed.payload.revision,id=issuer+':'+revision; + if(revisions.has(id)&&revisions.get(id)!==observed.canonical_sha256)fail('repair_status_conflict');revisions.set(id,observed.canonical_sha256); + for(const row of observed.payload.entries){const identity=[issuer,row.scope_kind,row.scope_id].join(':'),required=obligations.get(identity); + if(required&&row.status==='revoked'&&(row.operation_mask&required.operation_mask))fail('repair_authority_revoked'); + if(required&&row.minimum_document_revision>required.document_revision)fail('repair_status_revision'); + const held=floors.get(identity)??[];held.push([revision,row.minimum_document_revision]);floors.set(identity,held);} + } + for(const values of floors.values()){let min=0;for(const [,floor] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(floorv[0]>observed.payload.revision))fail('repair_status_rollback');} + if(wanted.size!==byRef.size||at>=Math.min(...deadlines))fail(); + const graph=Object.freeze({head,checkpoint,members:Object.freeze(setups),statuses:Object.freeze(statuses),obligations:Object.freeze([...obligations.values()].map(Object.freeze)),retain_until:Math.min(...deadlines),metadata_resource:setups[0].resources.metadata.active.payload.resource}); + if(!same(p.resource_refs,{metadata:graph.metadata_resource,dependencies:[]})||p.retain_until>graph.retain_until)fail(); + return Object.freeze({manifest:resolved,graph,custody:Object.freeze({ref:custody.ref,payload:p,get raw(){budget.output(custody.raw.length);return Uint8Array.from(custody.raw);}}),read_until:p.read_until,retain_until:p.retain_until}); +} diff --git a/clients/typescript/network/open-repair-mailbox-member.ts b/clients/typescript/network/open-repair-mailbox-member.ts new file mode 100644 index 0000000..aea9499 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-member.ts @@ -0,0 +1,136 @@ +/** Authenticate a retained mailbox member at its original admission time. + * Current permission, source custody and ciphertext are verified by the reader. + */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,LocalRawResolver,RawOriginal} from './open-repair-wire.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import type {DraftHistoryInputs} from './open-repair-history.ts'; +import {originalPublicDescriptor,verifyContactOriginals,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {verifyMailboxSlotOwnerInputs,verifyMailboxResourceInputs} from './open-repair-mailbox-authority.ts'; +import {verifyAckOfferBootstrapOriginal} from './open-repair-bound.ts'; +import {statusScope,verifyStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +type Obj=Record; +const ACK=['ack.root_authority','ack.write_grant','bootstrap.ack_offer','historical.status.ack_root','historical.status.ack_write','historical.status.ack_offer_bootstrap']; +const BASE=('mailbox.slot mailbox.read_grant mailbox.maintenance_root bootstrap.mailbox_feed resource.data_allocate resource.metadata_allocate resource.data_offer resource.metadata_offer resource.slot_activation resource.data_active resource.metadata_active source.descriptor historical.status.slot historical.status.read historical.status.maintenance historical.status.bootstrap contact.request delivery.attempt message.disclosure contact.policy contact.decision contact.store_grant contact.knock_lease contact.delivery_lease delivery.destination historical.status.disclosure historical.status.destination historical.status.data_resource historical.status.metadata_resource').split(' '); +const CAPS='max_live_bytes max_meta_bytes max_items max_requests max_pending max_replay_records max_jobs max_job_bytes'.split(' '); +const WINDOWS='admit_until read_until copy_until publish_until retain_until'.split(' '); +const INDEX_LIMITS={max_probe_bytes:8192,max_proof_bytes:4194304,max_proof_items:64,max_signature_checks:4096,max_requests:256,max_pending:8,max_replay_records:256,max_concurrent_handles:8,max_candidate_attempts:8}; +export interface MailboxMemberOptions{ + readonly expectedSlot:unknown;readonly expectedOwner:unknown;readonly expectedSender:unknown;readonly expectedTarget:unknown; + readonly targetStorageEpoch:string;readonly acceptedAt:number;readonly limitPolicy:unknown;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +function fail(code='repair_mailbox_activation_mismatch'):never{throw new RepairError(code);} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function same(a:unknown,b:unknown):boolean{ + if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false; + const names=Object.keys(a);return names.length===Object.keys(b).length&&names.every(n=>Object.hasOwn(b,n)&&same((a as Obj)[n],(b as Obj)[n])); +} +function dual(v:unknown,budget:RepairBudget):Obj{const k=fields(v,['signing_key','encryption_key']);originalPublicDescriptor(k.signing_key,budget);originalPublicDescriptor(k.encryption_key,budget,true);return {signing_key_id:k.signing_key.key_id,encryption_key_id:k.encryption_key.key_id};} +function opaque(v:unknown):void{if(typeof v!=='string'||/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.exec(v)?.[0]!==v)fail();} +function roleSet(roles:Obj,names:string[]):boolean{return same(Object.keys(roles).sort(),[...names].sort());} +function setup(resolved:DraftHistoryInputs,e:Obj,policy:RepairPolicy,budget:RepairBudget):Obj{ + const m=resolved.manifest.value as Obj,roles:Record=Object.create(null); + if(m.variant!=='mailbox_member'||!same(m.slot_key,e.slot)||!same(m.root_key,e.slot.root_key)||resolved.predecessors.length)fail(); + for(const row of resolved.roles){if(Object.hasOwn(roles,row.role))fail();roles[row.role]=row.original;} + const get=(role:string)=>{if(!Object.hasOwn(roles,role))fail('repair_original_missing');return {raw:roles[role].raw,ref:roles[role].ref};}; + const entries=Object.fromEntries(Object.entries({slot:'mailbox.slot',read:'mailbox.read_grant',maintenance:'mailbox.maintenance_root',bootstrap:'bootstrap.mailbox_feed',activation:'resource.slot_activation'}).map(([n,r])=>[n,get(r)])); + const resources=Object.fromEntries(['data','metadata'].map(n=>[n,Object.fromEntries(['allocate','offer','active'].map(p=>[p,get('resource.'+n+'_'+p)]))])); + const active=['data','metadata'].map(n=>fields(parseNewWire(resources[n].active.raw,policy,budget).value,['payload','proof']).payload); + const activated=u53(active[0]?.activated_at);if(active[1]?.activated_at!==activated||activated>e.at)fail(); + const originals=verifyMailboxSlotOwnerInputs(entries,{data:resources.data.offer,metadata:resources.metadata.offer},{expectedSlot:e.slot,expectedOwner:e.owner, + expectedTarget:e.target,targetStorageEpoch:e.epoch,limitPolicy:e.limits,at:activated,policy,budget}); + const slot=originals.slot.payload,activation=originals.activation.payload; + if(!same(slot.sender,dual(e.sender,budget))||!same(slot.recipient,dual(e.owner,budget)))fail(); + for(const name of ['slot','read','maintenance','bootstrap'] as const){const p=originals[name].payload; + if(!(p.issued_at<=e.at&&e.at=Math.min(p.windows.admit_until,p.windows.retain_until)))fail('repair_resource_expired');} + const checked:Obj={}; + for(const [name,purpose] of [['data','mailbox_data'],['metadata','feed_metadata']]){ + checked[name]=verifyMailboxResourceInputs({...resources[name],activation:entries.activation},{expectedRoot:e.slot.root_key,expectedOwner:e.owner,expectedTarget:e.target, + targetStorageEpoch:e.epoch,expectedPurpose:purpose,expectedScope:activation.scope,expectedAuthorityRefs:activation.authority_refs,expectedOfferRefs:activation.resource_offer_refs,at:e.at,policy,budget}); + if(!same(checked[name].active.payload.resource,slot[name+'_resource_ref']))fail(); + } + const contact=verifyContactOriginals(Object.fromEntries(Object.entries({node:'source.descriptor',request:'contact.request',policy:'contact.policy',decision:'contact.decision',grant:'contact.store_grant',knock_lease:'contact.knock_lease',delivery_lease:'contact.delivery_lease'}).map(([n,r])=>[n,get(r).raw])), + {senderKeyId:e.sender.signing_key.key_id,senderEncryptionKeyId:e.sender.encryption_key.key_id,recipientKeyId:e.owner.signing_key.key_id, + recipientEncryptionKeyId:e.owner.encryption_key.key_id,nodeKeyId:e.target.signing_key.key_id,storageEpoch:e.epoch,at:e.at,policy,budget}); + return {originals,resources:Object.freeze(checked),contact,roles:Object.freeze(roles),accepted_at:e.at}; +} +/** Each observation must cover its duty, and every scope it contains belongs to + * that signer. Reuse is bounded to exact bytes plus context in this invocation. */ +function historicalStatuses(roles:Obj,duties:Obj[],root:unknown,at:number,policy:RepairPolicy,budget:RepairBudget):readonly AuthenticatedStatusOriginal[]{ + const statuses:AuthenticatedStatusOriginal[]=[],cache=new Map(),revisions=new Map(),floors=new Map(); + for(const duty of duties){ + const permitted=duties.filter(v=>same(v.signer,duty.signer)),entry=roles['historical.status.'+duty.role];if(!entry)fail('repair_original_missing'); + const raw=entry.raw,preview=fields(parseNewWire(raw,policy,budget).value,['payload','proof']).payload; + if(!preview||!Array.isArray(preview.entries))fail('repair_invalid_status'); + const present=new Set(preview.entries.map((v:Obj)=>{if(!v||typeof v.scope_kind!=='string'||typeof v.scope_id!=='string')fail('repair_invalid_status');return v.scope_kind+':'+v.scope_id;})); + const required=permitted.filter(v=>present.has(v.scope_kind+':'+v.scope_id)).map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id,document_revision:v.document_revision,operation_mask:v.operation_mask})); + if(!required.some(v=>v.scope_kind===duty.scope_kind&&v.scope_id===duty.scope_id))fail('repair_status_missing'); + const context={expectedRoot:root,expectedSigningKey:duty.signer,at,allowedScopes:permitted.map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),required}; + const contextRaw=buildNewWire(context,policy,budget).raw,ref=rawRef(entry.ref); + if(ref.namespace!=='meta'||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + const identity=budget.hash(buildNewWire({ref,context},policy,budget).raw);let observed=cache.get(identity); + if(!observed){observed=verifyStatusOriginal({raw,ref},{...context,policy,budget});budget.retain(raw.length+contextRaw.length);cache.set(identity,observed);} + const rev=duty.signer.key_id+':'+observed.payload.revision,prior=revisions.get(rev); + if(prior!==undefined&&prior!==observed.canonical_sha256)fail('repair_status_conflict');revisions.set(rev,observed.canonical_sha256);statuses.push(observed); + for(const row of observed.payload.entries){const key=duty.signer.key_id+':'+row.scope_kind+':'+row.scope_id,held=floors.get(key)??[]; + held.push([observed.payload.revision,row.minimum_document_revision]);floors.set(key,held);} + } + for(const values of floors.values()){let minimum=0;for(const [,value] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(value({raw:roles[n].raw,ref:roles[n].ref}); + const checked=verifyAckOfferBootstrapOriginal(entry('bootstrap.ack_offer'),{root:entry('ack.root_authority'),write:entry('ack.write_grant')}, + {expectedAckSlot:write.ack_slot,expectedOwner:e.sender,expectedReceiptWriter:e.owner,expectedMessageId:m.message_id,expectedEnvelopeRef:m.envelope_ref,at:e.at,limitPolicy:INDEX_LIMITS,policy,budget}); + const root=checked.originals.root.payload.ack_slot.root_key,duties=[]; + for(const [name,role,mask] of [['root','ack_root',3],['write','ack_write',1],['bootstrap','ack_offer_bootstrap',3]] as const){const item=checked.originals[name]; + duties.push({role,scope_kind:'authority',scope_id:statusScope(root,'authority',{authority_kind:item.payload.kind,authority_sha256:item.ref.raw_sha256},policy,budget),document_revision:item.payload.revision,operation_mask:mask,signer:e.sender.signing_key});} + return Object.freeze({originals:checked.originals,statuses:historicalStatuses(roles,duties,root,e.at,policy,budget)}); +} +function member(resolved:DraftHistoryInputs,e:Obj,policy:RepairPolicy,budget:RepairBudget):Obj{ + const graph=setup(resolved,e,policy,budget),{roles,originals}=graph,m=resolved.manifest.value as Obj,root=m.root_key,key=m.slot_key,at=e.at; + if(!roleSet(roles,BASE)&&!roleSet(roles,[...BASE,...ACK]))fail(); + const control=(role:string,names:string,signer:Obj):Obj=>{const signed=fields(parseNewWire(roles[role].raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key',...names.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==role||u53(p.issued_at)>=u53(p.expires_at))fail(); + if(!(p.issued_at<=at&&atu53(dp.budget[n])>slot.budget[n])||WINDOWS.some(n=>u53(dp.windows[n])>slot.windows[n]||dp.windows[n]>dp.windows.retain_until)||at>=Math.min(dp.windows.admit_until,dp.windows.retain_until))fail(); + for(const [n,r] of [['contact_request_ref','contact.request'],['contact_policy_ref','contact.policy'],['contact_knock_lease_ref','contact.knock_lease'],['contact_decision_ref','contact.decision'],['store_grant_ref','contact.store_grant']])if(!same(dp[n],roles[r].ref))fail(); + u53(cp.revision,1);if(u53(cp.operation_mask)>127||rawRef(cp.envelope_ref).namespace!=='object')fail(); + if(!same(cp.root_key,root)||!same(cp.maintenance_root_ref,roles['mailbox.maintenance_root'].ref)||cp.issued_at>ap.issued_at||(cp.operation_mask&65)!==65|| + !(atMath.min(slot.windows.retain_until,maintenance.windows.retain_until))fail(); + const required=['contact.request','delivery.attempt','message.disclosure','authority.status.disclosure'],allowed=cp.allowed_roles; + if(!Array.isArray(allowed)||allowed.some((v:any,i:number)=>typeof v!=='string'||(i>0&&v<=allowed[i-1])||![...required,...ACK].includes(v))||required.some(n=>!allowed.includes(n)))fail('repair_status_disclosure'); + const returned=fields(cp.bootstrap_return,['subject','consumer','roles','until']); + if(!same(returned.subject,slot.recipient)||returned.consumer!=='mailbox_feed'||!same(returned.roles,['authority.status.disclosure','message.disclosure'])||!(at!allowed.includes(n)))fail('repair_status_disclosure');if(!same(ap.ack_grant_ref,roles['ack.write_grant'].ref))fail();ack=ackConfiguration(roles,e,m,policy,budget);} + const duties:Obj[]=[],add=(role:string,kind:string,subject:unknown,revision:number,mask:number,signer:Obj)=>duties.push({role,scope_kind:kind,scope_id:statusScope(root,kind,subject,policy,budget),document_revision:revision,operation_mask:mask,signer}); + add('slot','mailbox_slot',key,slot.revision,65,e.owner.signing_key); + for(const [name,role,p,mask,signer] of [['read','mailbox.read_grant',originals.read.payload,2,e.owner.signing_key],['maintenance','mailbox.maintenance_root',maintenance,65,e.owner.signing_key], + ['bootstrap','bootstrap.mailbox_feed',originals.bootstrap.payload,10,e.owner.signing_key],['destination','delivery.destination',dp,1,e.owner.signing_key],['disclosure','message.disclosure',cp,65,e.sender.signing_key]] as const){ + add(name,'authority',{authority_kind:p.kind,authority_sha256:roles[role].ref.raw_sha256},p.revision??1,mask,signer);} + for(const n of ['data','metadata']){const p=graph.resources[n].active.payload;add(n+'_resource','resource',p.resource,p.reservation_generation,65,e.target.signing_key);} + return Object.freeze({...graph,destination:dp,attempt:ap,disclosure:cp,statuses:historicalStatuses(roles,duties,root,at,policy,budget),obligations:Object.freeze(duties.map(Object.freeze)),ack_configuration:ack}); +} +export function verifyMailboxMemberInputs(manifest:unknown,resolver:LocalRawResolver,options:MailboxMemberOptions):Readonly{ + const a=fields(options,['expectedSlot','expectedOwner','expectedSender','expectedTarget','targetStorageEpoch','acceptedAt','limitPolicy','policy','budget']),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,owner:a.expectedOwner,sender:a.expectedSender,target:a.expectedTarget,epoch:a.targetStorageEpoch,at:a.acceptedAt,limits:a.limitPolicy},policy,budget).value as Obj; + u53(e.at);for(const k of [e.owner,e.sender,e.target])dual(k,budget); + const entry=fields(manifest,['raw','ref']),ref=rawRef(entry.ref),raw=parseNewWire(entry.raw,policy,budget).raw; + if(ref.namespace!=='meta'||ref.size!==raw.length||ref.raw_sha256!==budget.hash(raw))fail('repair_ref_mismatch'); + return member(resolveHistoricalInputs(raw,resolver,policy,budget),e,policy,budget); +} diff --git a/tests/test_open_repair_mailbox_member_typescript.py b/tests/test_open_repair_mailbox_member_typescript.py new file mode 100644 index 0000000..e0fc334 --- /dev/null +++ b/tests/test_open_repair_mailbox_member_typescript.py @@ -0,0 +1,105 @@ +"""Native admission graph consumes originals from a real HTTP mailbox commit.""" +import copy +import json +import subprocess +import unittest +from dataclasses import asdict +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_delivery_http as http_fixture +from tests.test_open_repair_probe_typescript import encoded +from tests.open_repair_ack_fixtures import reference +import memory_vault_open_repair_wire as wire +import memory_vault_open_repair_history as history +from memory_vault_open_repair_mailbox_snapshot import MailboxSnapshotSource +from memory_vault_open_repair_mailbox_activation import verify_mailbox_member_inputs + +DRIVER=runtime.DRIVER.split('const {Agent}',1)[0]+r''' +const {RepairBudget,LocalRawResolver}=await import('./open-repair-wire.ts'); +const {verifyMailboxMemberInputs}=await import('./open-repair-mailbox-member.ts'); +const {verifyMailboxFeedSourceEvent}=await import('./open-repair-mailbox-feed.ts'); +const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk); +const calls=JSON.parse(Buffer.concat(chunks).toString()),decode=e=>({raw:Buffer.from(e.raw,'base64'),ref:e.ref}),out=[]; +for(const v of calls){let budget;try{ + budget=new RepairBudget(v.policy);const resolver=new LocalRawResolver(v.policy,budget); + for(const p of v.packs){const e=decode(p);resolver.put(e.ref.namespace,e.ref.key,e.raw);} + if(v.operation==='feed'){const result=verifyMailboxFeedSourceEvent(decode(v.manifest),resolver,decode(v.custody),{...v.options,policy:v.policy,budget});out.push({ok:true,count:result.graph.head.count,statuses:result.graph.statuses.map(v=>v.canonical_sha256),retain_until:result.retain_until,metrics:budget.snapshot()});continue;} + const graph=verifyMailboxMemberInputs(decode(v.manifest),resolver,{...v.options,policy:v.policy,budget}); + out.push({ok:true,accepted_at:graph.accepted_at,refs:Object.fromEntries(Object.entries(graph.roles).map(([k,e])=>[k,e.ref])), + statuses:graph.statuses.map(v=>v.canonical_sha256),ack:graph.ack_configuration!==null,metrics:budget.snapshot()}); +}catch(e){out.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack),metrics:budget?.snapshot()});}} +process.stdout.write(JSON.stringify({out,subprocessCalls})); +''' + +class NativeMailboxMemberTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def native(self,calls): + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(calls).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=40) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout) + self.assertEqual(out['subprocessCalls'],0);return out['out'] + + def test_committed_http_member_originals_and_denied_cross_bindings(self): + h=http_fixture.MailboxStagingHTTPTests('test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources') + h.setUp();self.addCleanup(h.doCleanups);visited=[] + def inspect(staging,key,head): + visited.append(True);s=staging.source + rid=staging.db.execute('SELECT resource_id FROM open_repair_mailbox_roots').fetchone()[0] + held=MailboxSnapshotSource(staging).load(rid,key,head,max_bytes=2_000_000,max_items=256);part=held.parts[2] + def entry(ref):return dict(raw=held.read(ref.as_dict()),ref=ref.as_dict()) + packs=[entry(ref) for role,ref in part.transfer if role=='history.raw_pack'];manifest=entry(part.manifest) + core=next(ref for role,ref in part.transfer if role=='member.core');accepted=json.loads(held.read(core.as_dict()))['payload']['accepted_at'] + owner=json.loads(staging.db.execute('SELECT owner_keys FROM open_repair_mailbox_resources LIMIT 1').fetchone()[0]) + envelope=json.loads(held.read(held.envelopes[0].as_dict())) + sender=dict(signing_key=h.ai.public_descriptor(),encryption_key=envelope['payload']['context']['sender_encryption_key']) + options=dict(expectedSlot=key,expectedOwner=owner,expectedSender=sender,expectedTarget=s.target, + targetStorageEpoch=s.node['payload']['storage_epoch'],acceptedAt=accepted,limitPolicy=s.limits) + budget=wire.RepairBudget(s.policy);resolver=wire.LocalRawResolver(s.policy,budget) + for p in packs:resolver.put(p['ref']['namespace'],p['ref']['key'],p['raw']) + resolved=history.resolve_historical_inputs(manifest['raw'],resolver,s.policy,budget) + expected=verify_mailbox_member_inputs(resolved,expected_slot=key,expected_owner=owner,expected_sender=sender,expected_target=s.target, + target_storage_epoch=options['targetStorageEpoch'],accepted_at=accepted,limit_policy=s.limits,policy=s.policy,budget=budget) + call=dict(manifest=encoded(manifest),packs=[encoded(p) for p in packs],options=options,policy=asdict(s.policy)) + call=json.loads(json.dumps(call)) + calls=[call];labels=['exact'] + for name,value in [('expectedSender',owner),('targetStorageEpoch','different_epoch'),('acceptedAt',accepted+10000)]: + wrong=copy.deepcopy(call);wrong['options'][name]=value;calls.append(wrong);labels.append(name) + wrong=copy.deepcopy(call);wrong['manifest']['ref']['raw_sha256']='0'*64;calls.append(wrong);labels.append('manifest_ref') + wrong=copy.deepcopy(call);wrong['policy']['max_signature_checks']=3;calls.append(wrong);labels.append('signature_budget') + # Repack exact original documents under a wrong role. Every pack hash + # remains valid; typed admission checks must reject the substitution. + roles={row.role:dict(raw=row.original.raw,ref=row.original.ref.as_dict()) for row in resolved.roles} + for target,other in [('historical.status.disclosure','historical.status.slot'),('contact.decision','contact.policy'),('delivery.destination','delivery.attempt')]: + altered=dict(roles);altered[target]=roles[other];meter=wire.RepairBudget(s.policy) + pack=wire.build_raw_pack([v['raw'] for v in altered.values()],s.policy,meter) + indices={(v.raw_sha256,v.size):i for i,v in enumerate(pack.entries)} + value=dict(json.loads(manifest['raw']),roles=[dict(role=role,document_ref=v['ref'],pack_ref=pack.ref.as_dict(),entry_index=indices[(v['ref']['raw_sha256'],v['ref']['size'])]) for role,v in sorted(altered.items())]) + changed=history.build_historical_manifest(value,s.policy,meter) + wrong=copy.deepcopy(call);wrong['manifest']=encoded(dict(raw=changed.raw,ref=reference(changed.raw,'native_mutated_member'))) + wrong['packs']=[encoded(dict(raw=pack.raw,ref=pack.ref.as_dict()))];calls.append(wrong);labels.append(target) + results=self.native(calls);good=results[0];self.assertTrue(good['ok'],good) + self.assertEqual(good['accepted_at'],accepted);self.assertEqual(good['refs'],{k:v['ref'] for k,v in expected['roles'].items()}) + self.assertEqual(good['statuses'],[v.canonical_sha256 for v in expected['statuses']]);self.assertEqual(good['ack'],expected['ack_configuration'] is not None) + from memory_vault_open_repair_mailbox_activation import verify_mailbox_feed_source_event + feedpart=held.parts[1];feedpacks=[entry(ref) for role,ref in feedpart.transfer if role=='history.raw_pack'] + feedoptions={k:v for k,v in options.items() if k not in ('acceptedAt','targetStorageEpoch')} + feedcall=json.loads(json.dumps(dict(operation='feed',manifest=encoded(entry(feedpart.manifest)),custody=encoded(entry(feedpart.custody)),packs=[encoded(p) for p in feedpacks],options=feedoptions,policy=asdict(s.policy)))) + meter=wire.RepairBudget(s.policy);local=wire.LocalRawResolver(s.policy,meter) + for pack in feedpacks:local.put(pack['ref']['namespace'],pack['ref']['key'],pack['raw']) + feedexpected=verify_mailbox_feed_source_event(entry(feedpart.manifest),local,entry(feedpart.custody),expected_slot=key,expected_owner=owner,expected_sender=sender,expected_target=s.target,limit_policy=s.limits,policy=s.policy,budget=meter) + feedcalls=[feedcall] + wrong=copy.deepcopy(feedcall);wrong['options']['expectedSender']=owner;feedcalls.append(wrong) + wrong=copy.deepcopy(feedcall);wrong['custody']['ref']['raw_sha256']='0'*64;feedcalls.append(wrong) + feedresults=self.native(feedcalls);self.assertTrue(feedresults[0]['ok'],feedresults[0]) + self.assertEqual(feedresults[0]['count'],1);self.assertEqual(feedresults[0]['retain_until'],feedexpected['retain_until']) + self.assertEqual(feedresults[0]['statuses'],[v.canonical_sha256 for v in feedexpected['graph']['statuses']]) + for result in feedresults[1:]:self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error',result) + for label,result in zip(labels[1:],results[1:]): + with self.subTest(label=label):self.assertFalse(result['ok'],result);self.assertNotEqual(result['code'],'untyped_error',result) + h.inspect_committed_mailbox=inspect + h.test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources();self.assertEqual(visited,[True]) + +if __name__=='__main__':unittest.main() From d86d62083566a30804f3cae5ad6643da73e28f9a Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:45:55 +0800 Subject: [PATCH 09/24] Keep exact public SDK inventory aligned with native receipt return --- tests/test_network_packaging.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 427a99d..c207d07 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -49,7 +49,7 @@ "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-repair-probe.ts", - "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts")} + "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} TS_ENDPOINT_TESTS = {"tests/test_network_typescript_" + name + ".py" for name in ("nodes", "records", "vault", "peer", "peer_race", "transport", "setup", "retrieval_text", "retrieval", "agent", "agent_network", "topics")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 57) + self.assertEqual(len(TS_NETWORK), 59) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -125,6 +125,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", + "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", "tests/test_open_repair_proof.py", "tests/test_open_repair_proof_typescript.py", "tests/test_open_repair_access.py", @@ -192,7 +193,8 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", - "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts"}) + "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", + "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts"}) for name in repair_sources: self.assertTrue((ROOT / name).is_file()) for name in ("docs/NATIVE_OPEN_PROVIDER.md", "docs/CONTINUATION_TRIAL.md", "docs/OPEN_NETWORK_QUICKSTART.md"): From 08626c6bade2320a342b3a4ac48b299ffeb8ba74 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:58:49 +0800 Subject: [PATCH 10/24] Resume retained mailbox memory imports and saved receipts in native Agent --- clients/typescript/network/README.md | 20 +++ .../network/open-delivery-client.ts | 33 +++- .../network/open-repair-mailbox-inbox.ts | 38 +++++ .../network/open-repair-mailbox-read.ts | 151 ++++++++++++++++++ .../typescript/network/open-repair-receipt.ts | 2 +- clients/typescript/network/package.json | 7 +- scripts/build_client_plugin.py | 5 + scripts/build_release.py | 6 + scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 14 +- ...t_open_repair_mailbox_member_typescript.py | 98 +++++++++++- 11 files changed, 356 insertions(+), 22 deletions(-) create mode 100644 clients/typescript/network/open-repair-mailbox-inbox.ts create mode 100644 clients/typescript/network/open-repair-mailbox-read.ts diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index 31a286e..a378e96 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -277,3 +277,23 @@ in `open-repair-admin.ts`: `recover-ack`, `recover-empty`, and verified status observations across restarts, including failed recovery. See [request formats and commands](../../../docs/OPEN_ACK_RECOVERY.md#native-recovery-commands-development-after-alpha019). This command entry is development after the frozen alpha.0.19 archives. + +### Retained mailbox inbox recovery (development) + +A native `Agent.handle({op: "receive", limit: 1})` can resume an ordinary mailbox transfer +already durably staged by the Python client in the same protected inbox. It +rechecks the original owner/contact approvals, activated storage resources, +complete feed prefix, recipient-encrypted directory and admission core, and +READ observations recorded at receipt time. A retained revocation or missing +sender permission blocks the object read. The original memory share then +uses the existing Vault import and its idempotent transfer receipt. The saved +receipt remains available to an explicit `connect/return_receipt` request; +mailbox access does not grant an independent receipt return. + +`readMailboxIndex` and `readMailboxAdmission` accept a caller-authorized exact +reference reader. They support asynchronous reads without delegating to Python. +`verifyMailboxInboxEvidence` reopens the retained ordinary-mailbox format without +network access. This is historical local recovery, not a claim about current +source availability. Native discovery/download of a new remote mailbox and the +replica inbox format remain separate unfinished work; use the Python client for +those paths. These additions are development source, not yet a public release. diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index d69c7b8..5c28e98 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -2,6 +2,7 @@ * Original share bytes and author trust remain separate from network identity. */ import {randomBytes} from 'node:crypto'; +import {verifyMailboxInboxEvidence} from './open-repair-mailbox-inbox.ts'; import {performance} from 'node:perf_hooks'; import type {DatabaseSync} from 'node:sqlite'; import {canonicalBytes,document,objectFields,sha256,encodeBase64url,decodeBase64url,opaqueId,safeInteger, @@ -118,7 +119,8 @@ export class OpenDeliveryClient{ })); } private static keys(session:Obj):Obj{ - if(session.intent)session=session.intent.payload.authority; + if(Object.hasOwn(session,'mailbox'))session=session.authority; + else if(session.intent)session=session.intent.payload.authority; const request=session.request.payload,policy=session.policy.payload; return {sender_signing_key:request.signing_key,sender_encryption_key:request.encryption_key, recipient_signing_key:policy.signing_key,recipient_encryption_key:policy.encryption_key}; @@ -324,7 +326,7 @@ export class OpenDeliveryClient{ private stageInbox(messageId:string,sender:string,envelope:Uint8Array,body:Uint8Array,session:Obj):Obj{ originalDocument(envelope,{maximum:MAX_ENVELOPE_BYTES});const digest=sha256(envelope),content=validateContent(body); if(!['message','memory_transfer'].includes(content.kind))fail('open_delivery_invalid_content_kind'); - const proof=canonicalBytes(document(session,MAX_INBOX_SESSION_BYTES)); + const proof=canonicalBytes(OpenDeliveryClient.inboxSession(session)); return this.db(db=>transaction(db,()=>{const old=db.prepare('SELECT * FROM open_delivery_inbox WHERE message_id=?').get(messageId) as Obj|undefined; if(old){if(old.envelope_sha256!==digest||old.sender!==sender||!raw(old.body).equals(raw(body)))fail('network_inbox_identity_conflict');return old;} const size=this.size(db,'open_delivery_inbox');if(size.count>=4096||size.bytes+envelope.length+body.length+proof.length+2*MAX_RESULT_BYTES>MAX_LOCAL_BYTES)fail('network_inbox_capacity'); @@ -337,10 +339,23 @@ export class OpenDeliveryClient{ if(!row)fail('network_message_not_found');if(['saved','rejected'].includes(row.phase))return document(row.result,MAX_RESULT_BYTES); const result={message_id:messageId,state:'rejected',code};db.prepare("UPDATE open_delivery_inbox SET phase='rejected',result=? WHERE message_id=?").run(canonicalBytes(result),messageId);return result; }));} + private static inboxSession(value:DocumentInput):Obj{ + const session=document(value,8*1024*1024) as Obj; + if(!Object.hasOwn(session,'mailbox'))return document(session,MAX_INBOX_SESSION_BYTES) as Obj; + return objectFields(session,['mailbox','authority','source_node']); + } + private async verifyMailboxInbox(session:Obj,row:Obj):Promise{ + const verified=await verifyMailboxInboxEvidence(session.mailbox,row.envelope,{owner:{signing_key:validateSigningIdentity(this.participant.identity), + encryption_key:validateEncryptionIdentity(this.encryption)},encryptionIdentity:this.encryption,stagedAt:row.created_at}); + if(verified.core.message_id!==row.message_id||session.mailbox.sender.signing_key.key_id!==row.sender)fail('network_inbox_identity_conflict'); + const expected=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([name,role])=>[name,document(verified.setup.roles[role].raw)])); + if(!same(session.authority,expected))fail('network_inbox_identity_conflict'); + } private async finishInbox(messageId:string):Promise{ const row=this.inbox(messageId);if(!row)fail('network_message_not_found');if(['saved','rejected'].includes(row.phase))return document(row.result,MAX_RESULT_BYTES); - const session=document(row.session,MAX_INBOX_SESSION_BYTES) as Obj;if(sha256(row.envelope)!==row.envelope_sha256)fail('network_inbox_identity_conflict'); - verifyStorageReceipt(session.storage_receipt,{node:session.source_node,intent:session.intent});const keys=OpenDeliveryClient.keys(session); + const session=OpenDeliveryClient.inboxSession(row.session);if(sha256(row.envelope)!==row.envelope_sha256)fail('network_inbox_identity_conflict'); + if(Object.hasOwn(session,'mailbox'))await this.verifyMailboxInbox(session,row); + else verifyStorageReceipt(session.storage_receipt,{node:session.source_node,intent:session.intent});const keys=OpenDeliveryClient.keys(session); if(!same(keys.recipient_signing_key,validateSigningIdentity(this.participant.identity)))fail('open_delivery_key_binding_mismatch'); const reopened=await decryptEnvelope(row.envelope,{...keys,encryption_identity:this.encryption} as any); if(!raw(reopened).equals(raw(row.body)))fail('network_inbox_identity_conflict');const content=validateContent(row.body);let imported:Obj|null=null; @@ -373,10 +388,11 @@ export class OpenDeliveryClient{ } /** Read the actually saved inbox receipt for an explicitly requested return. * The caller cannot substitute receipt bytes or a saved-state flag. */ - savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):{raw:Uint8Array;ref:Obj}{ + async savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):Promise<{raw:Uint8Array;ref:Obj}>{ if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); - const keys=OpenDeliveryClient.keys(document(row.session,MAX_INBOX_SESSION_BYTES) as Obj); + const session=OpenDeliveryClient.inboxSession(row.session);if(Object.hasOwn(session,'mailbox'))await this.verifyMailboxInbox(session,row); + const keys=OpenDeliveryClient.keys(session); if(sha256(row.envelope)!==row.envelope_sha256||row.sender!==owner.signing_key.key_id||!same(envelopeRef(row.envelope),expectedEnvelope)|| !same(keys,{sender_signing_key:owner.signing_key,sender_encryption_key:owner.encryption_key, recipient_signing_key:validateSigningIdentity(this.participant.identity),recipient_encryption_key:validateEncryptionIdentity(this.encryption)}))fail('repair_saved_tuple_mismatch'); @@ -384,7 +400,7 @@ export class OpenDeliveryClient{ if(result.state!=='validated_saved'||result.message_id!==messageId||result.sender_key_id!==row.sender|| !['message','memory_transfer'].includes(result.content_kind))fail('repair_receipt_not_saved'); this.savedReceipt(messageId);const saved=this.inbox(messageId); - if(!saved||saved.phase!=='saved'||saved.sender!==row.sender||!raw(saved.envelope).equals(raw(row.envelope))||!raw(saved.body).equals(raw(row.body)))fail('repair_saved_tuple_mismatch'); + if(!saved||saved.phase!=='saved'||saved.sender!==row.sender||!raw(saved.envelope).equals(raw(row.envelope))||!raw(saved.body).equals(raw(row.body))||!raw(saved.session).equals(raw(row.session))||saved.created_at!==row.created_at)fail('repair_saved_tuple_mismatch'); const bytes=Buffer.from(saved.receipt); verifyRecipientReceipt(document(bytes,4096),{recipient_signing_key:validateSigningIdentity(this.participant.identity), sender_key_id:row.sender,message_id:messageId,envelope_ref:expectedEnvelope} as any); @@ -392,7 +408,8 @@ export class OpenDeliveryClient{ } private async sendReceipt(messageId:string,budget:DeliveryBudget,node?:SignedNode){ const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('open_delivery_not_saved');if(row.receipt_sent)return; - const receipt=this.savedReceipt(messageId);verifyRecipientReceipt(receipt,{recipient_signing_key:validateSigningIdentity(this.participant.identity), + const receipt=this.savedReceipt(messageId);if(Object.hasOwn(OpenDeliveryClient.inboxSession(row.session),'mailbox'))return; + verifyRecipientReceipt(receipt,{recipient_signing_key:validateSigningIdentity(this.participant.identity), sender_key_id:row.sender,message_id:messageId,envelope_ref:envelopeRef(row.envelope)}); if(!node){const session=document(row.session,MAX_INBOX_SESSION_BYTES) as Obj,child=budget.routing(); try{node=await this.sessionNode({node:session.source_node},child);}finally{budget.merge(child);}} diff --git a/clients/typescript/network/open-repair-mailbox-inbox.ts b/clients/typescript/network/open-repair-mailbox-inbox.ts new file mode 100644 index 0000000..c2d1c63 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-inbox.ts @@ -0,0 +1,38 @@ +/** Reopen an actually retained mailbox transfer at its recorded receipt time. + * No network operation or new permission is implied by local historical bytes. */ +import {buildNewWire,objectFields,rawRef,u53,RepairError,RepairBudget,LocalRawResolver} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import {decodeBase64url} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import {verifyMailboxFeedSourceEvent} from './open-repair-mailbox-feed.ts'; +import {readMailboxIndex,readMailboxAdmission} from './open-repair-mailbox-read.ts'; +type Obj=Record; +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +const key=(value:unknown)=>{const r=rawRef(value);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +function fail(code:string):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +export async function verifyMailboxInboxEvidence(evidence:unknown,envelope:Uint8Array,options:{owner:unknown;encryptionIdentity:EncryptionIdentityDocument;stagedAt:number}):Promise>{ + const a=fields(options,['owner','encryptionIdentity','stagedAt']); + const value=fields(evidence,['schema_version','received_at','slot','sender','target','limits','member','manifest_ref','custody_ref','originals','status_refs']); + if(value.schema_version!=='memory-vault-mailbox-inbox/v1')fail('repair_invalid_context'); + const at=u53(value.received_at);if(at>u53(a.stagedAt))fail('repair_invalid_context'); + const policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}),budget=new RepairBudget(policy); + const selected=buildNewWire({slot:value.slot,sender:value.sender,target:value.target,owner:a.owner,member:value.member,limits:value.limits,manifest_ref:value.manifest_ref,custody_ref:value.custody_ref,status_refs:value.status_refs},policy,budget).value as Obj; + const limits=fields(selected.limits,Object.keys(DEFAULT_REPAIR_CLIENT_LIMITS));for(const n of Object.keys(limits))u53(limits[n],1); + if(!Array.isArray(value.originals)||value.originals.length<1||value.originals.length>limits.max_proof_items||!Array.isArray(selected.status_refs)||selected.status_refs.length<1||selected.status_refs.length>16)fail('repair_over_budget'); + const originals=new Map;raw:Uint8Array}>(),resolver=new LocalRawResolver(policy,budget);let total=0; + for(const item of value.originals){const v=fields(item,['ref','raw_base64url']),ref=rawRef(v.ref),id=key(ref); + if(ref.namespace!=='meta'||originals.has(id)||ref.size>policy.max_document_bytes)fail('repair_ref_mismatch'); + const raw=decodeBase64url(v.raw_base64url,policy.max_document_bytes);total+=raw.length;if(total>limits.max_proof_bytes)fail('repair_over_budget'); + if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + if(!same(resolver.put(ref.namespace,ref.key,raw).ref,ref))fail('repair_ref_mismatch');originals.set(id,{raw,ref}); + } + const entry=(ref:unknown)=>{const v=originals.get(key(ref));if(!v)fail('repair_original_missing');return v;}; + const source=verifyMailboxFeedSourceEvent(entry(selected.manifest_ref),resolver,entry(selected.custody_ref),{expectedSlot:selected.slot,expectedOwner:selected.owner,expectedSender:selected.sender,expectedTarget:selected.target,limitPolicy:limits,policy,budget}); + if(!(source.custody.payload.stored_at<=at&&at[v.role,{ref:v.original.ref,raw:v.original.raw}])); + const members=await readMailboxIndex(roles.get('feed.head'),roles.get('feed.checkpoint'),{expectedSlot:selected.slot,expectedSigningKey:selected.target.signing_key,encryptionIdentity:a.encryptionIdentity,readOriginal:ref=>entry(ref).raw,at,maxMessages:65536,policy,budget}); + if(!members.some(v=>same(v,selected.member)))fail('repair_mailbox_member_mismatch'); + return readMailboxAdmission(selected.member,{expectedSlot:selected.slot,expectedSigningKey:selected.target.signing_key,expectedOwner:selected.owner,expectedSender:selected.sender,expectedTarget:selected.target, + encryptionIdentity:a.encryptionIdentity,readOriginal:ref=>ref.namespace==='object'?envelope:entry(ref).raw,at,currentStatuses:selected.status_refs.map(entry),knownStatuses:[],statusObligations:source.graph.obligations,limitPolicy:limits,policy,budget}); +} diff --git a/clients/typescript/network/open-repair-mailbox-read.ts b/clients/typescript/network/open-repair-mailbox-read.ts new file mode 100644 index 0000000..8f7c534 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-read.ts @@ -0,0 +1,151 @@ +/** Read and decrypt a complete mailbox directory through an authorized reader. + * Enumeration alone grants no message access; each admission still needs its + * original authority, source history and current status before object reads. */ +import {isUint8Array} from 'node:util/types'; +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,RawRef} from './open-repair-wire.ts'; +import {verifyBoundedControlSignature} from './open-repair-original.ts'; +import {emptyMailboxState,appendMailboxState} from './open-repair-mailbox-range.ts'; +import {decryptBytes,validateEncryptionIdentity} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +type Obj=Record; +export interface MailboxIndexReadOptions{ + readonly expectedSlot:unknown;readonly expectedSigningKey:unknown;readonly encryptionIdentity:EncryptionIdentityDocument; + readonly readOriginal:(reference:RawRef)=>Uint8Array|Promise; + readonly at:number;readonly maxMessages:number;readonly policy:RepairPolicy;readonly budget:RepairBudget; +} +function fail(code='repair_mailbox_range_mismatch'):never{throw new RepairError(code);} +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +export async function readMailboxIndex(headEntry:unknown,checkpointEntry:unknown,options:MailboxIndexReadOptions):Promise{ + const a=fields(options,['expectedSlot','expectedSigningKey','encryptionIdentity','readOriginal','at','maxMessages','policy','budget']),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,at:a.at,max:a.maxMessages,identity:a.encryptionIdentity},policy,budget).value as Obj; + const reader=a.readOriginal;u53(e.at);u53(e.max,1);if(e.max>65536||typeof reader!=='function')fail('repair_invalid_context'); + validateEncryptionIdentity(e.identity);const identity=e.identity as EncryptionIdentityDocument; + const checked=(raw:unknown,reference:unknown):Obj=>{const ref=rawRef(reference),doc=parseNewWire(raw as Uint8Array,policy,budget),held=doc.raw; + if(ref.namespace!=='meta'||held.length!==ref.size||budget.hash(held)!==ref.raw_sha256)fail('repair_ref_mismatch');return doc.value as Obj;}; + const event=(entry:unknown,kind:string,names:string):Obj=>{const v=fields(entry,['raw','ref']),signed=fields(checked(v.raw,v.ref),['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key','slot_key','committed_at','retain_until',...names.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==kind||!same(p.slot_key,e.slot))fail(); + verifyBoundedControlSignature(p,signed.proof,e.key,budget);if(!(u53(p.committed_at)<=e.at&&e.ate.max||count!==cp.count||!same(head.checkpoint_ref,fields(checkpointEntry,['raw','ref']).ref)||head.committed_at!==cp.committed_at||head.retain_until!==cp.retain_until)fail(); + const state=Object.fromEntries(['slot_binding','count','leaf_root','frontier'].map(n=>[n,cp[n]])); + let accumulated=emptyMailboxState(e.slot,policy,budget);const entries:Obj[]=[],seen=new Set(); + const load=async(reference:unknown):Promise=>{const ref=rawRef(reference);if(ref.namespace!=='meta')fail('repair_ref_mismatch');if(seen.has(ref.key))fail();seen.add(ref.key);return checked(await reader(ref),ref);}; + const walk=async(reference:unknown,level:number,start:number,end:number):Promise=>{ + const value=fields(await load(reference),['schema_version','kind','slot_key','start','end',...(level>=0?['level','children']:['sealed_page_ref','entries'])]); + if(value.schema_version!=='memory-vault-open-repair/v1'||!same(value.slot_key,e.slot)||u53(value.start)!==start||u53(value.end)!==end)fail(); + if(level>=0){ + if(value.kind!=='range.index'||u53(value.level)!==level||!Array.isArray(value.children)||value.children.length<1||value.children.length>16)fail();let cursor=start;const span=16**(level+1); + for(const child of value.children){fields(child,['start','end','ref']);const stop=Math.min(cursor+span,end);if(u53(child.start)!==cursor||u53(child.end)!==stop||stop<=cursor)fail();await walk(child.ref,level-1,cursor,stop);cursor=stop;} + if(cursor!==end)fail();return; + } + if(value.kind!=='range.repair_page'||!Array.isArray(value.entries)||value.entries.length!==end-start||end-start<1||end-start>16)fail(); + for(let i=0;iv!==privateRaw[i]))fail(); + for(const row of value.entries){accumulated=appendMailboxState(accumulated,row.sealed_core_ref.raw_sha256,e.slot,policy,budget);entries.push(row);} + }; + if(count){let level=0;while(count>16**(level+2))level++;await walk(head.range_root_ref,level,0,count);} + else if(head.range_root_ref!==null)fail(); + if(!same(accumulated,state))fail();return Object.freeze(entries); +} + +import {LocalRawResolver} from './open-repair-wire.ts'; +import {parseHistoricalManifest} from './open-repair-history.ts'; +import {verifyMailboxMemberInputs} from './open-repair-mailbox-member.ts'; +import {verifyMailboxInclusion} from './open-repair-mailbox-range.ts'; +import {authenticateStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +export interface MailboxAdmissionReadOptions extends Omit{ + readonly expectedOwner:unknown;readonly expectedSender:unknown;readonly expectedTarget:unknown;readonly limitPolicy:unknown; + readonly currentStatuses:readonly unknown[];readonly knownStatuses:readonly unknown[];readonly statusObligations:readonly unknown[]; + readonly onStatusAuthenticated?: (item:AuthenticatedStatusOriginal)=>void; +} +function currentRead(setup:Obj,entries:readonly unknown[],known:readonly unknown[],additional:readonly unknown[],at:number, + policy:RepairPolicy,budget:RepairBudget,onAuthenticated?: (item:AuthenticatedStatusOriginal)=>void):readonly AuthenticatedStatusOriginal[]{ + if(!Array.isArray(entries)||entries.length<1||entries.length>16||!Array.isArray(known)||known.length>32||!Array.isArray(additional)||additional.length>80|| + (onAuthenticated!==undefined&&typeof onAuthenticated!=='function'))fail('repair_invalid_status'); + const root=setup.originals.slot.payload.slot_key.root_key; + const obligations=setup.obligations.map((v:Obj)=>({...v,operation_mask:v.role==='destination'?0:v.role==='bootstrap'?10:2})); + for(const n of ['read','maintenance'])if((setup.originals[n].payload.operation_mask&2)!==2)fail('repair_authority_revoked'); + if((setup.disclosure.operation_mask&2)!==2)fail('repair_authority_revoked'); + const required=new Map(obligations.map((v:Obj)=>[[v.signer.key_id,v.scope_kind,v.scope_id].join(':'),v])); + const added=buildNewWire(additional,policy,budget).value as Obj[]; + for(const duty of added){fields(duty,['role','scope_kind','scope_id','document_revision','operation_mask','signer']);fields(duty.signer,['schema_version','algorithm','key_id','public_key']);} + const disclosed=[...obligations,...added]; + const authenticate=(entry:unknown,current:boolean):AuthenticatedStatusOriginal=>{const v=fields(entry,['raw','ref']),preview=fields(parseNewWire(v.raw,policy,budget).value,['payload','proof']).payload; + if(!preview||!preview.scope_key)fail('repair_invalid_status');const issuer=preview.scope_key.issuer_key_id,permitted=disclosed.filter(v=>v.signer.key_id===issuer); + if(!permitted.length||u53(preview.issued_at)>at)fail('repair_status_mismatch'); + const scopes=new Map();for(const duty of permitted)scopes.set(duty.scope_kind+':'+duty.scope_id,{scope_kind:duty.scope_kind,scope_id:duty.scope_id}); + return authenticateStatusOriginal(v,{expectedRoot:root,expectedSigningKey:permitted[0].signer,at:current?at:preview.issued_at, + allowedScopes:[...scopes].sort(([a],[b])=>ab?1:0).map(([,v])=>v),policy,budget},onAuthenticated);}; + const retained=[...setup.statuses,...known.map(v=>authenticate(v,false))],current=entries.map(v=>authenticate(v,true)); + const seen=new Map(),floors=new Map(),covered=new Set(); + for(const observed of [...retained,...current]){const p=observed.payload,issuer=p.scope_key.issuer_key_id,revision=p.revision,id=issuer+':'+revision; + if(seen.has(id)&&seen.get(id)!==observed.canonical_sha256)fail('repair_status_conflict');seen.set(id,observed.canonical_sha256); + for(const row of p.entries){const key=[issuer,row.scope_kind,row.scope_id].join(':'),wanted=required.get(key);if(!wanted)continue; + const mask=wanted.operation_mask;if(row.status==='revoked'&&(row.operation_mask&mask))fail('repair_authority_revoked'); + if(mask&&row.minimum_document_revision>wanted.document_revision)fail('repair_status_revision'); + const held=floors.get(key)??[];held.push([revision,row.minimum_document_revision]);floors.set(key,held);} + } + for(const values of floors.values()){let minimum=0;for(const [,floor] of values.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(floorv[0]>p.revision))fail('repair_status_rollback'); + const mask=wanted.operation_mask;if(mask&&row.status==='active'&&(row.operation_mask&mask)===mask)covered.add(key);}} + const expected=[...required].filter(([,v])=>v.operation_mask).map(([k])=>k);if(covered.size!==expected.length||expected.some(k=>!covered.has(k)))fail('repair_status_missing'); + return Object.freeze(current); +} +/** Authenticate/decrypt metadata and check present authority before fetching + * the envelope. The enclosing verified feed determines which member to select. */ +export async function readMailboxAdmission(member:unknown,options:MailboxAdmissionReadOptions):Promise>{ + const names=['expectedSlot','expectedSigningKey','encryptionIdentity','readOriginal','at','policy','budget','expectedOwner','expectedSender','expectedTarget','limitPolicy','currentStatuses','knownStatuses','statusObligations']; + const a=fields(options,Object.hasOwn(options,'onStatusAuthenticated')?[...names,'onStatusAuthenticated']:names),{policy,budget}=a; + const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,owner:a.expectedOwner,sender:a.expectedSender,target:a.expectedTarget,limits:a.limitPolicy,at:a.at,identity:a.encryptionIdentity,member},policy,budget).value as Obj; + if(typeof a.readOriginal!=='function')fail('repair_invalid_context');u53(e.at);validateEncryptionIdentity(e.identity); + if(!same(e.key,e.target.signing_key))fail('repair_mailbox_member_mismatch'); + const item=fields(e.member,['sequence','admission_link_ref','sealed_core_ref']),sequence=u53(item.sequence);if(sequence>=65536)fail('repair_invalid_context'); + const originals=new Map(); + const load=async(reference:unknown,namespace='meta'):Promise=>{const ref=rawRef(reference);if(ref.namespace!==namespace||(namespace==='object'&&ref.size>6291456))fail('repair_ref_mismatch'); + const input=await a.readOriginal(ref);let size:number;try{size=Object.getOwnPropertyDescriptor(Object.getPrototypeOf(Uint8Array.prototype),'byteLength')!.get!.call(input);}catch{fail('repair_invalid_bytes');} + if(size!==ref.size)fail('repair_ref_mismatch');budget.input(size);budget.retain(size);if(!isUint8Array(input))fail('repair_invalid_bytes'); + const proto=Object.getPrototypeOf(Uint8Array.prototype),buffer=Object.getOwnPropertyDescriptor(proto,'buffer')!.get!.call(input),offset=Object.getOwnPropertyDescriptor(proto,'byteOffset')!.get!.call(input); + const raw=Uint8Array.from(Buffer.from(buffer,offset,size)); + if(budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');const key=ref.namespace+':'+ref.key,prior=originals.get(key);if(prior&&!same(prior.ref,ref))fail('repair_ref_mismatch');originals.set(key,{ref,raw});return raw;}; + const event=async(reference:unknown,kind:string,names:string):Promise<[Uint8Array,Obj]>=>{const raw=await load(reference),signed=fields(parseNewWire(raw,policy,budget).value,['payload','proof']); + const p=fields(signed.payload,['schema_version','kind','signing_key','slot_key',...names.split(' ')]); + if(p.schema_version!=='memory-vault-open-repair/v1'||p.kind!==kind||!same(p.slot_key,e.slot))fail('repair_mailbox_member_mismatch');verifyBoundedControlSignature(p,signed.proof,e.key,budget);return [raw,p];}; + const [,link]=await event(item.admission_link_ref,'admission.link','sequence message_id envelope_ref core_ref sealed_core_ref historical_manifest_ref checkpoint_ref inclusion_path'); + if(u53(link.sequence)!==sequence||!same(link.sealed_core_ref,item.sealed_core_ref))fail('repair_mailbox_member_mismatch'); + const [coreRaw,core]=await event(link.core_ref,'admission.core','core_id sequence message_id envelope_ref attempt_ref historical_manifest_ref data_resource_ref metadata_resource_ref accepted_at object_until enum_until'); + for(const id of [core.core_id,core.message_id])if(typeof id!=='string'||/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.exec(id)?.[0]!==id)fail('repair_mailbox_member_mismatch'); + if(u53(core.sequence)!==sequence||['message_id','envelope_ref','historical_manifest_ref'].some(n=>!same(core[n],link[n]))||!(u53(core.accepted_at)<=e.at&&e.atv!==coreRaw[i]))fail('repair_mailbox_member_mismatch'); + const [,checkpoint]=await event(link.checkpoint_ref,'mailbox.checkpoint','slot_binding count leaf_root frontier committed_at retain_until'); + if(u53(checkpoint.count)!==sequence+1||!(core.accepted_at<=u53(checkpoint.committed_at)&&checkpoint.committed_at<=e.at)||u53(checkpoint.retain_until)[n,checkpoint[n]])),sequence,item.sealed_core_ref.raw_sha256,link.inclusion_path,e.slot,policy,budget); + const manifestRaw=await load(core.historical_manifest_ref),manifest=parseHistoricalManifest(manifestRaw,policy,budget),m=manifest.value as Obj; + if(m.variant!=='mailbox_member'||!same(m.root_key,e.slot.root_key)||!same(m.slot_key,e.slot)||['message_id','envelope_ref','attempt_ref'].some(n=>!same(m[n],core[n])))fail('repair_mailbox_member_mismatch'); + const resolver=new LocalRawResolver(policy,budget),packs=new Map(); + for(const row of m.roles){const ref=rawRef(row.pack_ref),prior=packs.get(ref.key);if(prior&&!same(prior,ref))fail('repair_ref_mismatch');if(!prior){resolver.put('meta',ref.key,await load(ref));packs.set(ref.key,ref);}} + const setup=verifyMailboxMemberInputs({raw:manifestRaw,ref:core.historical_manifest_ref},resolver,{expectedSlot:e.slot,expectedOwner:e.owner,expectedSender:e.sender,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,acceptedAt:core.accepted_at,limitPolicy:e.limits,policy,budget}); + for(const n of ['data','metadata'])if(!same(core[n+'_resource_ref'],setup.resources[n].active.payload.resource))fail('repair_mailbox_member_mismatch'); + const deadlines=[setup.disclosure.consent_until,setup.disclosure.bootstrap_return.until,setup.destination.windows.retain_until]; + for(const n of ['slot','read','maintenance','bootstrap']){const p=setup.originals[n].payload;deadlines.push(p.expires_at);if(p.windows)deadlines.push(p.windows.read_until,p.windows.retain_until);if(n==='bootstrap')deadlines.push(p.probe_until,p.proof_until,p.upload_until);} + for(const n of ['data','metadata']){const p=setup.resources[n].active.payload;deadlines.push(p.windows.read_until,p.windows.retain_until);} + if(core.enum_until>Math.min(...deadlines))fail('repair_mailbox_member_mismatch'); + const statuses=currentRead(setup,a.currentStatuses,a.knownStatuses,a.statusObligations,e.at,policy,budget,a.onStatusAuthenticated); + const envelope=await load(core.envelope_ref,'object'); + const retained=Object.freeze([...originals.values()].map(v=>Object.freeze({ref:v.ref,get raw(){budget.output(v.raw.length);return Uint8Array.from(v.raw);}}))); + return Object.freeze({core,link,checkpoint,setup,history:manifest,current_statuses:statuses,originals:retained,get envelope(){budget.output(envelope.length);return Uint8Array.from(envelope);}}); +} diff --git a/clients/typescript/network/open-repair-receipt.ts b/clients/typescript/network/open-repair-receipt.ts index af24445..e635810 100644 --- a/clients/typescript/network/open-repair-receipt.ts +++ b/clients/typescript/network/open-repair-receipt.ts @@ -94,7 +94,7 @@ export class SavedAckReceiptPublisher{ async publishSaved(baseUrl:string,value:unknown,timeout=30):Promise{ if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); const client=this.#client,policy=this.#policy,budget=new RepairBudget(policy),r=this.#request(value,budget); - const receipt=this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; + const receipt=await this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; const receiptPayload=(parseOriginalControl(receipt.raw,policy,budget).value as Obj).payload; const authorities=verifyAckOfferBootstrapOriginal(r.bootstrap_entry,{root:r.root_entry,write:r.write_entry}, {expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedReceiptWriter:client.subject,expectedMessageId:r.message_id, diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index b2d36c5..b8a5d5b 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -41,7 +41,12 @@ "./open-repair-wire": "./open-repair-wire.ts", "./open-repair-bound": "./open-repair-bound.ts", "./open-repair-empty": "./open-repair-empty.ts", - "./open-repair-occupied": "./open-repair-occupied.ts" + "./open-repair-occupied": "./open-repair-occupied.ts", + "./open-repair-mailbox-authority": "./open-repair-mailbox-authority.ts", + "./open-repair-mailbox-member": "./open-repair-mailbox-member.ts", + "./open-repair-mailbox-feed": "./open-repair-mailbox-feed.ts", + "./open-repair-mailbox-read": "./open-repair-mailbox-read.ts", + "./open-repair-mailbox-inbox": "./open-repair-mailbox-inbox.ts" }, "types": "./crypto.ts", "engines": { diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 268b2c4..31ae18e 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -142,6 +142,11 @@ "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", "clients/typescript/network/open-repair-mailbox-range.ts", + "clients/typescript/network/open-repair-mailbox-authority.ts", + "clients/typescript/network/open-repair-mailbox-member.ts", + "clients/typescript/network/open-repair-mailbox-feed.ts", + "clients/typescript/network/open-repair-mailbox-read.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 298113c..a0c2ab0 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -123,6 +123,7 @@ "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", "tests/test_open_repair_receipt_typescript.py", + "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", @@ -243,6 +244,11 @@ "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", "clients/typescript/network/open-repair-mailbox-range.ts", + "clients/typescript/network/open-repair-mailbox-authority.ts", + "clients/typescript/network/open-repair-mailbox-member.ts", + "clients/typescript/network/open-repair-mailbox-feed.ts", + "clients/typescript/network/open-repair-mailbox-read.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index a2c633b..e084862 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -54,7 +54,7 @@ "tests.test_open_repair_index_prepare", "tests.test_open_repair_index_prepare_admin", "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", - "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +244,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 427a99d..427923d 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -46,10 +46,10 @@ "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", - "open-repair-status.ts", "open-repair-mailbox-range.ts", + "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-repair-ack.ts", "open-repair-probe.ts", - "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts")} + "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} TS_ENDPOINT_TESTS = {"tests/test_network_typescript_" + name + ".py" for name in ("nodes", "records", "vault", "peer", "peer_race", "transport", "setup", "retrieval_text", "retrieval", "agent", "agent_network", "topics")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 57) + self.assertEqual(len(TS_NETWORK), 64) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -122,9 +122,10 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_status.py", "tests/test_open_repair_status_typescript.py", "tests/open_repair_ack_fixtures.py", "tests/test_open_repair_ack.py", "tests/test_open_repair_ack_typescript.py", "tests/test_open_repair_state.py", "tests/test_open_repair_mailbox_resources.py", "tests/test_open_repair_copy_resources.py", - "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", + "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", + "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", "tests/test_open_repair_proof.py", "tests/test_open_repair_proof_typescript.py", "tests/test_open_repair_access.py", @@ -187,12 +188,13 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-wire.ts", "clients/typescript/network/open-repair-history.ts", "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", - "clients/typescript/network/open-repair-mailbox-range.ts", + "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", - "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts"}) + "clients/typescript/network/open-repair-bound.ts", "clients/typescript/network/open-repair-empty.ts", "clients/typescript/network/open-repair-occupied.ts", + "clients/typescript/network/open-repair-offer-client.ts", "clients/typescript/network/open-repair-receipt.ts"}) for name in repair_sources: self.assertTrue((ROOT / name).is_file()) for name in ("docs/NATIVE_OPEN_PROVIDER.md", "docs/CONTINUATION_TRIAL.md", "docs/OPEN_NETWORK_QUICKSTART.md"): diff --git a/tests/test_open_repair_mailbox_member_typescript.py b/tests/test_open_repair_mailbox_member_typescript.py index e0fc334..4b67273 100644 --- a/tests/test_open_repair_mailbox_member_typescript.py +++ b/tests/test_open_repair_mailbox_member_typescript.py @@ -17,16 +17,26 @@ const {RepairBudget,LocalRawResolver}=await import('./open-repair-wire.ts'); const {verifyMailboxMemberInputs}=await import('./open-repair-mailbox-member.ts'); const {verifyMailboxFeedSourceEvent}=await import('./open-repair-mailbox-feed.ts'); +const {readMailboxIndex,readMailboxAdmission}=await import('./open-repair-mailbox-read.ts'); +const {createHash}=await import('node:crypto'); +const {verifyMailboxInboxEvidence}=await import('./open-repair-mailbox-inbox.ts'); const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk); const calls=JSON.parse(Buffer.concat(chunks).toString()),decode=e=>({raw:Buffer.from(e.raw,'base64'),ref:e.ref}),out=[]; -for(const v of calls){let budget;try{ +for(const v of calls){let budget,objectReads=0,observed=0;try{ budget=new RepairBudget(v.policy);const resolver=new LocalRawResolver(v.policy,budget); for(const p of v.packs){const e=decode(p);resolver.put(e.ref.namespace,e.ref.key,e.raw);} + if(v.operation==='inbox'){const result=await verifyMailboxInboxEvidence(v.evidence,Buffer.from(v.envelope,'base64'),v.options);out.push({ok:true,envelopeHash:createHash('sha256').update(result.envelope).digest('hex'),message:result.core.message_id});continue;} + if(v.operation==='admission'){const originals=new Map(v.originals.map(e=>{const d=decode(e);return [JSON.stringify(d.ref),d.raw];})); + const result=await readMailboxAdmission(v.member,{...v.options,currentStatuses:v.current.map(decode),knownStatuses:v.known.map(decode),statusObligations:[],onStatusAuthenticated:()=>{observed++;},policy:v.policy,budget,readOriginal:async ref=>{if(ref.namespace==='object')objectReads++;const raw=originals.get(JSON.stringify(ref));if(!raw)throw Error('missing exact original');return raw;}}); + out.push({ok:true,envelopeHash:createHash('sha256').update(result.envelope).digest('hex'),objectReads,observed,metrics:budget.snapshot()});continue;} + if(v.operation==='index'){let reads=0;const originals=new Map(v.originals.map(e=>{const d=decode(e);return [JSON.stringify(d.ref),d.raw];})); + const entries=await readMailboxIndex(decode(v.head),decode(v.checkpoint),{...v.options,policy:v.policy,budget,readOriginal:async ref=>{reads++;if(ref.namespace!=='meta')throw Error('premature object read');const bytes=originals.get(JSON.stringify(ref));if(!bytes)throw Error('missing exact ref');return bytes;}}); + out.push({ok:true,entries,reads,metrics:budget.snapshot()});continue;} if(v.operation==='feed'){const result=verifyMailboxFeedSourceEvent(decode(v.manifest),resolver,decode(v.custody),{...v.options,policy:v.policy,budget});out.push({ok:true,count:result.graph.head.count,statuses:result.graph.statuses.map(v=>v.canonical_sha256),retain_until:result.retain_until,metrics:budget.snapshot()});continue;} const graph=verifyMailboxMemberInputs(decode(v.manifest),resolver,{...v.options,policy:v.policy,budget}); out.push({ok:true,accepted_at:graph.accepted_at,refs:Object.fromEntries(Object.entries(graph.roles).map(([k,e])=>[k,e.ref])), statuses:graph.statuses.map(v=>v.canonical_sha256),ack:graph.ack_configuration!==null,metrics:budget.snapshot()}); -}catch(e){out.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack),metrics:budget?.snapshot()});}} +}catch(e){out.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack),objectReads,observed,metrics:budget?.snapshot()});}} process.stdout.write(JSON.stringify({out,subprocessCalls})); ''' @@ -35,6 +45,18 @@ class NativeMailboxMemberTests(unittest.TestCase): def setUpClass(cls): runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) (cls.fixture/'driver.mjs').write_text(DRIVER) + from tests.test_open_ack_typescript_agent import DRIVER as agent_driver + agent_driver=agent_driver.replace('process.stdout.write(JSON.stringify({results,calls,subprocessCalls}));',r''' +let savedReceipt; +if(input.saved_receipt){ + const {OpenNetworkClient}=await import('./open-client.ts'),{OpenDeliveryClient}=await import('./open-delivery-client.ts'),{loadClient}=await import('./client-config.ts'); + const network=new OpenNetworkClient(input.network_config),delivery=new OpenDeliveryClient(network.participant,network.encryption,loadClient(input.client_config)); + try{const item=await delivery.savedReceiptForAck(input.saved_receipt.message_id,input.saved_receipt.owner,input.saved_receipt.envelope_ref);savedReceipt={raw:Buffer.from(item.raw).toString('base64'),ref:item.ref};} + finally{delivery.close();network.close();} +} +process.stdout.write(JSON.stringify({results,calls,subprocessCalls,savedReceipt})); +''') + (cls.fixture/'agent-driver.mjs').write_text(agent_driver) def native(self,calls): result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, @@ -43,8 +65,26 @@ def native(self,calls): self.assertEqual(out['subprocessCalls'],0);return out['out'] def test_committed_http_member_originals_and_denied_cross_bindings(self): - h=http_fixture.MailboxStagingHTTPTests('test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources') + self.check_committed(False) + + def test_committed_http_member_with_independent_ack_configuration(self): + self.check_committed(True) + + def check_committed(self,with_ack): + method='test_ack_configuration_survives_mailbox_custody' if with_ack else 'test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources' + h=http_fixture.MailboxStagingHTTPTests(method) h.setUp();self.addCleanup(h.doCleanups);visited=[] + from memory_vault_trust import TrustStore + from memory_vault_client import ClientConfig + TrustStore(ClientConfig.load(h.b.client_config).trust_path).add(h.ai.public_descriptor()) + call=h.call + def share(agent,**request): + if agent is h.a and (request['op']=='send' or (request.get('invitation',{}).get('action')=='prepare' and request.get('invitation',{}).get('schema_version')=='memory-vault-open-ack-connect/v1')) and not (request.get('memory_ids') or request.get('invitation',{}).get('memory_ids')): + remembered=call(h.a,op='remember',request_id='req_native_cold_memory',kind='observation',text='Synthetic native cold recovery memory.') + if request['op']=='send':request=dict(request,memory_ids=[remembered['memory_id']]) + else:request=dict(request,invitation=dict(request['invitation'],memory_ids=[remembered['memory_id']])) + return call(agent,**request) + h.call=share def inspect(staging,key,head): visited.append(True);s=staging.source rid=staging.db.execute('SELECT resource_id FROM open_repair_mailbox_roots').fetchone()[0] @@ -97,9 +137,59 @@ def entry(ref):return dict(raw=held.read(ref.as_dict()),ref=ref.as_dict()) self.assertEqual(feedresults[0]['count'],1);self.assertEqual(feedresults[0]['retain_until'],feedexpected['retain_until']) self.assertEqual(feedresults[0]['statuses'],[v.canonical_sha256 for v in feedexpected['graph']['statuses']]) for result in feedresults[1:]:self.assertFalse(result['ok']);self.assertNotEqual(result['code'],'untyped_error',result) + from memory_vault_open_repair_client import read_mailbox_index + with h.b._network() as network:encryption=network.encryption + headref=wire.raw_ref(json.loads(held.read(feedpart.manifest.as_dict()))['feed_head_ref']) + cpref=wire.raw_ref(feedexpected['graph']['head']['checkpoint_ref']) + indexoptions=dict(expectedSlot=key,expectedSigningKey=s.target['signing_key'],encryptionIdentity=encryption.private_document(),at=feedexpected['custody'].payload['stored_at'],maxMessages=10) + indexcall=json.loads(json.dumps(dict(operation='index',head=encoded(entry(headref)),checkpoint=encoded(entry(cpref)),originals=[encoded(entry(ref)) for ref in held.originals],packs=[],options=indexoptions,policy=asdict(s.policy)))) + expectedindex=read_mailbox_index(entry(headref),entry(cpref),expected_slot=key,expected_signing_key=s.target['signing_key'],encryption_identity=encryption,read_original=held.read,at=indexoptions['at'],max_messages=10,policy=s.policy) + from memory_vault_network_crypto import EncryptionIdentity + wrong=copy.deepcopy(indexcall);wrong['options']['encryptionIdentity']=EncryptionIdentity.generate().private_document() + indexresults=self.native([indexcall,wrong]);self.assertTrue(indexresults[0]['ok'],indexresults[0]);self.assertEqual(indexresults[0]['entries'],list(expectedindex));self.assertGreater(indexresults[0]['reads'],0) + self.assertFalse(indexresults[1]['ok']);self.assertEqual(indexresults[1]['code'],'repair_mailbox_range_mismatch') + from memory_vault_open_repair_client import read_mailbox_admission + statuses=list({row.original.ref.raw_sha256:dict(raw=row.original.raw,ref=row.original.ref.as_dict()) for row in resolved.roles if row.role.startswith('historical.status.') and not row.role.startswith('historical.status.ack_')}.values()) + admissionoptions=dict(indexoptions,expectedOwner=owner,expectedSender=sender,expectedTarget=s.target,limitPolicy=s.limits);admissionoptions.pop('maxMessages') + admissioncall=json.loads(json.dumps(dict(operation='admission',member=expectedindex[0],originals=indexcall['originals'],packs=[],options=admissionoptions,current=[encoded(v) for v in statuses],known=[],policy=asdict(s.policy)))) + checked=read_mailbox_admission(expectedindex[0],expected_slot=key,expected_signing_key=s.target['signing_key'],expected_owner=owner,expected_sender=sender,expected_target=s.target,limit_policy=s.limits,current_statuses=statuses,encryption_identity=encryption,read_original=held.read,at=indexoptions['at'],policy=s.policy) + from memory_vault_open_provider import issue_status + from tests.test_open_repair_status import status_entry + owner_status=next(json.loads(v['raw'])['payload'] for v in statuses if json.loads(v['raw'])['payload']['signing_key']['key_id']==h.bi.key_id) + denied=status_entry(issue_status(h.bi,root=key['root_key'],revision=99,entries=[dict(row,status='revoked') for row in owner_status['entries']],issued_at=indexoptions['at']-10,valid_until=indexoptions['at']-1)) + wrong=copy.deepcopy(admissioncall);wrong['known']=[encoded(denied)] + missing=copy.deepcopy(admissioncall);missing['current']=[encoded(v) for v in statuses if json.loads(v['raw'])['payload']['signing_key']['key_id']!=h.ai.key_id] + admissionresults=self.native([admissioncall,wrong,missing]);self.assertTrue(admissionresults[0]['ok'],admissionresults[0]) + import hashlib + self.assertEqual(admissionresults[0]['envelopeHash'],hashlib.sha256(checked['envelope']).hexdigest());self.assertEqual(admissionresults[0]['objectReads'],1) + for result,code in zip(admissionresults[1:],['repair_authority_revoked','repair_status_missing']): + self.assertFalse(result['ok'],result);self.assertEqual(result['code'],code,result);self.assertEqual(result['objectReads'],0);self.assertGreater(result['observed'],0) + from memory_vault_open_repair_client import verify_mailbox_inbox_evidence + from memory_vault_network_crypto import b64url + evidence=dict(schema_version='memory-vault-mailbox-inbox/v1',received_at=indexoptions['at'],slot=key,sender=sender,target=s.target,limits=s.limits,member=expectedindex[0],manifest_ref=feedpart.manifest.as_dict(),custody_ref=feedpart.custody.as_dict(),originals=[dict(ref=ref.as_dict(),raw_base64url=b64url(held.read(ref.as_dict()))) for ref in held.originals if ref.namespace=='meta'],status_refs=[v['ref'] for v in statuses]) + verify_mailbox_inbox_evidence(evidence,checked['envelope'],owner=owner,encryption_identity=encryption,staged_at=indexoptions['at']) + import base64 + inboxcall=json.loads(json.dumps(dict(operation='inbox',packs=[],policy=asdict(s.policy),evidence=evidence,envelope=base64.b64encode(checked['envelope']).decode(),options=dict(owner=owner,encryptionIdentity=encryption.private_document(),stagedAt=indexoptions['at'])))) + wrong=copy.deepcopy(inboxcall);wrong['evidence']['received_at']+=1 + inboxresults=self.native([inboxcall,wrong]);self.assertTrue(inboxresults[0]['ok'],inboxresults[0]);self.assertEqual(inboxresults[0]['message'],checked['core']['message_id']);self.assertFalse(inboxresults[1]['ok']);self.assertEqual(inboxresults[1]['code'],'repair_invalid_context') + # The Python recipient stops after durable staging. A fresh native + # Agent resumes that inbox and imports the original memory bytes. + from memory_vault_open_delivery import decrypt_envelope + from memory_vault_network_crypto import document + plaintext=decrypt_envelope(checked['envelope'],encryption_identity=encryption,sender_signing_key=sender['signing_key'],sender_encryption_key=sender['encryption_key'],recipient_signing_key=owner['signing_key'],recipient_encryption_key=owner['encryption_key']) + session=dict(mailbox=evidence,authority={name:document(roles[role]['raw']) for name,role in [('request','contact.request'),('policy','contact.policy')]},source_node=s.node) + with h.b._network() as network: + row=network._delivery()._stage_inbox(message_id=checked['core']['message_id'],sender_key_id=h.ai.key_id,envelope=checked['envelope'],body=plaintext,session=session) + self.assertEqual(row['phase'],'staged') + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'agent-driver.mjs')],cwd=self.fixture,input=json.dumps(dict(client_config=str(h.b.client_config),network_config=str(h.b.network_config),requests=[dict(op='receive',limit=1),dict(op='receive',message_id=checked['core']['message_id'])],no_network=True,saved_receipt=dict(message_id=checked['core']['message_id'],owner=sender,envelope_ref=checked['core']['envelope_ref']))).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=40) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);native=json.loads(result.stdout);self.assertEqual(native['subprocessCalls'],0);self.assertEqual(native['calls'],[]) + for result in native['results']:self.assertTrue(result['ok'],result) + messages=native['results'][0]['result']['messages'];self.assertEqual(len(messages),1,messages);self.assertEqual(messages[0]['state'],'validated_saved');self.assertEqual(messages[0]['content_kind'],'memory_transfer');self.assertEqual(messages[0]['share']['records_added'],1) + with h.b._network() as network: + row=network._delivery()._inbox(checked['core']['message_id']);self.assertEqual(row['phase'],'saved');self.assertIsNotNone(row['receipt']);self.assertFalse(row['receipt_sent']);self.assertEqual(base64.b64decode(native['savedReceipt']['raw']),bytes(row['receipt'])) for label,result in zip(labels[1:],results[1:]): with self.subTest(label=label):self.assertFalse(result['ok'],result);self.assertNotEqual(result['code'],'untyped_error',result) h.inspect_committed_mailbox=inspect - h.test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources();self.assertEqual(visited,[True]) + getattr(h,method)();self.assertEqual(visited,[True]) if __name__=='__main__':unittest.main() From c052c5a784959d4001bfdd9b4c602bd0b7369ddf Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:14:40 +0800 Subject: [PATCH 11/24] Receive registered remote mailbox memories in native Agent --- clients/typescript/network/README.md | 49 +++--- clients/typescript/network/open-client.ts | 5 +- .../network/open-delivery-client.ts | 36 +++- .../typescript/network/open-mailbox-client.ts | 160 ++++++++++++++++++ .../network/open-mailbox-journal.ts | 41 +++++ .../network/open-mailbox-receivers.ts | 67 ++++++++ clients/typescript/network/package.json | 5 +- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 4 +- scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 8 +- tests/test_open_mailbox_client_typescript.py | 132 +++++++++++++++ 12 files changed, 478 insertions(+), 35 deletions(-) create mode 100644 clients/typescript/network/open-mailbox-client.ts create mode 100644 clients/typescript/network/open-mailbox-journal.ts create mode 100644 clients/typescript/network/open-mailbox-receivers.ts create mode 100644 tests/test_open_mailbox_client_typescript.py diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index a378e96..1266908 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -278,22 +278,33 @@ verified status observations across restarts, including failed recovery. See [request formats and commands](../../../docs/OPEN_ACK_RECOVERY.md#native-recovery-commands-development-after-alpha019). This command entry is development after the frozen alpha.0.19 archives. -### Retained mailbox inbox recovery (development) - -A native `Agent.handle({op: "receive", limit: 1})` can resume an ordinary mailbox transfer -already durably staged by the Python client in the same protected inbox. It -rechecks the original owner/contact approvals, activated storage resources, -complete feed prefix, recipient-encrypted directory and admission core, and -READ observations recorded at receipt time. A retained revocation or missing -sender permission blocks the object read. The original memory share then -uses the existing Vault import and its idempotent transfer receipt. The saved -receipt remains available to an explicit `connect/return_receipt` request; -mailbox access does not grant an independent receipt return. - -`readMailboxIndex` and `readMailboxAdmission` accept a caller-authorized exact -reference reader. They support asynchronous reads without delegating to Python. -`verifyMailboxInboxEvidence` reopens the retained ordinary-mailbox format without -network access. This is historical local recovery, not a claim about current -source availability. Native discovery/download of a new remote mailbox and the -replica inbox format remain separate unfinished work; use the Python client for -those paths. These additions are development source, not yet a public release. +### Registered mailbox recovery (development) + +A native `Agent.handle({op: "connect", invitation})` accepts the existing +`memory-vault-open-mailbox-connect/v1` `register`, `list`, and `remove` actions. +Registration retains an explicitly supplied owner slot, READ and maintenance +grants, feed bootstrap, expected sender/target keys, limits and signed source +endpoint. Registration itself performs no network requests. Python and native +clients share the same bounded receiver registry and retained status journal. + +`Agent.handle({op: "receive", limit: 1})` checks registered ordinary mailboxes, +downloads and authenticates the complete feed prefix, opens the recipient-encrypted +directory and admission core, then downloads the exact authorized body. It checks +current READ permissions before fetching the body and preserves authenticated +revocations across process restarts. Existing approvals and storage grants remain +required. A missing ordinary delivery lease does not prevent authorized recovery +from the mailbox's retained copy. + +Downloaded messages use the existing durable inbox and Vault import. A new +process can also resume an ordinary mailbox transfer staged by either client, +without network access, rechecking its original approvals and received-time +permissions. Memory transfers retain their idempotent import receipt. The saved +recipient receipt is available to explicit `connect/return_receipt` with separate +return authority; mailbox registration does not supply that authority. + +The native SDK exposes `MailboxFeedRecoveryClient` and `MailboxSetupJournal` for +bounded feed/body recovery, and `readMailboxIndex`, `readMailboxAdmission`, and +`verifyMailboxInboxEvidence` for exact-reference/local recovery. These paths do +not delegate to a Python process. Native mailbox provisioning and replica inbox +recovery remain unfinished; use the Python client for those paths. These additions +are development source, not yet a public release. diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index 082ab86..21176ca 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -6,6 +6,7 @@ import {absolutePath,readPrivate,NetworkError} from './io.ts'; import {loadClient} from './client-config.ts'; import type {Client} from './client-config.ts'; import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {RegisteredMailboxReceivers,MAILBOX_CONNECT_SCHEMA} from './open-mailbox-receivers.ts'; import {OpenParticipant} from './open-participant.ts'; import {OpenContactClient,CONNECT_SCHEMA} from './open-contact-client.ts'; import type {SignedNode} from './open-control.ts'; @@ -36,6 +37,8 @@ export class OpenNetworkClient{ private result(value:Record):Record{if(canonicalBytes(value).length>8192)fail('network_response_too_large');return value;} async connect(invitation?:unknown,requestId?:string):Promise>{ if(invitation!=null){ + if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA) + return this.result(new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).connect(invitation)); if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA) return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); if(typeof invitation!=='object'||Array.isArray(invitation)||(invitation as any).schema_version!==CONNECT_SCHEMA)fail('open_private_invitation_unsupported'); @@ -54,7 +57,7 @@ export class OpenNetworkClient{ return this.result(await delivery.send(requestId,recipients,text,memoryIds,control, (row,deadline)=>recoverPrepared(this.participant,this.encryption,delivery,row,deadline))); } - async receive(limit=4):Promise>{return this.result(await this.delivery().receive(limit));} + async receive(limit=4):Promise>{return this.result(await new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).receive(limit));} readMessage(messageId:string,offset=0):Record{return this.result(this.delivery().readMessage(messageId,offset));} respondTo(..._args:any[]):never{return fail('open_hint_exchange_unsupported');} readReceivedBatch(..._args:any[]):never{return fail('open_hint_exchange_unsupported');} diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 5c28e98..5aacd71 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -3,6 +3,8 @@ */ import {randomBytes} from 'node:crypto'; import {verifyMailboxInboxEvidence} from './open-repair-mailbox-inbox.ts'; +import {MailboxFeedRecoveryClient} from './open-mailbox-client.ts'; +import type {MailboxRecoverOptions} from './open-mailbox-client.ts'; import {performance} from 'node:perf_hooks'; import type {DatabaseSync} from 'node:sqlite'; import {canonicalBytes,document,objectFields,sha256,encodeBase64url,decodeBase64url,opaqueId,safeInteger, @@ -42,7 +44,8 @@ function preview(text:string):string{ const MAX_LOCAL_BYTES=256*1024*1024,MAX_SESSION_BYTES=32768,MAX_INBOX_SESSION_BYTES=65536, MAX_INTENT_BYTES=49152,MAX_RESULT_BYTES=16384; class DeliveryBudget{ - readonly deadline=performance.now()/1000+60; + readonly deadline:number; + constructor(deadline?:number){this.deadline=Math.min(performance.now()/1000+60,deadline??Infinity);if(!Number.isFinite(this.deadline))fail('open_delivery_budget_exhausted',true);} requests=0;requestBytes=0;responseBytes=0; check(){if(performance.now()/1000>=this.deadline||this.requests>256||this.requestBytes+this.responseBytes>64*1024*1024)fail('open_delivery_budget_exhausted',true);} request(bytes:number){this.check();if(this.requests>=256||this.requestBytes+this.responseBytes+bytes>64*1024*1024)fail('open_delivery_budget_exhausted',true);this.requests++;this.requestBytes+=bytes;} @@ -446,6 +449,28 @@ export class OpenDeliveryClient{ this.stageInbox(entry.message_id,session.request.payload.signing_key.key_id,envelope,body, {intent:original,storage_receipt:handle.storage_receipt,source_node:handle.source_node});return [await this.finishInbox(entry.message_id),true]; } + async receiveMailbox(reader:MailboxFeedRecoveryClient,baseUrl:string,options:MailboxRecoverOptions,limit=4):Promise{ + if(!(reader instanceof MailboxFeedRecoveryClient)||!same(reader.subject,{signing_key:validateSigningIdentity(this.participant.identity),encryption_key:validateEncryptionIdentity(this.encryption)}))fail('open_delivery_key_binding_mismatch'); + if(!Number.isSafeInteger(limit)||limit<1||limit>4)fail('network_invalid_receive_limit'); + return this.serial(async()=>{ + const deadline=performance.now()/1000+(options.timeout??60),feed=await reader.recover(baseUrl,options),messages:Obj[]=[]; + for(const member of feed.entries){if(messages.length>=limit)break; + const candidates=(feed.source.manifest.manifest.value as Obj).members.filter((v:Obj)=>same(v.admission_link_ref,member.admission_link_ref));if(candidates.length!==1)fail('repair_proof_mismatch'); + const selected=candidates[0],prior=this.inbox(selected.message_id);let result:Obj,fresh:boolean; + if(prior){if(prior.envelope_sha256!==selected.envelope_ref.raw_sha256)fail('network_inbox_identity_conflict');fresh=prior.phase==='staged';result=await this.finishInbox(selected.message_id);} + else{ + const remaining=Math.min(30,deadline-performance.now()/1000);if(remaining<=0)fail('open_delivery_budget_exhausted',true); + const recovered=await reader.readMember(baseUrl,feed,member,{knownStatuses:options.knownStatuses,timeout:remaining}),roles=recovered.setup.roles; + const authority=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([n,r])=>[n,document(roles[r].raw)])); + const session={authority,source_node:document((options.targetNodeEntry as Obj).raw),mailbox:reader.inboxEvidence(feed,member)},envelope=recovered.envelope; + const body=await decryptEnvelope(envelope,{...OpenDeliveryClient.keys(session),encryption_identity:this.encryption} as any); + this.stageInbox(selected.message_id,authority.request.payload.signing_key.key_id,envelope,body,session);result=await this.finishInbox(selected.message_id);fresh=true; + } + if(fresh)messages.push(result);if(result.state==='validated_saved')this.savedReceipt(selected.message_id); + } + return {messages,errors:[],state:'observed',body_transport:'mailbox_retained_copy',receipt_state:'retained_for_independent_return'}; + }); + } private pendingReceiptBatch():Obj[]{return this.db(db=>{ const select="SELECT message_id,phase,created_at FROM open_delivery_inbox WHERE (phase='staged' OR (phase='saved' AND receipt_sent=0))", order=' ORDER BY created_at,message_id LIMIT ?', @@ -460,13 +485,14 @@ export class OpenDeliveryClient{ const created=safeInteger(row.created_at),id=opaqueId(row.message_id); this.db(db=>db.prepare('INSERT OR REPLACE INTO open_delivery_retry_cursor VALUES(1,?,?)').run(created,id)); } - async receive(limit=4):Promise{if(!Number.isSafeInteger(limit)||limit<1||limit>4)fail('network_invalid_receive_limit');return this.serial(()=>this.receiveInternal(limit));} - private async receiveInternal(limit:number):Promise{ - const budget=new DeliveryBudget(),messages:Obj[]=[],errors:Obj[]=[]; - const pending=this.pendingReceiptBatch(); + async receive(limit=4,options:{pendingOnly?:boolean;skipPending?:boolean;deadline?:number}={}):Promise{if(!Number.isSafeInteger(limit)||limit<1||limit>4)fail('network_invalid_receive_limit');return this.serial(()=>this.receiveInternal(limit,options));} + private async receiveInternal(limit:number,options:{pendingOnly?:boolean;skipPending?:boolean;deadline?:number}):Promise{ + const budget=new DeliveryBudget(options.deadline),messages:Obj[]=[],errors:Obj[]=[]; + const pending=options.skipPending?[]:this.pendingReceiptBatch(); for(const row of pending){if(messages.length>=limit)break;try{const result=await this.finishInbox(row.message_id);if(row.phase==='staged')messages.push(result); if(result.state==='validated_saved')await this.sendReceipt(row.message_id,budget);}catch(error){errors.push({message_id:row.message_id,...errorData(error)});} finally{this.advancePendingReceipt(row);}} + if(options.pendingOnly)return {messages,errors:errors.slice(0,4),network_accessed:budget.requests>0}; for(const candidate of this.contactSessions(false)){if(messages.length>=limit)break; try{const session=await this.checkedSession(candidate,budget),node=session.node,leaseId=session.decision.payload.grant.payload.resource_lease.payload.lease_id; let [key,after]=this.cursor(session);const intent=readerIntent('list',{lease_id:leaseId,caller_key_id:this.participant.keyId}); diff --git a/clients/typescript/network/open-mailbox-client.ts b/clients/typescript/network/open-mailbox-client.ts new file mode 100644 index 0000000..f02cf05 --- /dev/null +++ b/clients/typescript/network/open-mailbox-client.ts @@ -0,0 +1,160 @@ +/** Bounded native HTTP recovery of an original mailbox feed and message body. */ +import {performance} from 'node:perf_hooks'; +import {OpenHTTPTransport,endpoint} from './open-transport.ts'; +import {RepairBudget,RepairError,buildNewWire,parseNewWire,objectFields,rawRef,u53,LocalRawResolver} from './open-repair-wire.ts'; +import type {RawOriginal,RawRef,RepairPolicy} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import {validateSigningIdentity,validateEncryptionIdentity,encodeBase64url} from './crypto.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {verifyMailboxFeedSourceEvent} from './open-repair-mailbox-feed.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {verifyBootstrapProofResponse,makeBootstrapChildRequest,makeMailboxBodyRequest} from './open-repair-proof.ts'; +import {authenticateStatusOriginal,verifyStatusOriginal} from './open-repair-status.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +import {readMailboxIndex,readMailboxAdmission} from './open-repair-mailbox-read.ts'; +import {MailboxSetupJournal} from './open-mailbox-journal.ts'; +type Obj=Record; +const monotonic=()=>performance.now()/1000; +const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); +function fail(code:string):never{throw new RepairError(code);} +function options(v:unknown,required:string[],optional:string[]):Obj{ + if(!v||typeof v!=='object')fail('repair_invalid_context');return fields(v,[...required,...optional.filter(n=>Object.hasOwn(v,n))]); +} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +const id=(v:unknown)=>{const r=rawRef(v);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +function snapshot(v:unknown,policy:RepairPolicy,budget:RepairBudget):RawOriginal{const e=fields(v,['raw','ref']),ref=rawRef(e.ref),raw=parseNewWire(e.raw,policy,budget).raw;if(ref.namespace!=='meta'||ref.size!==raw.length||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {ref,raw};} +function statusPreview(raw:Uint8Array,policy:RepairPolicy,budget:RepairBudget):Obj{ + const p=fields(fields(parseNewWire(raw,policy,budget).value,['payload','proof']).payload,['schema_version','kind','signing_key','scope_key','revision','issued_at','valid_until','entries']); + if(!Array.isArray(p.entries)||p.entries.length<1||p.entries.length>16)fail('repair_invalid_status');for(const row of p.entries)fields(row,['scope_kind','scope_id','status','minimum_document_revision','operation_mask']);return p; +} +function floors(retained:readonly Obj[],current:readonly Obj[],obligations:readonly Obj[]):void{ + const required=new Map(obligations.map(v=>[[v.signer.key_id,v.kind,v.scope_id].join(':'),v])),seen=new Map(),values=new Map(); + for(const observed of [...retained,...current]){const p=observed.payload,issuer=p.scope_key.issuer_key_id,revision=p.revision,key=issuer+':'+revision; + if(seen.has(key)&&seen.get(key)!==observed.canonical_sha256)fail('repair_status_conflict');seen.set(key,observed.canonical_sha256); + for(const row of p.entries){const k=[issuer,row.scope_kind,row.scope_id].join(':'),wanted=required.get(k); + if(wanted&&row.status==='revoked'&&(row.operation_mask&wanted.mask))fail('repair_authority_revoked'); + if(wanted&&row.minimum_document_revision>wanted.revision)fail('repair_status_revision'); + const held=values.get(k)??[];held.push([revision,row.minimum_document_revision]);values.set(k,held);} + } + for(const held of values.values()){let minimum=0;for(const [,v] of held.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(vv[0]>p.revision))fail('repair_status_rollback');} +} +export interface MailboxClientOptions{readonly policy?:RepairPolicy;readonly limitPolicy?:unknown;readonly allowLoopback?:boolean;readonly transport?:OpenHTTPTransport;readonly clock?:()=>number;readonly statusObserver?:(item:AuthenticatedStatusOriginal)=>void;readonly networkObserver?:()=>void;} +export interface MailboxRecoverOptions{readonly targetNodeEntry:unknown;readonly expectedTarget:unknown;readonly expectedSender:unknown;readonly expectedSlot:unknown;readonly slotEntries:unknown;readonly journal:MailboxSetupJournal;readonly knownStatuses?:readonly unknown[];readonly archiveStatuses?:readonly unknown[];readonly timeout?:number;} +export interface RecoveredMailboxFeed{readonly source:Readonly;readonly proof:Obj;readonly current_statuses:readonly AuthenticatedStatusOriginal[];readonly originals:readonly RawOriginal[];readonly entries:readonly Obj[];readonly metrics:Readonly;readonly base_url:string;} +export class MailboxFeedRecoveryClient{ + readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #limits:Obj; + readonly #transport:OpenHTTPTransport;readonly #ownTransport:boolean;readonly #loopback:boolean;readonly #clock:()=>number;readonly #observer?: (v:AuthenticatedStatusOriginal)=>void;readonly #networkObserver?:()=>void; + readonly #feeds=new WeakMap(); + constructor(identity:unknown,encryptionIdentity:unknown,value:MailboxClientOptions={}){ + const a=options(value,[],['policy','limitPolicy','allowLoopback','transport','clock','statusObserver','networkObserver']);const budget=new RepairBudget(a.policy??{...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512});this.#policy=budget.policy; + const held=buildNewWire({identity,encryption:encryptionIdentity,limits:a.limitPolicy??DEFAULT_REPAIR_CLIENT_LIMITS},this.#policy,budget).value as Obj; + this.#identity=held.identity;this.#encryption=held.encryption;this.#subject=buildNewWire({signing_key:validateSigningIdentity(held.identity),encryption_key:validateEncryptionIdentity(held.encryption)},this.#policy,budget).value as Obj; + this.#limits=fields(held.limits,Object.keys(DEFAULT_REPAIR_CLIENT_LIMITS));for(const n of Object.keys(this.#limits))u53(this.#limits[n],1); + if((a.allowLoopback!==undefined&&typeof a.allowLoopback!=='boolean')||(a.clock!==undefined&&typeof a.clock!=='function')||(a.statusObserver!==undefined&&typeof a.statusObserver!=='function')||(a.networkObserver!==undefined&&typeof a.networkObserver!=='function'))fail('repair_invalid_policy'); + this.#loopback=a.allowLoopback??false;this.#clock=a.clock??(()=>Date.now()/1000);this.#observer=a.statusObserver;this.#networkObserver=a.networkObserver;this.#transport=a.transport??new OpenHTTPTransport({allow_loopback:this.#loopback});this.#ownTransport=a.transport===undefined;Object.freeze(this); + } + get subject():Readonly{return this.#subject;} + get limits():Readonly{return this.#limits;} + inboxEvidence(feed:RecoveredMailboxFeed,member:unknown):Readonly{ + const c=this.#feeds.get(feed);if(!c||!feed.entries.some(v=>same(v,member)))fail('repair_invalid_context'); + const children=(c.held.manifest.value as Obj).children,reference=(role:string)=>{const found=children.filter((v:Obj)=>v.role===role);if(found.length!==1)fail('repair_proof_mismatch');return found[0].ref;}; + return Object.freeze({schema_version:'memory-vault-mailbox-inbox/v1',received_at:this.#now(),slot:c.e.slot,sender:c.e.sender,target:c.e.target,limits:this.#limits,member, + manifest_ref:reference('history.mailbox_feed'),custody_ref:reference('feed.custody'),originals:feed.originals.map(v=>({ref:v.ref,raw_base64url:encodeBase64url(v.raw)})),status_refs:feed.current_statuses.map(v=>v.ref)}); + } + close():void{if(this.#ownTransport)this.#transport.close();} + #now():number{return u53(Math.floor(this.#clock()));} + #timeout(v:unknown):number{if(typeof v!=='number'||!Number.isFinite(v)||v<=0||v>60)fail('repair_invalid_deadline');return v;} + #observe(journal:MailboxSetupJournal,key:string,item:AuthenticatedStatusOriginal):void{journal.observe(key,item);this.#observer?.(item);} + #known(entries:readonly unknown[],e:Obj,at:number,budget:RepairBudget,journal:MailboxSetupJournal,key:string):AuthenticatedStatusOriginal[]{ + if(!Array.isArray(entries)||entries.length>32)fail('repair_status_history_capacity');const result=[]; + for(const item of entries){const entry=snapshot(item,this.#policy,budget),p=statusPreview(entry.raw,this.#policy,budget),signer=p.signing_key,permitted=new Set(); + if(same(signer,this.#subject.signing_key)){permitted.add('mailbox_slot');permitted.add('authority');} + if(same(signer,e.sender.signing_key))permitted.add('authority');if(same(signer,e.target.signing_key))permitted.add('resource'); + if(!permitted.size||p.entries.some((v:Obj)=>!permitted.has(v.scope_kind))||u53(p.issued_at)>at)fail('repair_status_disclosure'); + const observed=authenticateStatusOriginal(entry,{expectedRoot:e.slot.root_key,expectedSigningKey:signer,at:p.issued_at,allowedScopes:p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),policy:this.#policy,budget},v=>this.#observe(journal,key,v)); + result.push(observed);if(p.entries.some((v:Obj)=>v.status==='revoked'&&(v.operation_mask&10)))fail('repair_authority_revoked'); + }return result; + } + async recover(baseUrl:string,value:MailboxRecoverOptions):Promise{ + const a=options(value,['targetNodeEntry','expectedTarget','expectedSender','expectedSlot','slotEntries','journal'],['knownStatuses','archiveStatuses','timeout']); + if(!(a.journal instanceof MailboxSetupJournal))fail('repair_invalid_context');const journal=a.journal as MailboxSetupJournal; + const timeout=this.#timeout(a.timeout??60),policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),deadline=monotonic()+timeout; + const e=buildNewWire({slot:a.expectedSlot,target:a.expectedTarget,sender:a.expectedSender},policy,budget).value as Obj; + const setup=verifyMailboxFeedBootstrap(a.slotEntries,{expectedSlot:e.slot,expectedOwner:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,limitPolicy:this.#limits,at:started,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:e.sender.signing_key.key_id,encryption_key_id:e.sender.encryption_key.key_id}))fail('repair_proof_mismatch'); + const nodeEntry=snapshot(a.targetNodeEntry,policy,budget),node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:e.target.signing_key,expectedStorageEpoch:e.slot.writer_storage_epoch,at:started,policy,budget}); + if(!same(endpoint(baseUrl,this.#loopback),endpoint(node.payload.base_url,this.#loopback)))fail('repair_proof_mismatch'); + const journalKey=journal.start({kind:'mailbox.feed_recovery',slot_key:e.slot,owner:this.#subject,sender:e.sender,target:e.target,entries:Object.fromEntries(Object.entries(setup).map(([n,v])=>[n,v.ref]))}); + const retained=new Map();for(const group of [a.knownStatuses??[],a.archiveStatuses??[],journal.statuses(journalKey)]){ + if(!Array.isArray(group)||group.length>32)fail('repair_status_history_capacity');for(const item of group){const entry=snapshot(item,policy,budget);retained.set(entry.ref.raw_sha256+':'+entry.ref.size,entry);}} + const known=this.#known([...retained.values()],e,started,budget,journal,journalKey),grant=setup.bootstrap.payload; + const expiry=Math.min(started+Math.min(60,Math.max(1,Math.floor(timeout))),node.payload.expires_at as number,grant.probe_until,grant.proof_until,...Object.values(setup).map(v=>v.payload.expires_at as number));if(expiry<=started)fail('repair_access_expired'); + const counts={requests:0,wire_bytes:0,proof_bytes:0}; + const request=async(raw:Uint8Array,child=false):Promise=>{if(counts.requests>=grant.limits.max_requests||monotonic()>=deadline)fail('repair_over_budget');counts.requests++; + this.#networkObserver?.();const reply=await this.#transport.requestRepair(baseUrl,raw,deadline,{child});counts.wire_bytes+=raw.length+reply.length;return reply;}; + const binding={expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,bootstrapGrantSha256:setup.bootstrap.ref.raw_sha256,selector:grant.selector,consumer:'mailbox_feed' as const,policy,budget}; + const outgoing=await makeBootstrapProbe(this.#identity,{...binding,at:started,expiresAt:expiry}),outgoingRaw=outgoing.original.raw;if(outgoingRaw.length>grant.limits.max_probe_bytes)fail('repair_over_budget'); + const challengeRaw=await request(outgoingRaw),digest=budget.hash(challengeRaw),challenge={raw:challengeRaw,ref:rawRef({namespace:'meta',key:digest,raw_sha256:digest,size:challengeRaw.length})}; + const preview=fields(parseNewWire(challengeRaw,policy,budget).value,['payload','proof']); + const answer=await solveBootstrapChallenge({raw:outgoingRaw,ref:outgoing.original.ref},challenge,{...binding,signer:this.#identity,encryptionIdentity:this.#encryption,targetNonce:outgoing.nonce,at:this.#now(),expiresAt:Math.min(expiry,u53(preview.payload?.expires_at))}); + const response=await request(answer.raw),held=verifyBootstrapProofResponse(response,{expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,selector:grant.selector,bootstrapGrantRef:setup.bootstrap.ref, + probeRef:outgoing.original.ref,challengeRef:challenge.ref,answerRef:answer.ref,at:this.#now(),maxProofItems:grant.limits.max_proof_items,maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'feed',consumer:'mailbox_feed',policy,budget}); + counts.proof_bytes=response.length+held.handle.raw.length+held.manifest.raw.length; + const originals=new Map(),roles=new Map(),available=new Map(); + for(const entry of [...Object.values(setup),...known,nodeEntry])available.set(id(entry.ref),entry.raw); + const children=(held.manifest.value as Obj).children; + for(const item of children){const reference=rawRef(item.ref);roles.set(item.role,[...(roles.get(item.role)??[]),reference]);} + const download=async(item:Obj):Promise=>{const ref=rawRef(item.ref),key=id(ref);if(originals.has(key))return; + if(ref.size>policy.max_document_bytes||counts.proof_bytes+ref.size>grant.limits.max_proof_bytes)fail('repair_over_budget');let raw=available.get(key); + if(raw)budget.input(raw.length);else{const chunks:Uint8Array[]=[];let offset=0;while(offset{for(const row of tree.roles){const key=id(row.original.ref),raw=row.original.raw,prior=available.get(key);if(prior&&!Buffer.from(prior).equals(Buffer.from(raw)))fail('repair_ref_conflict');available.set(key,raw);}for(const child of tree.predecessors)reuse(child);}; + for(const ref of roles.get('history.mailbox_feed')??[]){const tree=resolveHistoricalInputs(originals.get(id(ref))!.raw,packed,policy,budget);if((tree.manifest.value as Obj).variant!=='mailbox_feed')fail('repair_proof_mismatch');reuse(tree);} + for(const item of children)await download(item); + const entry=(ref:unknown)=>{const item=originals.get(id(ref));if(!item)fail('repair_original_missing');return item;}; + const one=(role:string)=>{const refs=roles.get(role);if(!refs||refs.length!==1)fail('repair_proof_mismatch');return entry(refs[0]);}; + const source=verifyMailboxFeedSourceEvent(one('history.mailbox_feed'),packed,one('feed.custody'),{expectedSlot:e.slot,expectedOwner:this.#subject,expectedSender:e.sender,expectedTarget:e.target,limitPolicy:this.#limits,policy,budget}); + for(const member of source.graph.members)for(const [name,parent] of Object.entries(setup)){const actual=member.originals[name];if(!same(actual.ref,parent.ref)||!Buffer.from(actual.raw).equals(Buffer.from(parent.raw)))fail('repair_proof_mismatch');} + const actual=new Set(),wanted=new Set();for(const [role,refs] of roles)if(!role.startsWith('current.status.'))for(const ref of refs)actual.add(role+'|'+id(ref)); + for(const tree of [source.manifest,...source.manifest.predecessors]){for(const row of tree.roles)wanted.add(row.role+'|'+id(row.original.ref));for(const row of tree.manifest.value.roles)wanted.add('history.raw_pack|'+id(row.pack_ref));} + for(const role of ['history.mailbox_feed','feed.custody'])wanted.add(role+'|'+id(one(role).ref));if(actual.size!==wanted.size||[...actual].some(k=>!wanted.has(k)))fail('repair_proof_mismatch'); + const duties=source.graph.obligations.map((v:Obj)=>({role:'current.status.'+v.role,kind:v.scope_kind,scope_id:v.scope_id,revision:v.document_revision,signer:v.signer,mask:['slot','maintenance','bootstrap'].includes(v.role)?10:2})); + const allowed=new Map();for(const member of source.graph.members)for(const v of member.obligations)allowed.set([v.signer.key_id,v.scope_kind,v.scope_id].join(':'),v); + const current:AuthenticatedStatusOriginal[]=[],covered=new Set(); + for(const [role,refs] of roles){if(!role.startsWith('current.status.'))continue;for(const ref of refs){const item=entry(ref),p=statusPreview(item.raw,policy,budget),present=new Set(p.entries.map((v:Obj)=>v.scope_kind+':'+v.scope_id)),permitted=duties.filter((v:Obj)=>same(v.signer,p.signing_key)),matched=permitted.filter((v:Obj)=>v.role===role&&present.has(v.kind+':'+v.scope_id)); + if(!matched.length)fail('repair_status_disclosure');const observed=verifyStatusOriginal(item,{expectedRoot:e.slot.root_key,expectedSigningKey:matched[0].signer,at:this.#now(),allowedScopes:[...allowed.values()].filter(v=>v.signer.key_id===p.signing_key.key_id).map(v=>({scope_kind:v.scope_kind,scope_id:v.scope_id})), + required:permitted.filter((v:Obj)=>present.has(v.kind+':'+v.scope_id)).map((v:Obj)=>({scope_kind:v.kind,scope_id:v.scope_id,document_revision:v.revision,operation_mask:v.mask})),policy,budget},v=>this.#observe(journal,journalKey,v)); + current.push(observed);for(const v of matched)covered.add([v.role,v.kind,v.scope_id].join(':'));}} + const expectedCoverage=new Set(duties.map((v:Obj)=>[v.role,v.kind,v.scope_id].join(':')));if(covered.size!==expectedCoverage.size||[...expectedCoverage].some(k=>!covered.has(k)))fail('repair_status_missing'); + const historic=[...source.graph.statuses,...source.graph.members.flatMap((v:Obj)=>v.statuses)];floors([...known,...historic],current,duties); + const members=await readMailboxIndex(one('feed.head'),one('feed.checkpoint'),{expectedSlot:e.slot,expectedSigningKey:e.target.signing_key,encryptionIdentity:this.#encryption as any,readOriginal:ref=>entry(ref).raw,at:this.#now(),maxMessages:setup.slot.payload.max_appends,policy,budget}); + const until=Math.min(expiry,held.handle.payload.expires_at as number,source.read_until,source.retain_until,...current.map(v=>v.payload.valid_until as number));if(this.#now()>=until||monotonic()>=deadline)fail('repair_access_expired'); + verifyMailboxFeedBootstrap(Object.fromEntries(Object.entries(setup).map(([n,v])=>[n,{ref:v.ref,raw:v.raw}])),{expectedSlot:e.slot,expectedOwner:this.#subject,expectedTarget:e.target,targetStorageEpoch:e.slot.writer_storage_epoch,limitPolicy:this.#limits,at:this.#now(),policy,budget}); + const result=Object.freeze({source,proof:held,current_statuses:Object.freeze(current),originals:Object.freeze([...originals.values()].map(v=>Object.freeze({ref:v.ref,get raw(){return Uint8Array.from(v.raw);}}))),entries:members,metrics:Object.freeze({...counts,...budget.snapshot()}),base_url:baseUrl}); + this.#feeds.set(result,{e,setup,source,held,originals,journal,journalKey,current,counts,until,budget});return result; + } + async readMember(baseUrl:string,feed:RecoveredMailboxFeed,member:unknown,value:{knownStatuses?:readonly unknown[];timeout?:number}={}):Promise>{ + const c=this.#feeds.get(feed),a=options(value,[],['knownStatuses','timeout']);if(!c||!same(endpoint(baseUrl,this.#loopback),endpoint(feed.base_url,this.#loopback)))fail('repair_proof_mismatch'); + const {e,source,held,originals,journal,journalKey,current,counts,budget}=c,policy=this.#policy,timeout=this.#timeout(a.timeout??30),deadline=monotonic()+timeout; + const selected=buildNewWire(member,policy,budget).value;if(!feed.entries.some(v=>same(v,selected)))fail('repair_invalid_context'); + const guard=()=>{if(this.#now()>=c.until||monotonic()>=deadline)fail('repair_access_expired');};guard(); + const known=a.knownStatuses??[];if(!Array.isArray(known)||known.length>32)fail('repair_status_history_capacity'); + const retained=new Map();for(const item of [...known,...journal.statuses(journalKey)]){const entry=snapshot(item,policy,budget);retained.set(entry.ref.raw_sha256+':'+entry.ref.size,entry);}if(retained.size>32)fail('repair_status_history_capacity'); + const item=selected as Obj;const link=originals.get(id(item.admission_link_ref));if(!link)fail('repair_original_missing');const linkValue=fields(parseNewWire(link.raw,policy,budget).value,['payload','proof']).payload; + const children=(held.manifest.value as Obj).children.filter((v:Obj)=>v.role==='member.core'&&same(v.ref,linkValue.core_ref));if(children.length!==1)fail('repair_proof_mismatch'); + const readOriginal=async(ref:RawRef):Promise=>{if(ref.namespace==='meta'){const original=originals.get(id(ref));if(!original)fail('repair_original_missing');return original.raw;} + guard();if(counts.proof_bytes+ref.size>c.setup.bootstrap.payload.limits.max_proof_bytes)fail('repair_over_budget');const chunks:Uint8Array[]=[];let offset=0; + while(offset=c.setup.bootstrap.payload.limits.max_requests)fail('repair_over_budget');const count=Math.min(65536,ref.size-offset),packet=makeMailboxBodyRequest(this.#identity,held,ref,{subject:this.#subject,target:e.target,at:this.#now(),expiresAt:held.handle.payload.expires_at as number,childIndex:children[0].index,offset,requestedBytes:count,policy,budget}); + counts.requests++;this.#networkObserver?.();const raw=await this.#transport.requestRepair(baseUrl,packet.raw,deadline,{child:true});counts.wire_bytes+=packet.raw.length+raw.length;if(raw.length!==count)fail('repair_ref_mismatch');budget.input(raw.length);chunks.push(raw);counts.proof_bytes+=raw.length;offset+=raw.length;} + guard();budget.output(ref.size);return Buffer.concat(chunks);}; + const result=await readMailboxAdmission(selected,{expectedSlot:e.slot,expectedSigningKey:e.target.signing_key,expectedOwner:this.#subject,expectedSender:e.sender,expectedTarget:e.target,encryptionIdentity:this.#encryption as any,readOriginal,at:this.#now(), + currentStatuses:current.map((v:AuthenticatedStatusOriginal)=>({ref:v.ref,raw:v.raw})),knownStatuses:[...retained.values()],statusObligations:source.graph.obligations,onStatusAuthenticated:v=>this.#observe(journal,journalKey,v),limitPolicy:this.#limits,policy,budget});guard();return result; + } +} diff --git a/clients/typescript/network/open-mailbox-journal.ts b/clients/typescript/network/open-mailbox-journal.ts new file mode 100644 index 0000000..f8d3a65 --- /dev/null +++ b/clients/typescript/network/open-mailbox-journal.ts @@ -0,0 +1,41 @@ +/** Bounded durable mailbox setup/status journal shared with the Python client. */ +import type {OpenParticipant} from './open-participant.ts'; +import type {AuthenticatedStatusOriginal} from './open-repair-status.ts'; +import {canonicalBytes,sha256} from './crypto.ts'; +import {RepairError} from './open-repair-wire.ts'; +import type {RawOriginal} from './open-repair-wire.ts'; +import {transaction} from './io.ts'; +type Obj=Record; +function fail(code:string):never{throw new RepairError(code);} +export class MailboxSetupJournal{ + readonly #participant:OpenParticipant; + constructor(participant:OpenParticipant){this.#participant=participant;participant.providerStorage(db=>db.exec(` + CREATE TABLE IF NOT EXISTS open_mailbox_setup_jobs(job_key TEXT PRIMARY KEY,plan BLOB NOT NULL,blocked INTEGER NOT NULL DEFAULT 0); + CREATE TABLE IF NOT EXISTS open_mailbox_setup_steps(job_key TEXT NOT NULL,stage TEXT NOT NULL,request BLOB NOT NULL,response BLOB,PRIMARY KEY(job_key,stage)); + CREATE TABLE IF NOT EXISTS open_mailbox_setup_statuses(job_key TEXT NOT NULL,digest TEXT NOT NULL,raw BLOB NOT NULL,PRIMARY KEY(job_key,digest));`));} + start(plan:unknown):string{ + const raw=canonicalBytes(plan,65536),key=sha256(raw); + this.#participant.providerStorage(db=>transaction(db,()=>{ + const old=db.prepare('SELECT plan,blocked FROM open_mailbox_setup_jobs WHERE job_key=?').get(key) as Obj|undefined; + if(old){if(!Buffer.from(old.plan).equals(Buffer.from(raw)))fail('repair_setup_journal_conflict');if(old.blocked)fail('repair_setup_journal_capacity');return;} + const size=db.prepare('SELECT count(*) AS n FROM open_mailbox_setup_jobs').get() as Obj;if(size.n>=16)fail('repair_setup_journal_capacity'); + db.prepare('INSERT INTO open_mailbox_setup_jobs(job_key,plan) VALUES(?,?)').run(key,raw); + }));return key; + } + #check(db:any,key:string):void{if(!/^[0-9a-f]{64}$/.test(key))fail('repair_invalid_context');const row=db.prepare('SELECT blocked FROM open_mailbox_setup_jobs WHERE job_key=?').get(key);if(!row||row.blocked)fail('repair_setup_journal_unavailable');} + statuses(key:string):readonly RawOriginal[]{return this.#participant.providerStorage(db=>{ + this.#check(db,key);const rows=db.prepare('SELECT digest,raw FROM open_mailbox_setup_statuses WHERE job_key=? ORDER BY digest LIMIT 33').all(key) as Obj[]; + if(rows.length>32||rows.reduce((n,v)=>n+v.raw.length,0)>262144)fail('repair_setup_journal_capacity'); + return rows.map(v=>{const raw=Uint8Array.from(v.raw);if(sha256(raw)!==v.digest)fail('repair_storage_corrupt');return {raw,ref:{namespace:'meta' as const,key:v.digest,raw_sha256:v.digest,size:raw.length}};}); + });} + observe(key:string,item:AuthenticatedStatusOriginal):void{ + const raw=item.raw,digest=item.raw_sha256;if(raw.length>16384||sha256(raw)!==digest)fail('repair_invalid_context'); + const saved=this.#participant.providerStorage(db=>transaction(db,()=>{ + this.#check(db,key);const old=db.prepare('SELECT raw FROM open_mailbox_setup_statuses WHERE job_key=? AND digest=?').get(key,digest) as Obj|undefined; + if(old){if(!Buffer.from(old.raw).equals(Buffer.from(raw)))fail('repair_storage_corrupt');return true;} + const size=db.prepare('SELECT count(*) AS n,coalesce(sum(length(raw)),0) AS size FROM open_mailbox_setup_statuses WHERE job_key=?').get(key) as Obj; + if(size.n>=32||size.size+raw.length>262144){db.prepare('UPDATE open_mailbox_setup_jobs SET blocked=1 WHERE job_key=?').run(key);return false;} + db.prepare('INSERT INTO open_mailbox_setup_statuses VALUES(?,?,?)').run(key,digest,raw);return true; + }));if(!saved)fail('repair_setup_journal_capacity'); + } +} diff --git a/clients/typescript/network/open-mailbox-receivers.ts b/clients/typescript/network/open-mailbox-receivers.ts new file mode 100644 index 0000000..f99cb55 --- /dev/null +++ b/clients/typescript/network/open-mailbox-receivers.ts @@ -0,0 +1,67 @@ +/** Existing explicit mailbox registration and ordinary Agent receive polling. */ +import {performance} from 'node:perf_hooks'; +import {canonicalBytes,document,objectFields,sha256,opaqueId,validateSigningIdentity,validateEncryptionIdentity} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {MailboxFeedRecoveryClient} from './open-mailbox-client.ts'; +import {MailboxSetupJournal} from './open-mailbox-journal.ts'; +import {RepairBudget,RepairError,parseNewWire,rawRef} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {endpoint} from './open-transport.ts'; +import {transaction} from './io.ts'; +type Obj=Record; +export const MAILBOX_CONNECT_SCHEMA='memory-vault-open-mailbox-connect/v1'; +function fail(code:string):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} +export class RegisteredMailboxReceivers{ + readonly #participant:OpenParticipant;readonly #encryption:EncryptionIdentityDocument;readonly #delivery:OpenDeliveryClient; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient){this.#participant=participant;this.#encryption=encryption;this.#delivery=delivery; + participant.providerStorage(db=>db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_receivers(receiver_id TEXT PRIMARY KEY,body BLOB NOT NULL,last_attempt INTEGER NOT NULL DEFAULT 0)'));} + #decode(config:Obj):Obj{ + const entry=(v:unknown)=>{const e=objectFields(v,['raw','ref']);if(typeof e.raw!=='string')fail('open_invalid_mailbox_receiver');return {raw:Buffer.from(e.raw,'utf8'),ref:e.ref};}; + const slots=objectFields(config.slot_entries,['slot','read','maintenance','bootstrap']); + return {expectedSlot:config.expected_slot,expectedSender:config.expected_sender,expectedTarget:config.expected_target,targetNodeEntry:entry(config.target_node_entry),slotEntries:Object.fromEntries(Object.entries(slots).map(([n,v])=>[n,entry(v)]))}; + } + #validate(value:Obj):Obj{ + objectFields(value,['schema_version','action','base_url','limit_policy','expected_slot','expected_sender','expected_target','target_node_entry','slot_entries']);if(value.schema_version!==MAILBOX_CONNECT_SCHEMA||value.action!=='register')fail('open_invalid_mailbox_receiver'); + const options=this.#decode(value),policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512},budget=new RepairBudget(policy),at=Math.floor(Date.now()/1000),owner={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)}; + const setup=verifyMailboxFeedBootstrap(options.slotEntries,{expectedSlot:value.expected_slot,expectedOwner:owner,expectedTarget:value.expected_target,targetStorageEpoch:value.expected_slot.writer_storage_epoch,limitPolicy:value.limit_policy,at,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:value.expected_sender.signing_key.key_id,encryption_key_id:value.expected_sender.encryption_key.key_id}))fail('open_invalid_mailbox_receiver'); + const ref=rawRef(options.targetNodeEntry.ref),raw=parseNewWire(options.targetNodeEntry.raw,policy,budget).raw; + if(ref.namespace!=='meta'||ref.size!==raw.length||budget.hash(raw)!==ref.raw_sha256)fail('open_invalid_mailbox_receiver'); + const node=verifySourceNodeOriginal(raw,{expectedSigningKey:value.expected_target.signing_key,expectedStorageEpoch:value.expected_slot.writer_storage_epoch,at,policy,budget}); + if(!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url,this.#participant.transport.allow_loopback)))fail('open_invalid_mailbox_receiver');return options; + } + connect(invitation:unknown):Obj{ + const value=document(invitation as any,65536) as Obj;if(value.schema_version!==MAILBOX_CONNECT_SCHEMA)fail('open_invalid_mailbox_receiver'); + if(value.action==='list'){objectFields(value,['schema_version','action']);const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id FROM open_mailbox_receivers ORDER BY receiver_id LIMIT 17').all()) as Obj[];if(rows.length>16)fail('open_mailbox_receiver_capacity');return {state:'configured',mailboxes:rows.map(v=>v.receiver_id),network_accessed:false};} + if(value.action==='remove'){objectFields(value,['schema_version','action','receiver_id']);opaqueId(value.receiver_id);this.#participant.providerStorage(db=>db.prepare('DELETE FROM open_mailbox_receivers WHERE receiver_id=?').run(value.receiver_id));return {state:'removed',receiver_id:value.receiver_id,network_accessed:false};} + this.#validate(value);const receiver='mailbox_'+sha256(canonicalBytes(value.expected_slot)),raw=canonicalBytes(value); + this.#participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?').get(receiver) as Obj|undefined; + if(old){if(!Buffer.from(old.body).equals(Buffer.from(raw)))fail('open_mailbox_receiver_conflict');return;} + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_receivers').get() as Obj).n>=16)fail('open_mailbox_receiver_capacity');db.prepare('INSERT INTO open_mailbox_receivers(receiver_id,body) VALUES(?,?)').run(receiver,raw); + }));return {state:'registered',receiver_id:receiver,network_accessed:false,receipt_return:'separate_authority_required'}; + } + async receive(limit:number):Promise{ + const deadline=performance.now()/1000+60,result=await this.#delivery.receive(limit,{pendingOnly:true,deadline}); + const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id,body FROM open_mailbox_receivers ORDER BY last_attempt,receiver_id LIMIT 17').all()) as Obj[]; + if(rows.length>16)fail('open_mailbox_receiver_capacity'); + for(const row of rows){let remaining=deadline-performance.now()/1000;if(result.messages.length>=limit||remaining<=0)break; + this.#participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receivers SET last_attempt=? WHERE receiver_id=?').run(BigInt(Date.now())*1000000n,row.receiver_id)); + let reader:MailboxFeedRecoveryClient|undefined; + try{ + const config=document(row.body,65536) as Obj;if(row.receiver_id!=='mailbox_'+sha256(canonicalBytes(config.expected_slot)))fail('open_invalid_mailbox_receiver'); + const options=this.#validate(config),journal=new MailboxSetupJournal(this.#participant); + reader=new MailboxFeedRecoveryClient(this.#participant.identity,this.#encryption,{limitPolicy:config.limit_policy,allowLoopback:this.#participant.transport.allow_loopback,transport:this.#participant.transport,networkObserver:()=>{result.network_accessed=true;}}); + remaining=deadline-performance.now()/1000;if(remaining<=0)fail('repair_access_expired');const received=await this.#delivery.receiveMailbox(reader,config.base_url,{...options,journal,timeout:Math.min(60,remaining)} as any,limit-result.messages.length); + result.messages.push(...received.messages);result.errors.push(...received.errors); + }catch(error){result.errors.push({receiver_id:row.receiver_id,code:(error as any)?.code??'network_storage_unavailable',retryable:(error as any)?.retryable===true});} + finally{reader?.close();} + } + if(result.messages.length({raw:Buffer.from(e.raw,'base64'),ref:e.ref}); +const calls=[],original=OpenHTTPTransport.prototype.requestRepair;let denyHttp=false; +OpenHTTPTransport.prototype.requestRepair=async function(...args){const p=JSON.parse(Buffer.from(args[1]).toString());calls.push(p.kind??p.payload?.kind);if(denyHttp)throw Error('network after retained revocation');return original.apply(this,args);}; +let network,client;const results=[]; +try{ + network=new OpenNetworkClient(v.network_config);const journal=new MailboxSetupJournal(network.participant); + const make=()=>new MailboxFeedRecoveryClient(network.identity,network.encryption,{policy:v.policy,limitPolicy:v.limits,allowLoopback:true});client=make(); + const opts={...v.options,targetNodeEntry:decode(v.node),slotEntries:Object.fromEntries(Object.entries(v.slots).map(([n,e])=>[n,decode(e)])),journal}; + const start=performance.now(),feed=await client.recover(v.base,opts),body=await client.readMember(v.base,feed,feed.entries[0]); + const plain=await decryptEnvelope(body.envelope,{sender_signing_key:v.options.expectedSender.signing_key,sender_encryption_key:v.options.expectedSender.encryption_key,recipient_signing_key:validateSigningIdentity(network.identity),recipient_encryption_key:validateEncryptionIdentity(network.encryption),encryption_identity:network.encryption}); + results.push({ok:true,count:feed.entries.length,envelopeHash:createHash('sha256').update(body.envelope).digest('hex'),plaintext:Buffer.from(plain).toString(),requests:feed.metrics.requests,seconds:(performance.now()-start)/1000}); + let code;const before=calls.length;try{await client.readMember(v.base,{...feed},feed.entries[0]);}catch(e){code=e.code;}results.push({forged:code,calls:calls.length-before}); + denyHttp=true;const count=calls.length;try{await client.recover(v.base,{...opts,knownStatuses:[decode(v.revoked)]});results.push({revoked:'accepted'});}catch(e){results.push({revoked:e.code,calls:calls.length-count});} + client.close();client=make();const prior=calls.length;try{await client.recover(v.base,opts);results.push({restart:'accepted'});}catch(e){results.push({restart:e.code,calls:calls.length-prior});} +}catch(e){results.push({ok:false,code:e.code??'untyped_error',detail:String(e.stack)});} +finally{client?.close();network?.close();} +process.stdout.write(JSON.stringify({results,calls,subprocessCalls})); +''' + +class NativeMailboxClientTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + from tests.test_open_ack_typescript_agent import DRIVER as agent_driver + (cls.fixture/'agent-driver.mjs').write_text(agent_driver) + + def test_native_http_feed_body_and_durable_denial(self): + self.check_http(False) + + def test_registered_agent_downloads_imports_and_retains_receipt(self): + self.check_http(True) + + def test_registered_agent_recovers_mailbox_with_independent_ack(self): + self.check_http(True,True) + + def check_http(self,agent_mode,with_ack=False): + method='test_ack_configuration_survives_mailbox_custody' if with_ack else 'test_actual_delivery_stages_exact_ciphertext_under_mailbox_resources' + h=fixture.MailboxStagingHTTPTests(method);h.setUp();self.addCleanup(h.doCleanups);visited=[] + if agent_mode: + from memory_vault_trust import TrustStore + from memory_vault_client import ClientConfig + TrustStore(ClientConfig.load(h.b.client_config).trust_path).add(h.ai.public_descriptor()) + original_call=h.call + def share(agent,**request): + if agent is h.a and (request['op']=='send' or (request.get('invitation',{}).get('action')=='prepare' and request.get('invitation',{}).get('schema_version')=='memory-vault-open-ack-connect/v1')): + remembered=original_call(h.a,op='remember',request_id='req_native_http_memory',kind='observation',text='Synthetic remotely recovered memory.') + if request['op']=='send':request=dict(request,memory_ids=[remembered['memory_id']]) + else:request=dict(request,invitation=dict(request['invitation'],memory_ids=[remembered['memory_id']])) + return original_call(agent,**request) + h.call=share + def inspect(staging,key,head): + visited.append(True);s=staging.source;rid=staging.db.execute('SELECT resource_id FROM open_repair_mailbox_roots').fetchone()[0] + snapshot=MailboxSnapshotSource(staging).load(rid,key,head,max_bytes=2_000_000,max_items=256) + part=snapshot.parts[2];budget=wire.RepairBudget(s.policy);resolver=wire.LocalRawResolver(s.policy,budget) + for role,ref in part.transfer: + if role=='history.raw_pack':resolver.put(ref.namespace,ref.key,snapshot.read(ref.as_dict())) + tree=history.resolve_historical_inputs(snapshot.read(part.manifest.as_dict()),resolver,s.policy,budget) + roles={v.role:dict(raw=v.original.raw,ref=v.original.ref.as_dict()) for v in tree.roles} + envelope=snapshot.read(snapshot.envelopes[0].as_dict());context=json.loads(envelope)['payload']['context'] + sender=dict(signing_key=h.ai.public_descriptor(),encryption_key=context['sender_encryption_key']) + from memory_vault_open_node import OpenParticipant,OpenHTTPServer + participant=OpenParticipant(s.identity,h.root/'node_0/transport',seeds=[],descriptor=s.node,encryption_identity=s.encryption_identity,allow_loopback=True,repair_policy=dict(enabled=True,limit_policy=s.limits),contact_policy=dict(enabled=True),delivery_policy=dict(enabled=True)) + server=OpenHTTPServer(('127.0.0.1',0),participant);thread=threading.Thread(target=server.serve_forever,kwargs=dict(poll_interval=.02),daemon=True);thread.start() + def close():server.shutdown();server.server_close();thread.join(timeout=3);participant.close() + self.addCleanup(close) + base='http://127.0.0.1:'+str(server.server_port) + node=signed_entry(dict(s.node['payload'],base_url=base,revision=s.node['payload']['revision']+1),s.identity,'synthetic_native_feed_node') + # The ordinary delivery transport cannot supply this message anymore. + staging.db.execute('DELETE FROM open_delivery_messages');staging.db.execute('DELETE FROM open_contact_resource_leases');staging.db.commit() + from memory_vault_open_provider import issue_status + from tests.test_open_repair_status import status_entry + p=json.loads(roles['historical.status.slot']['raw'])['payload'];now=int(time.time()) + revoked=status_entry(issue_status(h.bi,root=key['root_key'],revision=99,entries=[dict(v,status='revoked') for v in p['entries']],issued_at=now-10,valid_until=now-1)) + call=dict(network_config=str(h.b.network_config),base=base,node=encoded(node),slots={n:encoded(roles[r]) for n,r in [('slot','mailbox.slot'),('read','mailbox.read_grant'),('maintenance','mailbox.maintenance_root'),('bootstrap','bootstrap.mailbox_feed')]},options=dict(expectedSlot=key,expectedSender=sender,expectedTarget=s.target),policy=asdict(replace(s.policy,max_signature_checks=512)),limits=s.limits,revoked=encoded(revoked)) + if agent_mode: + invitation=dict(schema_version='memory-vault-open-mailbox-connect/v1',action='register',base_url=base,limit_policy=s.limits,expected_slot=key,expected_sender=sender,expected_target=s.target,target_node_entry=dict(raw=node['raw'].decode(),ref=node['ref']),slot_entries={n:dict(raw=roles[r]['raw'].decode(),ref=roles[r]['ref']) for n,r in [('slot','mailbox.slot'),('read','mailbox.read_grant'),('maintenance','mailbox.maintenance_root'),('bootstrap','bootstrap.mailbox_feed')]}) + message=json.loads(roles['delivery.attempt']['raw'])['payload']['message_id'] + def native(requests,no_network=False,allow_failure=False): + v=dict(client_config=str(h.b.client_config),network_config=str(h.b.network_config),requests=requests,no_network=no_network) + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'agent-driver.mjs')],cwd=self.fixture,input=json.dumps(v).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=75) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2000:]);out=json.loads(run.stdout);self.assertEqual(out['subprocessCalls'],0) + if no_network:self.assertEqual(out['calls'],[]) + if not allow_failure: + for row in out['results']:self.assertTrue(row['ok'],row) + return out + invalid=native([dict(op='connect',invitation=dict(invitation,base_url=base+'/wrong')),dict(op='connect',invitation=dict(invitation,expected_sender=s.target)),dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='list'))],True,True)['results'] + self.assertFalse(invalid[0]['ok']);self.assertFalse(invalid[1]['ok']);self.assertEqual(invalid[2]['result']['mailboxes'],[]) + registered=native([dict(op='connect',invitation=invitation)],True)['results'][0]['result'];self.assertEqual(registered['state'],'registered') + self.assertEqual(native([dict(op='connect',invitation=invitation)],True)['results'][0]['result'],registered) + received=native([dict(op='receive',limit=1)]) + result=received['results'][0]['result'];self.assertTrue(result['network_accessed']);self.assertEqual(result['errors'],[]);self.assertEqual(len(result['messages']),1,result) + saved=result['messages'][0];self.assertEqual(saved['state'],'validated_saved');self.assertEqual(saved['content_kind'],'memory_transfer');self.assertEqual(saved['share']['records_added'],1) + reopened=native([dict(op='receive',message_id=message),dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='list'))],True) + self.assertEqual(reopened['results'][1]['result']['mailboxes'],[registered['receiver_id']]) + with h.b._network() as network: + inbox=network._delivery()._inbox(message);self.assertEqual(inbox['phase'],'saved');self.assertEqual(inbox['receipt_sent'],0);self.assertIsNotNone(inbox['receipt']) + # Python reopens the exact native evidence and authenticates its saved receipt. + network._delivery()._verify_mailbox_inbox(network._delivery()._inbox_session(inbox['session']),inbox) + self.assertEqual(network._delivery()._saved_receipt(message),json.loads(inbox['receipt'])) + self.assertEqual(network._mailbox_saved_receiver(registered['receiver_id']),invitation) + removed=native([dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='remove',receiver_id=registered['receiver_id'])),dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='list'))],True) + self.assertEqual(removed['results'][1]['result']['mailboxes'],[]) + return + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture,input=json.dumps(call).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=75) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout);self.assertEqual(out['subprocessCalls'],0) + self.assertEqual(len(out['results']),4,out);good,forged,denied,restarted=out['results'];self.assertTrue(good['ok'],out);self.assertEqual(good['envelopeHash'],hashlib.sha256(envelope).hexdigest());self.assertIn('Synthetic mailbox staging message',good['plaintext']);self.assertEqual(good['count'],1) + self.assertIn('mailbox.body_read',out['calls']);self.assertEqual(forged,dict(forged='repair_proof_mismatch',calls=0));self.assertEqual(denied,dict(revoked='repair_authority_revoked',calls=0));self.assertEqual(restarted,dict(restart='repair_authority_revoked',calls=0)) + h.inspect_committed_mailbox=inspect;getattr(h,method)();self.assertEqual(visited,[True]) + +if __name__=='__main__':unittest.main() From 0614aa4b92bea70ddab308a14375f4ba750fba67 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:19:30 +0800 Subject: [PATCH 12/24] Prepare alpha36 native remote mailbox recovery candidate --- clients/typescript/network/README.md | 4 ++-- memory_vault.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index 1266908..a4314db 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -278,7 +278,7 @@ verified status observations across restarts, including failed recovery. See [request formats and commands](../../../docs/OPEN_ACK_RECOVERY.md#native-recovery-commands-development-after-alpha019). This command entry is development after the frozen alpha.0.19 archives. -### Registered mailbox recovery (development) +### Registered mailbox recovery (alpha.0.36 candidate) A native `Agent.handle({op: "connect", invitation})` accepts the existing `memory-vault-open-mailbox-connect/v1` `register`, `list`, and `remove` actions. @@ -307,4 +307,4 @@ bounded feed/body recovery, and `readMailboxIndex`, `readMailboxAdmission`, and `verifyMailboxInboxEvidence` for exact-reference/local recovery. These paths do not delegate to a Python process. Native mailbox provisioning and replica inbox recovery remain unfinished; use the Python client for those paths. These additions -are development source, not yet a public release. +are in the alpha.0.36 candidate; public release acceptance remains pending. diff --git a/memory_vault.py b/memory_vault.py index 0c1e1f5..0b5d092 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.35" +VERSION = "0.28.0-alpha.0.36" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" From 5945c9ae86355867ca1a23f09a76f0cf45d4ffd0 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:22:03 +0800 Subject: [PATCH 13/24] Bound the expanded native mailbox release inventory at 640 files --- scripts/release_source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/release_source.py b/scripts/release_source.py index e1f1f93..1c8c0fa 100644 --- a/scripts/release_source.py +++ b/scripts/release_source.py @@ -20,9 +20,10 @@ MAX_SOURCE_BYTES = 2 * 1024 * 1024 MAX_TREE_BYTES = 2 * 1024 * 1024 MAX_TREE_ENTRIES = 8192 -# The explicit native ACK runtime/review inventory now exceeds 608 files. +# The explicit native mailbox runtime/review inventory adds eleven files +# to the alpha35 inventory and now exceeds 624 selected source files. # Keep a finite source-count ceiling independent of the unchanged byte caps. -MAX_SELECTED_FILES = 624 +MAX_SELECTED_FILES = 640 MAX_SELECTED_BYTES = 32 * 1024 * 1024 GIT_TIMEOUT_SECONDS = 15 _SHA = re.compile(r"[0-9a-f]{40}") From bd3ca97ee56ca59a85c87eb1395e738e1a8c56ca Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:23:35 +0800 Subject: [PATCH 14/24] Align alpha36 plugin and installation guides with mailbox recovery --- packaging/CLIENT_README.md | 16 ++++++++++++++-- packaging/PROTOCOL_README.md | 16 ++++++++++++++-- packaging/RELEASE_NOTES.md | 18 +++++++++++++++--- packaging/REVIEW_README.md | 14 +++++++++++++- .../.codex-plugin/plugin.json | 2 +- 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 1bc2989..9704411 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 — authorized full client +# Memory Vault v0.28.0-alpha.0.36 — authorized full client + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected @@ -136,7 +148,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.35 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.36 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 15e70c4..0446a56 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 — independent protocol +# Memory Vault v0.28.0-alpha.0.36 — independent protocol + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected @@ -153,7 +165,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.35; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.36; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index f35a776..7af802a 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 — native saved receipt return and restart recovery +# Memory Vault v0.28.0-alpha.0.36 — native remote mailbox memory recovery + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected @@ -162,7 +174,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.35/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 6648812..e49b3c6 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,16 @@ -# Memory Vault v0.28.0-alpha.0.35 independent review kit +# Memory Vault v0.28.0-alpha.0.36 independent review kit + +Native TypeScript recipients can register an explicitly authorized ordinary +mailbox and recover messages and selected memories with normal Agent `receive`, +even after the old delivery records and leases are gone. The client verifies +original owner/contact/storage authority, the complete encrypted feed, current +READ permissions and the exact ciphertext before importing through its durable +inbox. Retained revocations survive restart. Python and TypeScript share receiver +configuration, status history and saved inbox evidence; either can resume an +interrupted import. The native path performs its own HTTP and cryptography. +Saved receipts remain available through independently authorized return requests. +Native provisioning and replica inbox recovery remain unfinished. See +`clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory receipt through the existing `connect/return_receipt` invitation. The protected diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index 6f856be..a74930e 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.35", + "version": "0.28.0-alpha.0.36", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" From b82d66bb607207f8328eae08bd85e5aa5e7e8a7a Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:50:38 +0800 Subject: [PATCH 15/24] Return retained cold mailbox receipts during native Agent receive --- clients/typescript/network/README.md | 24 ++++ clients/typescript/network/open-ack-client.ts | 1 + clients/typescript/network/open-client.ts | 5 +- .../network/open-delivery-client.ts | 15 ++- .../network/open-mailbox-receipts.ts | 113 +++++++++++++++++ .../network/open-mailbox-receivers.ts | 4 + .../typescript/network/open-repair-receipt.ts | 5 +- clients/typescript/network/package.json | 3 +- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 4 +- scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 8 +- tests/test_open_delivery_http.py | 3 + .../test_open_mailbox_receipts_typescript.py | 115 ++++++++++++++++++ 14 files changed, 289 insertions(+), 17 deletions(-) create mode 100644 clients/typescript/network/open-mailbox-receipts.ts create mode 100644 tests/test_open_mailbox_receipts_typescript.py diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index a4314db..b8ed1df 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -308,3 +308,27 @@ bounded feed/body recovery, and `readMailboxIndex`, `readMailboxAdmission`, and not delegate to a Python process. Native mailbox provisioning and replica inbox recovery remain unfinished; use the Python client for those paths. These additions are in the alpha.0.36 candidate; public release acceptance remains pending. + +### Retained receipt destinations (development after alpha.0.36) + +The native Agent accepts `memory-vault-open-ack-connect/v1` actions +`register_mailbox_receipt_return`, `list_mailbox_receipt_returns`, +`inspect_mailbox_receipt_return` and `remove_mailbox_receipt_return`. +Registration retains an exact `message_id`, `source_url`, `source_key_id` and +`repair_profile` (`receipt` or `receipt-index`) locally. It can happen before +reception; it grants no upload or disclosure permission. + +Ordinary `receive` attempts up to two ready, saved-message returns within its +shared deadline. It reopens the mailbox evidence and original independent ACK +configuration, proves the selected source, and persists the exact preparation +before returning the actual saved receipt. A lost reply or process restart +resumes the same original request. Authenticated original-root status history +remains durable; retained denials stop a pending upload before retransmission. +Expired uncertain returns become `reconciliation_required` and stop polling. + +The bounded selection and return journals use the same protected SQLite tables +as Python. Removing a selection preserves the saved message and original return +history. Re-registering a completed selection can reuse authenticated local +history without another upload. Direct `return_mailbox_receipt` supports the same +message and source selection. Native replica inbox support remains unfinished. +These additions are development source after the frozen alpha.0.36 candidate. diff --git a/clients/typescript/network/open-ack-client.ts b/clients/typescript/network/open-ack-client.ts index e5f1222..e25e7b9 100644 --- a/clients/typescript/network/open-ack-client.ts +++ b/clients/typescript/network/open-ack-client.ts @@ -21,6 +21,7 @@ type Obj=Record; export const ACK_CONNECT_SCHEMA='memory-vault-open-ack-connect/v1'; const PROFILES={receipt:{...DEFAULT_REPAIR_CLIENT_LIMITS,max_signature_checks:2048,max_proof_bytes:1048576}, 'receipt-index':{...DEFAULT_REPAIR_CLIENT_LIMITS,max_signature_checks:4096,max_proof_bytes:4194304,max_requests:256,max_replay_records:256}}; +export {PROFILES as ACK_REPAIR_PROFILES}; function fail(code:string):never{throw new NetworkError(code);} function decode(value:unknown):Obj{ const entry=objectFields(value,['raw','ref']); diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index 21176ca..a3abe78 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -7,6 +7,7 @@ import {loadClient} from './client-config.ts'; import type {Client} from './client-config.ts'; import {OpenDeliveryClient} from './open-delivery-client.ts'; import {RegisteredMailboxReceivers,MAILBOX_CONNECT_SCHEMA} from './open-mailbox-receivers.ts'; +import {MailboxReceiptJobs,MAILBOX_RECEIPT_ACTIONS,returnMailboxReceipt} from './open-mailbox-receipts.ts'; import {OpenParticipant} from './open-participant.ts'; import {OpenContactClient,CONNECT_SCHEMA} from './open-contact-client.ts'; import type {SignedNode} from './open-control.ts'; @@ -39,8 +40,10 @@ export class OpenNetworkClient{ if(invitation!=null){ if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA) return this.result(new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).connect(invitation)); - if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA) + if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA){ + if(MAILBOX_RECEIPT_ACTIONS.has((invitation as any).action))return this.result((invitation as any).action==='return_mailbox_receipt'?await returnMailboxReceipt(this.participant,this.encryption,this.delivery(),invitation):new MailboxReceiptJobs(this.participant,this.encryption,this.delivery()).connect(invitation)); return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); + } if(typeof invitation!=='object'||Array.isArray(invitation)||(invitation as any).schema_version!==CONNECT_SCHEMA)fail('open_private_invitation_unsupported'); const result=await new OpenContactClient(this.participant,this.encryption).dispatch(invitation,requestId); return this.result({...result,profile:'open-routing-v1',network_accessed:true,open_messaging_supported:true}); diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 5aacd71..8ddb1f3 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -347,12 +347,12 @@ export class OpenDeliveryClient{ if(!Object.hasOwn(session,'mailbox'))return document(session,MAX_INBOX_SESSION_BYTES) as Obj; return objectFields(session,['mailbox','authority','source_node']); } - private async verifyMailboxInbox(session:Obj,row:Obj):Promise{ + private async verifyMailboxInbox(session:Obj,row:Obj):Promise{ const verified=await verifyMailboxInboxEvidence(session.mailbox,row.envelope,{owner:{signing_key:validateSigningIdentity(this.participant.identity), encryption_key:validateEncryptionIdentity(this.encryption)},encryptionIdentity:this.encryption,stagedAt:row.created_at}); if(verified.core.message_id!==row.message_id||session.mailbox.sender.signing_key.key_id!==row.sender)fail('network_inbox_identity_conflict'); const expected=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([name,role])=>[name,document(verified.setup.roles[role].raw)])); - if(!same(session.authority,expected))fail('network_inbox_identity_conflict'); + if(!same(session.authority,expected))fail('network_inbox_identity_conflict');return verified; } private async finishInbox(messageId:string):Promise{ const row=this.inbox(messageId);if(!row)fail('network_message_not_found');if(['saved','rejected'].includes(row.phase))return document(row.result,MAX_RESULT_BYTES); @@ -389,8 +389,15 @@ export class OpenDeliveryClient{ db.prepare('UPDATE open_delivery_inbox SET receipt=? WHERE message_id=?').run(encoded,messageId);return receipt; })); } - /** Read the actually saved inbox receipt for an explicitly requested return. - * The caller cannot substitute receipt bytes or a saved-state flag. */ + /** Reauthenticate the saved mailbox evidence before deriving return authority. */ + async savedMailboxForAck(messageId:string):Promise{ + if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); + const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); + const session=OpenDeliveryClient.inboxSession(row.session);if(!Object.hasOwn(session,'mailbox'))fail('open_ack_mailbox_configuration_missing'); + const verified=await this.verifyMailboxInbox(session,row);if(!verified.setup.ack_configuration)fail('open_ack_mailbox_configuration_missing'); + return {owner:session.mailbox.sender,roles:Object.fromEntries(Object.entries(verified.setup.roles).map(([n,e]:[string,any])=>[n,{ref:e.ref,raw:e.raw}]))}; + } + /** Read the actual saved receipt; never accept substituted receipt bytes. */ async savedReceiptForAck(messageId:string,owner:Obj,expectedEnvelope:unknown):Promise<{raw:Uint8Array;ref:Obj}>{ if(!/^msg_[0-9a-f]{64}$/.test(messageId))fail('repair_saved_tuple_mismatch'); const row=this.inbox(messageId);if(!row||row.phase!=='saved')fail('repair_receipt_not_saved'); diff --git a/clients/typescript/network/open-mailbox-receipts.ts b/clients/typescript/network/open-mailbox-receipts.ts new file mode 100644 index 0000000..9be05aa --- /dev/null +++ b/clients/typescript/network/open-mailbox-receipts.ts @@ -0,0 +1,113 @@ +/** Explicit retained receipt destinations and original-authority cold returns. */ +import {performance} from 'node:perf_hooks'; +import {canonicalBytes,document,objectFields,sha256,encodeBase64url,decodeBase64url} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import {NetworkError,transaction} from './io.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import type {OpenDeliveryClient} from './open-delivery-client.ts'; +import {endpoint} from './open-transport.ts'; +import type {OpenHTTPTransport} from './open-transport.ts'; +import {coordinate,verifyNode} from './open-control.ts'; +import {LookupBudget} from './open-routing.ts'; +import {OpenProviderClient} from './open-provider-client.ts'; +import {AckReceiptClient} from './open-repair-offer-client.ts'; +import {SavedAckReceiptPublisher,SAVED_ACK_REQUEST_FIELDS} from './open-repair-receipt.ts'; +import {ACK_CONNECT_SCHEMA,ACK_REPAIR_PROFILES} from './open-ack-client.ts'; +import {OriginalAckStatusJournal} from './open-ack-status.ts'; +type Obj=Record; +const monotonic=()=>performance.now()/1000; +const FIELDS=['schema_version','action','message_id','source_url','source_key_id','repair_profile']; +const SCHEMA='memory-vault-open-saved-ack-request/v1',MAX_BUNDLE=1048576; +export const MAILBOX_RECEIPT_ACTIONS=new Set(['return_mailbox_receipt','register_mailbox_receipt_return','list_mailbox_receipt_returns','inspect_mailbox_receipt_return','remove_mailbox_receipt_return']); +function fail(code:string,retryable=false):never{throw new NetworkError(code,retryable);} +function message(v:unknown):string{if(typeof v!=='string'||!/^msg_[0-9a-f]{64}$/.test(v))fail('open_invalid_ack_request');return v;} +const equal=(a:unknown,b:unknown)=>Buffer.from(canonicalBytes(a)).equals(Buffer.from(canonicalBytes(b))); +function selection(v:unknown,participant:OpenParticipant,action:string):Obj{ + const r=objectFields(v,FIELDS);if(r.schema_version!==ACK_CONNECT_SCHEMA||r.action!==action)fail('open_invalid_ack_request'); + message(r.message_id);coordinate(r.source_key_id); + if(typeof r.repair_profile!=='string'||!Object.hasOwn(ACK_REPAIR_PROFILES,r.repair_profile))fail('open_invalid_repair_policy'); + endpoint(r.source_url,participant.transport.allow_loopback);return {...r,source_url:r.source_url.replace(/\/$/,'')}; +} +function decodedBundle(raw:Uint8Array,base:string,profile:string):Obj{ + const bundle=objectFields(document(raw,MAX_BUNDLE),['schema_version','base_url','repair_profile','request']); + if(bundle.schema_version!==SCHEMA||bundle.base_url!==base||bundle.repair_profile!==profile)fail('open_ack_mailbox_return_corrupt'); + const r=objectFields(bundle.request,SAVED_ACK_REQUEST_FIELDS),decode=(v:unknown)=>{const e=objectFields(v,['ref','raw_base64url']);return {ref:e.ref,raw:decodeBase64url(e.raw_base64url,MAX_BUNDLE)};}; + if(!Array.isArray(r.current_statuses)||r.current_statuses.length>7)fail('open_ack_mailbox_return_corrupt'); + return Object.fromEntries(Object.entries(r).map(([n,v])=>[n,n==='current_statuses'?v.map(decode):n.endsWith('_entry')?decode(v):v])); +} +export async function returnMailboxReceipt(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient,value:unknown, + options:{deadline?:number;networkObserver?:()=>void}={}):Promise{ + objectFields(options,[...(Object.hasOwn(options,'deadline')?['deadline']:[]),...(Object.hasOwn(options,'networkObserver')?['networkObserver']:[])]); + if((options.deadline!==undefined&&!Number.isFinite(options.deadline))||(options.networkObserver!==undefined&&typeof options.networkObserver!=='function'))fail('repair_invalid_deadline'); + const deadline=Math.min(monotonic()+60,options.deadline??Infinity),remaining=()=>{const n=deadline-monotonic();if(n<=0)fail('open_delivery_budget_exhausted',true);return Math.min(60,n);}; + remaining(); + const v=selection(document(value as any,4096),participant,'return_mailbox_receipt'),base=v.source_url; + const saved=await delivery.savedMailboxForAck(v.message_id),roles=saved.roles,owner=saved.owner; + const [root,write,bootstrap]=['ack.root_authority','ack.write_grant','bootstrap.ack_offer'].map(n=>roles[n]); + const wp=document(write.raw,65536).payload as Obj,binding=sha256(canonicalBytes({message_id:v.message_id,envelope_ref:wp.envelope_ref,source_url:base,source_key_id:v.source_key_id,repair_profile:v.repair_profile,write_ref:write.ref})); + participant.providerStorage(db=>db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_ack_returns(message_id TEXT PRIMARY KEY,binding TEXT NOT NULL,request BLOB NOT NULL,request_sha256 TEXT NOT NULL)')); + const held=participant.providerStorage(db=>db.prepare('SELECT * FROM open_mailbox_ack_returns WHERE message_id=?').get(v.message_id)) as Obj|undefined; + const decodeHeld=(row:Obj)=>{if(row.binding!==binding)fail('open_ack_mailbox_return_conflict');if(row.request.length>MAX_BUNDLE||sha256(row.request)!==row.request_sha256)fail('open_ack_mailbox_return_corrupt');return decodedBundle(row.request,base,v.repair_profile);}; + const statuses=new OriginalAckStatusJournal(participant,wp.ack_slot.root_key),authorityHistory=()=>statuses.load(new Set([owner.signing_key.key_id,v.source_key_id])); + let accessed=false;const observe=()=>{accessed=true;options.networkObserver?.();}; + const transport={requestRepair:async(url:string,raw:Uint8Array,until:number,o:unknown)=>{remaining();observe();return participant.transport.requestRepair(url,raw,Math.min(until,deadline),o as any);}} as OpenHTTPTransport; + const reader=new AckReceiptClient(participant.identity,encryption,{transport,statusObserver:item=>statuses.observe(item),allowLoopback:participant.transport.allow_loopback,limitPolicy:ACK_REPAIR_PROFILES[v.repair_profile as keyof typeof ACK_REPAIR_PROFILES]}); + try{ + let request:Obj; + if(held)request=decodeHeld(held); + else{ + remaining();observe();const node=(await participant.transport.requestNode(base,Math.min(deadline,monotonic()+15))).response,p=verifyNode(node); + if(p.status!=='active'||p.signing_key.key_id!==v.source_key_id||!equal(endpoint(p.base_url,participant.transport.allow_loopback),endpoint(base,participant.transport.allow_loopback)))fail('open_ack_mailbox_return_conflict'); + participant.acceptProviderControl(node);const targetRecord=await new OpenProviderClient(participant,encryption).proveTarget(node,new LookupBudget({maximum_seconds:Math.min(10,remaining())})); + const target={signing_key:targetRecord.payload.signing_key,encryption_key:targetRecord.payload.targetEncryptionKey},raw=canonicalBytes(node),digest=sha256(raw),nodeEntry={raw,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}}; + const statuses=new Map();for(const role of ['ack_root','ack_write','ack_offer_bootstrap']){const e=roles['historical.status.'+role];statuses.set(e.ref.raw_sha256,e);} + remaining();const prepared=await reader.prepareReturn(base,{targetNodeEntry:nodeEntry,expectedTarget:target,expectedAckSlot:wp.ack_slot,expectedOwner:owner,expectedMessageId:v.message_id,expectedEnvelopeRef:wp.envelope_ref,rootEntry:root,writeEntry:write,bootstrapEntry:bootstrap,knownStatuses:[...statuses.values()],archiveStatuses:authorityHistory(),timeout:30}); + request={message_id:v.message_id,envelope_ref:wp.envelope_ref,ack_slot:wp.ack_slot,owner,target,target_node_entry:nodeEntry,root_entry:root,write_entry:write,bootstrap_entry:bootstrap,binding_entry:{ref:prepared.source.binding.ref,raw:prepared.source.binding.raw},current_statuses:prepared.current_statuses.map(e=>({ref:e.ref,raw:e.raw})),read_until:prepared.source.read_until,retain_until:prepared.source.retain_until}; + const encode=(e:Obj)=>({ref:e.ref,raw_base64url:encodeBase64url(e.raw)}),bundle=canonicalBytes({schema_version:SCHEMA,base_url:base,repair_profile:v.repair_profile,request:Object.fromEntries(Object.entries(request).map(([n,e])=>[n,n==='current_statuses'?e.map(encode):n.endsWith('_entry')?encode(e):e]))},MAX_BUNDLE); + request=participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT * FROM open_mailbox_ack_returns WHERE message_id=?').get(v.message_id) as Obj|undefined;if(old)return decodeHeld(old); + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_ack_returns').get() as Obj).n>=16)fail('open_ack_mailbox_return_capacity');db.prepare('INSERT INTO open_mailbox_ack_returns VALUES(?,?,?,?)').run(v.message_id,binding,bundle,sha256(bundle));return request; + })); + } + remaining();const result=await new SavedAckReceiptPublisher(delivery,reader).publishSaved(base,request,30,{archiveStatuses:authorityHistory(),knownDisclosureStatuses:statuses.load(new Set([participant.identity.key_id]))}); + return {state:'retained_at_ack_source',message_id:v.message_id,receipt_ref:result.source.inputs.receipt.ref,commit_ref:result.source.commit.ref,from_local_history:result.from_local_history,network_accessed:accessed}; + }finally{reader.close();} +} +export class MailboxReceiptJobs{ + readonly participant:OpenParticipant;readonly encryption:EncryptionIdentityDocument;readonly delivery:OpenDeliveryClient; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient){ + this.participant=participant;this.encryption=encryption;this.delivery=delivery; + participant.providerStorage(db=>db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_receipt_jobs(message_id TEXT PRIMARY KEY,body BLOB NOT NULL,body_sha256 TEXT NOT NULL,phase TEXT NOT NULL,result BLOB,result_sha256 TEXT,last_attempt INTEGER NOT NULL DEFAULT 0)')); + } + #body(row:Obj,name:string):Obj{const raw=row[name];if(!raw||sha256(raw)!==row[name+'_sha256'])fail('open_ack_mailbox_return_corrupt');return document(raw,8192) as Obj;} + connect(value:unknown):Obj{ + const v=document(value as any,4096) as Obj;if(v.schema_version!==ACK_CONNECT_SCHEMA)fail('open_invalid_ack_request'); + if(v.action==='register_mailbox_receipt_return'){ + const body=selection(v,this.participant,v.action),raw=canonicalBytes(body,4096); + this.participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT message_id,body,body_sha256,phase,result,result_sha256 FROM open_mailbox_receipt_jobs WHERE message_id=?').get(v.message_id) as Obj|undefined; + if(old){if(!equal(this.#body(old,'body'),body))fail('open_ack_mailbox_return_conflict');return;} + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_receipt_jobs').get() as Obj).n>=16)fail('open_ack_mailbox_return_capacity');db.prepare("INSERT INTO open_mailbox_receipt_jobs(message_id,body,body_sha256,phase) VALUES(?,?,?,'pending')").run(v.message_id,raw,sha256(raw)); + }));return {state:'registered',message_id:v.message_id,network_accessed:false,receipt_return:'ordinary_receive_after_saved_original_authority_check'}; + } + if(v.action==='list_mailbox_receipt_returns'){objectFields(v,['schema_version','action']);const rows=this.participant.providerStorage(db=>db.prepare('SELECT message_id,phase FROM open_mailbox_receipt_jobs ORDER BY message_id LIMIT 17').all()) as Obj[]; + if(rows.length>16)fail('open_ack_mailbox_return_capacity');return {state:'configured',returns:rows.map(r=>({message_id:r.message_id,state:r.phase})),network_accessed:false};} + objectFields(v,['schema_version','action','message_id']);message(v.message_id); + if(v.action==='remove_mailbox_receipt_return'){this.participant.providerStorage(db=>db.prepare('DELETE FROM open_mailbox_receipt_jobs WHERE message_id=?').run(v.message_id));return {state:'removed',message_id:v.message_id,network_accessed:false};} + if(v.action!=='inspect_mailbox_receipt_return')fail('open_invalid_ack_request'); + const row=this.participant.providerStorage(db=>db.prepare('SELECT message_id,body,body_sha256,phase,result,result_sha256 FROM open_mailbox_receipt_jobs WHERE message_id=?').get(v.message_id)) as Obj|undefined;if(!row)fail('open_ack_mailbox_return_missing'); + return {state:row.phase,selection:this.#body(row,'body'),result:row.result===null?null:this.#body(row,'result'),network_accessed:false,source_rechecked:false}; + } + async poll(result:Obj,deadline:number,attempted:Set):Promise{ + if(attempted.size>=2||monotonic()>=deadline)return; + // Keep nanosecond scheduling timestamps in SQLite; they exceed JS safe integers. + const rows=this.participant.providerStorage(db=>db.prepare("SELECT j.message_id,j.body,j.body_sha256,j.phase,j.result,j.result_sha256 FROM open_mailbox_receipt_jobs AS j JOIN open_delivery_inbox AS i ON i.message_id=j.message_id WHERE j.phase='pending' AND i.phase='saved' ORDER BY j.last_attempt,j.message_id LIMIT 3").all()) as Obj[]; + const row=rows.find(r=>!attempted.has(r.message_id));if(!row)return;const mid=row.message_id;attempted.add(mid); + this.participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receipt_jobs SET last_attempt=? WHERE message_id=?').run(BigInt(Date.now())*1000000n,mid)); + const store=(phase:string,v:Obj)=>{const raw=canonicalBytes(v,8192);this.participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receipt_jobs SET phase=?,result=?,result_sha256=? WHERE message_id=? AND body_sha256=?').run(phase,raw,sha256(raw),mid,row.body_sha256));}; + try{ + const selected=selection(this.#body(row,'body'),this.participant,'register_mailbox_receipt_return');if(selected.message_id!==mid)fail('open_ack_mailbox_return_corrupt'); + const returned=await returnMailboxReceipt(this.participant,this.encryption,this.delivery,{...selected,action:'return_mailbox_receipt'},{deadline,networkObserver:()=>{result.network_accessed=true;}}); + store('complete',returned);(result.receipt_returns??=[]).push({message_id:mid,state:returned.state,from_local_history:returned.from_local_history});result.network_accessed ||=returned.network_accessed; + }catch(error){const e=error as any,code=e?.code??'network_storage_unavailable',retryable=e?.retryable===true,phase=['repair_reconciliation_required','repair_saved_reconciliation_required'].includes(code)?'reconciliation_required':'pending'; + store(phase,{state:phase,code,retryable});result.errors.push({message_id:mid,operation:'return_mailbox_receipt',code,retryable});} + } +} diff --git a/clients/typescript/network/open-mailbox-receivers.ts b/clients/typescript/network/open-mailbox-receivers.ts index f99cb55..faf0b7b 100644 --- a/clients/typescript/network/open-mailbox-receivers.ts +++ b/clients/typescript/network/open-mailbox-receivers.ts @@ -12,6 +12,7 @@ import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; import {verifySourceNodeOriginal} from './open-repair-original.ts'; import {endpoint} from './open-transport.ts'; import {transaction} from './io.ts'; +import {MailboxReceiptJobs} from './open-mailbox-receipts.ts'; type Obj=Record; export const MAILBOX_CONNECT_SCHEMA='memory-vault-open-mailbox-connect/v1'; function fail(code:string):never{throw new RepairError(code);} @@ -47,6 +48,8 @@ export class RegisteredMailboxReceivers{ } async receive(limit:number):Promise{ const deadline=performance.now()/1000+60,result=await this.#delivery.receive(limit,{pendingOnly:true,deadline}); + const jobs=new MailboxReceiptJobs(this.#participant,this.#encryption,this.#delivery),attempted=new Set(); + await jobs.poll(result,Math.min(deadline,performance.now()/1000+30),attempted); const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id,body FROM open_mailbox_receivers ORDER BY last_attempt,receiver_id LIMIT 17').all()) as Obj[]; if(rows.length>16)fail('open_mailbox_receiver_capacity'); for(const row of rows){let remaining=deadline-performance.now()/1000;if(result.messages.length>=limit||remaining<=0)break; @@ -62,6 +65,7 @@ export class RegisteredMailboxReceivers{ finally{reader?.close();} } if(result.messages.length!sameEntry(source.inputs[name as keyof typeof source.inputs],entries[name])))fail('repair_saved_corrupt'); return Object.freeze({source,originals:Object.freeze([...originals.values()].map(e=>Object.freeze({ref:e.ref,get raw(){return Uint8Array.from(e.raw);}}))),from_local_history:true}); } - async publishSaved(baseUrl:string,value:unknown,timeout=30):Promise{ + async publishSaved(baseUrl:string,value:unknown,timeout=30,history:{archiveStatuses?:readonly unknown[];knownDisclosureStatuses?:readonly unknown[]}={}):Promise{ + objectFields(history,[...(Object.hasOwn(history,'archiveStatuses')?['archiveStatuses']:[]),...(Object.hasOwn(history,'knownDisclosureStatuses')?['knownDisclosureStatuses']:[])]); if(typeof timeout!=='number'||!Number.isFinite(timeout)||timeout<=0||timeout>60)fail('repair_invalid_deadline'); const client=this.#client,policy=this.#policy,budget=new RepairBudget(policy),r=this.#request(value,budget); const receipt=await this.#delivery.savedReceiptForAck(r.message_id,r.owner,r.envelope_ref),now=client.timestamp; @@ -157,7 +158,7 @@ export class SavedAckReceiptPublisher{ db.prepare('UPDATE open_repair_saved_acks SET '+column+'=?,attempted=1 WHERE slot_digest=?').run(raw,slotDigest); })); }; - const options={currentStatuses:[...r.current_statuses,entries.status],readUntil:r.read_until,retainUntil:r.retain_until, + const options={currentStatuses:[...r.current_statuses,entries.status],archiveStatuses:history.archiveStatuses??[],knownDisclosureStatuses:history.knownDisclosureStatuses??[],readUntil:r.read_until,retainUntil:r.retain_until, targetNodeEntry:r.target_node_entry,expectedTarget:r.target,expectedAckSlot:r.ack_slot,expectedOwner:r.owner,expectedMessageId:r.message_id, expectedEnvelopeRef:r.envelope_ref,rootEntry:r.root_entry,writeEntry:r.write_entry,bootstrapEntry:r.bootstrap_entry,timeout,journal}; const result=resume===null?await client.put(baseUrl,entries.receipt,entries.disclosure,entries.put,options): diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index 709a521..41aaa86 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -49,7 +49,8 @@ "./open-repair-mailbox-inbox": "./open-repair-mailbox-inbox.ts", "./open-mailbox-client": "./open-mailbox-client.ts", "./open-mailbox-journal": "./open-mailbox-journal.ts", - "./open-mailbox-receivers": "./open-mailbox-receivers.ts" + "./open-mailbox-receivers": "./open-mailbox-receivers.ts", + "./open-mailbox-receipts": "./open-mailbox-receipts.ts" }, "types": "./crypto.ts", "engines": { diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 834b99f..184be1f 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -146,7 +146,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 11634fa..18887cf 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -123,7 +123,7 @@ "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", "tests/test_open_repair_receipt_typescript.py", - "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", + "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", @@ -248,7 +248,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index ce01db6..5ef5617 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -54,7 +54,7 @@ "tests.test_open_repair_index_prepare", "tests.test_open_repair_index_prepare_admin", "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", - "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +244,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 7740c28..6a19020 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -46,7 +46,7 @@ "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", - "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", + "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-repair-ack.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 67) + self.assertEqual(len(TS_NETWORK), 68) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -122,7 +122,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_status.py", "tests/test_open_repair_status_typescript.py", "tests/open_repair_ack_fixtures.py", "tests/test_open_repair_ack.py", "tests/test_open_repair_ack_typescript.py", "tests/test_open_repair_state.py", "tests/test_open_repair_mailbox_resources.py", "tests/test_open_repair_copy_resources.py", - "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", + "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", @@ -188,7 +188,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-wire.ts", "clients/typescript/network/open-repair-history.ts", "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", - "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", + "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", diff --git a/tests/test_open_delivery_http.py b/tests/test_open_delivery_http.py index 0e53e37..d0acd5a 100644 --- a/tests/test_open_delivery_http.py +++ b/tests/test_open_delivery_http.py @@ -822,6 +822,9 @@ def corrupt_envelope(reference): self.assertGreater(status_count,0) self.assertEqual(recipient_db.execute('SELECT count(*) FROM open_mailbox_setup_statuses').fetchone()[0],status_count) self.assertEqual(delivery._inbox(sent['message_id'])['phase'],'saved') + # This recovery session is finished before a different runtime + # reopens the same recipient state for independent receipt return. + recipient_context.close() def _remote_message_admission(self, source, owner_status, slot, slot_entries, owner, sender_encryption, sent, now, limits, envelope): import hashlib diff --git a/tests/test_open_mailbox_receipts_typescript.py b/tests/test_open_mailbox_receipts_typescript.py new file mode 100644 index 0000000..8f010ee --- /dev/null +++ b/tests/test_open_mailbox_receipts_typescript.py @@ -0,0 +1,115 @@ +"""Actual cold saved memory to native ordinary-receive receipt return.""" +import base64 +import json +import subprocess +import unittest +import sqlite3 +import time +from pathlib import Path +from unittest.mock import patch +from tests import test_open_delivery_http as fixture +from tests import test_open_ack_typescript_agent as runtime +from memory_vault_open_client import ACK_CONNECT_SCHEMA + +DRIVER=runtime.DRIVER.replace("const agent=new Agent", "if(input.clock_offset){const clock=Date.now;Date.now=()=>clock()+input.clock_offset*1000;}\nconst agent=new Agent").replace("if(name==='requestNode'", """if(name==='requestRepair'&&input.lose_reply&&JSON.parse(Buffer.from(args[1]).toString()).kind==='ack.put_request'&&JSON.parse(Buffer.from(result).toString()).kind==='ack.put_response'){ + const error=Error('synthetic lost ACK reply');error.code='synthetic_lost_ack_reply';throw error; + } + if(name==='requestNode'""") + +class NativeMailboxReceiptTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.NativeAckAgentTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def native(self,agent,requests,*,lose_reply=False,no_network=False,clock_offset=0): + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config),requests=requests,lose_reply=lose_reply,no_network=no_network,clock_offset=clock_offset)).encode(),stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=75) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2500:]);out=json.loads(run.stdout);self.assertEqual(out['subprocessCalls'],0) + state=Path(json.loads(Path(agent.network_config).read_bytes())['state_directory'])/'network.sqlite3' + db=sqlite3.connect(state) + try:self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(),[('ok',)]) + finally:db.close() + for row in out['results']:self.assertTrue(row['ok'],(row,run.stderr.decode(errors='replace')[-2000:])) + self.assertNotIn('network_storage_unavailable',json.dumps(out),(out,run.stderr.decode(errors='replace')[-2000:])) + if no_network:self.assertEqual(out['calls'],[]) + return out + + def test_native_ordinary_receive_returns_cold_memory_receipt_after_lost_reply(self): + self.check_return('resume') + + def test_expired_retained_revocation_stops_pending_native_return(self): + self.check_return('revoke') + + def test_expired_uncertain_return_stops_ordinary_polling(self): + self.check_return('expired') + + def check_return(self,mode): + h=fixture.MailboxStagingHTTPTests('test_cold_mailbox_returns_independent_receipt');h.setUp();self.addCleanup(h.doCleanups) + from memory_vault_trust import TrustStore + from memory_vault_client import ClientConfig + TrustStore(ClientConfig.load(h.b.client_config).trust_path).add(h.ai.public_descriptor()) + call=h.call + def share(agent,**request): + invitation=request.get('invitation',{}) + if agent is h.a and ((request['op']=='send' and not request.get('memory_ids')) or (invitation.get('action')=='prepare' and invitation.get('schema_version')==ACK_CONNECT_SCHEMA and not invitation.get('memory_ids'))): + m=call(h.a,op='remember',request_id='req_native_retained_return_memory',kind='observation',text='Synthetic cold mailbox return memory.') + if request['op']=='send':request=dict(request,memory_ids=[m['memory_id']]) + else:request=dict(request,invitation=dict(invitation,memory_ids=[m['memory_id']])) + return call(agent,**request) + h.call=share;visited=[] + def returned(host): + visited.append(True);write=json.loads(h.ack_configuration['ack.write_grant']['raw'])['payload'];mid=write['message_id'] + selected=dict(schema_version=ACK_CONNECT_SCHEMA,action='register_mailbox_receipt_return',message_id=mid,source_url=host.nodes[0]['payload']['base_url'],source_key_id=host.identities[0].key_id,repair_profile='receipt') + query=dict(schema_version=ACK_CONNECT_SCHEMA,action='inspect_mailbox_receipt_return',message_id=mid) + self.native(h.b,[dict(op='connect',invitation=selected),dict(op='connect',invitation=selected)],no_network=True) + first=self.native(h.b,[dict(op='receive',limit=1)],lose_reply=True)['results'][0]['result'] + self.assertTrue(any(e['code']=='synthetic_lost_ack_reply' for e in first['errors']),first);self.assertTrue(first['network_accessed']);self.assertFalse(first.get('receipt_returns')) + pending=self.native(h.b,[dict(op='connect',invitation=query)],no_network=True)['results'][0]['result'];self.assertEqual(pending['state'],'pending');self.assertEqual(pending['result']['code'],'synthetic_lost_ack_reply') + if mode=='expired': + expired=self.native(h.b,[dict(op='receive',limit=1)],clock_offset=120) + errors=expired['results'][0]['result']['errors'];self.assertTrue(any(e['code'] in {'repair_reconciliation_required','repair_saved_reconciliation_required'} for e in errors),errors) + self.assertFalse(any(c['kind']=='requestRepair' for c in expired['calls'])) + held=self.native(h.b,[dict(op='connect',invitation=query)],no_network=True)['results'][0]['result'];self.assertEqual(held['state'],'reconciliation_required') + stopped=self.native(h.b,[dict(op='receive',limit=1)],clock_offset=120);self.assertFalse(any(c['kind']=='requestRepair' for c in stopped['calls']));self.assertFalse(stopped['results'][0]['result'].get('receipt_returns')) + return + if mode=='revoke': + from memory_vault_open_provider import issue_status + from tests.test_open_repair_status import status_entry + from memory_vault_open_repair_status import authenticate_status_original + from memory_vault_open_repair_admin import _ReplicaStatusJournal + from memory_vault_open_repair_state import DEFAULT_POLICY + from memory_vault_open_repair_wire import RepairBudget + old=json.loads(h.ack_configuration['historical.status.ack_root']['raw'])['payload'];now=int(time.time()) + denied=status_entry(issue_status(h.ai,root=write['ack_slot']['root_key'],revision=99,entries=[dict(e,status='revoked') for e in old['entries']],issued_at=now-10,valid_until=now-1)) + checked=authenticate_status_original(denied,expected_root=write['ack_slot']['root_key'],expected_signing_key=h.ai.public_descriptor(),at=now-10,allowed_scopes=[dict(scope_kind=e['scope_kind'],scope_id=e['scope_id']) for e in old['entries']],policy=DEFAULT_POLICY,budget=RepairBudget(DEFAULT_POLICY)) + with h.b._network() as network: + with network.participant.state.db() as db:_ReplicaStatusJournal(db,write['ack_slot']['root_key'],original_source=True).observe(checked) + for _ in range(2): + refused=self.native(h.b,[dict(op='receive',limit=1)]) + result=refused['results'][0]['result'];self.assertFalse(result.get('receipt_returns'));self.assertTrue(any(e['code']=='repair_authority_revoked' for e in result['errors']),result) + self.assertFalse(any(c['kind']=='requestRepair' for c in refused['calls']),refused['calls']) + return + resumed=self.native(h.b,[dict(op='receive',limit=1)])['results'][0]['result'];self.assertEqual(resumed['receipt_returns'],[dict(message_id=mid,state='retained_at_ack_source',from_local_history=False)]) + completed=self.native(h.b,[dict(op='connect',invitation=query)],no_network=True)['results'][0]['result'];self.assertEqual(completed['state'],'complete') + again=self.native(h.b,[dict(op='receive',limit=1)]);self.assertFalse(again['results'][0]['result'].get('receipt_returns'));self.assertFalse(any(c['kind']=='requestRepair' for c in again['calls'])) + with h.b._network() as network: + inbox=network._delivery()._inbox(mid);saved=json.loads(inbox['result']);self.assertEqual(saved['content_kind'],'memory_transfer');self.assertEqual(saved['share']['records_added'],1) + inspected=network.connect(invitation=query);self.assertEqual(inspected,completed) + with patch.object(network.participant.transport,'request_repair',side_effect=AssertionError('completed native request repeated HTTP')): + py=network.connect(invitation=dict(selected,action='return_mailbox_receipt')) + self.assertTrue(py['from_local_history']);self.assertEqual(py['commit_ref'],completed['result']['commit_ref']) + chunks=[];cursor=None + while True: + q=dict(schema_version=ACK_CONNECT_SCHEMA,action='export_preparation',request_id=h.ack_preparation_request_id,part='owner_invitation') + if cursor is not None:q['cursor']=cursor + page=h.call(h.a,op='connect',invitation=q);chunks.append(base64.b64decode(page['bundle_chunk']));cursor=page['next_cursor'] + if cursor is None:break + recovered=h.call(h.a,op='connect',invitation=json.loads(b''.join(chunks)));self.assertEqual(recovered['message_id'],mid) + with h.a._network() as network: + with network.participant.state.db() as db:self.assertIsNotNone(db.execute('SELECT acknowledgement FROM open_delivery_outbox WHERE message_id=?',(mid,)).fetchone()[0]) + self.native(h.b,[dict(op='connect',invitation=dict(query,action='remove_mailbox_receipt_return')),dict(op='connect',invitation=selected)],no_network=True) + replay=self.native(h.b,[dict(op='receive',limit=1)]);self.assertTrue(replay['results'][0]['result']['receipt_returns'][0]['from_local_history']);self.assertFalse(any(c['kind']=='requestRepair' for c in replay['calls'])) + h._return_cold_ack=returned;h.test_cold_mailbox_returns_independent_receipt();self.assertEqual(visited,[True]) + +if __name__=='__main__':unittest.main() From 0b6b1c4f03c6496f1e9670a12c6c8acd31e67c20 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:59:09 +0800 Subject: [PATCH 16/24] Prepare alpha37 automatic native mailbox receipt release --- memory_vault.py | 2 +- packaging/CLIENT_README.md | 13 +++++++++++-- packaging/PROTOCOL_README.md | 13 +++++++++++-- packaging/RELEASE_NOTES.md | 15 ++++++++++++--- packaging/REVIEW_README.md | 11 ++++++++++- .../memory-vault-client/.codex-plugin/plugin.json | 2 +- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/memory_vault.py b/memory_vault.py index 0b5d092..d7b0e14 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.36" +VERSION = "0.28.0-alpha.0.37" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 9704411..0fa2a7f 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 — authorized full client +# Memory Vault v0.28.0-alpha.0.37 — authorized full client + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, @@ -148,7 +157,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.36 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.37 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 0446a56..4c230e7 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 — independent protocol +# Memory Vault v0.28.0-alpha.0.37 — independent protocol + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, @@ -165,7 +174,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.36; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.37; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index 7af802a..4b49e7a 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 — native remote mailbox memory recovery +# Memory Vault v0.28.0-alpha.0.37 — automatic native mailbox receipt returns + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, @@ -174,7 +183,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.36/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index e49b3c6..0374631 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.36 independent review kit +# Memory Vault v0.28.0-alpha.0.37 independent review kit + +Native TypeScript recipients can retain an independently authorized receipt +return for a cold-mailbox message. Ordinary Agent `receive` then returns the +actual durably saved receipt, keeping the exact upload across lost replies and +process restarts. Python and TypeScript share the same retained jobs, pending +requests and completed history. The sender can independently recover that +receipt and confirm its original send. Observed revocations block retries; +expired uncertain uploads stop for reconciliation. No new permission or source +selection is created by retrying. See `clients/typescript/network/README.md`. Native TypeScript recipients can register an explicitly authorized ordinary mailbox and recover messages and selected memories with normal Agent `receive`, diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index a74930e..fd07619 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.36", + "version": "0.28.0-alpha.0.37", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" From d7d17b2c85b8371e422b49ff028963a9f6c419b8 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:23:45 +0800 Subject: [PATCH 17/24] Authorize native mailbox admission with retained original permissions --- clients/typescript/network/README.md | 41 +++++++ .../typescript/network/open-contact-client.ts | 7 ++ .../network/open-mailbox-destination.ts | 99 ++++++++++++++++ .../network/open-mailbox-receivers.ts | 16 +++ clients/typescript/network/package.json | 3 +- docs/OPEN_NETWORK_CONTACT.md | 7 ++ memory_vault_open_repair_client.py | 44 +++++++- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 4 +- scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 8 +- ...est_open_mailbox_destination_typescript.py | 106 ++++++++++++++++++ 12 files changed, 330 insertions(+), 11 deletions(-) create mode 100644 clients/typescript/network/open-mailbox-destination.ts create mode 100644 tests/test_open_mailbox_destination_typescript.py diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index b8ed1df..4f2d260 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -332,3 +332,44 @@ history. Re-registering a completed selection can reuse authenticated local history without another upload. Direct `return_mailbox_receipt` supports the same message and source selection. Native replica inbox support remains unfinished. These additions are development source after the frozen alpha.0.36 candidate. + +### Receiver-issued mailbox authorization (development after alpha.0.37) + +After registering an already provisioned mailbox and explicitly approving the +sender's contact request, a native recipient can export the existing mailbox +admission invitation with ordinary `connect`: + +```ts +await agent.handle({op: 'connect', invitation: { + schema_version: 'memory-vault-open-mailbox-connect/v1', action: 'authorize', + receiver_id, contact_request_ref, expires_at, status_revision, status_until +}}); +``` + +Choose the expiry windows within the original contact and mailbox permissions. +The owner supplies and coordinates `status_revision`; a new selection must not +reuse or roll back an issued revision. Authorization binds the exact approved +contact, sender and recipient keys, mailbox slot, selected source and existing +storage resources. It does not reserve new storage or create ACK-return grants. +The native client signs with the recipient's own existing identity and persists +the exact destination and status originals before returning them. + +The response supplies a base64 `authorization_chunk` of at most 3,072 bytes, +`authorization_sha256`, `total_bytes`, `offset`, and `next_cursor`. Repeat the +same invitation with `cursor: next_cursor` until it is null, join decoded chunks +in order, and verify the complete byte count and SHA-256 before parsing the +sender's `retain` authorization. A cursor cannot be reused for different bytes. +`inspect` with `receiver_id` similarly pages `configuration_chunk` with +`configuration_sha256`; both operations reauthenticate the retained receiver +and perform no HTTP requests. + +Python and native clients share the bounded destination journal. Restarting or +switching clients returns the same signed originals; a conflicting request or +reused older revision fails. Both clients reauthenticate retained receive-status +observations inside the signing transaction. Revoked slot, parent permission or +storage-resource authority blocks both cached output and fresh issuance even +after that status expires; higher remembered document floors also remain in force. A completed local authorization does not prove that +the source is currently available or that a message was delivered. The sender +must still prepare and admit its exact ciphertext under current source checks; +these sender and provisioning operations currently use Python. The existing +native mailbox receive and independent receipt-return paths remain available. diff --git a/clients/typescript/network/open-contact-client.ts b/clients/typescript/network/open-contact-client.ts index bbcc9ad..2f79805 100644 --- a/clients/typescript/network/open-contact-client.ts +++ b/clients/typescript/network/open-contact-client.ts @@ -57,6 +57,13 @@ export class OpenContactClient{ if(!row||Buffer.from(row.body).equals(Buffer.from(LOCAL_RESERVATION)))throw new ContactError('contact_local_missing'); if(row.expires_at<=now())throw new ContactError('contact_expired');return document(row.body,24576); } + approvedIncomingOriginals(reference:string):Obj{ + const incoming=this.load('incoming',reference),decision=this.load('decision',reference).decision; + if(decision.payload.decision!=='approved')throw new ContactError('open_delivery_not_authorized'); + const grant=decision.payload.grant,docs={node:incoming.node,policy:incoming.policy,request:incoming.request, + decision,knock_lease:incoming.lease,grant,delivery_lease:grant.payload.resource_lease}; + return Object.fromEntries(Object.entries(docs).map(([name,value])=>[name,canonicalBytes(value)])); + } private savePolicy(reservationRef:string,leaseId:string,value:Obj,expires:number):void{ const raw=canonicalBytes(document(value,24576)); this.participant.contactStorage(db=>transaction(db,()=>{ diff --git a/clients/typescript/network/open-mailbox-destination.ts b/clients/typescript/network/open-mailbox-destination.ts new file mode 100644 index 0000000..c7fab85 --- /dev/null +++ b/clients/typescript/network/open-mailbox-destination.ts @@ -0,0 +1,99 @@ +/** Receiver-issued original mailbox admission, retained across client changes. */ +import {canonicalBytes,sha256,encodeBase64url,decodeBase64url,validateSigningIdentity,validateEncryptionIdentity} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import {transaction} from './io.ts'; +import {RepairBudget,RepairError,buildNewWire,parseNewWire,objectFields,rawRef,u53} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {originalPublicDescriptor,verifyContactOriginals,verifyBoundedControlSignature} from './open-repair-original.ts'; +import {signBoundedBootstrapOriginal} from './open-repair-probe.ts'; +import {MailboxSetupJournal} from './open-mailbox-journal.ts'; +import {authenticateStatusOriginal,statusScope} from './open-repair-status.ts'; +import {issueStatus} from './open-provider.ts'; +type Obj=Record; +function fail(code:string):never{throw new RepairError(code);} +/** Inputs are the locally reauthenticated receiver and approved contact originals. */ +export function prepareMailboxDestination(participant:OpenParticipant,encryption:EncryptionIdentityDocument,config:Obj,slots:Obj,contact:Obj, + value:{expires_at:number;status_revision:number;status_until:number}):Obj{ + objectFields(value,['expires_at','status_revision','status_until']); + const policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512},budget=new RepairBudget(policy),at=Math.floor(Date.now()/1000); + const owner={signing_key:validateSigningIdentity(participant.identity),encryption_key:validateEncryptionIdentity(encryption)}; + const encode=(entry:Obj)=>{const ref=rawRef(entry.ref),raw=parseNewWire(entry.raw,policy,budget).raw;if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {ref,raw_base64url:encodeBase64url(raw)};}; + const decode=(entry:unknown)=>{const e=objectFields(entry,['ref','raw_base64url']),ref=rawRef(e.ref),raw=decodeBase64url(e.raw_base64url,65536);if(raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {ref,raw};}; + const setup=verifyMailboxFeedBootstrap(slots,{expectedSlot:config.expected_slot,expectedOwner:owner,expectedTarget:config.expected_target, + targetStorageEpoch:config.expected_slot.writer_storage_epoch,limitPolicy:config.limit_policy,at,policy,budget}),slot=setup.slot.payload as Obj,root=slot.slot_key.root_key; + const sender=objectFields(config.expected_sender,['signing_key','encryption_key']); + originalPublicDescriptor(sender.signing_key,budget);originalPublicDescriptor(sender.encryption_key,budget,true); + if(sender.signing_key.key_id!==slot.sender.signing_key_id||sender.encryption_key.key_id!==slot.sender.encryption_key_id)fail('open_invalid_mailbox_receiver'); + const plan={root_key:root,slot_key:slot.slot_key,sender:slot.sender,target:config.expected_target,limits:config.limit_policy, + ...Object.fromEntries(['budget','windows','max_appends','max_live_items'].map(n=>[n,slot[n]]))}; + u53(value.status_revision,1);u53(value.expires_at);u53(value.status_until); + const binding=budget.hash(buildNewWire({plan,slots:Object.fromEntries(Object.entries(slots).map(([n,e])=>[n,encode(e)])), + contact:Object.fromEntries(Object.entries(contact).map(([n,raw])=>[n,budget.hash(raw as Uint8Array)])),...value},policy,budget).raw); + const rootDigest=budget.hash(buildNewWire(root,policy,budget).raw); + const journal=new MailboxSetupJournal(participant),journalKey=journal.start({kind:'mailbox.feed_recovery',slot_key:slot.slot_key,owner, + sender:config.expected_sender,target:config.expected_target,entries:Object.fromEntries(Object.entries(setup).map(([n,e])=>[n,e.ref]))}); + const required=new Map(); + for(const [name,mask] of [['slot',65],['read',2],['maintenance',65],['bootstrap',10]] as const){ + const e=setup[name],kind=name==='slot'?'mailbox_slot':'authority',scope=statusScope(root,kind,name==='slot'?slot.slot_key:{authority_kind:e.payload.kind,authority_sha256:e.ref.raw_sha256},policy,budget); + required.set(owner.signing_key.key_id+':'+kind+':'+scope,{revision:e.payload.revision as number,mask}); + } + for(const name of ['data','metadata'])required.set(config.expected_target.signing_key.key_id+':resource:'+statusScope(root,'resource',slot[name+'_resource_ref'],policy,budget),{revision:Infinity,mask:65}); + return participant.providerStorage(db=>transaction(db,()=>{ + const revisions=new Map();let observedOwnerRevision=-1; + for(const entry of journal.statuses(journalKey)){ + const signed=objectFields(parseNewWire(entry.raw,policy,budget).value,['payload','proof']),preview=signed.payload as Obj; + const signer=[owner.signing_key,config.expected_target.signing_key,config.expected_sender.signing_key].find(v=>v.key_id===preview.signing_key?.key_id); + if(!signer||u53(preview.issued_at)>at||!Array.isArray(preview.entries)||preview.entries.length>16)fail('repair_status_disclosure'); + const item=authenticateStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:signer,at:preview.issued_at, + allowedScopes:preview.entries.map((e:Obj)=>({scope_kind:e.scope_kind,scope_id:e.scope_id})),policy,budget}); + const status=item.payload as Obj,key=signer.key_id+':'+status.revision; + if(revisions.has(key)&&revisions.get(key)!==item.canonical_sha256)fail('repair_status_conflict');revisions.set(key,item.canonical_sha256); + if(signer.key_id===owner.signing_key.key_id)observedOwnerRevision=Math.max(observedOwnerRevision,status.revision); + for(const row of status.entries){const need=required.get(signer.key_id+':'+row.scope_kind+':'+row.scope_id);if(!need)continue; + if(row.status==='revoked'&&(row.operation_mask&need.mask))fail('repair_authority_revoked'); + if(row.minimum_document_revision>need.revision)fail('repair_status_revision'); + } + } + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_destinations(root_digest TEXT NOT NULL,revision INTEGER NOT NULL,binding TEXT NOT NULL,bundle BLOB NOT NULL,bundle_sha256 TEXT NOT NULL,PRIMARY KEY(root_digest,revision))'); + const old=db.prepare('SELECT binding,bundle,bundle_sha256 FROM open_mailbox_destinations WHERE root_digest=? AND revision=?').get(rootDigest,value.status_revision) as Obj|undefined; + if(old){ + if(old.binding!==binding)fail('repair_destination_conflict');if(old.bundle.length>65536||budget.hash(old.bundle)!==old.bundle_sha256)fail('repair_destination_journal_corrupt'); + const saved=objectFields(parseNewWire(old.bundle,policy,budget).value,['destination','owner_status']),result=Object.fromEntries(Object.entries(saved).map(([n,e])=>[n,decode(e)])); + for(const entry of Object.values(result)){const signed=objectFields(parseNewWire(entry.raw,policy,budget).value,['payload','proof']);verifyBoundedControlSignature(signed.payload,signed.proof,owner.signing_key,budget);} + return result; + } + const latest=(db.prepare('SELECT max(revision) AS revision FROM open_mailbox_destinations WHERE root_digest=?').get(rootDigest) as Obj).revision; + if(value.status_revision<=observedOwnerRevision||(latest!==null&&value.status_revision<=latest))fail('repair_destination_revision_rollback'); + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_destinations').get() as Obj).n>=128)fail('repair_destination_capacity'); + const held=verifyContactOriginals(contact,{senderKeyId:slot.sender.signing_key_id,senderEncryptionKeyId:slot.sender.encryption_key_id, + recipientKeyId:owner.signing_key.key_id,recipientEncryptionKeyId:owner.encryption_key.key_id,nodeKeyId:config.expected_target.signing_key.key_id, + storageEpoch:slot.slot_key.writer_storage_epoch,at,policy,budget}); + if(!(atsetup[n].payload.expires_at as number),held.originals.grant.payload.expires_at as number)) + ||!(ate.payload.expires_at as number))))fail('repair_resource_expired'); + const refs=Object.fromEntries([['contact_request_ref','request'],['contact_policy_ref','policy'],['contact_knock_lease_ref','knock_lease'],['contact_decision_ref','decision'],['store_grant_ref','grant']].map(([field,role])=>{ + const raw=held.originals[role].document.raw,digest=budget.hash(raw);return [field,{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}];})); + const destinationId='destination_'+sha256(Buffer.concat([Buffer.from('memory-vault-mailbox-setup/v1\0'),buildNewWire(root,policy,budget).raw,Buffer.from('\0destination')])); + const signed=signBoundedBootstrapOriginal({schema_version:'memory-vault-open-repair/v1',kind:'delivery.destination',signing_key:owner.signing_key, + issued_at:at,expires_at:value.expires_at,destination_id:destinationId,sender:slot.sender,recipient:slot.recipient,slot_key:slot.slot_key,slot_ref:setup.slot.ref,...refs, + ...Object.fromEntries(['data_resource_ref','data_resource_offer_ref','metadata_resource_ref','metadata_resource_offer_ref','read_grant_ref','maintenance_root_ref','budget','windows'].map(n=>[n,slot[n]]))},participant.identity,policy,budget); + const destination={raw:signed.raw,ref:signed.ref},scopes=[['destination',destination],...Object.entries(slots)].map(([name,entry])=>{ + const e=entry as Obj,p=(parseNewWire(e.raw,policy,budget).value as Obj).payload,kind=name==='slot'?'mailbox_slot':'authority'; + return {scope_kind:kind,scope_id:statusScope(root,kind,name==='slot'?slot.slot_key:{authority_kind:p.kind,authority_sha256:e.ref.raw_sha256},policy,budget), + minimum_document_revision:p.revision??1,status:'active',operation_mask:127}; + }).sort((a,b)=>a.scope_kindb.scope_kind?1:a.scope_idb.scope_id?1:0); + const raw=buildNewWire(issueStatus(participant.identity,{root,revision:value.status_revision,entries:scopes,issued_at:at,valid_until:value.status_until}),policy,budget).raw,digest=budget.hash(raw); + const result={destination,owner_status:{raw,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}}}; + const bundle=buildNewWire(Object.fromEntries(Object.entries(result).map(([n,e])=>[n,encode(e)])),policy,budget).raw;if(bundle.length>65536)fail('repair_destination_capacity'); + db.prepare('INSERT INTO open_mailbox_destinations VALUES(?,?,?,?,?)').run(rootDigest,value.status_revision,binding,bundle,budget.hash(bundle));return result; + })); +} +/** Finite output shares the existing Python cursor and exact authorization bytes. */ +export function mailboxPage(value:unknown,receiver:string,cursor:unknown,kind:'authorization'|'configuration'):Obj{ + const raw=canonicalBytes(value,65536),digest=sha256(raw);let offset=0; + if(cursor!==undefined&&cursor!==null){const v=objectFields(cursor,['sha256','offset']);offset=v.offset;if(v.sha256!==digest||!Number.isSafeInteger(offset)||offset<=0||offset>=raw.length||offset%3072)fail('open_invalid_mailbox_cursor');} + const end=Math.min(offset+3072,raw.length);return {state:'mailbox_'+kind,receiver_id:receiver,[kind+'_sha256']:digest,total_bytes:raw.length,offset, + [kind+'_chunk']:Buffer.from(raw.subarray(offset,end)).toString('base64'),next_cursor:end===raw.length?null:{sha256:digest,offset:end}, + network_accessed:false,source_rechecked:false,receipt_return:'separate_authority_required'}; +} diff --git a/clients/typescript/network/open-mailbox-receivers.ts b/clients/typescript/network/open-mailbox-receivers.ts index faf0b7b..d0bcf31 100644 --- a/clients/typescript/network/open-mailbox-receivers.ts +++ b/clients/typescript/network/open-mailbox-receivers.ts @@ -12,6 +12,8 @@ import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; import {verifySourceNodeOriginal} from './open-repair-original.ts'; import {endpoint} from './open-transport.ts'; import {transaction} from './io.ts'; +import {prepareMailboxDestination,mailboxPage} from './open-mailbox-destination.ts'; +import {OpenContactClient} from './open-contact-client.ts'; import {MailboxReceiptJobs} from './open-mailbox-receipts.ts'; type Obj=Record; export const MAILBOX_CONNECT_SCHEMA='memory-vault-open-mailbox-connect/v1'; @@ -40,6 +42,20 @@ export class RegisteredMailboxReceivers{ const value=document(invitation as any,65536) as Obj;if(value.schema_version!==MAILBOX_CONNECT_SCHEMA)fail('open_invalid_mailbox_receiver'); if(value.action==='list'){objectFields(value,['schema_version','action']);const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id FROM open_mailbox_receivers ORDER BY receiver_id LIMIT 17').all()) as Obj[];if(rows.length>16)fail('open_mailbox_receiver_capacity');return {state:'configured',mailboxes:rows.map(v=>v.receiver_id),network_accessed:false};} if(value.action==='remove'){objectFields(value,['schema_version','action','receiver_id']);opaqueId(value.receiver_id);this.#participant.providerStorage(db=>db.prepare('DELETE FROM open_mailbox_receivers WHERE receiver_id=?').run(value.receiver_id));return {state:'removed',receiver_id:value.receiver_id,network_accessed:false};} + if(value.action==='inspect'||value.action==='authorize'){ + objectFields(value,['schema_version','action','receiver_id',...(value.action==='authorize'?['contact_request_ref','expires_at','status_revision','status_until']:[]),...(Object.hasOwn(value,'cursor')?['cursor']:[])]); + opaqueId(value.receiver_id);const row=this.#participant.providerStorage(db=>db.prepare('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?').get(value.receiver_id)) as Obj|undefined; + if(!row)fail('open_mailbox_receiver_missing');const config=document(row.body,65536) as Obj; + if(value.receiver_id!=='mailbox_'+sha256(canonicalBytes(config.expected_slot)))fail('open_invalid_mailbox_receiver');const options=this.#validate(config); + if(value.action==='inspect')return mailboxPage(config,value.receiver_id,value.cursor,'configuration'); + const contact=new OpenContactClient(this.#participant,this.#encryption).approvedIncomingOriginals(value.contact_request_ref); + const bundle=prepareMailboxDestination(this.#participant,this.#encryption,config,options.slotEntries,contact, + {expires_at:value.expires_at,status_revision:value.status_revision,status_until:value.status_until}); + const encoded=(e:Obj)=>({raw:Buffer.from(e.raw).toString('utf8'),ref:e.ref}); + return mailboxPage({destination_entry:encoded(bundle.destination),owner_status_entry:encoded(bundle.owner_status), + slot_entries:Object.fromEntries(['slot','read','maintenance'].map(n=>[n,config.slot_entries[n]])),target:config.expected_target, + target_node_entry:config.target_node_entry,base_url:config.base_url},value.receiver_id,value.cursor,'authorization'); + } this.#validate(value);const receiver='mailbox_'+sha256(canonicalBytes(value.expected_slot)),raw=canonicalBytes(value); this.#participant.providerStorage(db=>transaction(db,()=>{const old=db.prepare('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?').get(receiver) as Obj|undefined; if(old){if(!Buffer.from(old.body).equals(Buffer.from(raw)))fail('open_mailbox_receiver_conflict');return;} diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index 41aaa86..be3a478 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -50,7 +50,8 @@ "./open-mailbox-client": "./open-mailbox-client.ts", "./open-mailbox-journal": "./open-mailbox-journal.ts", "./open-mailbox-receivers": "./open-mailbox-receivers.ts", - "./open-mailbox-receipts": "./open-mailbox-receipts.ts" + "./open-mailbox-receipts": "./open-mailbox-receipts.ts", + "./open-mailbox-destination": "./open-mailbox-destination.ts" }, "types": "./crypto.ts", "engines": { diff --git a/docs/OPEN_NETWORK_CONTACT.md b/docs/OPEN_NETWORK_CONTACT.md index 783ca92..76b3b20 100644 --- a/docs/OPEN_NETWORK_CONTACT.md +++ b/docs/OPEN_NETWORK_CONTACT.md @@ -197,6 +197,13 @@ signed destination and slot controls; the sender reuses its own frozen outbox ciphertext and contact originals. Preparation never invents permission or sends plaintext to the mailbox node. +Native TypeScript also supports receiver `authorize` and paginated `inspect` in +development source after alpha.0.37. It shares the exact destination journal with +Python, signs only with the receiver's own existing identity, and performs no HTTP +requests during authorization. Both clients apply authenticated retained +revocations before returning or issuing authorization. Source provisioning and +sender mailbox admission still use Python. + A receiver using the Agent interface can authorize from an existing registered mailbox and its locally approved contact, without opening a database or loading private key objects in the calling program: diff --git a/memory_vault_open_repair_client.py b/memory_vault_open_repair_client.py index 7b41212..e939b76 100644 --- a/memory_vault_open_repair_client.py +++ b/memory_vault_open_repair_client.py @@ -1348,6 +1348,47 @@ class MailboxDestinationStore: def __init__(self, db, identity, encryption_identity, *, policy=DEFAULT_POLICY): self.db,self.identity,self.encryption_identity,self.policy=db,identity,encryption_identity,policy + def _retained_authority(self, builder, slots, contact, at, budget): + """Authenticate durable receive observations before issuing more authority.""" + from memory_vault_open_repair_mailbox_activation import verify_mailbox_feed_bootstrap + p=builder.plan;root=p['root_key'];owner=builder.owner;target=p['target'] + request=original.parse_original_control(contact['request'],self.policy,budget).value['payload'] + sender=dict(signing_key=request['signing_key'],encryption_key=request['encryption_key']) + plan=wire.build_new_wire(dict(kind='mailbox.feed_recovery',slot_key=p['slot_key'],owner=owner,sender=sender,target=target, + entries={name:value['ref'] for name,value in slots.items()}),self.policy,budget).raw + key=budget._hash(plan) + exists=self.db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='open_mailbox_setup_jobs'").fetchone() + if exists is None or self.db.execute('SELECT 1 FROM open_mailbox_setup_jobs WHERE job_key=?',(key,)).fetchone() is None:return -1 + setup=verify_mailbox_feed_bootstrap(slots,expected_slot=p['slot_key'],expected_owner=owner, + expected_target=target,target_storage_epoch=p['slot_key']['writer_storage_epoch'], + limit_policy=p['limits'],at=at,policy=self.policy,budget=budget) + required={} + for name,bits in (('slot',65),('read',2),('maintenance',65),('bootstrap',10)): + entry=setup[name];kind='mailbox_slot' if name=='slot' else 'authority' + subject=p['slot_key'] if name=='slot' else dict(authority_kind=entry.payload['kind'],authority_sha256=entry.ref.raw_sha256) + required[(owner['signing_key']['key_id'],kind,status.status_scope(root,kind,subject,self.policy,budget))]=(entry.payload['revision'],bits) + for name in ('data','metadata'): + resource=setup['slot'].payload[name+'_resource_ref'] + required[(target['signing_key']['key_id'],'resource',status.status_scope(root,'resource',resource,self.policy,budget))]=(None,65) + revisions={};owner_revision=-1 + for entry in MailboxSetupJournal(self.db).statuses(key): + preview=original.parse_original_control(entry['raw'],self.policy,budget).value['payload'] + signer=next((v['signing_key'] for v in (owner,target,sender) if v['signing_key']==preview['signing_key']),None) + if signer is None or wire.u53(preview['issued_at'])>at:_fail('repair_status_disclosure') + observed=status.authenticate_status_original(entry,expected_root=root,expected_signing_key=signer,at=preview['issued_at'], + allowed_scopes=[dict(scope_kind=v['scope_kind'],scope_id=v['scope_id']) for v in preview['entries']],policy=self.policy,budget=budget) + value=observed.payload;identity=(signer['key_id'],value['revision']) + if identity in revisions and revisions[identity]!=observed.canonical_sha256:_fail('repair_status_conflict') + revisions[identity]=observed.canonical_sha256 + if signer==owner['signing_key']:owner_revision=max(owner_revision,value['revision']) + for row in value['entries']: + needed=required.get((signer['key_id'],row['scope_kind'],row['scope_id'])) + if needed is None:continue + revision,bits=needed + if row['status']=='revoked' and row['operation_mask']&bits:_fail('repair_authority_revoked') + if revision is not None and row['minimum_document_revision']>revision:_fail('repair_status_revision') + return owner_revision + def prepare(self, plan, slot_entries, contact_originals, *, at, expires_at, status_revision, status_until): from memory_vault_open_repair_bind import encode_entry,decode_entry import memory_vault_open_repair_resource as resource @@ -1365,6 +1406,7 @@ def prepare(self, plan, slot_entries, contact_originals, *, at, expires_at, stat if self.db.in_transaction:_fail('repair_destination_transaction_active') self.db.execute('BEGIN IMMEDIATE') try: + observed_revision=self._retained_authority(builder,slots,contact_originals,at,budget) self.db.execute('CREATE TABLE IF NOT EXISTS open_mailbox_destinations(root_digest TEXT NOT NULL,revision INTEGER NOT NULL,binding TEXT NOT NULL,bundle BLOB NOT NULL,bundle_sha256 TEXT NOT NULL,PRIMARY KEY(root_digest,revision))') row=self.db.execute('SELECT binding,bundle,bundle_sha256 FROM open_mailbox_destinations WHERE root_digest=? AND revision=?',(root,status_revision)).fetchone() if row is not None: @@ -1378,7 +1420,7 @@ def prepare(self, plan, slot_entries, contact_originals, *, at, expires_at, stat original._verify_control_signature(signed['payload'],signed['proof'],builder.owner['signing_key'],budget) else: latest=self.db.execute('SELECT max(revision) FROM open_mailbox_destinations WHERE root_digest=?',(root,)).fetchone()[0] - if latest is not None and status_revision<=latest:_fail('repair_destination_revision_rollback') + if status_revision<=observed_revision or (latest is not None and status_revision<=latest):_fail('repair_destination_revision_rollback') if self.db.execute('SELECT count(*) FROM open_mailbox_destinations').fetchone()[0]>=128:_fail('repair_destination_capacity') result=builder.destination_bundle(slots,contact_originals,at=at,expires_at=expires_at, status_revision=status_revision,status_until=status_until) diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 184be1f..47044f6 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -146,7 +146,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 18887cf..66f6d2d 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -123,7 +123,7 @@ "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", "tests/test_open_repair_receipt_typescript.py", - "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", + "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", @@ -248,7 +248,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 5ef5617..894d624 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -54,7 +54,7 @@ "tests.test_open_repair_index_prepare", "tests.test_open_repair_index_prepare_admin", "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", - "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_mailbox_destination_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +244,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 6a19020..b6c0564 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -46,7 +46,7 @@ "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", - "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", + "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-repair-ack.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 68) + self.assertEqual(len(TS_NETWORK), 69) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -122,7 +122,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_status.py", "tests/test_open_repair_status_typescript.py", "tests/open_repair_ack_fixtures.py", "tests/test_open_repair_ack.py", "tests/test_open_repair_ack_typescript.py", "tests/test_open_repair_state.py", "tests/test_open_repair_mailbox_resources.py", "tests/test_open_repair_copy_resources.py", - "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", + "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", @@ -188,7 +188,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-wire.ts", "clients/typescript/network/open-repair-history.ts", "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", - "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", + "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", diff --git a/tests/test_open_mailbox_destination_typescript.py b/tests/test_open_mailbox_destination_typescript.py new file mode 100644 index 0000000..cf898f3 --- /dev/null +++ b/tests/test_open_mailbox_destination_typescript.py @@ -0,0 +1,106 @@ +"""Native recipient authority drives real sender mailbox admission over HTTP.""" +import base64 +import json +import sqlite3 +import subprocess +import unittest +import time +from pathlib import Path +from tests import test_open_delivery_http as fixture +from tests import test_open_ack_typescript_agent as runtime + +class NativeMailboxDestinationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.NativeAckAgentTests.setUpClass.__func__(cls) + + def native(self,agent,requests): + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config),requests=requests,no_network=True)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=45) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2500:]) + result=json.loads(run.stdout);self.assertEqual(result['calls'],[]);self.assertEqual(result['subprocessCalls'],0) + state=Path(json.loads(Path(agent.network_config).read_bytes())['state_directory'])/'network.sqlite3' + db=sqlite3.connect(state) + try:self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(),[('ok',)]) + finally:db.close() + return result['results'] + + def test_native_authorizes_real_admission_and_python_reopens_exact_pages(self): + self.journey(False) + + def test_native_authorization_rejects_conflicts_expiry_and_changed_receiver(self): + self.journey(True) + + def test_expired_retained_revocation_blocks_cached_and_new_authorization(self): + self.journey(False,True) + + def journey(self,negative,revoked=False): + h=fixture.MailboxStagingHTTPTests('test_sender_admits_message_over_http');h.setUp();self.addCleanup(h.doCleanups) + original=h.call;visited=[];pages=[];first=[] + def dispatch(agent,**request): + invitation=request.get('invitation',{}) + if agent is not h.b or invitation.get('schema_version')!='memory-vault-open-mailbox-connect/v1' or invitation.get('action') not in ('register','authorize'): + return original(agent,**request) + rows=self.native(agent,[request]);self.assertTrue(rows[0]['ok'],rows[0]);result=rows[0]['result'] + if invitation['action']=='register': + self.assertEqual(result,original(agent,**request)) + inspected=self.native(agent,[dict(op='connect',invitation=dict(schema_version=invitation['schema_version'],action='inspect',receiver_id=result['receiver_id']))])[0] + self.assertTrue(inspected['ok'],inspected);self.assertEqual(inspected['result'],original(agent,op='connect',invitation=dict(schema_version=invitation['schema_version'],action='inspect',receiver_id=result['receiver_id']))) + return result + # Each page reopens in a new native process, then Python reads the same exact originals. + self.assertEqual(result,original(agent,**request));pages.append(base64.b64decode(result['authorization_chunk'],validate=True)) + if 'cursor' not in invitation: + visited.append(True);first.append(dict(invitation)) + if negative: + cases=[(dict(invitation,expires_at=invitation['expires_at']-1),'repair_destination_conflict'), + (dict(invitation,status_revision=1),'repair_destination_revision_rollback'), + (dict(invitation,status_revision=4,expires_at=1),'repair_resource_expired'), + (dict(invitation,status_revision=4,status_until=1),'repair_resource_expired'), + (dict(invitation,cursor=dict(sha256='0'*64,offset=3072)),'open_invalid_mailbox_cursor'), + (dict(invitation,receiver_id='mailbox_'+'0'*64),'open_mailbox_receiver_missing')] + for changed,code in cases: + denied=self.native(agent,[dict(op='connect',invitation=changed)])[0] + self.assertFalse(denied['ok'],denied);self.assertEqual(denied['error']['code'],code,denied) + state=Path(json.loads(Path(agent.network_config).read_bytes())['state_directory'])/'network.sqlite3' + db=sqlite3.connect(state) + try: + self.assertEqual(db.execute('SELECT revision FROM open_mailbox_destinations ORDER BY revision').fetchall(),[(2,),(3,)]) + row=db.execute('SELECT body FROM open_mailbox_receivers WHERE receiver_id=?',(invitation['receiver_id'],)).fetchone() + config=json.loads(row[0]);config['expected_sender']=config['expected_target'] + db.execute('UPDATE open_mailbox_receivers SET body=? WHERE receiver_id=?',(json.dumps(config).encode(),invitation['receiver_id']));db.commit() + finally:db.close() + wrong=self.native(agent,[request])[0];self.assertFalse(wrong['ok']);self.assertEqual(wrong['error']['code'],'open_invalid_mailbox_receiver') + db=sqlite3.connect(state) + try: + db.execute('UPDATE open_mailbox_receivers SET body=? WHERE receiver_id=?',(row[0],invitation['receiver_id']));db.commit() + finally:db.close() + self.assertEqual(self.native(agent,[request])[0]['result'],result) + return result + h.call=dispatch;h.test_sender_admits_message_over_http();self.assertEqual(visited,[True]);self.assertGreater(len(pages),1) + authorization=json.loads(b''.join(pages));self.assertEqual(authorization,h.sender_authorization) + # The fixture performs actual admission, feed/body recovery and authority denial checks. + self.assertIn('destination_entry',authorization);self.assertIn('owner_status_entry',authorization) + if revoked: + from memory_vault_open_provider import issue_status + from memory_vault_open_repair_status import status_scope,authenticate_status_original + from memory_vault_open_repair_state import DEFAULT_POLICY + from memory_vault_open_repair_wire import RepairBudget + from memory_vault_open_repair_client import MailboxSetupJournal + from tests.test_open_repair_status import status_entry + slot=json.loads(authorization['slot_entries']['slot']['raw'])['payload'];root=slot['slot_key']['root_key'];now=int(time.time()) + scope=dict(scope_kind='mailbox_slot',scope_id=status_scope(root,'mailbox_slot',slot['slot_key'],DEFAULT_POLICY,RepairBudget(DEFAULT_POLICY))) + entry=status_entry(issue_status(h.bi,root=root,revision=99,entries=[dict(scope,minimum_document_revision=slot['revision'],status='revoked',operation_mask=127)],issued_at=now-10,valid_until=now-1)) + observed=authenticate_status_original(entry,expected_root=root,expected_signing_key=h.bi.public_descriptor(),at=now-10,allowed_scopes=[scope],policy=DEFAULT_POLICY,budget=RepairBudget(DEFAULT_POLICY)) + with h.b._network() as network: + with network.participant.state.db() as db: + jobs=db.execute('SELECT job_key FROM open_mailbox_setup_jobs').fetchall();self.assertEqual(len(jobs),1) + MailboxSetupJournal(db).observe(jobs[0][0],observed) + for revision in (3,100): + denied=self.native(h.b,[dict(op='connect',invitation=dict(first[0],status_revision=revision))])[0] + self.assertFalse(denied['ok']);self.assertEqual(denied['error']['code'],'repair_authority_revoked',denied) + reopened=h.b.handle(dict(op='connect',invitation=dict(first[0],status_revision=revision))) + self.assertFalse(reopened['ok']);self.assertEqual(reopened['error']['code'],'repair_authority_revoked',reopened) + + +if __name__=='__main__':unittest.main() From 531f9ede2e4079bbb7c8951ecfb693600bf3cc7f Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:26:09 +0800 Subject: [PATCH 18/24] Prepare alpha38 native receiver authorization release --- memory_vault.py | 2 +- packaging/CLIENT_README.md | 13 +++++++++++-- packaging/PROTOCOL_README.md | 13 +++++++++++-- packaging/RELEASE_NOTES.md | 15 ++++++++++++--- packaging/REVIEW_README.md | 11 ++++++++++- .../memory-vault-client/.codex-plugin/plugin.json | 2 +- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/memory_vault.py b/memory_vault.py index d7b0e14..b8e6782 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.37" +VERSION = "0.28.0-alpha.0.38" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 0fa2a7f..0012950 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.37 — authorized full client +# Memory Vault v0.28.0-alpha.0.38 — authorized full client + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the @@ -157,7 +166,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.37 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.38 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index 4c230e7..c09dd74 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.37 — independent protocol +# Memory Vault v0.28.0-alpha.0.38 — independent protocol + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the @@ -174,7 +183,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.37; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.38; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index 4b49e7a..d6be29d 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.37 — automatic native mailbox receipt returns +# Memory Vault v0.28.0-alpha.0.38 — native receiver mailbox authorization + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the @@ -183,7 +192,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.37/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.38/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.38/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 0374631..0339b45 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,13 @@ -# Memory Vault v0.28.0-alpha.0.37 independent review kit +# Memory Vault v0.28.0-alpha.0.38 independent review kit + +Native TypeScript recipients can issue an original mailbox destination from +an explicitly approved contact and a registered mailbox. The receiver signs with +its own existing identity and exports bounded authorization pages for sender +admission. Python and native clients reopen the same persisted signed originals; +conflicting selections and stale revisions fail. Both clients enforce retained +parent/resource revocations before cached or fresh authorization, even after the +status expires. Authorization creates no new storage or receipt-return grants. +See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can retain an independently authorized receipt return for a cold-mailbox message. Ordinary Agent `receive` then returns the diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index fd07619..0f7caa2 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.37", + "version": "0.28.0-alpha.0.38", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" From 19f657c842b5e553cc1f340c6c6b7e156ea688b4 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:50:18 +0800 Subject: [PATCH 19/24] Retain sender mailbox messages in the native client --- clients/typescript/network/README.md | 15 +- clients/typescript/network/open-client.ts | 7 +- .../typescript/network/open-mailbox-send.ts | 188 ++++++++++++++++++ .../network/open-repair-mailbox-member.ts | 6 + clients/typescript/network/package.json | 3 +- docs/OPEN_NETWORK_CONTACT.md | 8 +- scripts/build_client_plugin.py | 2 +- scripts/build_release.py | 4 +- scripts/run_open_network_ci.py | 4 +- tests/test_network_packaging.py | 8 +- tests/test_open_mailbox_send_typescript.py | 78 ++++++++ 11 files changed, 307 insertions(+), 16 deletions(-) create mode 100644 clients/typescript/network/open-mailbox-send.ts create mode 100644 tests/test_open_mailbox_send_typescript.py diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index 4f2d260..917dadc 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -371,5 +371,16 @@ storage-resource authority blocks both cached output and fresh issuance even after that status expires; higher remembered document floors also remain in force. A completed local authorization does not prove that the source is currently available or that a message was delivered. The sender must still prepare and admit its exact ciphertext under current source checks; -these sender and provisioning operations currently use Python. The existing -native mailbox receive and independent receipt-return paths remain available. +source provisioning currently uses Python. The native sender supports the same +`prepare`, `admit`, and combined `retain` invitations in development after +alpha.0.38. It freezes the original outbox ciphertext, disclosure and attempt in +the shared Python/native journal before HTTP. A lost reply or restart reuses the +same request, and changed selections fail instead of silently renewing grants. + +Pass `ack_request_id` to `prepare` or `retain` only when selecting an existing +independently prepared receipt destination. Its six complete original controls +and statuses are authenticated and carried in the mailbox member. The node +still performs current permission, storage and resource checks before admission. +`retained_at_mailbox` means the source signed actual custody; +`recipient_acknowledged` remains false until a separate saved receipt is verified. +Neither local preparation nor source custody means the recipient saved a memory. diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index a3abe78..33a7e37 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -6,6 +6,7 @@ import {absolutePath,readPrivate,NetworkError} from './io.ts'; import {loadClient} from './client-config.ts'; import type {Client} from './client-config.ts'; import {OpenDeliveryClient} from './open-delivery-client.ts'; +import {MailboxSender,MAILBOX_SEND_ACTIONS} from './open-mailbox-send.ts'; import {RegisteredMailboxReceivers,MAILBOX_CONNECT_SCHEMA} from './open-mailbox-receivers.ts'; import {MailboxReceiptJobs,MAILBOX_RECEIPT_ACTIONS,returnMailboxReceipt} from './open-mailbox-receipts.ts'; import {OpenParticipant} from './open-participant.ts'; @@ -38,8 +39,10 @@ export class OpenNetworkClient{ private result(value:Record):Record{if(canonicalBytes(value).length>8192)fail('network_response_too_large');return value;} async connect(invitation?:unknown,requestId?:string):Promise>{ if(invitation!=null){ - if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA) - return this.result(new RegisteredMailboxReceivers(this.participant,this.encryption,this.delivery()).connect(invitation)); + if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA){ + const delivery=this.delivery(); + return this.result(MAILBOX_SEND_ACTIONS.has((invitation as any).action)?await new MailboxSender(this.participant,this.encryption).connect(invitation):new RegisteredMailboxReceivers(this.participant,this.encryption,delivery).connect(invitation)); + } if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA){ if(MAILBOX_RECEIPT_ACTIONS.has((invitation as any).action))return this.result((invitation as any).action==='return_mailbox_receipt'?await returnMailboxReceipt(this.participant,this.encryption,this.delivery(),invitation):new MailboxReceiptJobs(this.participant,this.encryption,this.delivery()).connect(invitation)); return this.result(await ((invitation as any).action==='return_receipt'?returnSavedReceipt:recoverReceipt)(this.participant,this.encryption,this.delivery(),invitation)); diff --git a/clients/typescript/network/open-mailbox-send.ts b/clients/typescript/network/open-mailbox-send.ts new file mode 100644 index 0000000..9fed662 --- /dev/null +++ b/clients/typescript/network/open-mailbox-send.ts @@ -0,0 +1,188 @@ +/** Sender-owned frozen mailbox drafts and actual remote admission. */ +import {performance} from 'node:perf_hooks'; +import {deflateSync} from 'node:zlib'; +import {canonicalBytes,document,sha256,opaqueId,encodeBase64url,decodeBase64url,validateSigningIdentity,validateEncryptionIdentity} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import {transaction} from './io.ts'; +import {RepairBudget,RepairError,buildNewWire,parseNewWire,objectFields,rawRef,u53} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {originalPublicDescriptor,verifyContactOriginals,verifyBoundedControlSignature,verifySourceNodeOriginal} from './open-repair-original.ts'; +import {signBoundedBootstrapOriginal} from './open-repair-probe.ts'; +import {verifyMailboxAckConfiguration} from './open-repair-mailbox-member.ts'; +import {verifyEnvelope} from './open-delivery.ts'; +import {statusScope} from './open-repair-status.ts'; +import {issueStatus} from './open-provider.ts'; +import {endpoint} from './open-transport.ts'; +type Obj=Record; +const SCHEMA='memory-vault-open-mailbox-connect/v1',REPAIR='memory-vault-open-repair/v1'; +export const MAILBOX_SEND_ACTIONS=new Set(['prepare','admit','retain']); +const SLOT_FIELDS={ + slot:'issued_at expires_at revision slot_key feed_ref sender recipient data_resource_ref data_resource_offer_ref metadata_resource_ref metadata_resource_offer_ref read_grant_ref maintenance_root_ref max_appends max_live_items budget windows', + read:'issued_at expires_at grant_id root_key slot_key reader serving_authority_id operation_mask budget windows revision', + maintenance:'issued_at expires_at root_key authority_id slot_key sender recipient maintainers operation_mask allowed_roles budget windows max_delegate_depth max_destinations_per_job max_concurrent_jobs revision'}; +const ACK_ROLES=['ack.root_authority','ack.write_grant','bootstrap.ack_offer','historical.status.ack_root','historical.status.ack_write','historical.status.ack_offer_bootstrap']; +function fail(code:string):never{throw new RepairError(code);} +function same(a:unknown,b:unknown):boolean{return Buffer.from(canonicalBytes(a)).equals(Buffer.from(canonicalBytes(b)));} +/** Shares Python's original-byte journals. Network retries never renew a grant. */ +export class MailboxSender{ + readonly #participant:OpenParticipant;readonly #encryption:EncryptionIdentityDocument; + readonly #policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument){this.#participant=participant;this.#encryption=encryption;} + #entry(value:unknown,budget:RepairBudget):Obj{ + const e=objectFields(value,['raw','ref']);if(typeof e.raw!=='string')fail('open_invalid_mailbox_request'); + return this.#checked(Buffer.from(e.raw,'utf8'),e.ref,budget); + } + #checked(raw:Uint8Array,reference:unknown,budget:RepairBudget):Obj{ + const ref=rawRef(reference);if(raw.length>524288||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch');return {raw,ref}; + } + #decode(value:unknown,budget:RepairBudget):Obj{const e=objectFields(value,['ref','raw_base64url']);return this.#checked(decodeBase64url(e.raw_base64url,524288),e.ref,budget);} + #encode(e:Obj):Obj{return {ref:e.ref,raw_base64url:encodeBase64url(e.raw)};} + #raw(raw:Uint8Array,budget:RepairBudget):Obj{const digest=budget.hash(raw);return {raw,ref:{namespace:'meta',key:digest,raw_sha256:digest,size:raw.length}};} + #dual(value:Obj,budget:RepairBudget):Obj{objectFields(value,['signing_key','encryption_key']);originalPublicDescriptor(value.signing_key,budget);originalPublicDescriptor(value.encryption_key,budget,true);return {signing_key_id:value.signing_key.key_id,encryption_key_id:value.encryption_key.key_id};} + #ack(requestId:string,messageId:string,budget:RepairBudget):Obj{ + opaqueId(requestId);const row=this.#participant.providerStorage(db=>{ + if(!db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='open_ack_agent_preparations'").get())return undefined; + return db.prepare('SELECT result,result_sha256 FROM open_ack_agent_preparations WHERE request_id=?').get(requestId); + }) as Obj|undefined; + if(!row?.result)fail('open_ack_preparation_incomplete');if(row.result.length>1048576||sha256(row.result)!==row.result_sha256)fail('open_ack_preparation_corrupt'); + const prepared=document(row.result,1048576) as Obj;if(prepared.message_id!==messageId)fail('open_ack_preparation_conflict'); + const artifact=objectFields(prepared.recipient_request,['schema_version','base_url','repair_profile','request']) as Obj; + if(artifact.schema_version!=='memory-vault-open-saved-ack-request/v1'||!['receipt','receipt-index'].includes(artifact.repair_profile))fail('repair_invalid_request_bundle'); + const bundle=objectFields(artifact.request,['message_id','envelope_ref','ack_slot','owner','target','target_node_entry','root_entry','write_entry','bootstrap_entry','binding_entry','current_statuses','read_until','retain_until']) as Obj,roles:Obj={}; + if(bundle.message_id!==messageId)fail('open_ack_preparation_conflict'); + for(const [name,field] of [['ack.root_authority','root_entry'],['ack.write_grant','write_entry'],['bootstrap.ack_offer','bootstrap_entry']])roles[name]=this.#decode(bundle[field],budget); + if(!Array.isArray(bundle.current_statuses)||bundle.current_statuses.length>64)fail('open_ack_preparation_corrupt'); + const statuses=bundle.current_statuses.map((v:Obj)=>this.#decode(v,budget)); + for(const [name,role] of [['ack.root_authority','ack_root'],['ack.write_grant','ack_write'],['bootstrap.ack_offer','ack_offer_bootstrap']]){ + const entry=roles[name],p=(parseNewWire(entry.raw,this.#policy,budget).value as Obj).payload; + const scope=statusScope(bundle.ack_slot.root_key,'authority',{authority_kind:p.kind,authority_sha256:entry.ref.raw_sha256},this.#policy,budget); + const matches=statuses.filter((v:Obj)=>(parseNewWire(v.raw,this.#policy,budget).value as Obj).payload.entries.some((r:Obj)=>r.scope_kind==='authority'&&r.scope_id===scope)); + if(matches.length!==1)fail('open_ack_configuration_status_missing');roles['historical.status.'+role]=matches[0]; + } + return roles; + } + #prepare(value:Obj):Obj{ + objectFields(value,['schema_version','action','message_id','slot_entries','destination_entry','attempt_until','consent_until',...(Object.hasOwn(value,'ack_request_id')?['ack_request_id']:[])]); + opaqueId(value.message_id);const budget=new RepairBudget(this.#policy),at=Math.floor(Date.now()/1000); + const row=this.#participant.providerStorage(db=>db.prepare('SELECT envelope,session FROM open_delivery_outbox WHERE message_id=?').get(value.message_id)) as Obj|undefined; + if(!row?.envelope||!row.session)fail('open_delivery_outbox_missing'); + const session=document(row.session,65536) as Obj,request=session.request.payload,recipient={signing_key:session.policy.payload.signing_key,encryption_key:session.policy.payload.encryption_key}; + const sender={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)},senderIds=this.#dual(sender,budget),recipientIds=this.#dual(recipient,budget); + const envelope=verifyEnvelope(row.envelope,{sender_signing_key:sender.signing_key,sender_encryption_key:sender.encryption_key,recipient_signing_key:recipient.signing_key,recipient_encryption_key:recipient.encryption_key,now:at}); + if(!same(request.signing_key,sender.signing_key)||!same(request.encryption_key,sender.encryption_key)||envelope.context.message_id!==value.message_id)fail('repair_message_mismatch'); + const reference={...this.#raw(row.envelope,budget).ref,namespace:'object',key:envelope.context.object_ref.key}; + objectFields(value.slot_entries,['slot','read','maintenance']);const slots=Object.fromEntries(Object.entries(value.slot_entries).map(([n,e])=>[n,this.#entry(e,budget)])); + const destinationEntry=this.#entry(value.destination_entry,budget); + const check=(e:Obj,kind:string,names:string):Obj=>{ + if(e.ref.namespace!=='meta')fail('repair_ref_mismatch');const v=objectFields(parseNewWire(e.raw,this.#policy,budget).value,['payload','proof']),p=objectFields(v.payload,['schema_version','kind','signing_key',...names.split(' ')]) as Obj; + if(p.schema_version!==REPAIR||p.kind!==kind)fail('repair_invalid_message');verifyBoundedControlSignature(p,v.proof,recipient.signing_key,budget);if(u53(p.issued_at)>=u53(p.expires_at))fail('repair_resource_expired');return p; + }; + const slot=check(slots.slot,'mailbox.slot',SLOT_FIELDS.slot),read=check(slots.read,'mailbox.read_grant',SLOT_FIELDS.read),maintenance=check(slots.maintenance,'mailbox.maintenance_root',SLOT_FIELDS.maintenance); + const key=slot.slot_key,root=key.root_key;objectFields(key,['root_key','slot_id','writer','writer_storage_epoch']);opaqueId(key.slot_id);opaqueId(key.writer_storage_epoch); + objectFields(root,['root_kind','owner','owner_epoch','root_id','anchor_ref']);opaqueId(root.owner_epoch);opaqueId(root.root_id);objectFields(root.anchor_ref,['namespace','key']); + if(root.root_kind!=='mailbox'||root.anchor_ref.namespace!=='anchor'||!/^[a-f0-9]{64}$/.test(root.anchor_ref.key))fail('repair_message_mismatch'); + const destination=check(destinationEntry,'delivery.destination','issued_at expires_at destination_id sender recipient contact_request_ref contact_policy_ref contact_knock_lease_ref contact_decision_ref store_grant_ref slot_key slot_ref data_resource_ref data_resource_offer_ref metadata_resource_ref metadata_resource_offer_ref read_grant_ref maintenance_root_ref budget windows'); + if(!same(root.owner,recipientIds)||[slot,maintenance,destination].some(p=>!same(p.sender,senderIds)||!same(p.recipient,recipientIds))||!same(read.reader,recipientIds)|| + [read,maintenance,destination].some(p=>!same(p.slot_key,key))||!same(read.root_key,root)||!same(maintenance.root_key,root)||read.serving_authority_id!==maintenance.authority_id|| + !same(destination.slot_ref,slots.slot.ref)||!same(slot.read_grant_ref,slots.read.ref)||!same(slot.maintenance_root_ref,slots.maintenance.ref))fail('repair_message_mismatch'); + for(const n of ['data_resource_ref','data_resource_offer_ref','metadata_resource_ref','metadata_resource_offer_ref','read_grant_ref','maintenance_root_ref'])if(!same(destination[n],slot[n]))fail('repair_message_mismatch'); + for(const [field,names] of [['budget','max_live_bytes max_meta_bytes max_items max_requests max_pending max_replay_records max_jobs max_job_bytes'],['windows','admit_until read_until copy_until publish_until retain_until']]){ + objectFields(destination[field],names.split(' '));for(const n of names.split(' '))if(u53(destination[field][n])>u53(slot[field][n])||(field==='windows'&&destination[field][n]>destination.windows.retain_until))fail('repair_message_mismatch'); + } + const docs:Obj=Object.fromEntries(['node','policy','request','decision'].map(n=>[n,session[n]]));docs.knock_lease=session.lease;docs.grant=session.decision.payload.grant;docs.delivery_lease=docs.grant.payload.resource_lease; + const contact=Object.fromEntries(Object.entries(docs).map(([n,v])=>[n,canonicalBytes(v)])); + const held=verifyContactOriginals(contact,{senderKeyId:sender.signing_key.key_id,senderEncryptionKeyId:sender.encryption_key.key_id,recipientKeyId:recipient.signing_key.key_id,recipientEncryptionKeyId:recipient.encryption_key.key_id,nodeKeyId:key.writer.signing_key_id,storageEpoch:key.writer_storage_epoch,at:destination.issued_at,policy:this.#policy,budget}); + for(const [n,role] of [['contact_request_ref','request'],['contact_policy_ref','policy'],['contact_knock_lease_ref','knock_lease'],['contact_decision_ref','decision'],['store_grant_ref','grant']]){ + const ref=rawRef(destination[n]),raw=held.originals[role].document.raw;if(ref.raw_sha256!==budget.hash(raw)||ref.size!==raw.length)fail('repair_ref_mismatch'); + } + u53(value.attempt_until);u53(value.consent_until);const ack=Object.hasOwn(value,'ack_request_id')?this.#ack(value.ack_request_id,value.message_id,budget):undefined; + if(ack)verifyMailboxAckConfiguration(ack,{sender,recipient,messageId:value.message_id,envelopeRef:reference,at,policy:this.#policy,budget}); + const encodedAck=ack?Object.fromEntries(Object.entries(ack).map(([n,e])=>[n,this.#encode(e)])):undefined; + const binding=sha256(canonicalBytes({envelope:reference,recipient,slot_refs:Object.fromEntries(Object.entries(slots).map(([n,e])=>[n,e.ref])),destination_ref:destinationEntry.ref, + contact_hashes:Object.fromEntries(Object.entries(contact).map(([n,raw])=>[n,sha256(raw)])),attempt_until:value.attempt_until,consent_until:value.consent_until,...(ack?{ack_configuration:encodedAck}:{})})); + const raw=this.#participant.providerStorage(db=>transaction(db,()=>{ + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_message_drafts(sender TEXT NOT NULL,message_id TEXT NOT NULL,binding TEXT NOT NULL,root_digest TEXT NOT NULL,status_revision INTEGER NOT NULL,envelope BLOB NOT NULL,originals BLOB NOT NULL,PRIMARY KEY(sender,message_id))'); + const old=db.prepare('SELECT binding,envelope,originals FROM open_mailbox_message_drafts WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined; + if(old){if(old.binding!==binding||!Buffer.from(old.envelope).equals(Buffer.from(row.envelope)))fail('repair_message_conflict');return old.originals;} + if((db.prepare('SELECT count(*) AS n FROM open_mailbox_message_drafts').get() as Obj).n>=16)fail('repair_message_capacity'); + if(!(envelope.context.created_at<=at&&at!(p.issued_at<=at&&atsignBoundedBootstrapOriginal({schema_version:REPAIR,kind,signing_key:sender.signing_key,issued_at:at,expires_at:until,...fields},this.#participant.identity,this.#policy,budget); + const consent=sign('message.disclosure',{consent_id:'consent_'+binding,root_key:root,slot_key:key,sender:senderIds,recipient:recipientIds,envelope_ref:reference,maintenance_root_ref:slots.maintenance.ref, + allowed_roles:['contact.request','delivery.attempt','message.disclosure','authority.status.disclosure',...(ack?ACK_ROLES:[])].sort(),operation_mask:127,consent_until:value.consent_until, + bootstrap_return:{subject:recipientIds,consumer:'mailbox_feed',roles:['authority.status.disclosure','message.disclosure'],until:value.consent_until},revision:1},value.consent_until); + const attempt=sign('delivery.attempt',{attempt_id:'attempt_'+binding,message_id:value.message_id,envelope_ref:reference,sender:senderIds,recipient:recipientIds,destination_ref:destinationEntry.ref,slot_key:key, + operation:'message.store',disclosure_ref:consent.ref,ack_grant_ref:ack?.['ack.write_grant'].ref??null},value.attempt_until); + const scope=statusScope(root,'authority',{authority_kind:'message.disclosure',authority_sha256:consent.ref.raw_sha256},this.#policy,budget); + const observed=this.#raw(buildNewWire(issueStatus(this.#participant.identity,{root,revision,entries:[{scope_kind:'authority',scope_id:scope,minimum_document_revision:1,status:'active',operation_mask:127}],issued_at:at,valid_until:value.consent_until}),this.#policy,budget).raw,budget); + const bundle={disclosure:this.#encode(consent),disclosure_status:this.#encode(observed),attempt:this.#encode(attempt),destination:this.#encode(destinationEntry),slot:Object.fromEntries(Object.entries(slots).map(([n,e])=>[n,this.#encode(e)])), + contact:Object.fromEntries(Object.entries(contact).map(([n,bytes])=>[n,this.#encode(this.#raw(bytes,budget))])),...(ack?{ack_configuration:encodedAck}:{})}; + const encoded=buildNewWire(bundle,this.#policy,budget).raw;if(encoded.length>131072)fail('repair_message_capacity'); + db.prepare('INSERT OR REPLACE INTO open_mailbox_message_status_revisions VALUES(?,?,?)').run(sender.signing_key.key_id,rootDigest,revision); + db.prepare('INSERT INTO open_mailbox_message_drafts VALUES(?,?,?,?,?,?,?)').run(sender.signing_key.key_id,value.message_id,binding,rootDigest,revision,row.envelope,encoded);return encoded; + })); + const saved=parseNewWire(raw,this.#policy,new RepairBudget(this.#policy)).value as Obj; + return {state:'mailbox_draft_saved',message_id:value.message_id,attempt_ref:this.#decode(saved.attempt,budget).ref,disclosure_ref:this.#decode(saved.disclosure,budget).ref,network_accessed:false}; + } + async #admit(value:Obj):Promise{ + objectFields(value,['schema_version','action','base_url','message_id','target','target_node_entry','owner_status_entry','expires_at','object_until','enum_until']);opaqueId(value.message_id); + const budget=new RepairBudget(this.#policy),at=Math.floor(Date.now()/1000),sender={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)}; + const nodeEntry=this.#entry(value.target_node_entry,budget),nodePreview=(parseNewWire(nodeEntry.raw,this.#policy,budget).value as Obj).payload; + const node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:value.target.signing_key,expectedStorageEpoch:nodePreview.storage_epoch,at,policy:this.#policy,budget}); + if(nodeEntry.ref.namespace!=='meta'||!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url as string,this.#participant.transport.allow_loopback)))fail('repair_probe_mismatch'); + const packet=this.#participant.providerStorage(db=>transaction(db,()=>{ + if(!db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='open_mailbox_message_drafts'").get())fail('repair_message_delivery_missing'); + const row=db.prepare('SELECT originals FROM open_mailbox_message_drafts WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined;if(!row)fail('repair_message_delivery_missing'); + const raw=Buffer.from(row.originals);if(raw.length>131072)fail('repair_message_capacity');const draft=parseNewWire(raw,this.#policy,budget).value as Obj; + const slot=this.#decode(draft.slot.slot,budget),key=(parseNewWire(slot.raw,this.#policy,budget).value as Obj).payload.slot_key; + if(!same(this.#dual(value.target,budget),key.writer)||node.payload.storage_epoch!==key.writer_storage_epoch)fail('repair_probe_mismatch'); + const plain={raw:raw.toString('utf8'),ref:this.#raw(raw,budget).ref};let encoded:Obj=plain; + if(draft.ack_configuration){const compressed=deflateSync(raw);budget.output(compressed.length);encoded={codec:'zlib-base64url-v1',compressed_size:compressed.length,raw_base64url:encodeBase64url(compressed),ref:plain.ref};} + const payload={schema_version:REPAIR,kind:'mailbox.source_message',signing_key:sender.signing_key,subject:sender,target:value.target,target_storage_epoch:key.writer_storage_epoch,slot_key:key,message_id:value.message_id, + draft:encoded,owner_status:this.#encode(this.#entry(value.owner_status_entry,budget)),object_until:value.object_until,enum_until:value.enum_until}; + const binding=budget.hash(buildNewWire({...payload,draft:plain},this.#policy,budget).raw); + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_message_requests(sender TEXT NOT NULL,message_id TEXT NOT NULL,binding TEXT NOT NULL,raw BLOB NOT NULL,PRIMARY KEY(sender,message_id))'); + const prior=db.prepare('SELECT binding,raw FROM open_mailbox_message_requests WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined; + let requestRaw:Uint8Array; + if(prior){if(prior.binding!==binding)fail('repair_message_conflict');requestRaw=prior.raw;} + else{ + if(!(at65536)fail('repair_remote_setup_too_large');db.prepare('INSERT INTO open_mailbox_message_requests VALUES(?,?,?,?)').run(sender.signing_key.key_id,value.message_id,binding,requestRaw); + } + return {raw:requestRaw,payload,draft}; + })); + const response=await this.#participant.transport.requestRepair(value.base_url,packet.raw,performance.now()/1000+30); + if(response.length>65536)fail('repair_remote_setup_too_large');const signed=objectFields(parseNewWire(response,this.#policy,budget).value,['payload','proof']); + const p=objectFields(signed.payload,['schema_version','kind','signing_key','request_sha256','slot_key','message_id','stored_at','originals']) as Obj; + if(p.schema_version!==REPAIR||p.kind!=='mailbox.source_message_stored'||p.request_sha256!==budget.hash(packet.raw)||!same(p.slot_key,packet.payload.slot_key)||p.message_id!==value.message_id||!same(p.signing_key,value.target.signing_key))fail('repair_message_mismatch'); + u53(p.stored_at);verifyBoundedControlSignature(p,signed.proof,value.target.signing_key,budget); + const kinds={core:'admission.core',link:'admission.link',custody:'message.custody',head:'mailbox.feed_head',feed_custody:'feed.custody'},entries:Obj={},payloads:Obj={};objectFields(p.originals,Object.keys(kinds)); + for(const [n,kind] of Object.entries(kinds)){ + const e=this.#decode(p.originals[n],budget),v=objectFields(parseNewWire(e.raw,this.#policy,budget).value,['payload','proof']),item=v.payload as Obj; + if(item.schema_version!==REPAIR||item.kind!==kind||!same(item.slot_key,packet.payload.slot_key))fail('repair_message_mismatch');verifyBoundedControlSignature(item,v.proof,value.target.signing_key,budget);entries[n]=e;payloads[n]=item; + } + const core=payloads.core,custody=payloads.custody,attempt=this.#decode(packet.draft.attempt,budget),ap=(parseNewWire(attempt.raw,this.#policy,budget).value as Obj).payload; + if(core.message_id!==value.message_id||custody.message_id!==value.message_id||core.object_until!==value.object_until||core.enum_until!==value.enum_until||!same(core.attempt_ref,attempt.ref)||!same(core.envelope_ref,ap.envelope_ref)|| + !same(custody.envelope_ref,core.envelope_ref)||!same(payloads.link.envelope_ref,core.envelope_ref)||payloads.link.message_id!==value.message_id||!same(payloads.feed_custody.feed_head_ref,entries.head.ref)||!same(custody.admission_link_ref,entries.link.ref)||!same(custody.feed_head_ref,entries.head.ref))fail('repair_message_mismatch'); + this.#participant.providerStorage(db=>transaction(db,()=>{ + db.exec('CREATE TABLE IF NOT EXISTS open_mailbox_message_results(sender TEXT NOT NULL,message_id TEXT NOT NULL,raw BLOB NOT NULL,PRIMARY KEY(sender,message_id))'); + const old=db.prepare('SELECT raw FROM open_mailbox_message_results WHERE sender=? AND message_id=?').get(sender.signing_key.key_id,value.message_id) as Obj|undefined; + if(old&&!Buffer.from(old.raw).equals(Buffer.from(response)))fail('repair_message_conflict');db.prepare('INSERT OR IGNORE INTO open_mailbox_message_results VALUES(?,?,?)').run(sender.signing_key.key_id,value.message_id,response); + })); + return {state:'retained_at_mailbox',message_id:value.message_id,stored_at:p.stored_at,network_accessed:true,custody_ref:entries.custody.ref,feed_custody_ref:entries.feed_custody.ref,recipient_acknowledged:false}; + } + async connect(invitation:unknown):Promise{ + const v=document(invitation as any,65536) as Obj;if(v.schema_version!==SCHEMA)fail('open_invalid_mailbox_request'); + if(v.action==='prepare')return this.#prepare(v);if(v.action==='admit')return this.#admit(v); + objectFields(v,['schema_version','action','message_id','authorization','attempt_until','consent_until','expires_at','object_until','enum_until',...(Object.hasOwn(v,'ack_request_id')?['ack_request_id']:[])]); + if(v.action!=='retain')fail('open_invalid_mailbox_request');const a=objectFields(v.authorization,['destination_entry','owner_status_entry','slot_entries','target','target_node_entry','base_url']); + this.#prepare({schema_version:SCHEMA,action:'prepare',message_id:v.message_id,slot_entries:a.slot_entries,destination_entry:a.destination_entry,attempt_until:v.attempt_until,consent_until:v.consent_until,...(Object.hasOwn(v,'ack_request_id')?{ack_request_id:v.ack_request_id}:{})}); + return this.#admit({schema_version:SCHEMA,action:'admit',message_id:v.message_id,...Object.fromEntries(['base_url','target','target_node_entry','owner_status_entry'].map(n=>[n,a[n]])),...Object.fromEntries(['expires_at','object_until','enum_until'].map(n=>[n,v[n]]))}); + } +} diff --git a/clients/typescript/network/open-repair-mailbox-member.ts b/clients/typescript/network/open-repair-mailbox-member.ts index aea9499..937dfab 100644 --- a/clients/typescript/network/open-repair-mailbox-member.ts +++ b/clients/typescript/network/open-repair-mailbox-member.ts @@ -90,6 +90,12 @@ function ackConfiguration(roles:Obj,e:Obj,m:Obj,policy:RepairPolicy,budget:Repai duties.push({role,scope_kind:'authority',scope_id:statusScope(root,'authority',{authority_kind:item.payload.kind,authority_sha256:item.ref.raw_sha256},policy,budget),document_revision:item.payload.revision,operation_mask:mask,signer:e.sender.signing_key});} return Object.freeze({originals:checked.originals,statuses:historicalStatuses(roles,duties,root,e.at,policy,budget)}); } +/** Sender preparation verifies the same complete independent receipt originals. */ +export function verifyMailboxAckConfiguration(roles:Obj,options:{sender:Obj;recipient:Obj;messageId:string;envelopeRef:unknown;at:number;policy:RepairPolicy;budget:RepairBudget}):Obj{ + if(!roleSet(roles,ACK))fail('repair_original_missing'); + return ackConfiguration(roles,{sender:options.sender,owner:options.recipient,at:options.at}, + {message_id:options.messageId,envelope_ref:options.envelopeRef},options.policy,options.budget); +} function member(resolved:DraftHistoryInputs,e:Obj,policy:RepairPolicy,budget:RepairBudget):Obj{ const graph=setup(resolved,e,policy,budget),{roles,originals}=graph,m=resolved.manifest.value as Obj,root=m.root_key,key=m.slot_key,at=e.at; if(!roleSet(roles,BASE)&&!roleSet(roles,[...BASE,...ACK]))fail(); diff --git a/clients/typescript/network/package.json b/clients/typescript/network/package.json index be3a478..5a8d4ed 100644 --- a/clients/typescript/network/package.json +++ b/clients/typescript/network/package.json @@ -51,7 +51,8 @@ "./open-mailbox-journal": "./open-mailbox-journal.ts", "./open-mailbox-receivers": "./open-mailbox-receivers.ts", "./open-mailbox-receipts": "./open-mailbox-receipts.ts", - "./open-mailbox-destination": "./open-mailbox-destination.ts" + "./open-mailbox-destination": "./open-mailbox-destination.ts", + "./open-mailbox-send": "./open-mailbox-send.ts" }, "types": "./crypto.ts", "engines": { diff --git a/docs/OPEN_NETWORK_CONTACT.md b/docs/OPEN_NETWORK_CONTACT.md index 76b3b20..1c3dc52 100644 --- a/docs/OPEN_NETWORK_CONTACT.md +++ b/docs/OPEN_NETWORK_CONTACT.md @@ -201,8 +201,12 @@ Native TypeScript also supports receiver `authorize` and paginated `inspect` in development source after alpha.0.37. It shares the exact destination journal with Python, signs only with the receiver's own existing identity, and performs no HTTP requests during authorization. Both clients apply authenticated retained -revocations before returning or issuing authorization. Source provisioning and -sender mailbox admission still use Python. +revocations before returning or issuing authorization. Source provisioning still +uses Python. In development after alpha.0.38, native senders also support the `prepare`, `admit` and `retain` invitations below. They share +the frozen ciphertext, disclosure, exact HTTP request and verified custody +result with Python. Restarting or changing clients does not renew authority. +An optional `ack_request_id` carries only the explicitly selected, independently +prepared original receipt grants; custody does not imply recipient acknowledgement. A receiver using the Agent interface can authorize from an existing registered mailbox and its locally approved contact, without opening a database or loading diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index 47044f6..e689022 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -146,7 +146,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 66f6d2d..7f1ce3a 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -123,7 +123,7 @@ "tests/test_open_repair_roundtrip.py", "tests/test_open_repair_receipt.py", "tests/test_open_repair_receipt_typescript.py", - "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", + "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", "tests/test_open_mailbox_send_typescript.py", "tests/open_repair_index_fixtures.py", "tests/test_open_repair_stage.py", "tests/test_open_repair_index.py", @@ -248,7 +248,7 @@ "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", - "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", + "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 894d624..6295227 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -54,7 +54,7 @@ "tests.test_open_repair_index_prepare", "tests.test_open_repair_index_prepare_admin", "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", - "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_mailbox_destination_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_mailbox_destination_typescript", "tests.test_open_mailbox_send_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +244,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-mailbox-send.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index b6c0564..e4dc618 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -46,7 +46,7 @@ "open-repair-original.ts", "open-repair-resource.ts", "open-repair-bootstrap.ts", - "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", + "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-mailbox-send.ts", "open-repair-ack.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 69) + self.assertEqual(len(TS_NETWORK), 70) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -122,7 +122,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "tests/test_open_repair_status.py", "tests/test_open_repair_status_typescript.py", "tests/open_repair_ack_fixtures.py", "tests/test_open_repair_ack.py", "tests/test_open_repair_ack_typescript.py", "tests/test_open_repair_state.py", "tests/test_open_repair_mailbox_resources.py", "tests/test_open_repair_copy_resources.py", - "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", + "tests/test_open_repair_copy_prepare.py", "tests/test_open_repair_copy_state.py", "tests/test_open_repair_copy_service.py", "tests/test_open_repair_copy_upload.py", "tests/test_open_repair_copy_empty.py", "tests/test_open_repair_mailbox_activation.py", "tests/test_open_repair_mailbox_range.py", "tests/test_open_repair_mailbox_authority_typescript.py", "tests/test_open_repair_mailbox_member_typescript.py", "tests/test_open_mailbox_client_typescript.py", "tests/test_open_mailbox_receipts_typescript.py", "tests/test_open_mailbox_destination_typescript.py", "tests/test_open_mailbox_send_typescript.py", "tests/test_open_repair_mailbox_root.py", "tests/test_open_repair_mailbox_status.py", "tests/test_open_repair_mailbox_source.py", "tests/test_open_repair_mailbox_snapshot.py", "tests/test_open_repair_mailbox_copy.py", "tests/test_open_repair_mailbox_copy_authority.py", "tests/test_open_repair_mailbox_copy_upload.py", "tests/test_open_repair_mailbox_reservation.py", "tests/test_open_repair_mailbox_consent.py", "tests/test_open_mailbox_replica_receive.py", "tests/test_open_mailbox_receipt_jobs.py", "tests/test_open_ack_replica_send.py", "tests/test_open_mailbox_copy_jobs.py", "tests/test_open_repair_mailbox_copy_admin.py", "tests/test_open_repair_mailbox_feed_copy.py", "tests/test_open_repair_mailbox_message_copy.py", "tests/test_open_repair_probe.py", "tests/test_open_repair_probe_typescript.py", "tests/test_open_repair_offer_client_typescript.py", "tests/test_open_repair_put_client_typescript.py", "tests/test_open_repair_receipt_typescript.py", @@ -188,7 +188,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-wire.ts", "clients/typescript/network/open-repair-history.ts", "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", - "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", + "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", diff --git a/tests/test_open_mailbox_send_typescript.py b/tests/test_open_mailbox_send_typescript.py new file mode 100644 index 0000000..64f8292 --- /dev/null +++ b/tests/test_open_mailbox_send_typescript.py @@ -0,0 +1,78 @@ +"""Native sender retains exact drafts, submits actual HTTP and reopens in Python.""" +import json +import sqlite3 +import subprocess +import unittest +from pathlib import Path +from tests import test_open_delivery_http as fixtures +from tests import test_open_ack_typescript_agent as runtime + +class NativeMailboxSenderTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.NativeAckAgentTests.setUpClass.__func__(cls) + driver=runtime.DRIVER.replace("const {Agent}", "const {NetworkError}=await import('./io.ts');\nconst {Agent}",1) + driver=driver.replace("const result=await original.apply(this,args);", "const result=await original.apply(this,args);\n if(name==='requestRepair'&&input.lose_reply)throw new NetworkError('open_network_unavailable',true);") + (cls.fixture/'driver.mjs').write_text(driver) + + def native(self,agent,request,lose=False,no_network=False): + run=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(client_config=str(agent.client_config),network_config=str(agent.network_config),requests=[request],lose_reply=lose,no_network=no_network)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=45) + self.assertEqual(run.returncode,0,run.stderr.decode(errors='replace')[-2500:]) + result=json.loads(run.stdout);self.assertEqual(result['subprocessCalls'],0) + if no_network:self.assertEqual(result['calls'],[]) + return result['results'][0] + + def test_native_prepare_admit_restart_and_python_reuse(self): + self.journey(False) + + def test_native_sender_preserves_independent_receipt_permission(self): + self.journey(True) + + def test_native_compressed_receipt_draft_admitted_over_http(self): + self.journey(True,True) + + def journey(self,ack,remote=False): + name='test_cold_mailbox_returns_independent_receipt' if ack else 'test_sender_admits_message_over_http' + if remote:name='test_ack_configuration_admitted_over_http' + h=fixtures.MailboxStagingHTTPTests(name);h.setUp();self.addCleanup(h.doCleanups) + original=h.a.handle;actions=[];lost=[];snapshots={} + def state():return Path(json.loads(Path(h.a.network_config).read_bytes())['state_directory'])/'network.sqlite3' + def saved(message): + db=sqlite3.connect(state()) + try: + self.assertEqual(db.execute('PRAGMA integrity_check').fetchall(),[('ok',)]) + draft=db.execute('SELECT originals,envelope FROM open_mailbox_message_drafts WHERE message_id=?',(message,)).fetchone() + if db.execute("SELECT 1 FROM sqlite_master WHERE name='open_mailbox_message_requests'").fetchone(): + packet=db.execute('SELECT raw FROM open_mailbox_message_requests WHERE message_id=?',(message,)).fetchone() + else:packet=None + return draft,packet + finally:db.close() + def dispatch(request): + value=request.get('invitation',{}) + if value.get('schema_version')!='memory-vault-open-mailbox-connect/v1' or value.get('action') not in ('prepare','admit','retain'): + return original(request) + action=value['action'];lose=action=='retain' and not lost + result=self.native(h.a,request,lose,action=='prepare');actions.append(action) + if lose: + lost.append(True);self.assertFalse(result['ok'],result);snapshots[value['message_id']]=saved(value['message_id']);return result + self.assertTrue(result['ok'],result) + before=saved(value['message_id']) + if value['message_id'] in snapshots:self.assertEqual(before,snapshots[value['message_id']]) + # Reopen the same originals through Python and compare response and persisted bytes. + reopened=original(request);self.assertTrue(reopened['ok'],reopened);self.assertEqual(reopened['result'],result['result']);self.assertEqual(saved(value['message_id']),before) + if action=='prepare': + changed=dict(value,consent_until=value['consent_until']-1) + denied=self.native(h.a,dict(request,invitation=changed),no_network=True);self.assertFalse(denied['ok']);self.assertEqual(denied['error']['code'],'repair_message_conflict') + self.assertEqual(saved(value['message_id']),before) + if action=='admit': + changed=dict(value,enum_until=value['enum_until']+1) + denied=self.native(h.a,dict(request,invitation=changed),no_network=True);self.assertFalse(denied['ok']);self.assertEqual(denied['error']['code'],'repair_message_conflict') + return result + h.a.handle=dispatch + getattr(h,name)() + self.assertIn('prepare',actions) + if not ack or remote:self.assertIn('admit',actions);self.assertEqual(lost,[True]) + +if __name__=='__main__':unittest.main() From 3df5cf8e58c94c0c2f703d574321f3c2561ffcec Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:51:11 +0800 Subject: [PATCH 20/24] Prepare alpha39 native sender mailbox release --- memory_vault.py | 2 +- packaging/CLIENT_README.md | 18 +++++++++++++++-- packaging/PROTOCOL_README.md | 18 +++++++++++++++-- packaging/RELEASE_NOTES.md | 20 ++++++++++++++++--- packaging/REVIEW_README.md | 16 ++++++++++++++- .../.codex-plugin/plugin.json | 2 +- 6 files changed, 66 insertions(+), 10 deletions(-) diff --git a/memory_vault.py b/memory_vault.py index b8e6782..9554cbe 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.38" +VERSION = "0.28.0-alpha.0.39" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index 0012950..b110254 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,18 @@ -# Memory Vault v0.28.0-alpha.0.38 — authorized full client +# Memory Vault v0.28.0-alpha.0.39 — authorized full client + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. Native TypeScript recipients can issue an original mailbox destination from an explicitly approved contact and a registered mailbox. The receiver signs with @@ -166,7 +180,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.38 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.39 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index c09dd74..b9d7953 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,18 @@ -# Memory Vault v0.28.0-alpha.0.38 — independent protocol +# Memory Vault v0.28.0-alpha.0.39 — independent protocol + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. Native TypeScript recipients can issue an original mailbox destination from an explicitly approved contact and a registered mailbox. The receiver signs with @@ -183,7 +197,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.38; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.39; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index d6be29d..86301ad 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,18 @@ -# Memory Vault v0.28.0-alpha.0.38 — native receiver mailbox authorization +# Memory Vault v0.28.0-alpha.0.39 — native sender mailbox admission + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. Native TypeScript recipients can issue an original mailbox destination from an explicitly approved contact and a registered mailbox. The receiver signs with @@ -192,7 +206,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.38/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.38/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.39/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.39/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 0339b45..7418368 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,18 @@ -# Memory Vault v0.28.0-alpha.0.38 independent review kit +# Memory Vault v0.28.0-alpha.0.39 independent review kit + +Native TypeScript senders can prepare and retain an existing outgoing message +in the recipient-authorized mailbox through Agent `connect`. The sender keeps +the original ciphertext and persists its exact disclosure, attempt and upload +request before HTTP. A lost reply, restart or switch to Python reuses those same +bytes. The source returns signed custody that the client verifies and retains; +this does not claim the recipient saved the message or acknowledged it. + +An optional `ack_request_id` selects an already prepared independent receipt +permission. Complete signed controls and statuses travel with the message, +including bounded compressed transport for the larger draft. No new permission +or storage allocation is inferred. Native source provisioning and replica inbox +recovery still use Python. See `clients/typescript/network/README.md` and +`docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. Native TypeScript recipients can issue an original mailbox destination from an explicitly approved contact and a registered mailbox. The receiver signs with diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index 0f7caa2..46f92bb 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.38", + "version": "0.28.0-alpha.0.39", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" From e6a7adf4a40087b2868e924bbf35db5780b805e6 Mon Sep 17 00:00:00 2001 From: smartyue Date: Fri, 2 Oct 2026 21:55:47 +0800 Subject: [PATCH 21/24] Distribute bounded delivery resources and verify native message replicas --- .well-known/agent-memory.json | 22 +- AI_START_HERE.md | 4 +- README.md | 16 +- clients/typescript/network/README.md | 2 + clients/typescript/network/open-ack-status.ts | 8 +- clients/typescript/network/open-client.ts | 2 +- .../network/open-delivery-client.ts | 80 +- .../network/open-mailbox-receivers.ts | 16 +- .../network/open-mailbox-replica-client.ts | 112 + .../network/open-mailbox-replicas.ts | 69 + .../network/open-repair-mailbox-inbox.ts | 2 + .../network/open-repair-mailbox-read.ts | 16 +- .../open-repair-mailbox-replica-inbox.ts | 27 + .../network/open-repair-mailbox-replica.ts | 170 ++ .../typescript/network/open-repair-status.ts | 4 + clients/typescript/network/package.json | 6 +- docs/CAPACITY_HANDOFF.md | 158 ++ docs/DELIVERY_CAPACITY.md | 176 ++ docs/HTTP_CAPACITY.md | 59 + docs/RELEASE.md | 4 + docs/ROUTING_CAPACITY.md | 135 ++ docs/STATUS.md | 4 + docs/evidence/capacity-alpha40/after.json | 1818 +++++++++++++++++ docs/evidence/capacity-alpha40/before.json | 1714 ++++++++++++++++ .../capacity-alpha40/native-replica.json | 52 + .../capacity-alpha40/replica-profile.json | 115 ++ llms.txt | 12 +- memory_vault.py | 2 +- memory_vault_network.py | 40 +- memory_vault_open_delivery_client.py | 107 +- memory_vault_open_index.py | 31 +- memory_vault_open_node.py | 167 +- .../.codex-plugin/plugin.json | 2 +- scripts/benchmark_native_runtime.py | 92 + scripts/benchmark_open_delivery.py | 254 +++ scripts/benchmark_open_http.py | 171 ++ scripts/benchmark_open_neighbors.py | 259 +++ scripts/build_client_plugin.py | 4 + scripts/build_release.py | 10 + scripts/release_source.py | 8 +- scripts/run_open_network_ci.py | 3 +- tests/test_capacity_replica_closure.py | 169 ++ tests/test_network_packaging.py | 6 +- tests/test_open_delivery_distribution.py | 214 ++ ...t_open_delivery_distribution_typescript.py | 115 ++ tests/test_open_mailbox_replica_typescript.py | 54 + tests/test_open_neighbors.py | 130 ++ tests/test_open_read_concurrency.py | 144 ++ 48 files changed, 6685 insertions(+), 100 deletions(-) create mode 100644 clients/typescript/network/open-mailbox-replica-client.ts create mode 100644 clients/typescript/network/open-mailbox-replicas.ts create mode 100644 clients/typescript/network/open-repair-mailbox-replica-inbox.ts create mode 100644 clients/typescript/network/open-repair-mailbox-replica.ts create mode 100644 docs/CAPACITY_HANDOFF.md create mode 100644 docs/DELIVERY_CAPACITY.md create mode 100644 docs/HTTP_CAPACITY.md create mode 100644 docs/ROUTING_CAPACITY.md create mode 100644 docs/evidence/capacity-alpha40/after.json create mode 100644 docs/evidence/capacity-alpha40/before.json create mode 100644 docs/evidence/capacity-alpha40/native-replica.json create mode 100644 docs/evidence/capacity-alpha40/replica-profile.json create mode 100644 scripts/benchmark_native_runtime.py create mode 100644 scripts/benchmark_open_delivery.py create mode 100644 scripts/benchmark_open_http.py create mode 100644 scripts/benchmark_open_neighbors.py create mode 100644 tests/test_capacity_replica_closure.py create mode 100644 tests/test_open_delivery_distribution.py create mode 100644 tests/test_open_delivery_distribution_typescript.py create mode 100644 tests/test_open_mailbox_replica_typescript.py create mode 100644 tests/test_open_neighbors.py create mode 100644 tests/test_open_read_concurrency.py diff --git a/.well-known/agent-memory.json b/.well-known/agent-memory.json index 7dc1687..dee7bbc 100644 --- a/.well-known/agent-memory.json +++ b/.well-known/agent-memory.json @@ -1,20 +1,19 @@ { "schema_version": "memory-vault-discovery/v1", "name": "Memory Vault", - "implementation_version": "0.28.0-alpha.0.37", + "implementation_version": "0.28.0-alpha.0.40", "implementation_status": "alpha_preview", "license": "Apache-2.0", "repository": "https://github.com/qh-work/memory-vault-sync", - "release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37", - "release_publication_verified": true, + "release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40", "release_assets": { - "client": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-client-v0.28.0-alpha.0.37.zip", - "protocol": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-protocol-v0.28.0-alpha.0.37.zip", - "review": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-review-v0.28.0-alpha.0.37.zip", - "network_trial": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-network-test-v0.28.0-alpha.0.37.zip", - "checksums": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/SHA256SUMS" + "client": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-client-v0.28.0-alpha.0.40.zip", + "protocol": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-protocol-v0.28.0-alpha.0.40.zip", + "review": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-review-v0.28.0-alpha.0.40.zip", + "network_trial": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-network-test-v0.28.0-alpha.0.40.zip", + "checksums": "https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/SHA256SUMS" }, - "previous_release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.36", + "previous_release": "https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37", "relative_links_base": "this discovery document", "start_here": "../AI_START_HERE.md", "specification": "../PROTOCOL.md", @@ -46,7 +45,7 @@ "automatic_sync_default": false, "readme_is_installation": false, "runtime_tests_run_for_release": true, - "runtime_test_scope": "Twelve checks passed using the extracted alpha.0.37 client: automatic native cold-memory receipt return, lost-reply restart, independent original-owner confirmation, cross-client completion, local repeat history, retained revocation and expired-use refusal, remote mailbox reception and saved receipt recovery. Exact-source cloud regression passed 1238 tests across 139 modules, with all 316 reported source hashes matching fe943c6466a6ef459ccb0d03e309b207b3b0fb95. All three Python 3.10 suites and cross-platform conformance passed. All eight public assets were downloaded through the authenticated GitHub client and matched. These results do not establish independent external adoption, global reliability or thousand-agent capacity.", + "runtime_test_scope": "63 local targeted cases passed; isolated native 48-message/2-per-second trial completed 39 -> 48 chains. Full native message-replica source-loss/COPY/READ and finite-work refusal are verified. See release CI and capacity report for final exact-source validation. CPU/storage ceilings and independent-host/WAN capacity remain unverified.", "runtime_test_report": "../docs/RELEASE.md", "full_runtime_test_suite_run": true, "runtime_parity_verified": false, @@ -124,5 +123,6 @@ "runnable_package": "review", "source_owned_reads_required": true, "external_model_trial_verified": false - } + }, + "publication_verification": "Use the published GitHub release record and exact-source release manifest; this static discovery document is not live publication evidence." } diff --git a/AI_START_HERE.md b/AI_START_HERE.md index d7f880e..008fe1a 100644 --- a/AI_START_HERE.md +++ b/AI_START_HERE.md @@ -1,5 +1,7 @@ # Memory Vault: connect, remember, exchange, continue +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](docs/CAPACITY_HANDOFF.md). + Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](clients/typescript/network/README.md). Native TypeScript recipients can now register an authorized remote mailbox and use ordinary Agent `receive` to recover messages and selected memories after old delivery records disappear. Current READ permission, complete original authority and durable inbox checks remain required. Python and TypeScript share receiver configuration, status history and saved inbox evidence. See [native mailbox recovery](clients/typescript/network/README.md). @@ -10,7 +12,7 @@ Sources can now republish an original ACK location through another independently Python and native TypeScript senders can now locate an original-source ACK receipt through an explicitly selected directory using `connect/recover_discovered_receipt`. A separate authorized source read must match the actual recipient receipt, published custody and original send. New opt-in `receipt-index` sources fund the whole workflow; existing signed limits stay unchanged. See [directory receipt recovery](docs/OPEN_ACK_DIRECTORY.md#agent-directory-recovery-development-after-alpha032). -Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). +Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and native ACK/root/feed replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). Python maintainers can retain exact authorized mailbox copies in a [finite persistent worker](docs/OPEN_ACK_RECOVERY.md). Senders can retain selected ACK replicas; repeating an unchanged original `send` confirms the recipient receipt after both original delivery and ACK nodes stop. Exact requests and signed refusal history survive restart. Destination selection and independent permissions remain explicit. See [ACK replica confirmation](docs/OPEN_ACK_PROVISIONING.md). diff --git a/README.md b/README.md index 8c92fc3..b392414 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Memory Vault — an agent communication and memory network +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](docs/CAPACITY_HANDOFF.md). + Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](clients/typescript/network/README.md). Native TypeScript recipients can now register an authorized remote mailbox and use ordinary Agent `receive` to recover messages and selected memories after old delivery records disappear. Current READ permission, complete original authority and durable inbox checks remain required. Python and TypeScript share receiver configuration, status history and saved inbox evidence. See [native mailbox recovery](clients/typescript/network/README.md). @@ -10,7 +12,7 @@ Sources can now republish an original ACK location through another independently Python and native TypeScript senders can now locate an original-source ACK receipt through an explicitly selected directory using `connect/recover_discovered_receipt`. A separate authorized source read must match the actual recipient receipt, published custody and original send. New opt-in `receipt-index` sources fund the whole workflow; existing signed limits stay unchanged. See [directory receipt recovery](docs/OPEN_ACK_DIRECTORY.md#agent-directory-recovery-development-after-alpha032). -Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). +Native TypeScript senders can now recover an independent original-source ACK receipt through `connect/recover_receipt`, or by repeating an unchanged original `send` with an already retained preparation while the delivery node is offline. Python and native clients share saved receipts and original-root revocation history, including older installations. Preparation and native ACK/root/feed replica reception still use Python; native ordinary mailbox reception and explicit saved receipt return are available. See [native ACK confirmation](docs/OPEN_ACK_PROVISIONING.md#native-agent-receipt-recovery-source-after-alpha031). Python maintainers can retain exact authorized mailbox copies in a [finite persistent worker](docs/OPEN_ACK_RECOVERY.md). Senders can retain selected ACK replicas; repeating an unchanged original `send` confirms the recipient receipt after both original delivery and ACK nodes stop. Exact requests and signed refusal history survive restart. Destination selection and independent permissions remain explicit. See [ACK replica confirmation](docs/OPEN_ACK_PROVISIONING.md). @@ -74,7 +76,7 @@ provenance and exchange contract in their preferred language and storage. ## Join the open network: encrypted messages and shared memories **[Open-network quickstart](docs/OPEN_NETWORK_QUICKSTART.md)** is the entry for -agents and node operators. Python and native TypeScript in **0.28.0-alpha.0.37** support +agents and node operators. Python and native TypeScript in **0.28.0-alpha.0.40** support encrypted `send`, `receive` and local message reads after explicit first-contact approval. An agent can send text or select original memories for sharing. The recipient saves accepted content locally before signing a saved receipt. @@ -178,15 +180,15 @@ The pre-existing MCP memory interface remains for existing users. ## Download the current preview -Use the assets for **[v0.28.0-alpha.0.37](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37)**. +Use the assets for **[v0.28.0-alpha.0.40](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40)**. The [open-network quickstart](docs/OPEN_NETWORK_QUICKSTART.md) works from the full client archive without installing a plugin. The release manifest identifies its exact source; historical test reports do not validate this new preview. -- **[Protocol-only package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-protocol-v0.28.0-alpha.0.37.zip):** specification, schemas and synthetic examples; no executable. -- **[Full plugin package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-client-v0.28.0-alpha.0.37.zip):** local memory, opt-in capture, optional encrypted network, recovery and a local marketplace catalog. -- **[Independent review kit](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-review-v0.28.0-alpha.0.37.zip):** public source and synthetic tests; nothing runs automatically. -- **[Synthetic network trial](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.37/memory-vault-network-test-v0.28.0-alpha.0.37.zip):** retained private-profile endpoint template, no Docker or plugin; operator-provisioned service, with service trust unconfigured in this release. +- **[Protocol-only package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-protocol-v0.28.0-alpha.0.40.zip):** specification, schemas and synthetic examples; no executable. +- **[Full plugin package](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-client-v0.28.0-alpha.0.40.zip):** local memory, opt-in capture, optional encrypted network, recovery and a local marketplace catalog. +- **[Independent review kit](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-review-v0.28.0-alpha.0.40.zip):** public source and synthetic tests; nothing runs automatically. +- **[Synthetic network trial](https://github.com/qh-work/memory-vault-sync/releases/download/v0.28.0-alpha.0.40/memory-vault-network-test-v0.28.0-alpha.0.40.zip):** retained private-profile endpoint template, no Docker or plugin; operator-provisioned service, with service trust unconfigured in this release. - **Core source:** [`memory_vault.py`](memory_vault.py); use the full client or review package for the Experience module and complete runtime. Alpha.3 makes current records deterministically rank before superseded/resolved diff --git a/clients/typescript/network/README.md b/clients/typescript/network/README.md index 917dadc..ef1d15b 100644 --- a/clients/typescript/network/README.md +++ b/clients/typescript/network/README.md @@ -1,5 +1,7 @@ # Independent network-v1 endpoint candidate +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](../../../docs/CAPACITY_HANDOFF.md). + This optional Node TypeScript package implements the existing Memory Vault network-v1 envelope without invoking Python. It is separate from the zero dependency HTTP SDK in the parent directory. It uses `jose` 6.2.10 for X25519 diff --git a/clients/typescript/network/open-ack-status.ts b/clients/typescript/network/open-ack-status.ts index 966d3ea..e4ce1d6 100644 --- a/clients/typescript/network/open-ack-status.ts +++ b/clients/typescript/network/open-ack-status.ts @@ -14,9 +14,9 @@ function fail(code:string):never{throw new NetworkError(code);} * Transactions are synchronous and finish before any network await. */ export class OriginalAckStatusJournal{ readonly root:Buffer;readonly key:string;private readonly participant:OpenParticipant; - constructor(participant:OpenParticipant,root:unknown){ + constructor(participant:OpenParticipant,root:unknown,originalSource=true){ this.participant=participant; - this.root=bytes(root);this.key=hash(Buffer.concat([Buffer.from('original-source\0'),this.root])); + this.root=bytes(root);this.key=hash(originalSource?Buffer.concat([Buffer.from('original-source\0'),this.root]):this.root); participant.providerStorage(db=>db.exec(` CREATE TABLE IF NOT EXISTS open_ack_replica_roots(root_id TEXT PRIMARY KEY,root BLOB NOT NULL,blocked INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS open_ack_replica_statuses(root_id TEXT NOT NULL,ref_id TEXT NOT NULL,issuer TEXT NOT NULL,raw BLOB NOT NULL,ref BLOB NOT NULL,PRIMARY KEY(root_id,ref_id));`)); @@ -79,3 +79,7 @@ export class OriginalAckStatusJournal{ if(full)fail('repair_status_history_capacity'); } } +/** Exact Python replica domain; selection removal cannot erase observations. */ +export class ReplicaStatusJournal extends OriginalAckStatusJournal{ + constructor(participant:OpenParticipant,root:unknown){super(participant,root,false);} +} diff --git a/clients/typescript/network/open-client.ts b/clients/typescript/network/open-client.ts index 33a7e37..53b23f3 100644 --- a/clients/typescript/network/open-client.ts +++ b/clients/typescript/network/open-client.ts @@ -41,7 +41,7 @@ export class OpenNetworkClient{ if(invitation!=null){ if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===MAILBOX_CONNECT_SCHEMA){ const delivery=this.delivery(); - return this.result(MAILBOX_SEND_ACTIONS.has((invitation as any).action)?await new MailboxSender(this.participant,this.encryption).connect(invitation):new RegisteredMailboxReceivers(this.participant,this.encryption,delivery).connect(invitation)); + return this.result(MAILBOX_SEND_ACTIONS.has((invitation as any).action)?await new MailboxSender(this.participant,this.encryption).connect(invitation):await new RegisteredMailboxReceivers(this.participant,this.encryption,delivery).connect(invitation)); } if(typeof invitation==='object'&&!Array.isArray(invitation)&&(invitation as any).schema_version===ACK_CONNECT_SCHEMA){ if(MAILBOX_RECEIPT_ACTIONS.has((invitation as any).action))return this.result((invitation as any).action==='return_mailbox_receipt'?await returnMailboxReceipt(this.participant,this.encryption,this.delivery(),invitation):new MailboxReceiptJobs(this.participant,this.encryption,this.delivery()).connect(invitation)); diff --git a/clients/typescript/network/open-delivery-client.ts b/clients/typescript/network/open-delivery-client.ts index 8ddb1f3..d1e1b29 100644 --- a/clients/typescript/network/open-delivery-client.ts +++ b/clients/typescript/network/open-delivery-client.ts @@ -3,6 +3,8 @@ */ import {randomBytes} from 'node:crypto'; import {verifyMailboxInboxEvidence} from './open-repair-mailbox-inbox.ts'; +import {MailboxMessageReplicaRecoveryClient} from './open-mailbox-replica-client.ts'; +import type {MessageReplicaRecoverOptions} from './open-mailbox-replica-client.ts'; import {MailboxFeedRecoveryClient} from './open-mailbox-client.ts'; import type {MailboxRecoverOptions} from './open-mailbox-client.ts'; import {performance} from 'node:perf_hooks'; @@ -188,7 +190,7 @@ export class OpenDeliveryClient{ const checked=verifyDecision(session.decision,{request:session.request,policy:session.policy,lease:session.lease,node}); if(checked.decision!=='approved'||!checked.grant)fail('open_contact_approval_required');return {...session,node}; } - private async sendingSession(recipient:string,budget:DeliveryBudget):Promise{ + private async sendingSession(recipient:string,budget:DeliveryBudget,row?:Obj):Promise{ let sessions=this.contactSessions(true,recipient); if(!sessions.length){ const rows=this.db(db=>db.prepare("SELECT body FROM open_contact_local WHERE category='outgoing' AND expires_at>? ORDER BY expires_at DESC,reference LIMIT 128").all(now())) as Obj[]; @@ -208,8 +210,58 @@ export class OpenDeliveryClient{ }finally{budget.merge(child);}break; }sessions=this.contactSessions(true,recipient); } - for(const session of sessions){try{return await this.checkedSession(session,budget);}catch(error){if(!['contact_expired','contact_policy_mismatch'].includes((error as any)?.code))throw error;}} - fail('open_contact_approval_required'); + const eligible:Obj[]=[],unavailable:unknown[]=[]; + for(const session of sessions){try{const checked=await this.checkedSession(session,budget); + if(!row)return checked;eligible.push(checked); + }catch(error){const code=(error as any)?.code; + if(['contact_expired','contact_policy_mismatch'].includes(code))continue; + if(row&&code==='contact_resource_unresolved'){unavailable.push(error);continue;}throw error;}} + if(!eligible.length){if(unavailable.length)throw unavailable[0];fail('open_contact_approval_required');} + const frozen=await this.freezeDispatch(row!,eligible);return document(frozen.session,MAX_SESSION_BYTES) as Obj; + } + private static sessionScope(session:Obj):string{ + // Exact signed resource lease, identical to Python. Aliases share a quota. + return sha256(canonicalBytes(session.decision.payload.grant.payload.resource_lease)); + } + private async freezeDispatch(row:Obj,sessions:Obj[]):Promise{ + const keys=OpenDeliveryClient.keys(sessions[0]); + sessions=sessions.filter(session=>same(OpenDeliveryClient.keys(session),keys)); + const envelope=await createEnvelope(row.body,{signer:this.participant.identity, + sender_encryption_key:validateEncryptionIdentity(this.encryption), + recipient_signing_key:keys.recipient_signing_key,recipient_encryption_key:keys.recipient_encryption_key, + message_id:row.message_id,object_key:randomBytes(32).toString('hex'),created_at:row.created_at}); + const encoded=canonicalBytes(envelope,MAX_ENVELOPE_BYTES); + return this.db(db=>transaction(db,()=>{ + const prior=db.prepare('SELECT * FROM open_delivery_outbox WHERE request_id=?').get(row.request_id) as Obj|undefined; + if(!prior)fail('open_delivery_outbox_missing');if(prior.envelope)return prior; + const used=new Map(); + // Admission already limits the local outbox to 1024 records. + for(const saved of db.prepare('SELECT session,length(envelope) AS bytes FROM open_delivery_outbox WHERE session IS NOT NULL LIMIT 1024').all() as Obj[]){ + const scope=OpenDeliveryClient.sessionScope(document(saved.session,MAX_SESSION_BYTES) as Obj),count=used.get(scope)??{items:0,bytes:0}; + used.set(scope,{items:count.items+1,bytes:count.bytes+(saved.bytes||MAX_ENVELOPE_BYTES)}); + } + const available=sessions.map(session=>{const lease=session.decision.payload.grant.payload.resource_lease.payload, + scope=OpenDeliveryClient.sessionScope(session),count=used.get(scope)??{items:0,bytes:0}; + return {session,scope,lease,count}; + }).filter(v=>v.count.itemsencoded.length>session.decision.payload.grant.payload.resource_lease.payload.max_bytes))fail('open_delivery_approval_bytes_insufficient'); + fail('open_delivery_approval_capacity');} + available.sort((a,b)=>a.count.items/a.lease.max_items-b.count.items/b.lease.max_items|| + a.count.bytes/a.lease.max_bytes-b.count.bytes/b.lease.max_bytes||(a.scopeb.scope?1:0)); + const proof=canonicalBytes(document(available[0].session,MAX_SESSION_BYTES)),size=this.size(db,'open_delivery_outbox'); + if(size.bytes-MAX_ENVELOPE_BYTES-MAX_SESSION_BYTES+encoded.length+proof.length>MAX_LOCAL_BYTES)fail('network_outbox_capacity'); + db.prepare('UPDATE open_delivery_outbox SET envelope=?,session=? WHERE request_id=?').run(encoded,proof,row.request_id); + return db.prepare('SELECT * FROM open_delivery_outbox WHERE request_id=?').get(row.request_id) as Obj; + })); + } + private receivingSessions():Obj[]{ + const sessions=this.contactSessions(false),row=this.db(db=>db.prepare("SELECT body FROM open_delivery_client_state WHERE key='receive_session_cursor'").get()) as Obj|undefined; + if(row){const previous=raw(row.body).toString('ascii'),index=sessions.findIndex(session=>OpenDeliveryClient.sessionScope(session)===previous); + if(index>=0)return [...sessions.slice(index+1),...sessions.slice(0,index+1)];} + return sessions; + } + private advanceReceivingSession(session:Obj):void{ + this.db(db=>db.prepare("INSERT INTO open_delivery_client_state VALUES('receive_session_cursor',?) ON CONFLICT(key) DO UPDATE SET body=excluded.body").run(Buffer.from(OpenDeliveryClient.sessionScope(session),'ascii'))); } private uploadIntent(row:Obj,session:Obj):Obj{ if(row.intent){const prior=document(row.intent,MAX_INTENT_BYTES) as Obj;if(prior.payload.expires_at>now())return prior;} @@ -281,7 +333,7 @@ export class OpenDeliveryClient{ } private async sendInternal(requestId:string,recipient:string,hash:string,text:string,memoryIds:string[],recoverAck?:AckRecovery):Promise{ let row=this.prepareOutbox(requestId,recipient,hash,text,memoryIds);const budget=new DeliveryBudget();let session:Obj; - if(!row.session){session=await this.sendingSession(recipient,budget);row=await this.encryptOutbox(row,session);} + if(!row.session){session=await this.sendingSession(recipient,budget,row);row=this.outbox(requestId)!;} session=document(row.session,MAX_SESSION_BYTES) as Obj;row=await this.encryptOutbox(row,session); let recovery:Obj={network_accessed:false}; if(row.result&&!row.acknowledgement&&recoverAck){ @@ -478,6 +530,21 @@ export class OpenDeliveryClient{ return {messages,errors:[],state:'observed',body_transport:'mailbox_retained_copy',receipt_state:'retained_for_independent_return'}; }); } + async receiveMailboxReplica(reader:MailboxMessageReplicaRecoveryClient,baseUrl:string,options:MessageReplicaRecoverOptions):Promise{ + if(!(reader instanceof MailboxMessageReplicaRecoveryClient)||!same(reader.subject,{signing_key:validateSigningIdentity(this.participant.identity),encryption_key:validateEncryptionIdentity(this.encryption)}))fail('open_delivery_key_binding_mismatch'); + return this.serial(async()=>{ + const deadline=performance.now()/1000+(options.timeout??60),recovered=await reader.recover(baseUrl,options),selected=recovered.replica.source.message_scope.envelope_ref; + const message=recovered.replica.source.graph.members.find((v:Obj)=>same(v.attempt.envelope_ref,selected))?.attempt.message_id??(recovered.replica.source.custody.payload.message_id as string); + const prior=this.inbox(message);let result:Obj,fresh:boolean; + if(prior){if(prior.envelope_sha256!==selected.raw_sha256)fail('network_inbox_identity_conflict');fresh=prior.phase==='staged';result=await this.finishInbox(message);} + else{ + const remaining=Math.min(30,deadline-performance.now()/1000);if(remaining<=0)fail('open_delivery_budget_exhausted',true);const verified=await reader.readMessage(baseUrl,recovered,remaining),roles=verified.setup.roles; + const authority=Object.fromEntries([['request','contact.request'],['policy','contact.policy']].map(([n,r])=>[n,document(roles[r].raw)])),session={authority,source_node:document((options.targetNodeEntry as Obj).raw),mailbox:reader.inboxEvidence(recovered)},envelope=verified.envelope; + const body=await decryptEnvelope(envelope,{...OpenDeliveryClient.keys(session),encryption_identity:this.encryption} as any);this.stageInbox(message,authority.request.payload.signing_key.key_id,envelope,body,session);result=await this.finishInbox(message);fresh=true; + } + if(result.state==='validated_saved')this.savedReceipt(message);return {messages:fresh?[result]:[],errors:[],state:'observed',network_accessed:true,body_transport:'mailbox_message_replica',receipt_state:'retained_for_independent_return'}; + }); + } private pendingReceiptBatch():Obj[]{return this.db(db=>{ const select="SELECT message_id,phase,created_at FROM open_delivery_inbox WHERE (phase='staged' OR (phase='saved' AND receipt_sent=0))", order=' ORDER BY created_at,message_id LIMIT ?', @@ -500,7 +567,7 @@ export class OpenDeliveryClient{ if(result.state==='validated_saved')await this.sendReceipt(row.message_id,budget);}catch(error){errors.push({message_id:row.message_id,...errorData(error)});} finally{this.advancePendingReceipt(row);}} if(options.pendingOnly)return {messages,errors:errors.slice(0,4),network_accessed:budget.requests>0}; - for(const candidate of this.contactSessions(false)){if(messages.length>=limit)break; + for(const candidate of this.receivingSessions()){if(messages.length>=limit)break; try{const session=await this.checkedSession(candidate,budget),node=session.node,leaseId=session.decision.payload.grant.payload.resource_lease.payload.lease_id; let [key,after]=this.cursor(session);const intent=readerIntent('list',{lease_id:leaseId,caller_key_id:this.participant.keyId}); const prepared=await this.call(node,'list.prepare',{lease_id:leaseId},budget),challenge=prepared.challenge,answer=await this.answer(challenge,intent,node); @@ -511,7 +578,8 @@ export class OpenDeliveryClient{ const [result,fresh]=await this.receiveEntry(session,entry,budget);after=entry.sequence;this.advance(key,after);if(fresh)messages.push(result); if(result.state==='validated_saved')try{await this.sendReceipt(entry.message_id,budget,node);}catch(error){errors.push({message_id:entry.message_id,...errorData(error)});}} if(safeInteger(listing.next_sequence)!==after)fail('open_delivery_sequence_mismatch'); - }catch(error){errors.push(errorData(error));if(errors.length>=4)break;}} + }catch(error){errors.push(errorData(error));if(errors.length>=4)break;} + finally{this.advanceReceivingSession(candidate);}} return {messages,errors:errors.slice(0,4),network_accessed:budget.requests>0}; } readMessage(messageId:string,offset=0):Obj{ diff --git a/clients/typescript/network/open-mailbox-receivers.ts b/clients/typescript/network/open-mailbox-receivers.ts index d0bcf31..89554dd 100644 --- a/clients/typescript/network/open-mailbox-receivers.ts +++ b/clients/typescript/network/open-mailbox-receivers.ts @@ -15,6 +15,7 @@ import {transaction} from './io.ts'; import {prepareMailboxDestination,mailboxPage} from './open-mailbox-destination.ts'; import {OpenContactClient} from './open-contact-client.ts'; import {MailboxReceiptJobs} from './open-mailbox-receipts.ts'; +import {RegisteredMailboxReplicas} from './open-mailbox-replicas.ts'; type Obj=Record; export const MAILBOX_CONNECT_SCHEMA='memory-vault-open-mailbox-connect/v1'; function fail(code:string):never{throw new RepairError(code);} @@ -38,8 +39,9 @@ export class RegisteredMailboxReceivers{ const node=verifySourceNodeOriginal(raw,{expectedSigningKey:value.expected_target.signing_key,expectedStorageEpoch:value.expected_slot.writer_storage_epoch,at,policy,budget}); if(!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url,this.#participant.transport.allow_loopback)))fail('open_invalid_mailbox_receiver');return options; } - connect(invitation:unknown):Obj{ + async connect(invitation:unknown):Promise{ const value=document(invitation as any,65536) as Obj;if(value.schema_version!==MAILBOX_CONNECT_SCHEMA)fail('open_invalid_mailbox_receiver'); + if(['register_replica','receive_replica','list_replicas','remove_replica','inspect_replica'].includes(value.action))return new RegisteredMailboxReplicas(this.#participant,this.#encryption,this.#delivery).connect(value); if(value.action==='list'){objectFields(value,['schema_version','action']);const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id FROM open_mailbox_receivers ORDER BY receiver_id LIMIT 17').all()) as Obj[];if(rows.length>16)fail('open_mailbox_receiver_capacity');return {state:'configured',mailboxes:rows.map(v=>v.receiver_id),network_accessed:false};} if(value.action==='remove'){objectFields(value,['schema_version','action','receiver_id']);opaqueId(value.receiver_id);this.#participant.providerStorage(db=>db.prepare('DELETE FROM open_mailbox_receivers WHERE receiver_id=?').run(value.receiver_id));return {state:'removed',receiver_id:value.receiver_id,network_accessed:false};} if(value.action==='inspect'||value.action==='authorize'){ @@ -66,12 +68,18 @@ export class RegisteredMailboxReceivers{ const deadline=performance.now()/1000+60,result=await this.#delivery.receive(limit,{pendingOnly:true,deadline}); const jobs=new MailboxReceiptJobs(this.#participant,this.#encryption,this.#delivery),attempted=new Set(); await jobs.poll(result,Math.min(deadline,performance.now()/1000+30),attempted); - const rows=this.#participant.providerStorage(db=>db.prepare('SELECT receiver_id,body FROM open_mailbox_receivers ORDER BY last_attempt,receiver_id LIMIT 17').all()) as Obj[]; - if(rows.length>16)fail('open_mailbox_receiver_capacity'); + const replicas=new RegisteredMailboxReplicas(this.#participant,this.#encryption,this.#delivery); + const rows=this.#participant.providerStorage(db=>db.prepare(`SELECT receiver_id,body,receiver_kind,body_sha256 FROM (SELECT receiver_id,body,last_attempt,'original' AS receiver_kind,NULL AS body_sha256 FROM open_mailbox_receivers UNION ALL + SELECT receiver_id,body,last_attempt,'replica' AS receiver_kind,body_sha256 FROM open_mailbox_replica_receivers AS r WHERE NOT EXISTS( + SELECT 1 FROM open_delivery_inbox AS i WHERE i.envelope_sha256=r.envelope_sha256 AND i.phase!='staged')) ORDER BY last_attempt,receiver_id LIMIT 4`).all()) as Obj[]; for(const row of rows){let remaining=deadline-performance.now()/1000;if(result.messages.length>=limit||remaining<=0)break; - this.#participant.providerStorage(db=>db.prepare('UPDATE open_mailbox_receivers SET last_attempt=? WHERE receiver_id=?').run(BigInt(Date.now())*1000000n,row.receiver_id)); + this.#participant.providerStorage(db=>db.prepare('UPDATE '+(row.receiver_kind==='replica'?'open_mailbox_replica_receivers':'open_mailbox_receivers')+' SET last_attempt=? WHERE receiver_id=?').run(BigInt(Date.now())*1000000n,row.receiver_id)); let reader:MailboxFeedRecoveryClient|undefined; try{ + if(row.receiver_kind==='replica'){ + if(sha256(row.body)!==row.body_sha256)fail('open_invalid_mailbox_receiver'); + const received=await replicas.receive(document(row.body,65536) as Obj,deadline,()=>{result.network_accessed=true;});result.messages.push(...received.messages);result.errors.push(...received.errors);continue; + } const config=document(row.body,65536) as Obj;if(row.receiver_id!=='mailbox_'+sha256(canonicalBytes(config.expected_slot)))fail('open_invalid_mailbox_receiver'); const options=this.#validate(config),journal=new MailboxSetupJournal(this.#participant); reader=new MailboxFeedRecoveryClient(this.#participant.identity,this.#encryption,{limitPolicy:config.limit_policy,allowLoopback:this.#participant.transport.allow_loopback,transport:this.#participant.transport,networkObserver:()=>{result.network_accessed=true;}}); diff --git a/clients/typescript/network/open-mailbox-replica-client.ts b/clients/typescript/network/open-mailbox-replica-client.ts new file mode 100644 index 0000000..bf0c48f --- /dev/null +++ b/clients/typescript/network/open-mailbox-replica-client.ts @@ -0,0 +1,112 @@ +/** Native bounded replica recovery. All controls and original ciphertext are + * obtained through the selected endpoint, with independent dual possession. */ +import {performance} from 'node:perf_hooks'; +import {OpenHTTPTransport,endpoint} from './open-transport.ts'; +import {RepairBudget,buildNewWire,parseNewWire,rawRef,u53,LocalRawResolver,parseRawPack} from './open-repair-wire.ts'; +import type {RawOriginal,RepairPolicy} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY,DEFAULT_REPAIR_CLIENT_LIMITS} from './open-repair-client.ts'; +import {validateSigningIdentity,validateEncryptionIdentity,encodeBase64url} from './crypto.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {resolveHistoricalInputs} from './open-repair-history.ts'; +import {makeBootstrapProbe,solveBootstrapChallenge} from './open-repair-probe.ts'; +import {verifyBootstrapProofResponse,makeBootstrapChildRequest,makeMailboxBodyRequest} from './open-repair-proof.ts'; +import {authenticateStatusOriginal,statusScope} from './open-repair-status.ts'; +import {readMailboxIndex,verifyMailboxAdmissionMetadata} from './open-repair-mailbox-read.ts'; +import {ReplicaStatusJournal} from './open-ack-status.ts'; +import {replicaFields as fields,replicaFail as fail,replicaSame as same,replicaRefKey as id,replicaEntry,replicaFloors,verifyMailboxMessageReplica,checkMailboxMessageReturn} from './open-repair-mailbox-replica.ts'; +type Obj=Record; +export const MAILBOX_REPLICA_LIMITS=Object.freeze({...DEFAULT_REPAIR_CLIENT_LIMITS,max_signature_checks:4096,max_proof_bytes:4194304,max_requests:256,max_replay_records:256,max_proof_items:128}); +const monotonic=()=>performance.now()/1000; +function boundedTimeout(v:unknown):number{if(typeof v!=='number'||!Number.isFinite(v)||v<=0||v>60)fail('repair_invalid_deadline');return v;} +/** Authenticate retained observations before considering current access. */ +export function knownReplicaStatuses(entries:unknown,e:Obj,owner:Obj,at:number,policy:RepairPolicy,budget:RepairBudget,observe?: (v:any)=>void):Obj[]{ + if(!Array.isArray(entries)||entries.length>32)fail('repair_status_history_capacity');const parties={...e,owner},signers=new Map(Object.values(parties).filter((v:Obj)=>v?.signing_key).map((v:Obj)=>[v.signing_key.key_id,v.signing_key])),result:Obj[]=[];let denial:unknown; + for(const input of entries)try{const entry=replicaEntry(input,policy,budget),p=(parseNewWire(entry.raw,policy,budget).value as Obj).payload,signer=p.signing_key; + if(!same(signers.get(signer.key_id),signer)||u53(p.issued_at)>at||!Array.isArray(p.entries))fail('repair_status_disclosure'); + const permitted=new Set();if(same(signer,owner.signing_key)){permitted.add('authority');permitted.add('mailbox_slot');}if(same(signer,e.sender.signing_key)||same(signer,e.source.signing_key)||same(signer,e.maintainer.signing_key))permitted.add('authority');if(same(signer,e.source.signing_key)||same(signer,e.target.signing_key))permitted.add('resource');if(same(signer,e.maintainer.signing_key))permitted.add('assignment'); + if(p.entries.some((v:Obj)=>!permitted.has(v.scope_kind)))fail('repair_status_disclosure'); + result.push(authenticateStatusOriginal(entry,{expectedRoot:e.slot.root_key,expectedSigningKey:signer,at:p.issued_at,allowedScopes:p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),policy,budget},observe)); + }catch(error){denial??=error;} + if(denial)throw denial;return result; +} +export interface MessageReplicaRecoverOptions{targetNodeEntry:unknown;expectedTarget:unknown;expectedSender:unknown;expectedSlot:unknown;expectedSource:unknown;sourceStorageEpoch:string;expectedMaintainer:unknown;expectedEnvelopeRef:unknown;slotEntries:unknown;journal:ReplicaStatusJournal;knownStatuses?:readonly unknown[];archiveStatuses?:readonly unknown[];timeout?:number;} +export class MailboxMessageReplicaRecoveryClient{ + readonly #identity:Obj;readonly #encryption:Obj;readonly #subject:Obj;readonly #policy:RepairPolicy;readonly #transport:OpenHTTPTransport;readonly #loopback:boolean;readonly #clock:()=>number;readonly #observer?:()=>void; + readonly #recovered=new WeakMap(); + constructor(identity:unknown,encryption:unknown,a:{transport:OpenHTTPTransport;allowLoopback:boolean;clock?:()=>number;networkObserver?:()=>void}){ + this.#policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512});const budget=new RepairBudget(this.#policy),e=buildNewWire({identity,encryption},this.#policy,budget).value as Obj; + this.#identity=e.identity;this.#encryption=e.encryption;this.#subject={signing_key:validateSigningIdentity(e.identity),encryption_key:validateEncryptionIdentity(e.encryption)};this.#transport=a.transport;this.#loopback=a.allowLoopback;this.#clock=a.clock??(()=>Date.now()/1000);this.#observer=a.networkObserver;Object.freeze(this); + } + get subject():Readonly{return this.#subject;} + close():void{} + #now():number{return u53(Math.floor(this.#clock()));} + async recover(baseUrl:string,a:MessageReplicaRecoverOptions):Promise>{ + if(!(a.journal instanceof ReplicaStatusJournal))fail('repair_invalid_context'); + const policy=this.#policy,budget=new RepairBudget(policy),started=this.#now(),timeout=boundedTimeout(a.timeout??60),deadline=monotonic()+timeout; + const e=buildNewWire({slot:a.expectedSlot,sender:a.expectedSender,target:a.expectedTarget,source:a.expectedSource,maintainer:a.expectedMaintainer},policy,budget).value as Obj; + if(e.slot.writer_storage_epoch!==a.sourceStorageEpoch)fail('repair_proof_mismatch');const envelopeRef=rawRef(a.expectedEnvelopeRef); + const setup=verifyMailboxFeedBootstrap(a.slotEntries,{expectedSlot:e.slot,expectedOwner:this.#subject,expectedTarget:e.source,targetStorageEpoch:a.sourceStorageEpoch,limitPolicy:MAILBOX_REPLICA_LIMITS,at:started,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:e.sender.signing_key.key_id,encryption_key_id:e.sender.encryption_key.key_id}))fail('repair_proof_mismatch'); + const nodeEntry=replicaEntry(a.targetNodeEntry,policy,budget),np=(parseNewWire(nodeEntry.raw,policy,budget).value as Obj).payload,node=verifySourceNodeOriginal(nodeEntry.raw,{expectedSigningKey:e.target.signing_key,expectedStorageEpoch:np.storage_epoch,at:started,policy,budget}); + if(!same(endpoint(baseUrl,this.#loopback),endpoint(node.payload.base_url,this.#loopback)))fail('repair_proof_mismatch');const epoch=node.payload.storage_epoch as string; + const retained=new Map();for(const group of [a.knownStatuses??[],a.archiveStatuses??[],a.journal.load(new Set([this.#subject,...[e.sender,e.target,e.source,e.maintainer]].map(v=>v.signing_key.key_id)))]){if(!Array.isArray(group)||group.length>32)fail('repair_status_history_capacity');for(const item of group){const entry=replicaEntry(item,policy,budget);retained.set(id(entry.ref),entry);}} + const known=knownReplicaStatuses([...retained.values()],e,this.#subject,started,policy,budget,v=>a.journal.observe(v)); + const probeNeeds=['slot','read','maintenance','bootstrap'].map(n=>({signer:this.#subject.signing_key,scope_kind:n==='slot'?'mailbox_slot':'authority',scope_id:statusScope(e.slot.root_key,n==='slot'?'mailbox_slot':'authority',n==='slot'?e.slot:{authority_kind:setup[n].payload.kind,authority_sha256:setup[n].ref.raw_sha256},policy,budget),revision:setup[n].payload.revision,mask:n==='read'?2:10}));replicaFloors(known,[],probeNeeds); + const grant=setup.bootstrap.payload,expiry=Math.min(started+Math.min(60,Math.max(1,Math.floor(timeout))),node.payload.expires_at as number,grant.probe_until,grant.proof_until,...Object.values(setup).map(v=>v.payload.expires_at as number));if(expiry<=started)fail('repair_access_expired'); + const counts={requests:0,wire_bytes:0,proof_bytes:0},request=async(raw:Uint8Array,child=false)=>{if(counts.requests>=grant.limits.max_requests||monotonic()>=deadline)fail('repair_over_budget');counts.requests++;this.#observer?.();const reply=await this.#transport.requestRepair(baseUrl,raw,deadline,{child});counts.wire_bytes+=raw.length+reply.length;return reply;}; + const binding={expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:epoch,bootstrapGrantSha256:setup.bootstrap.ref.raw_sha256,selector:grant.selector,consumer:'mailbox_feed' as const,policy,budget}; + const outgoing=await makeBootstrapProbe(this.#identity,{...binding,at:started,expiresAt:expiry});if(outgoing.original.raw.length>grant.limits.max_probe_bytes)fail('repair_over_budget'); + const challengeRaw=await request(outgoing.original.raw),digest=budget.hash(challengeRaw),challenge={raw:challengeRaw,ref:rawRef({namespace:'meta',key:digest,raw_sha256:digest,size:challengeRaw.length})}; + const preview=(parseNewWire(challengeRaw,policy,budget).value as Obj).payload,answer=await solveBootstrapChallenge({raw:outgoing.original.raw,ref:outgoing.original.ref},challenge,{...binding,signer:this.#identity,encryptionIdentity:this.#encryption,targetNonce:outgoing.nonce,at:this.#now(),expiresAt:Math.min(expiry,u53(preview.expires_at))}); + const response=await request(answer.raw),held=verifyBootstrapProofResponse(response,{expectedSubject:this.#subject,expectedTarget:e.target,targetStorageEpoch:epoch,selector:grant.selector,bootstrapGrantRef:setup.bootstrap.ref,probeRef:outgoing.original.ref,challengeRef:challenge.ref,answerRef:answer.ref,at:this.#now(),maxProofItems:grant.limits.max_proof_items,maxProofBytes:grant.limits.max_proof_bytes,expectedSourceState:'replica_message',consumer:'mailbox_feed',policy,budget}); + counts.proof_bytes=response.length+held.handle.raw.length+held.manifest.raw.length; + const originals=new Map(),roles=new Map(),available=new Map(),children=(held.manifest.value as Obj).children; + for(const item of [...Object.values(setup),...known])available.set(id(item.ref),item.raw);for(const item of children)roles.set(item.role,[...(roles.get(item.role)??[]),rawRef(item.ref)]); + const download=async(item:Obj)=>{const ref=rawRef(item.ref),k=id(ref);if(originals.has(k))return;if(ref.size>policy.max_document_bytes||counts.proof_bytes+ref.size>grant.limits.max_proof_bytes)fail('repair_over_budget');let raw=available.get(k); + if(raw)budget.input(raw.length);else{let offset=0;const parts:Uint8Array[]=[];while(offsetv.role==='replica.read_pack');if(packedChildren.length!==1)fail('repair_proof_mismatch');await download(packedChildren[0]);const packedItem=originals.get(id(packedChildren[0].ref))!,pack=parseRawPack(packedItem.raw,packedItem.ref,policy,budget); + const packRoles=new Set(['replica.manifest','replica.custody','copy.allocation','copy.offer','copy.assignment','copy.reservation_consent','copy.owner_disclosure','copy.source_disclosure','copy.current_status','return.owner','return.source','return.maintainer','current.status.replica_read','copy.sender_reservation_consent','copy.sender_disclosure','return.sender']); + const wanted=new Map();for(const item of children)if(packRoles.has(item.role)){const ref=rawRef(item.ref);wanted.set(ref.raw_sha256+':'+ref.size,ref);} + if(wanted.size!==pack.entries.length||pack.entries.some(v=>!wanted.has(v.raw_sha256+':'+v.size)))fail('repair_proof_mismatch'); + for(const [i,item] of pack.entries.entries()){const ref=wanted.get(item.raw_sha256+':'+item.size),raw=pack.entry(i,ref).raw,old=available.get(id(ref));if(old&&!Buffer.from(old).equals(Buffer.from(raw)))fail('repair_ref_conflict');available.set(id(ref),raw);} + for(const item of children)if(['history.raw_pack','history.mailbox_feed'].includes(item.role))await download(item); + const packed=new LocalRawResolver(policy,budget);for(const ref of roles.get('history.raw_pack')??[]){const item=originals.get(id(ref))!;if(!same(packed.put(ref.namespace,ref.key,item.raw).ref,ref))fail('repair_ref_mismatch');} + const reuse=(tree:Obj)=>{for(const row of tree.roles){const ref=row.original.ref,raw=row.original.raw,old=available.get(id(ref));if(old&&!Buffer.from(old).equals(Buffer.from(raw)))fail('repair_ref_conflict');available.set(id(ref),raw);}for(const prior of tree.predecessors)reuse(prior);}; + for(const ref of roles.get('history.mailbox_feed')??[]){const tree=resolveHistoricalInputs(originals.get(id(ref))!.raw,packed,policy,budget);if((tree.manifest.value as Obj).variant!=='mailbox_feed')fail('repair_proof_mismatch');reuse(tree);}for(const item of children)await download(item); + const entry=(ref:unknown)=>{const v=originals.get(id(ref));if(!v)fail('repair_original_missing');return v;},one=(role:string)=>{const refs=roles.get(role);if(refs?.length!==1)fail('repair_proof_mismatch');return entry(refs[0]);}; + const resolver=new LocalRawResolver(policy,budget);for(const item of originals.values())if(!same(resolver.put(item.ref.namespace,item.ref.key,item.raw).ref,item.ref))fail('repair_ref_mismatch'); + const context={expectedSlot:e.slot,expectedOwner:this.#subject,expectedSender:e.sender,expectedSource:e.source,sourceStorageEpoch:a.sourceStorageEpoch,expectedMaintainer:e.maintainer,expectedTarget:e.target,targetStorageEpoch:epoch,expectedEnvelopeRef:envelopeRef,limitPolicy:MAILBOX_REPLICA_LIMITS,policy,budget}; + const replica=verifyMailboxMessageReplica(one('replica.manifest'),resolver,one('replica.custody'),context); + for(const member of replica.source.graph.members)for(const [n,parent] of Object.entries(setup)){if(!same(member.originals[n].ref,parent.ref)||!Buffer.from(member.originals[n].raw).equals(Buffer.from(parent.raw)))fail('repair_proof_mismatch');} + const extras=new Set(['replica.manifest','replica.custody','replica.read_pack','return.owner','return.source','return.maintainer','return.sender','current.status.replica_read']),actual=new Set(),expected=new Set(); + for(const [role,refs] of roles)if(!extras.has(role))for(const ref of refs)actual.add(role+'|'+id(ref));for(const [role,items] of replica.entries)for(const item of items)expected.add(role+'|'+id(item.ref));if(actual.size!==expected.size||[...expected].some(v=>!actual.has(v)))fail('repair_proof_mismatch'); + const consents=Object.fromEntries(['owner','source','maintainer','sender'].map(n=>[n,one('return.'+n)])),current=(roles.get('current.status.replica_read')??[]).map(entry),permission=checkMailboxMessageReturn(replica,consents,{...context,currentStatuses:current,at:this.#now(),action:'proof',onObserved:(v:any)=>a.journal.observe(v)}); + const previous=[...replica.source.graph.statuses,...replica.source.graph.members.flatMap((v:Obj)=>v.statuses),...replica.authority.statuses,...known];replicaFloors(previous,permission.statuses,permission.obligations); + const members=await readMailboxIndex(one('feed.head'),one('feed.checkpoint'),{expectedSlot:e.slot,expectedSigningKey:e.source.signing_key,encryptionIdentity:this.#encryption as any,readOriginal:ref=>entry(ref).raw,at:this.#now(),maxMessages:setup.slot.payload.max_appends,policy,budget}); + const until=Math.min(expiry,held.handle.payload.expires_at as number,permission.expires_at);if(this.#now()>=until||monotonic()>=deadline)fail('repair_access_expired'); + const archive=new Map([...previous,...permission.statuses].map(v=>[id(v.ref),v]));if(archive.size>32)fail('repair_status_history_capacity'); + const result=Object.freeze({replica,proof:held,current_statuses:permission.statuses,archive_statuses:[...archive.values()],entries:members,metrics:{...counts,...budget.snapshot()},base_url:baseUrl});this.#recovered.set(result,{e,context,setup,held,originals,consents,current,archive:[...archive.values()],counts,until,journal:a.journal});return result; + } + async readMessage(baseUrl:string,recovered:object,timeout=30):Promise>{ + const c=this.#recovered.get(recovered);if(!c||!same(endpoint(baseUrl,this.#loopback),endpoint((recovered as Obj).base_url,this.#loopback)))fail('repair_invalid_context');const deadline=monotonic()+boundedTimeout(timeout),policy=this.#policy,budget=new RepairBudget(policy),context={...c.context,policy,budget}; + const resolver=new LocalRawResolver(policy,budget);for(const item of c.originals.values())if(!same(resolver.put(item.ref.namespace,item.ref.key,item.raw).ref,item.ref))fail('repair_ref_mismatch'); + const child=(role:string)=>{const rows=(c.held.manifest.value as Obj).children.filter((v:Obj)=>v.role===role);if(rows.length!==1)fail('repair_proof_mismatch');return c.originals.get(id(rows[0].ref));}; + const replica=verifyMailboxMessageReplica(child('replica.manifest'),resolver,child('replica.custody'),context),permission=checkMailboxMessageReturn(replica,c.consents,{...context,currentStatuses:c.current,at:this.#now(),action:'child',onObserved:(v:any)=>c.journal.observe(v)}); + const retained=knownReplicaStatuses(c.journal.load(new Set([this.#subject,c.e.sender,c.e.target,c.e.source,c.e.maintainer].map(v=>v.signing_key.key_id))),c.e,this.#subject,this.#now(),policy,budget,v=>c.journal.observe(v));replicaFloors([...c.archive,...retained],permission.statuses,permission.obligations); + const checked=await verifyMailboxAdmissionMetadata(replica.source.message_member,{expectedSlot:c.e.slot,expectedSigningKey:c.e.source.signing_key,expectedOwner:this.#subject,expectedSender:c.e.sender,expectedTarget:c.e.source,encryptionIdentity:this.#encryption as any,readOriginal:ref=>{const v=c.originals.get(id(ref));if(!v)fail('repair_original_missing');return v.raw;},at:this.#now(),limitPolicy:MAILBOX_REPLICA_LIMITS,policy,budget,currentStatuses:[],knownStatuses:[],statusObligations:[]}); + const ref=rawRef(context.expectedEnvelopeRef),children=(c.held.manifest.value as Obj).children.filter((v:Obj)=>v.role==='member.core'&&same(v.ref,checked.link.core_ref));if(children.length!==1||!same(ref,checked.core.envelope_ref))fail('repair_proof_mismatch'); + if(ref.size+c.counts.proof_bytes>MAILBOX_REPLICA_LIMITS.max_proof_bytes)fail('repair_over_budget');let offset=0;const parts:Uint8Array[]=[]; + const guard=()=>{if(this.#now()>=Math.min(c.until,permission.expires_at)||monotonic()>=deadline)fail('repair_access_expired');}; + while(offset=MAILBOX_REPLICA_LIMITS.max_requests)fail('repair_over_budget');const count=Math.min(65536,ref.size-offset),packet=makeMailboxBodyRequest(this.#identity,c.held,ref,{subject:this.#subject,target:c.e.target,at:this.#now(),expiresAt:c.held.handle.payload.expires_at,childIndex:children[0].index,offset,requestedBytes:count,policy,budget});c.counts.requests++;this.#observer?.();const raw=await this.#transport.requestRepair(baseUrl,packet.raw,deadline,{child:true});if(raw.length!==count)fail('repair_ref_mismatch');budget.input(raw.length);parts.push(raw);offset+=count;} + guard();budget.output(ref.size);const envelope=Buffer.concat(parts);if(budget.hash(envelope)!==ref.raw_sha256)fail('repair_ref_mismatch');return Object.freeze({...checked,envelope,replica,current_statuses:permission.statuses}); + } + inboxEvidence(recovered:object):Readonly{ + const c=this.#recovered.get(recovered);if(!c)fail('repair_invalid_context');const reference=(role:string)=>{const found=(c.held.manifest.value as Obj).children.filter((v:Obj)=>v.role===role);if(found.length!==1)fail('repair_proof_mismatch');return found[0].ref;}; + const selected=new Map();for(const items of (recovered as Obj).replica.entries.values())for(const item of items)selected.set(id(item.ref),item); + for(const role of ['replica.manifest','replica.custody','return.owner','return.source','return.maintainer','return.sender']){const ref=reference(role);selected.set(id(ref),c.originals.get(id(ref)));} + for(const item of [...(recovered as Obj).current_statuses,...c.archive])selected.set(id(item.ref),item); + return Object.freeze({schema_version:'memory-vault-mailbox-replica-inbox/v1',received_at:this.#now(),slot:c.e.slot,sender:c.e.sender,source:c.e.source,target:c.e.target,maintainer:c.e.maintainer,target_storage_epoch:c.context.targetStorageEpoch,envelope_ref:c.context.expectedEnvelopeRef,limits:MAILBOX_REPLICA_LIMITS,manifest_ref:reference('replica.manifest'),custody_ref:reference('replica.custody'),consent_refs:Object.fromEntries(['owner','source','maintainer','sender'].map(n=>[n,reference('return.'+n)])),originals:[...selected.values()].sort((a,b)=>id(a.ref)({ref:v.ref,raw_base64url:encodeBase64url(v.raw)})),status_refs:(recovered as Obj).current_statuses.map((v:Obj)=>v.ref),archive_status_refs:c.archive.map((v:Obj)=>v.ref)}); + } +} diff --git a/clients/typescript/network/open-mailbox-replicas.ts b/clients/typescript/network/open-mailbox-replicas.ts new file mode 100644 index 0000000..23bf65a --- /dev/null +++ b/clients/typescript/network/open-mailbox-replicas.ts @@ -0,0 +1,69 @@ +/** Explicit finite replica placements, shared with Python's protected state. + * No global membership, authority minting, or automatic unconsented movement. */ +import {performance} from 'node:perf_hooks'; +import {canonicalBytes,document,sha256,opaqueId,validateSigningIdentity,validateEncryptionIdentity,validateSigningPublic,validateEncryptionPublic,encodeBase64url} from './crypto.ts'; +import type {EncryptionIdentityDocument} from './crypto.ts'; +import type {OpenParticipant} from './open-participant.ts'; +import {transaction} from './io.ts'; +import {verifyNode} from './open-control.ts'; +import {endpoint} from './open-transport.ts'; +import {RepairBudget,parseNewWire} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifySourceNodeOriginal} from './open-repair-original.ts'; +import {statusScope} from './open-repair-status.ts'; +import {verifyMailboxFeedBootstrap} from './open-repair-mailbox-authority.ts'; +import {MailboxMessageReplicaRecoveryClient,MAILBOX_REPLICA_LIMITS,knownReplicaStatuses} from './open-mailbox-replica-client.ts'; +import {ReplicaStatusJournal} from './open-ack-status.ts'; +import {replicaFields as fields,replicaFail as fail,replicaSame as same,replicaEntry,replicaFloors} from './open-repair-mailbox-replica.ts'; +import type {OpenDeliveryClient} from './open-delivery-client.ts'; +type Obj=Record; +const SCHEMA='memory-vault-open-mailbox-connect/v1',TABLE='open_mailbox_replica_receivers'; +const REQUEST='expected_slot expected_sender expected_target expected_source source_storage_epoch expected_maintainer expected_envelope_ref target_node_entry slot_entries known_statuses archive_statuses'.split(' '); +const decode=(v:unknown)=>{const e=fields(v,['raw','ref']);if(typeof e.raw!=='string')fail('open_invalid_mailbox_receiver');return {raw:Buffer.from(e.raw,'utf8'),ref:e.ref};}; +export class RegisteredMailboxReplicas{ + readonly #participant:OpenParticipant;readonly #encryption:EncryptionIdentityDocument;readonly #delivery:OpenDeliveryClient; + constructor(participant:OpenParticipant,encryption:EncryptionIdentityDocument,delivery:OpenDeliveryClient){this.#participant=participant;this.#encryption=encryption;this.#delivery=delivery; + participant.providerStorage(db=>db.exec(`CREATE TABLE IF NOT EXISTS ${TABLE}(receiver_id TEXT PRIMARY KEY,selection_digest TEXT NOT NULL,envelope_sha256 TEXT NOT NULL,body BLOB NOT NULL,body_sha256 TEXT NOT NULL,last_attempt INTEGER NOT NULL DEFAULT 0)`));} + #checked(value:Obj,current=true):Obj{ + fields(value,['schema_version','action','base_url','repair_profile','request']);if(value.schema_version!==SCHEMA||value.action!=='register_replica'||value.repair_profile!=='mailbox')fail('open_invalid_mailbox_receiver');const request=fields(value.request,REQUEST); + const policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}),budget=new RepairBudget(policy),at=Math.floor(Date.now()/1000),owner={signing_key:validateSigningIdentity(this.#participant.identity),encryption_key:validateEncryptionIdentity(this.#encryption)}; + const e={slot:request.expected_slot,sender:request.expected_sender,target:request.expected_target,source:request.expected_source,maintainer:request.expected_maintainer}; + for(const keys of [owner,e.sender,e.target,e.source,e.maintainer]){fields(keys,['signing_key','encryption_key']);validateSigningPublic(keys.signing_key);validateEncryptionPublic(keys.encryption_key);} + if(same(e.target,e.source)||same(e.target,e.maintainer)||request.source_storage_epoch!==e.slot.writer_storage_epoch)fail('open_invalid_mailbox_receiver'); + fields(request.slot_entries,['slot','read','maintenance','bootstrap']);const slots=Object.fromEntries(Object.entries(request.slot_entries).map(([n,v])=>[n,decode(v)])); + const setup=verifyMailboxFeedBootstrap(slots,{expectedSlot:e.slot,expectedOwner:owner,expectedTarget:e.source,targetStorageEpoch:request.source_storage_epoch,limitPolicy:MAILBOX_REPLICA_LIMITS,at,policy,budget}); + if(!same(setup.slot.payload.sender,{signing_key_id:e.sender.signing_key.key_id,encryption_key_id:e.sender.encryption_key.key_id})||!setup.maintenance.payload.maintainers.some((v:Obj)=>same(v,{signing_key_id:e.maintainer.signing_key.key_id,encryption_key_id:e.maintainer.encryption_key.key_id})))fail('open_invalid_mailbox_receiver'); + const envelope=request.expected_envelope_ref;if(envelope.namespace!=='object')fail('open_invalid_mailbox_receiver');const entry=replicaEntry(decode(request.target_node_entry),policy,budget),p=(parseNewWire(entry.raw,policy,budget).value as Obj).payload; + const node=verifySourceNodeOriginal(entry.raw,{expectedSigningKey:e.target.signing_key,expectedStorageEpoch:p.storage_epoch,at:current?at:p.issued_at,policy,budget});if(!same(endpoint(value.base_url,this.#participant.transport.allow_loopback),endpoint(node.payload.base_url,this.#participant.transport.allow_loopback)))fail('open_invalid_mailbox_receiver'); + for(const [n,max] of [['known_statuses',16],['archive_statuses',32]] as const)if(!Array.isArray(request[n])||request[n].length>max)fail('repair_status_history_capacity'); + const journal=new ReplicaStatusJournal(this.#participant,e.slot.root_key),observe=(v:any)=>journal.observe(v); + // The independent journal survives invalid later inputs and removals. + const retained=journal.load(new Set([owner,e.sender,e.target,e.source,e.maintainer].map(v=>v.signing_key.key_id))),known=knownReplicaStatuses([...request.known_statuses.map(decode),...request.archive_statuses.map(decode),...retained],e,owner,at,policy,budget,observe); + return {e,owner,setup,node,slots,journal,known,policy,budget,request}; + } + async connect(value:Obj):Promise{ + if(value.action==='list_replicas'){fields(value,['schema_version','action']);const rows=this.#participant.providerStorage(db=>db.prepare(`SELECT receiver_id FROM ${TABLE} ORDER BY receiver_id LIMIT 17`).all()) as Obj[];if(rows.length>16)fail('open_mailbox_receiver_capacity');return {state:'configured',replicas:rows.map(v=>v.receiver_id),network_accessed:false};} + if(['remove_replica','inspect_replica'].includes(value.action)){ + fields(value,['schema_version','action','receiver_id',...(Object.hasOwn(value,'cursor')?['cursor']:[])]);opaqueId(value.receiver_id); + if(value.action==='remove_replica'){this.#participant.providerStorage(db=>db.prepare(`DELETE FROM ${TABLE} WHERE receiver_id=?`).run(value.receiver_id));return {state:'removed',receiver_id:value.receiver_id,network_accessed:false};} + const row=this.#participant.providerStorage(db=>db.prepare(`SELECT body,body_sha256 FROM ${TABLE} WHERE receiver_id=?`).get(value.receiver_id)) as Obj|undefined;if(!row)fail('open_mailbox_receiver_missing');if(sha256(row.body)!==row.body_sha256)fail('open_invalid_mailbox_receiver'); + let offset=0;if(value.cursor!=null){fields(value.cursor,['sha256','offset']);offset=value.cursor.offset;if(value.cursor.sha256!==row.body_sha256||!Number.isSafeInteger(offset)||offset<=0||offset>=row.body.length||offset%3072)fail('open_invalid_mailbox_cursor');}const stop=Math.min(offset+3072,row.body.length); + return {state:'mailbox_replica_configuration',receiver_id:value.receiver_id,replica_configuration_sha256:row.body_sha256,total_bytes:row.body.length,offset,replica_configuration_chunk:Buffer.from(row.body.subarray(offset,stop)).toString('base64'),next_cursor:stoptransaction(db,()=>{const old=db.prepare(`SELECT body FROM ${TABLE} WHERE receiver_id=?`).get(receiver) as Obj|undefined;if(old){if(!Buffer.from(old.body).equals(Buffer.from(raw)))fail('open_mailbox_receiver_conflict');return;} + if((db.prepare(`SELECT count(*) AS n FROM ${TABLE}`).get() as Obj).n>=16||(db.prepare(`SELECT count(*) AS n FROM ${TABLE} WHERE envelope_sha256=?`).get(c.request.expected_envelope_ref.raw_sha256) as Obj).n>=2)fail('open_mailbox_receiver_capacity'); + db.prepare(`INSERT INTO ${TABLE}(receiver_id,selection_digest,envelope_sha256,body,body_sha256) VALUES(?,?,?,?,?)`).run(receiver,selection,c.request.expected_envelope_ref.raw_sha256,raw,sha256(raw));})); + return {state:'registered',receiver_id:receiver,network_accessed:false,recovery:'ordinary_receive',receipt_return:'separate_authority_required'}; + } + async receive(value:Obj,deadline:number,accessed?:()=>void):Promise{ + const c=this.#checked(value,false),{e,owner,setup,journal,known,policy,budget}=c; + const needs=['slot','read','maintenance','bootstrap'].map(n=>({signer:owner.signing_key,scope_kind:n==='slot'?'mailbox_slot':'authority',scope_id:statusScope(e.slot.root_key,n==='slot'?'mailbox_slot':'authority',n==='slot'?e.slot:{authority_kind:setup[n].payload.kind,authority_sha256:setup[n].ref.raw_sha256},policy,budget),revision:setup[n].payload.revision,mask:n==='read'?2:10}));replicaFloors(known,[],needs); + const saved=this.#participant.providerStorage(db=>db.prepare("SELECT 1 FROM open_delivery_inbox WHERE envelope_sha256=? AND phase!='staged'").get(c.request.expected_envelope_ref.raw_sha256));if(saved)return {messages:[],errors:[],state:'observed',network_accessed:false,body_transport:'mailbox_message_replica'}; + accessed?.();const introduction=(await this.#participant.transport.requestNode(value.base_url,deadline)).response,node=verifyNode(introduction); + if(node.status!=='active'||!same(node.signing_key,e.target.signing_key)||node.storage_epoch!==c.node.payload.storage_epoch||node.revision; const fields=(v:unknown,n:readonly string[]):Obj=>objectFields(v,n); const key=(value:unknown)=>{const r=rawRef(value);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; function fail(code:string):never{throw new RepairError(code);} function same(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&same((a as Obj)[k],(b as Obj)[k]));} export async function verifyMailboxInboxEvidence(evidence:unknown,envelope:Uint8Array,options:{owner:unknown;encryptionIdentity:EncryptionIdentityDocument;stagedAt:number}):Promise>{ + if((evidence as Obj)?.schema_version==='memory-vault-mailbox-replica-inbox/v1')return verifyMailboxReplicaInboxEvidence(evidence,envelope,options); const a=fields(options,['owner','encryptionIdentity','stagedAt']); const value=fields(evidence,['schema_version','received_at','slot','sender','target','limits','member','manifest_ref','custody_ref','originals','status_refs']); if(value.schema_version!=='memory-vault-mailbox-inbox/v1')fail('repair_invalid_context'); diff --git a/clients/typescript/network/open-repair-mailbox-read.ts b/clients/typescript/network/open-repair-mailbox-read.ts index 8f7c534..b9cbfcf 100644 --- a/clients/typescript/network/open-repair-mailbox-read.ts +++ b/clients/typescript/network/open-repair-mailbox-read.ts @@ -105,6 +105,18 @@ function currentRead(setup:Obj,entries:readonly unknown[],known:readonly unknown /** Authenticate/decrypt metadata and check present authority before fetching * the envelope. The enclosing verified feed determines which member to select. */ export async function readMailboxAdmission(member:unknown,options:MailboxAdmissionReadOptions):Promise>{ + const metadata=await verifyMailboxAdmissionMetadata(member,options); + const {policy,budget}=options; + const statuses=currentRead(metadata.setup,options.currentStatuses,options.knownStatuses,options.statusObligations,options.at,policy,budget,options.onStatusAuthenticated); + const ref=rawRef(metadata.core.envelope_ref),raw=await options.readOriginal(ref); + if(!isUint8Array(raw)||ref.namespace!=='object'||ref.size>6291456||raw.length!==ref.size||budget.hash(raw)!==ref.raw_sha256)fail('repair_ref_mismatch'); + budget.input(raw.length);budget.retain(raw.length);const envelope=Uint8Array.from(raw); + return Object.freeze({...metadata,current_statuses:statuses,originals:Object.freeze([...metadata.originals,{ref,raw:envelope}]),get envelope(){budget.output(envelope.length);return Uint8Array.from(envelope);}}); +} +/** Original source authentication and recipient decryption, without asserting + * current source READ. Replica callers must establish independent live READ + * before fetching the envelope; this function performs no object fetch. */ +export async function verifyMailboxAdmissionMetadata(member:unknown,options:MailboxAdmissionReadOptions):Promise>{ const names=['expectedSlot','expectedSigningKey','encryptionIdentity','readOriginal','at','policy','budget','expectedOwner','expectedSender','expectedTarget','limitPolicy','currentStatuses','knownStatuses','statusObligations']; const a=fields(options,Object.hasOwn(options,'onStatusAuthenticated')?[...names,'onStatusAuthenticated']:names),{policy,budget}=a; const e=buildNewWire({slot:a.expectedSlot,key:a.expectedSigningKey,owner:a.expectedOwner,sender:a.expectedSender,target:a.expectedTarget,limits:a.limitPolicy,at:a.at,identity:a.encryptionIdentity,member},policy,budget).value as Obj; @@ -144,8 +156,6 @@ export async function readMailboxAdmission(member:unknown,options:MailboxAdmissi for(const n of ['slot','read','maintenance','bootstrap']){const p=setup.originals[n].payload;deadlines.push(p.expires_at);if(p.windows)deadlines.push(p.windows.read_until,p.windows.retain_until);if(n==='bootstrap')deadlines.push(p.probe_until,p.proof_until,p.upload_until);} for(const n of ['data','metadata']){const p=setup.resources[n].active.payload;deadlines.push(p.windows.read_until,p.windows.retain_until);} if(core.enum_until>Math.min(...deadlines))fail('repair_mailbox_member_mismatch'); - const statuses=currentRead(setup,a.currentStatuses,a.knownStatuses,a.statusObligations,e.at,policy,budget,a.onStatusAuthenticated); - const envelope=await load(core.envelope_ref,'object'); const retained=Object.freeze([...originals.values()].map(v=>Object.freeze({ref:v.ref,get raw(){budget.output(v.raw.length);return Uint8Array.from(v.raw);}}))); - return Object.freeze({core,link,checkpoint,setup,history:manifest,current_statuses:statuses,originals:retained,get envelope(){budget.output(envelope.length);return Uint8Array.from(envelope);}}); + return Object.freeze({core,link,checkpoint,setup,history:manifest,originals:retained}); } diff --git a/clients/typescript/network/open-repair-mailbox-replica-inbox.ts b/clients/typescript/network/open-repair-mailbox-replica-inbox.ts new file mode 100644 index 0000000..41a4b92 --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-replica-inbox.ts @@ -0,0 +1,27 @@ +/** Verify the exact replica proof retained with an inbox entry at receipt time. + * Local reopening grants no later network access or renewed COPY/READ rights. */ +import {RepairBudget,LocalRawResolver,buildNewWire,parseNewWire,rawRef,u53} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {decodeBase64url} from './crypto.ts'; +import {authenticateStatusOriginal} from './open-repair-status.ts'; +import {verifyMailboxAdmissionMetadata} from './open-repair-mailbox-read.ts'; +import {MAILBOX_REPLICA_LIMITS} from './open-mailbox-replica-client.ts'; +import {replicaFields as fields,replicaFail as fail,replicaSame as same,replicaRefKey as key,replicaFloors,verifyMailboxMessageReplica,checkMailboxMessageReturn} from './open-repair-mailbox-replica.ts'; +type Obj=Record; +export async function verifyMailboxReplicaInboxEvidence(evidence:unknown,envelope:Uint8Array,a:Obj):Promise>{ + fields(a,['owner','encryptionIdentity','stagedAt']);const v=fields(evidence,'schema_version received_at slot sender source target maintainer target_storage_epoch envelope_ref limits manifest_ref custody_ref consent_refs originals status_refs archive_status_refs'.split(' ')); + if(v.schema_version!=='memory-vault-mailbox-replica-inbox/v1'||u53(v.received_at)>u53(a.stagedAt))fail('repair_invalid_context');const at=v.received_at,policy=Object.freeze({...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512}),budget=new RepairBudget(policy); + const limits=buildNewWire(v.limits,policy,budget).value as Obj;fields(limits,Object.keys(MAILBOX_REPLICA_LIMITS));for(const [name,maximum] of Object.entries(MAILBOX_REPLICA_LIMITS))if(u53(limits[name],1)>maximum)fail('repair_over_budget'); + if(!Array.isArray(v.originals)||v.originals.length<1||v.originals.length>limits.max_proof_items||!Array.isArray(v.status_refs)||v.status_refs.length<1||v.status_refs.length>16||!Array.isArray(v.archive_status_refs)||v.archive_status_refs.length>32)fail('repair_over_budget'); + const resolver=new LocalRawResolver(policy,budget),originals=new Map();let total=envelope.length; + for(const item of v.originals){fields(item,['ref','raw_base64url']);const ref=rawRef(item.ref);if(ref.namespace!=='meta'||ref.size>policy.max_document_bytes||originals.has(key(ref)))fail('repair_ref_mismatch');const raw=decodeBase64url(item.raw_base64url,policy.max_document_bytes);total+=raw.length;if(total>limits.max_proof_bytes)fail('repair_over_budget');if(!same(resolver.put(ref.namespace,ref.key,raw).ref,ref))fail('repair_ref_mismatch');originals.set(key(ref),{ref,raw});} + const entry=(ref:unknown)=>{const item=originals.get(key(ref));if(!item)fail('repair_original_missing');return item;}; + const context={expectedSlot:v.slot,expectedOwner:a.owner,expectedSender:v.sender,expectedSource:v.source,sourceStorageEpoch:v.slot.writer_storage_epoch,expectedMaintainer:v.maintainer,expectedTarget:v.target,targetStorageEpoch:v.target_storage_epoch,expectedEnvelopeRef:v.envelope_ref,limitPolicy:limits,policy,budget}; + const replica=verifyMailboxMessageReplica(entry(v.manifest_ref),resolver,entry(v.custody_ref),context);if(replica.custody.payload.stored_at>at)fail('repair_access_expired');fields(v.consent_refs,['owner','source','maintainer','sender']); + const permission=checkMailboxMessageReturn(replica,Object.fromEntries(Object.entries(v.consent_refs).map(([n,ref])=>[n,entry(ref)])),{...context,currentStatuses:v.status_refs.map(entry),at,action:'child'}); + const signers=new Map([a.owner,v.sender,v.source,v.maintainer,v.target].map(v=>[v.signing_key.key_id,v.signing_key])),archive:Obj[]=[]; + for(const ref of v.archive_status_refs){const saved=entry(ref),p=(parseNewWire(saved.raw,policy,budget).value as Obj).payload;if(!same(signers.get(p.signing_key.key_id),p.signing_key)||u53(p.issued_at)>at)fail('repair_status_disclosure');archive.push(authenticateStatusOriginal(saved,{expectedRoot:v.slot.root_key,expectedSigningKey:p.signing_key,at:p.issued_at,allowedScopes:p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id})),policy,budget}));} + replicaFloors(archive,permission.statuses,permission.obligations); + const checked=await verifyMailboxAdmissionMetadata(replica.source.message_member,{expectedSlot:v.slot,expectedSigningKey:v.source.signing_key,encryptionIdentity:a.encryptionIdentity,readOriginal:ref=>entry(ref).raw,at,policy,budget,expectedOwner:a.owner,expectedSender:v.sender,expectedTarget:v.source,limitPolicy:limits,currentStatuses:[],knownStatuses:[],statusObligations:[]}); + const ref=rawRef(checked.core.envelope_ref);if(!same(ref,v.envelope_ref)||envelope.length!==ref.size||budget.hash(envelope)!==ref.raw_sha256)fail('repair_ref_mismatch');return Object.freeze({...checked,envelope,originals:[...checked.originals,{ref,raw:envelope}],current_statuses:permission.statuses}); +} diff --git a/clients/typescript/network/open-repair-mailbox-replica.ts b/clients/typescript/network/open-repair-mailbox-replica.ts new file mode 100644 index 0000000..df51ade --- /dev/null +++ b/clients/typescript/network/open-repair-mailbox-replica.ts @@ -0,0 +1,170 @@ +/** Complete original message COPY closure and independent present READ. + * A target custody signature alone never authorizes disclosure or decryption. + * This verifier has no storage, transport, admission or publication effects. */ +import {buildNewWire,parseNewWire,objectFields,rawRef,u53,RepairError} from './open-repair-wire.ts'; +import type {RepairPolicy,RepairBudget,RawOriginal,LocalRawResolver} from './open-repair-wire.ts'; +import {verifyBoundedControlSignature} from './open-repair-original.ts'; +import {statusScope,authenticateStatusOriginal} from './open-repair-status.ts'; +import {verifyMailboxFeedSourceEvent} from './open-repair-mailbox-feed.ts'; +type Obj=Record; +const COMMON=['schema_version','kind','signing_key'],SCHEMA='memory-vault-open-repair/v1'; +const BUDGET='max_live_bytes max_meta_bytes max_items max_requests max_pending max_replay_records max_jobs max_job_bytes'.split(' '); +const WINDOWS='admit_until read_until copy_until publish_until retain_until'.split(' '); +const ALLOCATE='issued_at expires_at request_id target_node_key_id target_storage_epoch intent intent_sha256'; +const OFFER='issued_at reservation_until offer_id allocation_request_ref intent intent_sha256 resource target_encryption_key reservation_generation budget windows'; +const ASSIGNMENT='issued_at expires_at assignment_id job_id root_key parent_root_ref parent_assignment_ref depth subject target_node_key_id target_storage_epoch operation_mask scope resource_intent_sha256 resource_offer_ref resource bootstrap_grant_refs budget windows'; +const RESERVATION='issued_at expires_at consent_id revision variant root_authority_ref source_custody_ref historical_manifest_ref maintainer target target_storage_epoch reservation_disclosure'; +const DISCLOSURE='issued_at expires_at consent_id revision variant root_key assignment_ref source_custody_ref historical_manifest_ref target target_storage_epoch disclosure'; +const RETURN='issued_at expires_at consent_id revision variant root_key source_custody_ref historical_manifest_ref assignment_ref subject target target_storage_epoch bootstrap_grant_ref return_permission'; +const CUSTODY='root_key scope original_custody_ref replica_manifest_ref assignment_ref resource_offer_ref resource reservation_generation stored_at read_until retain_until'; +export const replicaFields=objectFields; +export function replicaFail(code='repair_copy_authority_mismatch'):never{throw new RepairError(code);} +export function replicaSame(a:unknown,b:unknown):boolean{if(a===b)return true;if(!a||!b||typeof a!=='object'||typeof b!=='object'||Array.isArray(a)!==Array.isArray(b))return false;const keys=Object.keys(a);return keys.length===Object.keys(b).length&&keys.every(k=>Object.hasOwn(b,k)&&replicaSame((a as Obj)[k],(b as Obj)[k]));} +export const replicaRefKey=(v:unknown)=>{const r=rawRef(v);return [r.namespace,r.key,r.raw_sha256,r.size].join(':');}; +const pair=(role:string,ref:unknown)=>role+'|'+replicaRefKey(ref); +const sorted=(values:T[],key:(v:T)=>string)=>values.sort((a,b)=>key(a)key(b)?1:0); +const require=(v:unknown,code='repair_copy_authority_mismatch')=>{if(!v)replicaFail(code);}; +const opaque=(v:unknown)=>{require(typeof v==='string'&&/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.exec(v)?.[0]===v);return v;}; +const ids=(v:Obj)=>({signing_key_id:v.signing_key.key_id,encryption_key_id:v.encryption_key.key_id}); +function windows(v:Obj,at?:number):void{objectFields(v,WINDOWS);for(const n of WINDOWS){u53(v[n]);require(v[n]<=v.retain_until&&(at===undefined||v[n]>at));}} +function capacity(v:Obj):void{objectFields(v,BUDGET);for(const n of BUDGET)u53(v[n]);} +function lifetime(v:Obj,at?:number):void{u53(v.issued_at);u53(v.expires_at);require(v.issued_atstatusScope(root,'authority',{authority_kind:item.payload.kind,authority_sha256:item.ref.raw_sha256},policy,budget); +const assignment=(root:unknown,item:Obj,policy:RepairPolicy,budget:RepairBudget)=>statusScope(root,'assignment',{assignment_kind:'maintenance.assignment',assignment_sha256:item.ref.raw_sha256},policy,budget); +const duty=(signer:Obj,scope_kind:string,scope_id:string,revision:number,mask:number)=>({signer,scope_kind,scope_id,revision,mask}); +const scopes=(values:Obj[])=>sorted([...new Map(values.map(v=>[v.scope_kind+':'+v.scope_id,{scope_kind:v.scope_kind,scope_id:v.scope_id}])).values()],v=>v.scope_kind+':'+v.scope_id); +/** Conflict, monotonically increasing floors, stale current views, and known + * revocations are checked across every authentic observation. */ +export function replicaFloors(previous:readonly Obj[],current:readonly Obj[],obligations:readonly Obj[]=[]):void{ + const required=new Map(obligations.map(v=>[[v.signer.key_id,v.scope_kind,v.scope_id].join(':'),v])),seen=new Map(),values=new Map(); + for(const item of [...previous,...current]){const p=item.payload,issuer=p.scope_key.issuer_key_id,revision=p.revision,k=issuer+':'+revision; + if(seen.has(k)&&seen.get(k)!==item.canonical_sha256)replicaFail('repair_status_conflict');seen.set(k,item.canonical_sha256); + for(const row of p.entries){const key=[issuer,row.scope_kind,row.scope_id].join(':'),need=required.get(key); + if(need&&row.status==='revoked'&&(row.operation_mask&need.mask))replicaFail('repair_authority_revoked'); + if(need&&row.minimum_document_revision>need.revision)replicaFail('repair_status_revision'); + const held=values.get(key)??[];held.push([revision,row.minimum_document_revision]);values.set(key,held);}} + for(const held of values.values()){let minimum=0;for(const [,v] of held.sort((a,b)=>a[0]-b[0]||a[1]-b[1])){if(vv[0]>p.revision))replicaFail('repair_status_rollback');} +} +function observations(entries:unknown,parties:Obj,allowed:Map,needs:Obj[],root:unknown,at:number,previous:Obj[],policy:RepairPolicy,budget:RepairBudget,onObserved?: (v:Obj)=>void):Obj[]{ + if(!Array.isArray(entries)||entries.length<1||entries.length>16)replicaFail('repair_status_missing');const checked:Obj[]=[],signers=new Map(Object.values(parties).map((v:Obj)=>[v.signing_key.key_id,v.signing_key]));let denial:unknown; + // Persist each authentic status even when a later supplied document is bad. + for(const entry of entries)try{const p=(parseNewWire((entry as Obj).raw,policy,budget).value as Obj).payload,key=p.signing_key.key_id; + if(!signers.has(key))replicaFail('repair_status_disclosure');authenticateStatusOriginal(entry,{expectedRoot:root,expectedSigningKey:signers.get(key),at,allowedScopes:scopes(allowed.get(key)??[]),policy,budget},v=>{checked.push(v);onObserved?.(v);}); + }catch(error){denial??=error;} + if(new Set(checked.map(v=>replicaRefKey(v.ref))).size!==checked.length)denial??=new RepairError('repair_duplicate_status'); + try{replicaFloors(previous,checked);}catch(error){denial??=error;} + for(const need of needs){const rows=checked.filter(v=>replicaSame(v.payload.signing_key,need.signer)).flatMap(v=>v.payload.entries).filter(v=>v.scope_kind===need.scope_kind&&v.scope_id===need.scope_id); + if(!rows.length)denial??=new RepairError('repair_status_missing');for(const row of rows){if(row.status==='revoked')denial??=new RepairError('repair_authority_revoked');else if(row.minimum_document_revision>need.revision)denial??=new RepairError('repair_status_revision');else if((row.operation_mask&need.mask)!==need.mask)denial??=new RepairError('repair_status_operation');}} + if(denial)throw denial;return checked; +} +function inventory(source:Obj,reservations:Obj,policy:RepairPolicy,budget:RepairBudget):Obj[]{ + const rows=new Map(),add=(role:string,item:Obj)=>{const p=(parseNewWire(item.raw,policy,budget).value as Obj).payload;rows.set(pair(role,item.ref),{role,original:item,issuer:p?.signing_key.key_id??''});}; + for(const tree of [source.manifest,...source.manifest.predecessors])for(const row of tree.roles)add(row.role,row.original); + add('feed.custody',source.feed_custody);add('message.custody',source.custody);add('copy.reservation_consent',reservations.owner);add('copy.sender_reservation_consent',reservations.sender); + return sorted([...rows.values()],v=>pair(v.role,v.original.ref)); +} +function permissions(rows:Obj[],issuer:string,root:unknown,mode:'copy'|'return',policy:RepairPolicy,budget:RepairBudget):Obj{ + const selected=rows.filter(v=>v.issuer===issuer),allowed:Obj[]=[]; + for(const row of selected){const p=(parseNewWire(row.original.raw,policy,budget).value as Obj).payload,item={...row.original,payload:p}; + if(p.kind==='authority.status')allowed.push(...p.entries.map((v:Obj)=>({scope_kind:v.scope_kind,scope_id:v.scope_id}))); + if(['copy.reservation_consent','copy.sender_reservation_consent'].includes(row.role)||mode==='return'&&['mailbox.message_copy_disclosure','mailbox.feed_copy_disclosure'].includes(p.kind))allowed.push({scope_kind:'authority',scope_id:authority(root,item,policy,budget)}); + if(mode==='return'&&p.kind==='maintenance.assignment')allowed.push({scope_kind:'assignment',scope_id:assignment(root,item,policy,budget)});} + return {originals:selected.map(v=>({role:v.role,ref:v.original.ref})),status_scopes:scopes(allowed)}; +} +export interface MessageReplicaOptions {expectedSlot:unknown;expectedOwner:unknown;expectedSender:unknown;expectedSource:unknown;sourceStorageEpoch:string;expectedMaintainer:unknown;expectedTarget:unknown;targetStorageEpoch:string;expectedEnvelopeRef:unknown;limitPolicy:unknown;policy:RepairPolicy;budget:RepairBudget;} +export function verifyMailboxMessageReplica(manifestEntry:unknown,resolver:LocalRawResolver,custodyEntry:unknown,a:MessageReplicaOptions):Readonly{ + const {policy,budget}=a;const e=buildNewWire({slot:a.expectedSlot,owner:a.expectedOwner,sender:a.expectedSender,source:a.expectedSource,maintainer:a.expectedMaintainer,target:a.expectedTarget},policy,budget).value as Obj; + const envelope=rawRef(a.expectedEnvelopeRef);require(envelope.namespace==='object'&&envelope.size>0&&envelope.size<=6291456); + const custody=signed(custodyEntry,e.target.signing_key,'replica.custody',CUSTODY,policy,budget),p=custody.payload;resource(p.resource); + const at=u53(p.stored_at);require(at0&&v.original_roles.length<=128,'repair_copy_commit_mismatch'); + const entries=new Map();let last=''; + for(const row of v.original_roles){objectFields(row,['role','ref']);require(typeof row.role==='string');const ref=rawRef(row.ref),k=pair(row.role,ref);require(!last||k>last,'repair_copy_commit_mismatch');last=k; + if(row.role==='message.envelope'){require(replicaSame(ref,envelope),'repair_copy_commit_mismatch');continue;} + const item=resolver.resolve(ref);entries.set(row.role,[...(entries.get(row.role)??[]),{raw:item.raw,ref:item.ref}]);} + const one=(role:string):RawOriginal=>{const values=entries.get(role);if(values?.length!==1)replicaFail('repair_copy_commit_mismatch');return values[0];}; + const feed=verifyMailboxFeedSourceEvent(one('history.mailbox_feed'),resolver,one('feed.custody'),{expectedSlot:e.slot,expectedOwner:e.owner,expectedSender:e.sender,expectedTarget:e.source,limitPolicy:a.limitPolicy,policy,budget}); + const selected=(feed.manifest.manifest.value as Obj).members.filter((v:Obj)=>replicaSame(v.envelope_ref,envelope));require(selected.length===1);const member=selected[0]; + const role=(name:string,ref:unknown):RawOriginal=>{const rows=feed.manifest.roles.filter((v:Obj)=>v.role===name&&replicaSame(v.original.ref,ref));require(rows.length===1);return rows[0].original;}; + const messageCustody=role('member.custody',member.source_custody_ref),mc=(parseNewWire(messageCustody.raw,policy,budget).value as Obj).payload,core=(parseNewWire(role('member.core',member.admission_core_ref).raw,policy,budget).value as Obj).payload; + const setup=feed.graph.members[member.sequence];require(setup.attempt.message_id===member.message_id&&replicaSame(mc.envelope_ref,envelope)); + const active=setup.resources.data.active.payload,readUntil=Math.min(feed.read_until,core.object_until,active.windows.read_until),retainUntil=Math.min(feed.retain_until,core.object_until,active.windows.retain_until); + const data=setup.obligations.find((v:Obj)=>v.role==='data_resource');require(data); + const scope={kind:'mailbox_member',slot_key:e.slot,attempt_ref:core.attempt_ref,envelope_ref:envelope,admission_link_ref:member.admission_link_ref,source_custody_ref:member.source_custody_ref}; + const source={...feed,feed_custody:feed.custody,custody:{...messageCustody,payload:mc},message_scope:scope,message_history_ref:member.historical_manifest_ref,message_core_ref:member.admission_core_ref, + message_member:{sequence:member.sequence,admission_link_ref:member.admission_link_ref,sealed_core_ref:(parseNewWire(role('member.link',member.admission_link_ref).raw,policy,budget).value as Obj).payload.sealed_core_ref},graph:{...feed.graph,obligations:[...feed.graph.obligations,{...data,operation_mask:70}]},read_until:Math.min(readUntil,retainUntil),retain_until:retainUntil}; + const originals=feed.graph.members[0].originals,parent=originals.maintenance,read=originals.read,bootstrap=originals.bootstrap,slot=originals.slot,root=e.slot.root_key,pr=parent.payload; + require(e.slot.writer_storage_epoch===a.sourceStorageEpoch&&pr.maintainers.some((v:Obj)=>replicaSame(v,ids(e.maintainer)))&&(pr.operation_mask&78)===78&&pr.max_delegate_depth>=2&&pr.max_destinations_per_job>0&&pr.max_concurrent_jobs>0&&!replicaSame(ids(e.target),ids(e.source))&&!replicaSame(ids(e.target),ids(e.maintainer))); + const allocation=signed(one('copy.allocation'),e.maintainer.signing_key,'resource.allocate',ALLOCATE,policy,budget),offer=signed(one('copy.offer'),e.target.signing_key,'resource.offer',OFFER,policy,budget),assign=signed(one('copy.assignment'),e.maintainer.signing_key,'maintenance.assignment',ASSIGNMENT,policy,budget),ap=allocation.payload,op=offer.payload,mp=assign.payload; + lifetime(ap,at);lifetime(mp,at);for(const [n,v] of [['request_id',ap],['offer_id',op],['assignment_id',mp]] as const)opaque(v[n]); + const intent=objectFields(ap.intent,'kind allocation_id job_id root_key caller target target_storage_epoch purpose scope historical_manifest_ref budget windows'.split(' '));capacity(intent.budget);windows(intent.windows,at);resource(op.resource); + for(const n of ['allocation_id','job_id','target_storage_epoch'])opaque(intent[n]);const digest=budget.hash(buildNewWire(intent,policy,budget).raw); + require(intent.kind==='resource.copy_intent'&&intent.purpose==='message_replica'&&intent.budget.max_live_bytes>=envelope.size&&replicaSame(intent.caller,e.maintainer)&&replicaSame(intent.target,e.target)&&intent.target_storage_epoch===a.targetStorageEpoch&&replicaSame(intent.root_key,root)&&replicaSame(intent.scope,scope)&&replicaSame(intent.historical_manifest_ref,member.historical_manifest_ref)&&replicaSame(op.intent,intent)&&ap.intent_sha256===digest&&op.intent_sha256===digest&&replicaSame(op.allocation_request_ref,allocation.ref)&&replicaSame(op.target_encryption_key,e.target.encryption_key)&&op.resource.node_key_id===e.target.signing_key.key_id&&op.resource.storage_epoch===a.targetStorageEpoch&&u53(op.reservation_generation,1)===1&&replicaSame(op.budget,intent.budget)&&replicaSame(op.windows,intent.windows)&&ap.issued_at<=u53(op.issued_at)&&op.issued_at<=mp.issued_at&&atintent.budget[n]<=control.payload.budget[n])&&WINDOWS.every(n=>intent.windows[n]<=control.payload.windows[n])); + const parentUntil=Math.min(...[slot,parent,read,bootstrap].map(v=>v.payload.expires_at),read.payload.windows.read_until,bootstrap.payload.probe_until,bootstrap.payload.proof_until,bootstrap.payload.upload_until); + require(mp.job_id===intent.job_id&&replicaSame(mp.root_key,root)&&replicaSame(mp.parent_root_ref,parent.ref)&&mp.parent_assignment_ref===null&&u53(mp.depth)===2&&replicaSame(mp.subject,ids(e.target))&&u53(mp.operation_mask)===70&&replicaSame(mp.scope,scope)&&mp.resource_intent_sha256===digest&&replicaSame(mp.resource_offer_ref,offer.ref)&&replicaSame(mp.resource,op.resource)&&replicaSame(mp.bootstrap_grant_refs,[bootstrap.ref])&&replicaSame(mp.budget,op.budget)&&replicaSame(mp.windows,op.windows)&&Object.values(mp.windows).every(v=>u53(v)<=mp.expires_at)&&mp.expires_at<=parentUntil); + for(const v of [ap,mp])require(v.target_node_key_id===e.target.signing_key.key_id&&v.target_storage_epoch===a.targetStorageEpoch); + let maximum=Math.min(source.read_until,source.retain_until,parentUntil,active.windows.copy_until,slot.payload.windows.copy_until,pr.windows.copy_until,intent.windows.copy_until); + for(const item of feed.graph.members){const d=item.disclosure;require((d.operation_mask&70)===70&&['maintenance','slot','read','bootstrap'].every(n=>replicaSame(item.originals[n].ref,originals[n].ref)));maximum=Math.min(maximum,d.consent_until,d.expires_at);} + const reservations:Obj={};for(const variant of ['owner','sender']){const item=signed(one(variant==='owner'?'copy.reservation_consent':'copy.sender_reservation_consent'),e[variant].signing_key,'mailbox.message_copy_reservation_consent',RESERVATION,policy,budget),c=item.payload;lifetime(c,at);opaque(c.consent_id);u53(c.revision,1);const d=objectFields(c.reservation_disclosure,['intent_sha256','until']); + require(c.variant===variant&&replicaSame(c.root_authority_ref,parent.ref)&&replicaSame(c.source_custody_ref,source.custody.ref)&&replicaSame(c.historical_manifest_ref,intent.historical_manifest_ref)&&replicaSame(c.maintainer,ids(e.maintainer))&&replicaSame(c.target,e.target)&&c.target_storage_epoch===a.targetStorageEpoch&&Math.max(mc.stored_at,feed.custody.payload.stored_at)<=c.issued_at&&c.issued_at<=ap.issued_at&&d.intent_sha256===digest&&at(),consents:Obj[]=[];for(const name of ['owner','sender','source','maintainer'])allowed.set(e[name].signing_key.key_id,[]);let until=Math.min(maximum,mp.expires_at); + for(const variant of ['owner','source','sender']){const item=signed(one('copy.'+variant+'_disclosure'),e[variant].signing_key,'mailbox.message_copy_disclosure',DISCLOSURE,policy,budget),c=item.payload;lifetime(c,at);opaque(c.consent_id);u53(c.revision,1); + require(c.variant===variant&&replicaSame(c.root_key,root)&&replicaSame(c.assignment_ref,assign.ref)&&replicaSame(c.source_custody_ref,source.custody.ref)&&replicaSame(c.historical_manifest_ref,intent.historical_manifest_ref)&&replicaSame(c.target,e.target)&&c.target_storage_epoch===a.targetStorageEpoch&&mp.issued_at<=c.issued_at); + const d=objectFields(c.disclosure,['originals','status_scopes','until']),expected=permissions(rows,e[variant].signing_key.key_id,root,'copy',policy,budget);require(replicaSame(d.originals,expected.originals)&&replicaSame(d.status_scopes,expected.status_scopes)&&atduty(v.signer,v.scope_kind,v.scope_id,v.document_revision,v.operation_mask|(['slot','maintenance','disclosure','metadata_resource','data_resource'].includes(v.role)?4:0))); + for(const variant of ['owner','sender']){const item=reservations[variant],id=authority(root,item,policy,budget);allowed.get(e[variant].signing_key.key_id)!.push({scope_kind:'authority',scope_id:id});needs.push(duty(e[variant].signing_key,'authority',id,item.payload.revision,4));} + needs.push(duty(e.owner.signing_key,'authority',authority(root,parent,policy,budget),pr.revision,78));const as=assignment(root,assign,policy,budget);allowed.get(e.maintainer.signing_key.key_id)!.push({scope_kind:'assignment',scope_id:as});needs.push(duty(e.maintainer.signing_key,'assignment',as,0,70)); + for(const [i,variant] of ['owner','source','sender'].entries())needs.push(duty(e[variant].signing_key,'authority',authority(root,consents[i],policy,budget),consents[i].payload.revision,4)); + const previous=[...feed.graph.statuses,...feed.graph.members.flatMap((v:Obj)=>v.statuses)],statuses=observations(entries.get('copy.current_status'),{owner:e.owner,sender:e.sender,source:e.source,maintainer:e.maintainer},allowed,needs,root,at,previous,policy,budget); + const read_until=Math.min(until,intent.windows.read_until,...statuses.map(v=>v.payload.valid_until)),retain_until=Math.min(until,intent.windows.retain_until);require(at(),edges=new Map(),add=(role:string,ref:unknown)=>{const r=rawRef(ref);closure.set(pair(role,r),{role,ref:r});},edge=(parent:unknown,relation:string,child:unknown)=>{const v={parent_ref:rawRef(parent),relation,child_ref:rawRef(child)};edges.set(replicaRefKey(parent)+'|'+relation+'|'+replicaRefKey(child),v);}; + for(const row of rows)add(row.role,row.original.ref);for(const [role,item] of [['copy.allocation',allocation],['copy.offer',offer],['copy.assignment',assign],...consents.map((item,i)=>['copy.'+['owner','source','sender'][i]+'_disclosure',item])] as [string,Obj][])add(role,item.ref); + add('message.envelope',envelope);for(const item of statuses)add('copy.current_status',item.ref); + for(const tree of [feed.manifest,...feed.manifest.predecessors]){ + let ref=one('history.mailbox_feed').ref; + if(tree!==feed.manifest){const rows=feed.manifest.roles.filter((v:Obj)=>v.role==='history.member'&&Buffer.from(v.original.raw).equals(Buffer.from(tree.manifest.raw)));require(rows.length===1,'repair_copy_commit_mismatch');ref=rows[0].original.ref;} + add(tree===feed.manifest?'history.mailbox_feed':'history.member',ref); + for(const row of tree.manifest.value.roles){add('history.raw_pack',row.pack_ref);edge(ref,'manifest-member',row.document_ref);edge(ref,'manifest-member',row.pack_ref);} + } + const fm=feed.manifest.manifest.value;edge(fm.slot_ref,'slot-feed',fm.feed_head_ref);edge(fm.feed_head_ref,'head-checkpoint',feed.graph.head.checkpoint_ref);edge(fm.feed_head_ref,'head-range',feed.graph.head.range_root_ref); + for(const item of fm.members){edge(item.admission_link_ref,'link-core',item.admission_core_ref);edge(item.admission_link_ref,'link-history',item.historical_manifest_ref);edge(item.source_custody_ref,'custody-link',item.admission_link_ref);} + for(const row of feed.manifest.roles)if(['range.index','range.repair_page'].includes(row.role)){const p=(parseNewWire(row.original.raw,policy,budget).value as Obj);if(row.role==='range.index')for(const child of p.children)edge(row.original.ref,'range-child',child.ref);else{edge(row.original.ref,'page-sealed',p.sealed_page_ref);for(const child of p.entries)edge(row.original.ref,'page-link',child.admission_link_ref);}} + edge(member.admission_core_ref,'core-envelope',envelope); + const expected={schema_version:SCHEMA,kind:'replica.manifest',root_key:root,scope,original_roles:sorted([...closure.values()],v=>pair(v.role,v.ref)),physical_objects:sorted([...closure.values()],v=>pair(v.role,v.ref)),edges:sorted([...edges.values()],v=>replicaRefKey(v.parent_ref)+'|'+v.relation+'|'+replicaRefKey(v.child_ref))}; + require(replicaSame(v,expected)&&new Set(v.original_roles.map((v:Obj)=>replicaRefKey(v.ref))).size<=op.budget.max_items,'repair_copy_commit_mismatch'); + return Object.freeze({state:'historical_replica',custody,source,authority:plan,entries}); +} +export function checkMailboxMessageReturn(replica:Obj,consentEntries:Obj,a:Obj):Readonly{ + const {policy,budget}=a,source=replica.source,plan=replica.authority,custody=replica.custody,root=plan.assignment.payload.root_key,setup=source.graph.members[0].originals,bootstrap=setup.bootstrap; + require(['challenge','proof','child'].includes(a.action),'repair_access_action');u53(a.at); + const e=buildNewWire({owner:a.expectedOwner,sender:a.expectedSender,source:a.expectedSource,maintainer:a.expectedMaintainer,target:a.expectedTarget},policy,budget).value as Obj,variants=['owner','source','maintainer','sender'];objectFields(consentEntries,variants); + const allowed=new Map();for(const p of Object.values(e) as Obj[])allowed.set(p.signing_key.key_id,[]);const needs:Obj[]=[],consents:Obj[]=[]; + let expires=Math.min(custody.payload.read_until,custody.payload.retain_until,source.read_until,bootstrap.payload.expires_at,bootstrap.payload[a.action==='challenge'?'probe_until':'proof_until']); + for(const item of source.graph.members){for(const n of ['slot','read','maintenance']){const p=item.originals[n].payload;expires=Math.min(expires,p.expires_at,p.windows.read_until,p.windows.retain_until);}const d=item.disclosure;expires=Math.min(expires,d.expires_at,d.consent_until,d.bootstrap_return.until);} + const rows=new Map(plan.originals.map((v:Obj)=>[pair(v.role,v.original.ref),v]));for(const [role,item] of [['copy.allocation',plan.allocation],['copy.offer',plan.offer],['copy.assignment',plan.assignment],...plan.disclosures.map((v:Obj,i:number)=>['copy.'+['owner','source','sender'][i]+'_disclosure',v])] as [string,Obj][])rows.set(pair(role,item.ref),{role,original:item,issuer:item.payload.signing_key.key_id}); + for(const variant of variants){const item=signed(consentEntries[variant],e[variant].signing_key,'mailbox.message_replica_return_consent',RETURN,policy,budget),p=item.payload;lifetime(p,a.at);opaque(p.consent_id);u53(p.revision,1); + require(p.variant===variant&&replicaSame(p.root_key,root)&&replicaSame(p.source_custody_ref,source.custody.ref)&&replicaSame(p.historical_manifest_ref,plan.allocation.payload.intent.historical_manifest_ref)&&replicaSame(p.assignment_ref,plan.assignment.ref)&&replicaSame(p.subject,ids(e.owner))&&replicaSame(p.target,e.target)&&p.target_storage_epoch===a.targetStorageEpoch&&replicaSame(p.bootstrap_grant_ref,bootstrap.ref)&&p.issued_at>=plan.assignment.payload.issued_at); + const d=objectFields(p.return_permission,['originals','status_scopes','until']),expected=permissions(sorted([...rows.values()],v=>pair(v.role,v.original.ref)),e[variant].signing_key.key_id,root,'return',policy,budget); + require(replicaSame(d.originals,expected.originals)&&replicaSame(d.status_scopes,expected.status_scopes)&&p.issued_atv.scope_kind+':'+v.scope_id));if(item.payload.entries.some((v:Obj)=>!permitted.has(v.scope_kind+':'+v.scope_id)))replicaFail('repair_status_disclosure');} + const previous=[...source.graph.statuses,...plan.statuses,...source.graph.members.flatMap((v:Obj)=>v.statuses)],statuses=observations(a.currentStatuses,e,allowed,needs,root,a.at,previous,policy,budget,a.onObserved);expires=Math.min(expires,...statuses.map(v=>v.payload.valid_until));require(a.at=22.19; `MEMORY_VAULT_NODE` may select it. The tests/adapter +accept `MEMORY_VAULT_JOSE_MODULE` for an existing locked jose distribution. +Dependency installation needs an approved source. For comparison, restore a +second baseline checkout and copy only the three files in `baseline-adapters.json`; +leave production code unchanged. Measure separately from regressions. + +## Remaining work + +- Frozen-message recovery requires retained original COPY/READ and independent + return authority. It is not automatic universal replication, authority renewal, + silent frozen-send relocation or an unlimited backlog. +- This native verifier covers message replicas. Native ACK, root and whole-feed + replica clients remain separate work. +- Further old demand needs independently funded targets. Restart cannot reset + quota. Profiling/caching must preserve exact immutable context, current status, + revocation floors, epochs and transaction snapshots. +- Next: bounded higher-rate sweep, first-contact/SQLite/CPU attribution, a second + independently consented old-message replica, and independent hosts/WAN. A local + trial is not a maximum supported-agent count or global reliability result. +- No paid API, real model, host deployment or real grant/trust changes are needed. + Repository publication follows the authorized normal CI/merge/release workflow. + Existing gpt-6.1-sol/high task settings were accepted earlier; tools do not expose + independently verifiable actual inference model metadata. diff --git a/docs/DELIVERY_CAPACITY.md b/docs/DELIVERY_CAPACITY.md new file mode 100644 index 0000000..9b4984a --- /dev/null +++ b/docs/DELIVERY_CAPACITY.md @@ -0,0 +1,176 @@ +> This records the batch3 Python measurement. The native completion and existing +> full native replica recovery and fresh alpha.0.39 comparison are documented in +> [CAPACITY_HANDOFF.md](CAPACITY_HANDOFF.md). + +# Approved resource distribution for real message delivery + +The Python Agent `send/receive` path now distributes **new** messages over +independently approved, currently valid contact resource leases. Selection and +ciphertext/session freezing happen in one SQLite writer transaction. All frozen +outbox records, including unfinished uploads, count against their exact signed +resource lease. Among slots with enough item/byte capacity, selection uses relative +reserved item count, then byte usage and a stable scope tie-breaker. The server +still independently enforces its live policy, grant, resource budget, possession +proof and per-node capacity. Aliases of one lease cannot multiply its quota. + +The pool includes only local, actual recipient-approved decisions for the same +sender and recipient encryption identities. Discovering a node does not authorize +sending it messages. The recipient must explicitly obtain that node's resource, +receive the sender's first-contact request and approve its finite grant. No real +user permission or signing/encryption identity is created by the dispatch change. +The synthetic fixture makes those separate owner decisions using existing protocol +operations. New grants are learned by the ordinary explicit result workflow. + +Existing frozen messages keep their immutable envelope, node/epoch, exact authority +and request/message IDs on retries, interruption or revocation. The implementation +does not silently migrate a frozen send after ambiguous remote commit. A known +revoked old node can be excluded for a **new** send when another independent valid +grant exists; the old frozen send is still bound to its original node. Existing +repair callers retain their former first-valid-session behavior. + +Receive polling rotates approved sessions with one persisted bounded cursor. +Per-resource sequence cursors, signature and possession checks, receipt identity, +content validation, explicit Memory selection/import and author trust stay intact. +The same Agent API reads the saved message and recalls imported Memory. Approval +does not infer author trust: the fixture separately and explicitly trusts A's +Memory author key in B's test TrustStore. No model calls participate. + +## Fixed-load complete-chain comparison + +Mac mini / Apple M1 / eight CPU cores / 8 GiB / macOS 27.2. Python 3.12.0b4, +SQLite 3.42.0, cryptography 50.0.1. Two independent loopback HTTP processes share +one physical computer/SSD and one source group. This is not proof of independent +failure domains or hardware scalability. + +Both runs offer **48 messages at 2/s for 24 seconds**, without dropping or pausing +scheduled offers. There is one bounded pipeline worker; measured latency starts +at the intended offer time, so driver queueing and join delays are included. Each +message selects a different pre-created signed Memory observation. Local source +Memory creation is outside the offer window; export, encryption, remote storage, +decryption/validation, recipient Memory import, signed saved receipt, message read +and Memory recall are inside the measured chain. Success requires every one of +those stages, not just a server accepting ciphertext. + +R0 initially has a separately approved lease for 24 items and 2 MiB. R1 starts at +second 6, joins using R0 and receives another explicitly approved lease for exactly +24 items and 2 MiB. A and B only have R0 in their initial network configuration. +The observer never assigns individual messages to nodes. Offers continue while +the new resource becomes usable. Both runs use the same fixture, timing, payload +shape, limits and protocol; synthetic key material is newly generated per run. +Baseline is preserved batch2 source with the original first-session selection. +After is the measured final dispatch implementation. + +| Whole 24-second offer window, including drain | Before | After | +| --- | ---: | ---: | +| Offered messages | 48 | 48 | +| Complete verified chains | 40 | 48 | +| Success rate | 83.33% | 100.00% | +| Complete chains/s | 1.667 | 2.000 | +| Successful p95, ms | 398.24 | 406.25 | +| Successful p99, ms | 400.94 | 417.96 | +| All outcomes p95, ms | 388.23 | 406.25 | +| All outcomes p99, ms | 400.94 | 417.96 | +| Offer queue lag p99, ms | 10.34 | 10.16 | +| Recipient saved inbox + sent receipt | 40 | 48 | +| Imported and recalled distinct Memories | 40 | 48 | + +Elapsed including drain: 24.001678s before, 24.003349s after. R1 approval was usable at 7.953s / 7.946s. + +| Node | Stored/read/saved-receipt chains before / after | Stored ciphertext bytes before / after | CPU seconds before / after | Peak HTTP requests/s before / after | +| --- | ---: | ---: | ---: | ---: | +| 0 | 16 / 24 | 113234 / 169866 | 2.199 / 2.785 | 32 / 33 | +| 1 | 24 / 24 | 169896 / 169896 | 2.960 / 2.897 | 26 / 26 | + +Before has eight `open_delivery_quota` failures: the newest first session fills R1's +24-item grant while eight R0 slots remain unused. After uses all 24+24 individually +approved slots, and all 48 chains complete. The additional storage/receipt/read work +on R0 is visible in node counters and actual rows; R1 also stores 24 complete +messages. This proves finite authorized storage capacity is usable across nodes +with automatic assignment. It does **not** measure higher CPU or SQLite maximum +throughput. There are zero HTTP admission/queue refusals in either delivery run, +and observed peak traffic is below the unchanged 64/s per-source cap. + +Tail latency increases slightly. Before's successful-only percentiles exclude its +eight failed messages; all-outcome percentiles and every stage failure are retained. +This result describes all 48 offers, including joining; it is not a best tail window. +The new node's first-contact setup has two `contact_unavailable` attempts before a +successful third attempt in each run. These are visible in setup_attempts and wire +counts. Fixture retries are at most six, 0.5 seconds apart, use the same logical +request ID, and do not auto-approve anything. They are not hidden successful message +chains. The eight-message pilot had one capacity failure before its second resource +was approved; it is excluded from the selected capacity comparison. + +## Interpreting the prior routing result and reducing discovery failures + +The prior 80/s public-query workload exceeded one node's 64/s same-source HTTP +admission allowance. Distributing queries among node-level rate gates improved +admission, not proven compute/storage throughput. Those guards remain per-node; +there is no invented network-wide identity rate service. Routing/source bounds and +the lookup/application request and byte budgets were not increased. Message grants +remain independently node/epoch bound, and their finite quotas are never multiplied +by routing to another node. A retargeted, freshly signed sender request still gets +an exact signed `contact_wrong_node` denial and no upload challenge/storage. + +The routing API consumed a five-second probe delay even when a reply had no new +neighbor. It now consumes the delay only for an actual eligible challenge, with the +same maximum of one new proof per five seconds, at most three find attempts and the +same shared finite lookup budget. It does not add an unbounded retry queue. + +| Full public-query run: 1920 offers, 80/s, 24s, join at 6s | Batch2 | Final fix | +| --- | ---: | ---: | +| Successful queries | 1766 | 1828 | +| Failures | 154 | 92 | +| Success rate | 91.98% | 95.21% | +| Successful p95, ms | 29.76 | 30.04 | +| Successful p99, ms | 44.91 | 37.24 | + +First useful new-peer offer times from window start: + +| Joining peer | Batch2, seconds | Final, seconds | +| --- | ---: | ---: | +| R1 | 15.088 | 6.112 | +| R2 | 10.038 | 11.125 | + +The remaining 92 failures are explicit node0 HTTP admission refusals before enough +proven peers are available. A single allowed 64/s entry cannot complete a sustained +80/s offer without deferral or rejection. The fix reduces unnecessary convergence +delay; it does not hide errors, relax rate gates or claim that all startup offers +succeed. A finite caller retry/admission policy would need a separately measured +latency and request budget; it is not added to disguise overload here. + +## Verification, delivery and remaining scope + +The final-code combined regression run passed all 51 distinct cases (225.143s), +including Python/native contact state interoperability, existing mailbox HTTP +flows, delivery, consent, revocation, quotas, saved receipts and repair +compatibility. Focused intermediate logs are also preserved. The final run covers +actual HTTP, late independent approval, all-chain Memory saving/recall, receive +fairness, atomic concurrent freezing, duplicate aliases, capacity refusal, policy +and known node revocation, immutable retries, and foreign-node grant rejection. The final measured protocol +runtime hashes match the delivered code. The benchmark's added finite watchdog +caps the offer window plus 30s drain and stops only its disposable child processes. +That watchdog branch was not triggered in measured runs. + +```sh +python -m unittest tests.test_open_delivery_distribution tests.test_open_delivery_http tests.test_open_repair_provision tests.test_open_neighbors tests.test_open_contact_state -v +python scripts/benchmark_open_delivery.py --messages 48 --rate 2 --join-after 12 --output after.json +python scripts/benchmark_open_neighbors.py --rate 80 --seconds 24 --join-after 6 --output discovery-after.json +``` + +The bundle plus cumulative patch gives the complete source without fetching Git. +All selected per-offer traces, setup failures, RPC/blob counts, stored bytes, +receipts, resource leases, CPU/RSS and source hashes are in delivery-evidence/. +Earlier two batches remain archived in this same Library artifact/version history. + +This implementation covers the Python common Agent direct delivery carrier. Native +TypeScript selection/receive fairness was unchanged in that batch3 measurement; +the completion batch aligns it with Python and records its own full fixed-load run. It does not move an +already frozen message to a replica, dynamically copy all stored Memory, prove a +multi-host compute/storage capacity increase, or demonstrate thousand-agent scale. +Independent hosts/SSDs with owner-operated HTTPS are needed for WAN and resource +scaling validation, not for the local implementation delivered here. Real-model +conversation/semantic reuse remains untested; no paid API was called. No persistent +real credential, new user grant, trust change, push, merge or deployment occurred. +Actual deployment/new real grants or credential setup need user authorization. +The thread's gpt-6.1-sol/high setting was accepted earlier; the runtime does not +expose actual inference model metadata for independent confirmation. diff --git a/docs/HTTP_CAPACITY.md b/docs/HTTP_CAPACITY.md new file mode 100644 index 0000000..dc59d32 --- /dev/null +++ b/docs/HTTP_CAPACITY.md @@ -0,0 +1,59 @@ +> Recorded batch1 transport/index measurements. Later code and complete message-chain results are in DELIVERY_CAPACITY.md. + +# Bounded HTTP directory read capacity — development evidence + +Baseline: `c840617eb50e8856695234b482fed5560c0c72bf` (main, rechecked against the remote). Development branch: `feat/bounded-directory-read-capacity`. No version bump, commit, push, deployment or live user configuration change was performed. + +## Implemented behavior + +- Each HTTP node uses eight fixed daemon execution workers and a queue of at most sixteen accepted sockets. No worker is created per incoming connection. Queue wait consumes the original three-second whole-input deadline; a queued request never receives a fresh input window. Expired/overflow work receives a best-effort 503 with Retry-After. Transport resets can still occur on an overloaded socket; existing transport errors remain retryable. Active repair-response deadlines retain their existing separate finite bound. +- An initialized transport database verifies its durable configuration binding on every open without acquiring the writer lock. First initialization and legacy migration still acquire and recheck under the writer lock. Existing protected-path and sidecar validation remain on each open. +- Directory `get` uses a consistent WAL read snapshot. Lease/floor/replay expiry and capacity checks use the same logical live rows without mutating storage. The next allocation prunes expired rows; reads cannot create unbounded state. Committed revocation/conflict floors remain effective, and an uncommitted concurrent writer is not treated as committed authority. + +The eight execution slots, global 128 requests/s gate and per-source 64 requests/s gate remain. Owner consent, signed identity/node/epoch binding, retry receipts, resource ceilings and the six Agent operations retain their existing contracts. + +## Actual experiment + +One Mac mini, Apple M1, eight cores, 8 GiB RAM; macOS 27.2 (26B5091g). Python 3.12.0b4, SQLite 3.42.0, cryptography 50.0.1, joserfc 1.7.5, httpx 0.28.1. Node 22.19.0 / jose 6.2.10 were used for selected native SDK regression. All keys, contacts, messages and storage were disposable synthetic fixtures. No real model API was called. + +`scripts/benchmark_open_http.py` runs real independent node processes and actual signed HTTP requests/responses. There are 64 deterministic synthetic contact owners, 32 concurrent requester threads and four rounds of 64 directory reads per topology (256 requests each). The same owners select their explicit directory shard by XOR distance over the configured experiment nodes. The observer's full node list is not installed into any routing participant; all server participants have empty initial introductions. Placement in this benchmark is explicit, not automatic migration or a global discovery acceptance claim. Per-node contact counts were `[64]`, `[34,30]`, and `[8,14,32,10]`. + +| Node processes | Successful reads | Error rate | Burst service reads/s | Mean of per-round successful-read p95, ms | +| --- | --- | --- | --- | --- | +| 1 | 32/256 → 117/256 | 87.50% → 54.30% | 99.1 → 195.3 | 79.4 → 125.1 | +| 2 | 64/256 → 256/256 | 75.00% → 0.00% | 199.5 → 271.6 | 78.4 → 139.1 | +| 4 | 140/256 → 256/256 | 45.31% → 0.00% | 281.6 → 299.0 | 100.2 → 135.6 | + +Burst service rate excludes setup and the 1.1-second quiet intervals required to keep the ordinary rate gates intact. It is not a sustained requests/s capacity claim. Successful-read latency omits failed requests, so the baseline percentiles describe a smaller surviving subset. More admitted work increases queue latency while greatly increasing completion. Two to four processes do not show linear throughput growth on this one host and one requester process. The experiment does not isolate a final CPU bottleneck. + +A separate paced single-node control submitted 256 requests at 48 requests/s without inter-round quiet intervals (about 5.3 seconds). Both versions completed 256/256. Per-round p95 was approximately 14 ms in both; there is no demonstrated steady-load latency gain. JSON evidence retains every round, status count, source hash and measurement definition. Timing samples that overlapped regression workloads were discarded. + +## Validation and limits + +- Existing/index/network-race/transport/multihop/routing/contact regression: 68 reported tests, OK with one initially skipped native-state class, in 126.067 seconds. This includes real eight-process multihop lookup after bootstrap exit and independent committed owner revocation. It is a targeted campaign, not the full repository suite. +- Final boundary campaign: six tests pass. They exercise real WAL readers with an uncommitted writer then committed revocation, expired-row invisibility and later reclamation, binding changes on later opens, bounded live socket overload, preserved whole-input timeout after queuing, and bind-failure cleanup. +- Selected native Agent campaign: two tests pass in 9.314 seconds with real Node / JOSE and real Python HTTP resources. Each Node driver prevents subprocess delegation before importing production SDK code. Native approval/encrypted delivery/saved receipts and cold-sender/owner/resource restart are exercised. Python Agent separately saves selected canonical memory, restarts, confirms the original receipt and recalls the original evidence. + +The broad and native campaigns preceded the final small listener-construction/cleanup guard; the final boundary campaign and the published performance JSON pin the delivered runtime sources. There is no combined whole-suite certification on one exact source. No load tests have been run against the unmerged alpha39 branch; its versions of the three touched runtime files match the baseline and the independent patch is applicable. + +This batch improves finite burst admission and reader/writer coexistence. Single-node overflow still occurs. Process-local first-contact CPU slots, other service writer transactions, hot-shard migration, automatic placement/rebalancing, multi-host fault domains, long-duration demand growth and 1,000-model communication remain unverified or unfinished. Additional nodes demonstrably serve independently selected useful directory reads here; automatic redistribution is not implemented by this batch. + +## Agent access and the next acceptance step + +Existing Python hosts call `Agent(client_config, network_config).handle(request)` or use the six-operation NDJSON `serve` interface in `docs/OPEN_NETWORK_QUICKSTART.md`. Native Node hosts call `await Agent.handle(request)` using the same six-operation objects. A model host must expose that already-authorized endpoint as its tool and preserve the actual signed identities, independent recipient consent and current resource permissions. SDK tests do not establish a model's understanding or autonomous follow-through. No installed MCP/capture configuration was altered. + +A harmless local inventory found no Ollama CLI and no listener on its ordinary loopback port. No model was downloaded and no external paid API was used. To run a real-model sustained acceptance next, select authorized existing Agent/model sessions or local models and the endpoints/resources they may use, with explicit permission for that integration. Keep new spending and external deployment separately authorized. + +## Reproduce + +Use an isolated Python environment with the repository's network/server dependency files. From the checkout, run: + +```sh +python scripts/benchmark_open_http.py --output before-or-after.json +python scripts/benchmark_open_http.py --arrival-rate 48 --nodes 1 --output paced.json +python -m unittest -v tests.test_open_read_concurrency +``` + +The base bundle permits offline cloning of exact main and alpha36/alpha37 history. Apply `capacity.patch` in that main clone; new source files are included by the patch. `delivery-manifest.json` records SHA256 for all changed sources and evidence. The old dirty Mac checkout and separate unmerged alpha39 worktree remain intact. + +Model selection was explicitly submitted and accepted for this same task using `gpt-6.1-sol` with high reasoning. The execution API does not return actual inference-model metadata, so independent runtime confirmation of the selected model is unavailable; no different model was silently selected. diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 884e53a..740c335 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,5 +1,9 @@ # Distribution scope and publication gates +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](CAPACITY_HANDOFF.md). + +Alpha.0.40 candidate assets are validated against their exact source commit by `release-manifest.json` and `SHA256SUMS`; use the [release record](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40) for publication and CI evidence. Earlier release results below remain historical. + Published **[alpha.0.37](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37)** is fixed at `fe943c6466a6ef459ccb0d03e309b207b3b0fb95`. Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](../clients/typescript/network/README.md). diff --git a/docs/ROUTING_CAPACITY.md b/docs/ROUTING_CAPACITY.md new file mode 100644 index 0000000..49c19a8 --- /dev/null +++ b/docs/ROUTING_CAPACITY.md @@ -0,0 +1,135 @@ +> Recorded batch2 routing measurements. DELIVERY_CAPACITY.md documents the later probe fix and the separate real delivery implementation. + +# Bounded public routing query distribution + +`OpenParticipant.find_neighbors(target, view="general", budget=None)` distributes +one public routing step across a bounded pool of endpoint-proven routers. A caller +starts with its configured one or two introductions, receives a signed `find` +response and can challenge one returned new router with `hello` at most once per +five seconds. Future calls rotate proven peers. At most three peers are tried per +operation; any extra challenge uses the same finite lookup budget. New keys have +probe priority over temporarily failed old peers. Existing routing source, bucket, +expiry and durable rollback/revocation/fork limits apply. No global roster or new +admission authority is introduced, and the first entry is not required after other +peers have been proven. + +```python +hints = await participant.find_neighbors(contact_key(owner_key_id), view="directory") +# hints["nodes"] are introductions. They do not grant access or certify a contact. +contact = await participant.find_contact(owner_key_id) +``` + +This is a public neighborhood query API, not a replacement for iterative lookup. +`find_contact` still observes all selected shards before choosing an eligible +contact, so a later shard's revocation or conflict cannot be bypassed by choosing a +convenient replica. The current Python/TypeScript message send and storage paths +have not been switched to this new API. This batch does not move memory records, +replicate ciphertext, redistribute private reads/writes, or raise the eight-worker +HTTP limit. It does not establish full contact resolution or message delivery +capacity. A future client can use these hints in a routing stage, but that complete +path needs separate implementation and measurement. + +## Controlled before/after workload + +The benchmark uses deterministic synthetic Ed25519 identities and real HTTP on +127.0.0.1. All three nodes run on one Mac mini (Apple M1, eight CPU cores, 8 GiB RAM, +macOS 27.2). The driver has sixteen threads and sends **80 queries/s for 24 seconds**, +1920 offered queries per run, with no removed idle gaps. Both runs have the same +three server processes. Nodes 1 and 2 enter the network at second 6 using existing +join and maintenance exchanges. The client receives only node 0; the observer's +full node list is used only to label measurements. New node descriptors are not +injected into the client. Requests query the same routing coordinate, and responses +are signature/request/node verified. No real memories, paid model calls or external +services participate. + +Baseline is the prior one-hop `find` call at a fixed entry, executed on the preserved +batch1 runtime. After uses the new `find_neighbors` API. **This baseline is not the +existing complete `find_contact` operation.** The core source hashes are retained +in both reports. Success means a verified, non-error public routing response; hints +remain unverified until separately challenged. Endpoint proofs, maintenance and +failed attempts are additional wire work and are recorded. + +| Full offer window plus drain | Before | After | +| --- | ---: | ---: | +| Offered queries | 1920 | 1920 | +| Successful queries | 1509 | 1766 | +| Success rate | 78.59% | 91.98% | +| Successful queries/s (including drain) | 62.85 | 73.55 | +| Successful p95, ms | 23.89 | 29.76 | +| Successful p99, ms | 27.80 | 44.91 | +| All outcomes p95, ms | 23.69 | 29.52 | +| All outcomes p99, ms | 27.47 | 44.66 | + +Elapsed including drain: 24.0098s before, 24.0113s after. + +| Last eight-second offered cohort (seconds 16..24) | Before | After | +| --- | ---: | ---: | +| Total / offered | 640 | 640 | +| Successful queries | 504 | 640 | +| Success rate | 78.75% | 100.00% | +| Successful queries/s | 63.00 | 80.00 | +| Successful p95, ms | 23.75 | 33.05 | +| Successful p99, ms | 26.43 | 45.56 | + +Completions physically inside seconds 16..24: 504 before and 640 after (the same 8-second completion rate). Offered-cohort tail latencies include queueing, scheduling delay and final drain; every failed outcome is retained. + +| Node | Driver successes before / after | Total server find work before / after | CPU seconds before / after | Max RSS MiB before / after | +| --- | ---: | ---: | ---: | ---: | +| 0 | 1509 / 1088 | 1522 / 1106 | 7.348 / 5.302 | 40.17 / 39.81 | +| 1 | 0 / 238 | 16 / 254 | 0.867 / 2.012 | 38.47 / 38.78 | +| 2 | 0 / 440 | 18 / 456 | 0.911 / 2.974 | 39.11 / 39.70 | + +Driver CPU seconds: 17.948 before, 26.027 after. The last offered cohort uses nodes 0/1/2 for {'2': 214, '0': 213, '1': 213}. + +## Interpretation and boundaries + +The additional routers carry real client requests after being discovered through +normal protocol traffic. They reduce the original node's client workload without +raising its policy limit. At this offered rate the baseline's main observed +constraint is the unchanged per-source 64/s HTTP admission policy, plus maintenance +traffic sharing that allowance. This is **policy-bound public query capacity**; +CPU saturation, SQLite write capacity, wide-area limits and maximum sustainable +throughput have not been established. The new API increases client CPU work and +p95/p99 latency. During join/proof convergence the full after run still has 154 +failed queries (8.02%); the final 12 seconds have zero failures, and the final eight +seconds distribute successful requests 213/213/214 across nodes 0/1/2. This short +finite observation is not a long-run reliability guarantee. + +The trace shows the first useful query on node 2 around second 10 and node 1 around +second 15. These are bounded client probes, not immediate topology convergence. +A router can return a different local neighborhood than another router; this API +only returns hints and never labels them globally closest or authoritative. + +No contact/control permission was weakened. Known revocation and conflict remain +blocking. Independently authorized replica placement and end-to-end storage or +message load balancing remain separate work. The next useful environment is at +least three independently hosted nodes with owner-operated HTTPS and explicit +finite resource grants, followed by a measured complete authorized read/write or +mailbox path. One shared Mac cannot demonstrate independent CPU/storage/failure +domains or thousand-agent capacity. Native SDK integration also remains. + +## Validation and reproducibility + +29 existing/new routing, real HTTP, directory revocation/conflict and concurrency +regression cases passed in 109.716 seconds before the final newcomer-priority +correction. Five neighbor boundary cases passed with the final correction; 30 +distinct cases are covered. The latter includes corrupted signatures, late join, +old-peer failure, current revocation, failover and shared budget enforcement. +The final measured runtime hashes match the delivered source. The alpha39 native +mailbox branch (`3df5cf8e58c94c0c2f703d574321f3c2561ffcec`) changes none of the four +runtime files involved here; the cumulative patch applies without changing that +original worktree. Its new mailbox paths have not been performance certified here. + +```sh +python scripts/benchmark_open_neighbors.py --rate 80 --seconds 24 --join-after 6 --output after.json +python -m unittest tests.test_open_neighbors -v +``` + +For baseline, use the batch1 source in the delivery and the same benchmark script; +it detects absence of `find_neighbors` and uses the prior fixed-entry signed find. +Per-request traces, four-second windows, all failure attempts, CPU/RSS and wire byte +counts are in the delivery's `neighbor-evidence/before.json` and `after.json`. +Pilot and pre-correction runs are excluded from the selected evidence. The runtime +model setting was accepted as `gpt-6.1-sol` with high reasoning; actual inference +model metadata is not exposed for independent confirmation. No model-switch +request was repeated in this batch. diff --git a/docs/STATUS.md b/docs/STATUS.md index cbf66a1..bf41769 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,9 @@ # Memory Vault development status +Alpha.0.40 adds bounded HTTP admission, WAL read concurrency, separately approved resource distribution and native full message-replica COPY/READ recovery. The isolated 48-Memory/2-per-second workload completes 39 → 48 chains; this proves authorized quota use, not a CPU/SQLite ceiling. Replica cold reads remain slow and stop at the existing finite work allowance. See [capacity evidence and limits](CAPACITY_HANDOFF.md). + +Alpha.0.40 candidate assets are validated against their exact source commit by `release-manifest.json` and `SHA256SUMS`; use the [release record](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.40) for publication and CI evidence. Earlier release results below remain historical. + Published **[alpha.0.37](https://github.com/qh-work/memory-vault-sync/releases/tag/v0.28.0-alpha.0.37)** is fixed at `fe943c6466a6ef459ccb0d03e309b207b3b0fb95`. Native TypeScript recipients can now retain an explicitly authorized receipt destination for a cold-mailbox message. Ordinary Agent `receive` returns its actual saved receipt, reusing the exact upload after a lost reply or restart. Python and TypeScript share jobs and completed history; revocation stops retries and expired uncertain uploads stop for reconciliation. See [native mailbox receipt returns](../clients/typescript/network/README.md). diff --git a/docs/evidence/capacity-alpha40/after.json b/docs/evidence/capacity-alpha40/after.json new file mode 100644 index 0000000..832b03e --- /dev/null +++ b/docs/evidence/capacity-alpha40/after.json @@ -0,0 +1,1818 @@ +{ + "synthetic": true, + "real_models": 0, + "physical_hosts": 1, + "cpu_count": 8, + "python": "3.12.0b4 (v3.12.0b4:97a6a41816, Jul 11 2023, 11:19:02) [Clang 13.0.0 (clang-1300.0.29.30)]", + "platform": "macOS-27.2-arm64-arm-64bit", + "scope": "independently approved direct message storage, encrypted read, recipient saved receipt, Vault import/recall", + "source_hashes": { + "memory_vault_network.py": "a83d457d41d305048f1277f735a29af061607ef53c6e4eca5c585d812ea5063b", + "memory_vault_open_node.py": "aad46af7bbdfef59b9317bb9732322a323afbbd9783dad36d1a6bbe359ce38a4", + "memory_vault_open_delivery_client.py": "14a72a6926163e41a9ae0a051d0e39992921382a49e1796e5a99c0cc77e4fcad", + "memory_vault_open_delivery_state.py": "cc473f5b102523b16cc3352bf25042e8ae0b86b80eb3f4f641b84d54be4d8d07", + "memory_vault_open_index.py": "c637cc96eba5a83653d2d368b4198906d7448a3f30c388f563af99452dd13367", + "clients/typescript/network/agent.ts": "4e2610e314c63fb6e33c3baae96ff6c96d33808277614c52001ddc7164b296b9", + "clients/typescript/network/client-config.ts": "c414ccdb6c95d099c670c5c8fb5582239d7ec9d179e7a318069cc997cf113817", + "clients/typescript/network/content.ts": "dd214415cf8eee1a26d5f0c6b3e08ca8f5b63066fc0ab0f627f6671611402212", + "clients/typescript/network/control.ts": "0ad7cf773ba5be25ab67b31c11dcbe65d89f38d6d8cd0c9664e8c548b49e0f1c", + "clients/typescript/network/crypto.ts": "7b538dd6d334cb2c42ea24103e6930d38063440d16eafc455716b3570fee438a", + "clients/typescript/network/hints.ts": "cc42d41c87544ae3374359cbcba4b3ab14e7d82738fab2629c5991d2d3c96562", + "clients/typescript/network/io.ts": "8be85ce42d1d68ba9169e1a639e10c77f3dffa274094638446bd3821215a805b", + "clients/typescript/network/nodes.ts": "f77c5b901ef43b3d8d4f2813af6c9c0ad61372900c07e416cd5232b23c71988b", + "clients/typescript/network/open-ack-client.ts": "d923bda9bd53462c22432d5c3e0f202f8426924189a341037456a25165fbfe3a", + "clients/typescript/network/open-ack-discovery.ts": "8afad511e4eb1645b62caafc5c22fc2284c94c965e8f15f5e6ea0390fc530a21", + "clients/typescript/network/open-ack-status.ts": "ab56e0bb37057060910da4b140e2f278c74ff67243d989c23299e80e2afddde0", + "clients/typescript/network/open-blob.ts": "00d9f845f34d8d309ed8910c46177df3c0aeb1939e35fb26a6dfa23584433fbd", + "clients/typescript/network/open-capacity.ts": "514a163013cc1bf20551dba9bd858a56c0fa1abd720054e61048e3101d4522e8", + "clients/typescript/network/open-client.ts": "087b28d765c6b946ee6a38d428082db0f7a63f36d376e92c8b758515cb79c4eb", + "clients/typescript/network/open-contact-client.ts": "bbd520337e54c1e5fea656129cabc8c0e6879ee4f03326ff920ccf665ab70068", + "clients/typescript/network/open-contact-state.ts": "d7deef9625f140edc8c31d2b7b1480db3da9f5967820a29ca7a605de0592167c", + "clients/typescript/network/open-contact.ts": "38fc88aea51e9fbcaf3f7c19c25a5cf0704660dcbd54fca02191d2ec629371da", + "clients/typescript/network/open-control.ts": "863f9a0a29d9bd85cec16dfd87f331198fd44ef7dce8e93620f6cd15a39d2b0e", + "clients/typescript/network/open-delivery-client.ts": "9aba3143c937cdfe806a2528241cd82b4f1a36becffee4607de85f684de41885", + "clients/typescript/network/open-delivery-control.ts": "839fd6106e851e2a722e9c394c09e7acab9d2505d0c40eb76e91085656b9e970", + "clients/typescript/network/open-delivery.ts": "6582e3de1909ceb65c1dbe35aefe9f08cee0270b745927459ecbaf3c1df43892", + "clients/typescript/network/open-mailbox-client.ts": "01a27d6f8abd8a9ebe229b11b7dfcd08b95c38b74d6ceec0373a82c476c41dc9", + "clients/typescript/network/open-mailbox-destination.ts": "7f5c3c515f51c7bfb676068abbb43db7b1beb696ec748f788f636cc28adaec09", + "clients/typescript/network/open-mailbox-journal.ts": "0929088f3f4c1b647b6599f57be23cb921f70db2e5fc97dc36a853217979b3a0", + "clients/typescript/network/open-mailbox-receipts.ts": "a3df0b7aac2701001b6672dde6d11ce5e885c7f2d9554d431b5a2bbe8cb4c86e", + "clients/typescript/network/open-mailbox-receivers.ts": "6b7e7cf14f43e4be27a61bb8d97221e73d216b8c2c96bad28a8cded908230fb1", + "clients/typescript/network/open-mailbox-replica-client.ts": "57c869cfbb88dadb2f10e525a20b2068cdddc91e39ce6f9bcb3c729cbdd35cc4", + "clients/typescript/network/open-mailbox-replicas.ts": "58130254ff4bafcf9cd190f5f9d19c73a22ad8a810adc73552115702aa71adaa", + "clients/typescript/network/open-mailbox-send.ts": "560ad4a9b49a6070cb7381ef851fe7f555b45745f1cdb97f2ce8d09749164cae", + "clients/typescript/network/open-node.ts": "4d5326f2ebb63fee44f268125d25697ccb47b448357cd6ea1a7209bb497de417", + "clients/typescript/network/open-participant.ts": "20327310486a282274b501a29f2ec67794df25bfe97b0820b1fe7e312bc1cc67", + "clients/typescript/network/open-provider-client.ts": "0dda9dc2e48f3d05fc9cb77a02f75267c5b1d9b5cabc4924ec821c64fac3cc05", + "clients/typescript/network/open-provider.ts": "7f382152585b133d4ef91491b238b109b8fe8b1648ebf923d0ee038b606b37ee", + "clients/typescript/network/open-repair-ack.ts": "312e27ff0e0729a3ee34306f44ad3de3414323cf54b8902b74d035613005ab01", + "clients/typescript/network/open-repair-admin.ts": "3bf9f2c4826b9cf4b305cdfb36e6d1e077435bc9899e0fac61067dcb71716b64", + "clients/typescript/network/open-repair-bootstrap.ts": "b8831454095299d7ed34c8d56e43aaf55cf6f38ec4e94fad57886e506eaf82ca", + "clients/typescript/network/open-repair-bound.ts": "e762e43fb2c0d570227f38b20dbcbebf078134c414693472c6ff2d18a6857fbb", + "clients/typescript/network/open-repair-client.ts": "73f439160f9a597f2b94a43a9d5abda734efcf236b14eae5e421b8898144e4ee", + "clients/typescript/network/open-repair-empty.ts": "309547b4b281b34ac8a9ce84f5d434d3e5cb72dccba0585ddca0a57d6897ef93", + "clients/typescript/network/open-repair-history.ts": "5f88d2335c552b4edb3eaa84f397ab6ce294261f1a98e97810800346914c7bf2", + "clients/typescript/network/open-repair-mailbox-authority.ts": "2098c3eaa3af6636751781c2f9129e8ab96b660437cc080742f68a98bf4dd1be", + "clients/typescript/network/open-repair-mailbox-feed.ts": "3996efc4e76fa199ae0ea109e31cc5702a65db282d54334d1777d1ef8d9d4ad6", + "clients/typescript/network/open-repair-mailbox-inbox.ts": "89f3ebc2d98233786b80713228dc02adc746b3c011d2bb89988060aed600b9ba", + "clients/typescript/network/open-repair-mailbox-member.ts": "25d16b4437bed2c3984116487088e94b51a75259ab9a9bc36a0408e7274c7784", + "clients/typescript/network/open-repair-mailbox-range.ts": "c6d6b6078f8632784abc5ee8b52dd2ed7e531d1631b68491c568c9a5646c9527", + "clients/typescript/network/open-repair-mailbox-read.ts": "4498b2b33d5d0a02d028ac80d3c3c733f28620f2545bdf5f2b14e0a1bacb3039", + "clients/typescript/network/open-repair-mailbox-replica-inbox.ts": "ea4394a6e2abcaf9c79c50fc4b363ef8fbc6daa5de9fa043556c6ba7ee5ae70c", + "clients/typescript/network/open-repair-mailbox-replica.ts": "61d90adcf745e864359566c22e9a0512af1a1fce98f4c7d67fd6e5f014c88533", + "clients/typescript/network/open-repair-occupied.ts": "9c43836e6957dfe7319dad7deffabe87730ec4190c3fc921b94b75c0239c6fb1", + "clients/typescript/network/open-repair-offer-client.ts": "8e0ec6d63b4c26274d709c76ca9f396d0c6863292324bb9c05b179c208265d21", + "clients/typescript/network/open-repair-original.ts": "d289733c05d0b6a0691fdfe7458aac4fb0b577a9297676d05cafe245eff4aa01", + "clients/typescript/network/open-repair-probe.ts": "3720d5a45b1354a4c0e7f6cb8cbb4e8a8d6ec998c1ac52f883fe90621420eb7b", + "clients/typescript/network/open-repair-proof.ts": "40ee00f74fd3147434bb54a7f978340d06da9ef448177b4d2d2b6aa288b88502", + "clients/typescript/network/open-repair-receipt.ts": "25e345006f8e5cd1f8a807c024c2319ef1aad9177582975ea2d91a05b7bd0a36", + "clients/typescript/network/open-repair-resource.ts": "b42c85fddd32672687cb3d2a613b11b223dcc04d9326b70967da43fc20484b43", + "clients/typescript/network/open-repair-status.ts": "21880a57c7523664e58b43c705558df5a4aaee0dd18aab2730b037709a8def36", + "clients/typescript/network/open-repair-wire.ts": "82f92918da5ba4975e22a9021e5985ed7760ba9145ba359c6578e311f7c02862", + "clients/typescript/network/open-routing.ts": "ce59007a2b3703fab81d97a01d298ea825e34884ef0b694600f6af206a3e880b", + "clients/typescript/network/open-state.ts": "00d37b59b8db9b5d7749249d535e53ea2f2e0a0d0722ab9dae03579f9065801c", + "clients/typescript/network/open-transport.ts": "64b765bb76427375ad07e84d8ae2e3ce17d580593ef5e89ef2def6ab3fba949f", + "clients/typescript/network/peer.ts": "1e6e1614aa51584db194582f819af6cbc8d2179c3e803da8406c4dd58c4ac781", + "clients/typescript/network/privacy.ts": "b4a21f3d95f2fdb16518e881dc930be764f8cd52b96991e6eab41feb231d52af", + "clients/typescript/network/ranking_math.ts": "b509bec4360d23c39429d17d491766ed172334f10ca71667659eb9b7e5c1af6b", + "clients/typescript/network/records.ts": "6e0f2d66f128db7593e9ca4b53bba2b037ae682f32cc852660c1cfe5af76408c", + "clients/typescript/network/retrieval.ts": "ee5baa5fddbf6d780ce5a3ef27bb91cd98985693040852bf484b425e5b6cd47d", + "clients/typescript/network/retrieval_text.ts": "98e007279ad065aea7bdc29ebbb846a1709205016143cc75860b583feaca041f", + "clients/typescript/network/setup.ts": "5bce583aaf60f164f5fad2e694c0f88f6cfe4cc5be3c3e2402ed20754425da44", + "clients/typescript/network/topics.ts": "b1ae2e39490b3126f2b2090a6fbea3dcf3828192dab8545f88193f909e7e3fae", + "clients/typescript/network/transport-state.ts": "814ea3808a1242cba010df02272c4a12b836e515820b82c6b0af21663ef63323", + "clients/typescript/network/transport.ts": "ff986ed1378bcb531773257e5857798d58d3bf3fb80178a36fdcaf98d42430df", + "clients/typescript/network/vault.ts": "8a0465bbd6aa7dae73d80b3ccc0e6ee43ae1690587024e46c60c7f141be42adc" + }, + "result": { + "mode": "approved_resource_distribution", + "agent_runtime": "native", + "native_before": [ + { + "cpu": { + "user": 704530, + "system": 97998 + }, + "rss": 139575296, + "subprocessCalls": 0, + "distribution": true + }, + { + "cpu": { + "user": 469817, + "system": 23781 + }, + "rss": 122912768, + "subprocessCalls": 0, + "distribution": true + } + ], + "native_after": [ + { + "cpu": { + "user": 7143337, + "system": 596788 + }, + "rss": 159645696, + "subprocessCalls": 0, + "distribution": true + }, + { + "cpu": { + "user": 5167527, + "system": 418393 + }, + "rss": 129499136, + "subprocessCalls": 0, + "distribution": true + } + ], + "offered_messages": 48, + "offered_messages_per_second": 2, + "offer_window_seconds": 24.0, + "elapsed_including_drain": 24.01034599999548, + "successful_chains": 48, + "success_rate": 1.0, + "successful_chains_per_second": 1.9991382048392405, + "statuses": { + "ok": 48 + }, + "successful_p95_ms": 365.48066698014736, + "successful_p99_ms": 386.7010419780854, + "all_p95_ms": 365.48066698014736, + "all_p99_ms": 386.7010419780854, + "offer_lag_p99_ms": 10.209166997810826, + "successful_by_node": { + "0": 24, + "1": 24 + }, + "recipient_saved_inbox": 48, + "recipient_vault_memories": 48, + "per_node": [ + { + "node": 0, + "counts": { + "http_requests": 363, + "second_1790947809": 0, + "admitted": 363, + "rpc_lease": 0, + "request_bytes": 1099546, + "response_bytes": 1047124, + "rpc_policy.put": 0, + "rpc_find": 14, + "rpc_put": 1, + "rpc_get": 2, + "rpc_policy.get": 0, + "rpc_challenge": 0, + "rpc_submit": 0, + "rpc_poll": 0, + "rpc_decide": 0, + "rpc_result": 0, + "second_1790947810": 12, + "rpc_prepare": 24, + "rpc_start": 24, + "blob_upload.chunk": 24, + "blob_input_bytes": 169602, + "blob_output_bytes": 169602, + "rpc_commit": 24, + "rpc_receipt.get": 48, + "rpc_list.prepare": 48, + "rpc_list": 48, + "rpc_read.prepare": 24, + "rpc_read.start": 24, + "blob_download.chunk": 24, + "rpc_receipt.put": 24, + "second_1790947811": 24, + "second_1790947812": 24, + "second_1790947813": 24, + "second_1790947814": 24, + "second_1790947815": 24, + "second_1790947816": 29, + "rpc_hello": 10, + "second_1790947817": 30, + "second_1790947818": 6, + "second_1790947819": 4, + "second_1790947820": 6, + "second_1790947821": 4, + "second_1790947822": 5, + "second_1790947823": 5, + "second_1790947824": 4, + "second_1790947825": 16, + "second_1790947826": 14, + "second_1790947827": 16, + "second_1790947828": 14, + "second_1790947829": 16, + "second_1790947830": 14, + "second_1790947831": 15, + "second_1790947832": 15, + "second_1790947833": 16, + "second_1790947834": 2 + }, + "peak_http_requests_per_second": 30, + "stored_items": 24, + "stored_ciphertext_bytes": 169602, + "saved_receipts": 24, + "delivery_leases": [ + { + "lease_id": "lease_e47eb55d44a406d04303aa271a93f541f99e3b5aa80c5950e669434444399130", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 2.4404559999999997, + "system_cpu_seconds": 0.49677099999999996, + "max_rss_bytes": 46104576 + }, + { + "node": 1, + "counts": { + "http_requests": 342, + "second_1790947816": 4, + "admitted": 342, + "rpc_lease": 2, + "request_bytes": 1059323, + "response_bytes": 1001747, + "rpc_policy.put": 1, + "rpc_find": 12, + "rpc_put": 1, + "second_1790947817": 16, + "rpc_hello": 10, + "rpc_get": 1, + "rpc_policy.get": 1, + "rpc_challenge": 2, + "rpc_submit": 1, + "rpc_poll": 1, + "rpc_decide": 1, + "rpc_list.prepare": 34, + "rpc_list": 34, + "rpc_result": 1, + "second_1790947818": 24, + "rpc_prepare": 24, + "rpc_start": 24, + "blob_upload.chunk": 24, + "blob_input_bytes": 169632, + "blob_output_bytes": 169632, + "rpc_commit": 24, + "rpc_receipt.get": 48, + "rpc_read.prepare": 24, + "rpc_read.start": 24, + "blob_download.chunk": 24, + "rpc_receipt.put": 24, + "second_1790947819": 26, + "second_1790947820": 24, + "second_1790947821": 26, + "second_1790947822": 24, + "second_1790947823": 26, + "second_1790947824": 24, + "second_1790947825": 16, + "second_1790947826": 14, + "second_1790947827": 16, + "second_1790947828": 14, + "second_1790947829": 16, + "second_1790947830": 14, + "second_1790947831": 16, + "second_1790947832": 14, + "second_1790947833": 16, + "second_1790947834": 12 + }, + "peak_http_requests_per_second": 26, + "stored_items": 24, + "stored_ciphertext_bytes": 169632, + "saved_receipts": 24, + "delivery_leases": [ + { + "lease_id": "lease_ccdebdd9c93d367f1e9204c096bd961256586d5b3cede101a631ca1827abc994", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 2.5197089999999998, + "system_cpu_seconds": 0.49668999999999996, + "max_rss_bytes": 46399488 + } + ], + "join": { + "started_seconds": 6.0085583749751095, + "authorized_seconds": 7.262500332988566 + }, + "join_errors": [], + "setup_attempts": [ + { + "node": 0, + "attempt": 1, + "ok": true, + "code": null, + "time": 251363.438165 + }, + { + "node": 1, + "attempt": 1, + "ok": false, + "code": "contact_unavailable", + "time": 251370.853902333 + }, + { + "node": 1, + "attempt": 2, + "ok": true, + "code": null, + "time": 251371.562567041 + } + ], + "requests": [ + { + "index": 0, + "offered_seconds": 0.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.23058298393152654, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8f68eae307b23e62a9e746c40719579102c5c7b04228839cbbc8ca844440991a", + "latency_ms": 334.8842079867609, + "completed_seconds": 0.3348869579785969 + }, + { + "index": 1, + "offered_seconds": 0.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.003916988149285, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a20adb9b55fd9960821d04bf9d57bc908624fc7d78e73ae09eacc8031b007b33", + "latency_ms": 330.7371669798158, + "completed_seconds": 0.8307382499915548 + }, + { + "index": 2, + "offered_seconds": 1.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.948791997274384, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_51079f314b955ec3103a3f847d027de7e735306ad8ff18099f00b8cffdcfb7b9", + "latency_ms": 318.36741699953564, + "completed_seconds": 1.3183682499802671 + }, + { + "index": 3, + "offered_seconds": 1.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.6570419936906546, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_dfbed31a8f6f121ff0808ee23c757724d64c3a189cc7fa9ccbeb232724af3de2", + "latency_ms": 311.82333297329023, + "completed_seconds": 1.8118242499767803 + }, + { + "index": 4, + "offered_seconds": 2.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.732582977041602, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3edd694af8280f7edef9f9a62308979ba9109356e4b51e47036e817f64a5d842", + "latency_ms": 320.5242079857271, + "completed_seconds": 2.3205252079933416 + }, + { + "index": 5, + "offered_seconds": 2.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.429207984590903, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e13f90410bdd501b9ebd568f46207f63e59e6ee81f98c32ecd70b979dae22c61", + "latency_ms": 319.3186249991413, + "completed_seconds": 2.8193196669744793 + }, + { + "index": 6, + "offered_seconds": 3.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.502374998992309, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_1d876059381da1350cad3c0f33f85388d1004b9d5cc62019057eadc2860690b5", + "latency_ms": 314.9886249739211, + "completed_seconds": 3.314989666978363 + }, + { + "index": 7, + "offered_seconds": 3.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.13999999850057, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a06f50a0f1662bf7153ff79a54583330a944209c6d3ca7f25d1ae0a5594b16c8", + "latency_ms": 307.65933298971504, + "completed_seconds": 3.8076602079963777 + }, + { + "index": 8, + "offered_seconds": 4.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.20937497378327, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_437d398340428d1bc9cc96c72a59151431fd9b9c460cb4dca53cd9e962003c23", + "latency_ms": 323.81549998535775, + "completed_seconds": 4.32381637499202 + }, + { + "index": 9, + "offered_seconds": 4.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.7387080006301403, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_74d30a671f3e7932eb972ed34a79e81633a07b0539a645a6037054ad4c514763", + "latency_ms": 310.2499999804422, + "completed_seconds": 4.810250916983932 + }, + { + "index": 10, + "offered_seconds": 5.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.471124980133027, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_666863c7897d3ef1d9bfa220f8678d2e4f132881abb9b856b18a7026c67ade8d", + "latency_ms": 317.6245829963591, + "completed_seconds": 5.317625457973918 + }, + { + "index": 11, + "offered_seconds": 5.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.97562499367632, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ab0e453dcbc6ba073f28d6c3e56b800062303ac20ee77072a566fd3cb157ccfa", + "latency_ms": 314.9044999736361, + "completed_seconds": 5.8149053329834715 + }, + { + "index": 12, + "offered_seconds": 6.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.762874989770353, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_185bc8a1475c8286182ae633ab3ab9622dedede9cc537618df2604e3a62e3c60", + "latency_ms": 436.67341698892415, + "completed_seconds": 6.436678541998845 + }, + { + "index": 13, + "offered_seconds": 6.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.6159169899765402, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_eccee2dba147170d42d905510d4819fe56a69d6dd3787f2bd62fa755f359df98", + "latency_ms": 286.38141698320396, + "completed_seconds": 6.786382332997164 + }, + { + "index": 14, + "offered_seconds": 7.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.336291997693479, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d457b6417293e17eaf841912b3633f7d0993675137b3a24ec2bc2b16995fe1fc", + "latency_ms": 375.04083299427293, + "completed_seconds": 7.375041624996811 + }, + { + "index": 15, + "offered_seconds": 7.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.000207977602258, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_9a82c3138b7ab7cde245d54f6e54a2939bd9611f2f1493881d8e1c642d5848fc", + "latency_ms": 341.68499999213964, + "completed_seconds": 7.84168591699563 + }, + { + "index": 16, + "offered_seconds": 8.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.140082973521203, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_db4116acef9632bc77f6dc15d7796d5233605daab42a4c90bb0477c5e0185b44", + "latency_ms": 354.4743749953341, + "completed_seconds": 8.354475332977017 + }, + { + "index": 17, + "offered_seconds": 8.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.117749974597245, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d7bc3a1c9907d5b186d1eb26f873d807b6177f84b633c1d2cc362d06058dee2a", + "latency_ms": 348.2023329997901, + "completed_seconds": 8.8482033329783 + }, + { + "index": 18, + "offered_seconds": 9.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.097541991854087, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d1374b1b2b4e2fd9bbbab84c290cd381b0f8a7ce0a146887c2b4711de84d4ca6", + "latency_ms": 347.0013329933863, + "completed_seconds": 9.347002291993704 + }, + { + "index": 19, + "offered_seconds": 9.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.111124982358888, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f2a1e86139017dc37294a4cb1c89f363c439c3e3ee44fe5b6df7b0d28743da7e", + "latency_ms": 349.66454198001884, + "completed_seconds": 9.849665666988585 + }, + { + "index": 20, + "offered_seconds": 10.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.102916974574327, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a3e6c6bf3fc10432ced2c9fbe7bf2fde9c36d9ab62b41d3fa1b8079537d5ded1", + "latency_ms": 350.2269579912536, + "completed_seconds": 10.350228041992523 + }, + { + "index": 21, + "offered_seconds": 10.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.118999984115362, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_6e87e5a518bc8cdbea2b2a8220b32db0a094912e260ecde73d1a62641388afff", + "latency_ms": 351.75004199845716, + "completed_seconds": 10.851750874979189 + }, + { + "index": 22, + "offered_seconds": 11.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.16408299235627, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_7f227c080877002396eb0e9a83f17f1b76c66491e8c8c801598565d89f3330c6", + "latency_ms": 341.4394579885993, + "completed_seconds": 11.341440124990186 + }, + { + "index": 23, + "offered_seconds": 11.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.105499979341403, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d71270a81ecc6d594150ed1669067a2937709e19882afdf9d9a81d690a54e04b", + "latency_ms": 351.6734579752665, + "completed_seconds": 11.851674374978757 + }, + { + "index": 24, + "offered_seconds": 12.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.098082973854616, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f952404541296c0a4036fab199361db41f6146c345411608652b374939113b60", + "latency_ms": 338.92299997387454, + "completed_seconds": 12.33892383298371 + }, + { + "index": 25, + "offered_seconds": 12.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.5290829760488123, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a0db30f5dca9e51aee7fe8151d23666aada67ad9e15f2e4f47164ad6c48900fe", + "latency_ms": 321.081541973399, + "completed_seconds": 12.82108229197911 + }, + { + "index": 26, + "offered_seconds": 13.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.068291972856969, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f0cf4c7bc0c9756d98a011bc8eae4ff89c8f56c729141ff095760f3dfd045871", + "latency_ms": 323.6810419766698, + "completed_seconds": 13.323682082991581 + }, + { + "index": 27, + "offered_seconds": 13.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.209166997810826, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_6506b47c00f7685c1bff3a0658ff88859ca2a0c75bc50c2ab83fc8ab27a5d3d2", + "latency_ms": 355.78766698017716, + "completed_seconds": 13.855788499990012 + }, + { + "index": 28, + "offered_seconds": 14.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.111332987435162, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c1dabd6c8184279b63ffc1c7b7cb888e34103b506cc704d4f2c18744137c93a5", + "latency_ms": 347.3817919730209, + "completed_seconds": 14.347383041982539 + }, + { + "index": 29, + "offered_seconds": 14.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.191874995129183, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_46c69a1fa56547ea5c353ba2e210733f5dfad0f0c2d0567d5583c03cb63fd5c5", + "latency_ms": 312.5214579922613, + "completed_seconds": 14.812522291991627 + }, + { + "index": 30, + "offered_seconds": 15.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.433541995240375, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_6805fc3863c762d2988954bb5b2588403550cefc514d1fbafede2404940ccb39", + "latency_ms": 311.48958299309015, + "completed_seconds": 15.311490457999753 + }, + { + "index": 31, + "offered_seconds": 15.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.173249989748001, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c1de781e9558e17e27a59a984b779d93cfb83c5a5251c376db6f708173bd1e03", + "latency_ms": 365.48066698014736, + "completed_seconds": 15.865481666987762 + }, + { + "index": 32, + "offered_seconds": 16.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.7221669948194176, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_bdf3be969ae3e676cec9ced950867c0f130076956d4c6221f8b951836dda5daa", + "latency_ms": 337.5981249846518, + "completed_seconds": 16.337599041988142 + }, + { + "index": 33, + "offered_seconds": 16.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 3.206166991731152, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_4d350be3cf68164789467efc6d25f65324683058239b9f7d8169bb3bbdb655bb", + "latency_ms": 323.84795797406696, + "completed_seconds": 16.82384895798168 + }, + { + "index": 34, + "offered_seconds": 17.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.369875001022592, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8176c9a2c7f13b73b53353445943e51aee0ff76a96e021f3169ac49a468339ce", + "latency_ms": 351.5385829959996, + "completed_seconds": 17.35153962500044 + }, + { + "index": 35, + "offered_seconds": 17.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.737457992741838, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c771ff07892bceeac3c7b648d07f68b41296fdeaab47172cfe9d39f08194e571", + "latency_ms": 356.5667919756379, + "completed_seconds": 17.85656783299055 + }, + { + "index": 36, + "offered_seconds": 18.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 5.107457982376218, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e5935d16e1a9053be726018013e44f3f454079369c914a3e2b9027b233069e2c", + "latency_ms": 350.6079579819925, + "completed_seconds": 18.350609332992462 + }, + { + "index": 37, + "offered_seconds": 18.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.139416990568861, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_49897b1f107b99444bf140af62fa23635698b6e720c23288525fbf73811a4045", + "latency_ms": 361.6230829793494, + "completed_seconds": 18.861624041979667 + }, + { + "index": 38, + "offered_seconds": 19.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.8077919774223119, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3f89f102c6a4dff46454691697648ddb1639e22dc846ab0db4177ae8c477a831", + "latency_ms": 346.1954579979647, + "completed_seconds": 19.346196250000503 + }, + { + "index": 39, + "offered_seconds": 19.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.125082975719124, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c51c7f08e63f50cc1e9b8ae7dea171b9febb7e396c6a75456e0e7ec0d33aae5c", + "latency_ms": 360.6920829915907, + "completed_seconds": 19.86069299999508 + }, + { + "index": 40, + "offered_seconds": 20.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.09474997408688, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e2271f1b38b54cb74d21e99e28af38ecfa549ee4a6a18183fbb3fb1f9a02389c", + "latency_ms": 360.6581249914598, + "completed_seconds": 20.36065904199495 + }, + { + "index": 41, + "offered_seconds": 20.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.5711249725427479, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_b32986ba3cd178923c9f1dfb711eb22d3c927691473949aa17ab641bdc174fba", + "latency_ms": 346.64495798642747, + "completed_seconds": 20.84664570799214 + }, + { + "index": 42, + "offered_seconds": 21.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.437416981905699, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f40a17272dc974685cff2229dab923559598c5930a1c6629616bb039ecbb4cb4", + "latency_ms": 386.7010419780854, + "completed_seconds": 21.38670187498792 + }, + { + "index": 43, + "offered_seconds": 21.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.386374975088984, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_9a2e7d1fb91c564012172c1287a8405da148a336d0680e9f306a39a8b7d44665", + "latency_ms": 344.43137497873977, + "completed_seconds": 21.8444319999835 + }, + { + "index": 44, + "offered_seconds": 22.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.440041972789913, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_dbc8bee6be4d2cedbe47fcfc5ce72f54284f27b85d91320403bd62f0e4380256", + "latency_ms": 356.54854198219255, + "completed_seconds": 22.356549457996152 + }, + { + "index": 45, + "offered_seconds": 22.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.5702499949838966, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_79bf32db7b9431366d577281c686f5f40e7699b700c76ff2695d34e2d148ec38", + "latency_ms": 347.0553749939427, + "completed_seconds": 22.847056332975626 + }, + { + "index": 46, + "offered_seconds": 23.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.759124982636422, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_d1780a59fe8cb4327cc3be7d8789c3cdd97bb390d18c362028c36a81fa2d069f", + "latency_ms": 360.992832982447, + "completed_seconds": 23.36099383299006 + }, + { + "index": 47, + "offered_seconds": 23.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.816749992547557, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c563d592118349ab32caea80477955621fd04f8b5ce12e5d961781f5c0519384", + "latency_ms": 350.80141699290834, + "completed_seconds": 23.850802332977764 + } + ], + "driver": { + "user_cpu_seconds": 0.14187100000000002, + "system_cpu_seconds": 0.07647099999999998, + "max_rss_bytes": 44007424 + } + } +} diff --git a/docs/evidence/capacity-alpha40/before.json b/docs/evidence/capacity-alpha40/before.json new file mode 100644 index 0000000..a1c13f4 --- /dev/null +++ b/docs/evidence/capacity-alpha40/before.json @@ -0,0 +1,1714 @@ +{ + "synthetic": true, + "real_models": 0, + "physical_hosts": 1, + "cpu_count": 8, + "python": "3.12.0b4 (v3.12.0b4:97a6a41816, Jul 11 2023, 11:19:02) [Clang 13.0.0 (clang-1300.0.29.30)]", + "platform": "macOS-27.2-arm64-arm-64bit", + "scope": "independently approved direct message storage, encrypted read, recipient saved receipt, Vault import/recall", + "source_hashes": { + "memory_vault_network.py": "5f1f8d6c6ca08eadc63efe13ff7e77077707f01d2b2dc7791e691c5ef474e60a", + "memory_vault_open_node.py": "b6d05a333e173331f3d0466d2c8f2ba4a8454295a554a8b708d93faf081f4562", + "memory_vault_open_delivery_client.py": "73fb48e1d30e120e7d852202ffdb097cc83f4d77753cd8a40abdb781ccc4e0a5", + "memory_vault_open_delivery_state.py": "cc473f5b102523b16cc3352bf25042e8ae0b86b80eb3f4f641b84d54be4d8d07", + "memory_vault_open_index.py": "f13616488e3c0af507e1cb011a7faea10fea1346446213803138be0b369dfcc0", + "clients/typescript/network/agent.ts": "4e2610e314c63fb6e33c3baae96ff6c96d33808277614c52001ddc7164b296b9", + "clients/typescript/network/client-config.ts": "c414ccdb6c95d099c670c5c8fb5582239d7ec9d179e7a318069cc997cf113817", + "clients/typescript/network/content.ts": "dd214415cf8eee1a26d5f0c6b3e08ca8f5b63066fc0ab0f627f6671611402212", + "clients/typescript/network/control.ts": "0ad7cf773ba5be25ab67b31c11dcbe65d89f38d6d8cd0c9664e8c548b49e0f1c", + "clients/typescript/network/crypto.ts": "7b538dd6d334cb2c42ea24103e6930d38063440d16eafc455716b3570fee438a", + "clients/typescript/network/hints.ts": "cc42d41c87544ae3374359cbcba4b3ab14e7d82738fab2629c5991d2d3c96562", + "clients/typescript/network/io.ts": "8be85ce42d1d68ba9169e1a639e10c77f3dffa274094638446bd3821215a805b", + "clients/typescript/network/nodes.ts": "f77c5b901ef43b3d8d4f2813af6c9c0ad61372900c07e416cd5232b23c71988b", + "clients/typescript/network/open-ack-client.ts": "d923bda9bd53462c22432d5c3e0f202f8426924189a341037456a25165fbfe3a", + "clients/typescript/network/open-ack-discovery.ts": "8afad511e4eb1645b62caafc5c22fc2284c94c965e8f15f5e6ea0390fc530a21", + "clients/typescript/network/open-ack-status.ts": "cd5f45005a32f0cedde730b078d9a5127879719170a89eea213e1b8d77401bdc", + "clients/typescript/network/open-blob.ts": "00d9f845f34d8d309ed8910c46177df3c0aeb1939e35fb26a6dfa23584433fbd", + "clients/typescript/network/open-capacity.ts": "514a163013cc1bf20551dba9bd858a56c0fa1abd720054e61048e3101d4522e8", + "clients/typescript/network/open-client.ts": "060c4db30804867636da447f1273fcd425ba9258e3fc3b7845322dd42164c236", + "clients/typescript/network/open-contact-client.ts": "bbd520337e54c1e5fea656129cabc8c0e6879ee4f03326ff920ccf665ab70068", + "clients/typescript/network/open-contact-state.ts": "d7deef9625f140edc8c31d2b7b1480db3da9f5967820a29ca7a605de0592167c", + "clients/typescript/network/open-contact.ts": "38fc88aea51e9fbcaf3f7c19c25a5cf0704660dcbd54fca02191d2ec629371da", + "clients/typescript/network/open-control.ts": "863f9a0a29d9bd85cec16dfd87f331198fd44ef7dce8e93620f6cd15a39d2b0e", + "clients/typescript/network/open-delivery-client.ts": "009b61e3df8ede4fdbb47702061f241b664190c5d123dcc6c1d86653816d87f8", + "clients/typescript/network/open-delivery-control.ts": "839fd6106e851e2a722e9c394c09e7acab9d2505d0c40eb76e91085656b9e970", + "clients/typescript/network/open-delivery.ts": "6582e3de1909ceb65c1dbe35aefe9f08cee0270b745927459ecbaf3c1df43892", + "clients/typescript/network/open-mailbox-client.ts": "01a27d6f8abd8a9ebe229b11b7dfcd08b95c38b74d6ceec0373a82c476c41dc9", + "clients/typescript/network/open-mailbox-destination.ts": "7f5c3c515f51c7bfb676068abbb43db7b1beb696ec748f788f636cc28adaec09", + "clients/typescript/network/open-mailbox-journal.ts": "0929088f3f4c1b647b6599f57be23cb921f70db2e5fc97dc36a853217979b3a0", + "clients/typescript/network/open-mailbox-receipts.ts": "a3df0b7aac2701001b6672dde6d11ce5e885c7f2d9554d431b5a2bbe8cb4c86e", + "clients/typescript/network/open-mailbox-receivers.ts": "1b7f63d6e8583d48c768b1924191c9a0a0c3949e534f7655d93e09b6554900d3", + "clients/typescript/network/open-mailbox-send.ts": "560ad4a9b49a6070cb7381ef851fe7f555b45745f1cdb97f2ce8d09749164cae", + "clients/typescript/network/open-node.ts": "4d5326f2ebb63fee44f268125d25697ccb47b448357cd6ea1a7209bb497de417", + "clients/typescript/network/open-participant.ts": "20327310486a282274b501a29f2ec67794df25bfe97b0820b1fe7e312bc1cc67", + "clients/typescript/network/open-provider-client.ts": "0dda9dc2e48f3d05fc9cb77a02f75267c5b1d9b5cabc4924ec821c64fac3cc05", + "clients/typescript/network/open-provider.ts": "7f382152585b133d4ef91491b238b109b8fe8b1648ebf923d0ee038b606b37ee", + "clients/typescript/network/open-repair-ack.ts": "312e27ff0e0729a3ee34306f44ad3de3414323cf54b8902b74d035613005ab01", + "clients/typescript/network/open-repair-admin.ts": "3bf9f2c4826b9cf4b305cdfb36e6d1e077435bc9899e0fac61067dcb71716b64", + "clients/typescript/network/open-repair-bootstrap.ts": "b8831454095299d7ed34c8d56e43aaf55cf6f38ec4e94fad57886e506eaf82ca", + "clients/typescript/network/open-repair-bound.ts": "e762e43fb2c0d570227f38b20dbcbebf078134c414693472c6ff2d18a6857fbb", + "clients/typescript/network/open-repair-client.ts": "73f439160f9a597f2b94a43a9d5abda734efcf236b14eae5e421b8898144e4ee", + "clients/typescript/network/open-repair-empty.ts": "309547b4b281b34ac8a9ce84f5d434d3e5cb72dccba0585ddca0a57d6897ef93", + "clients/typescript/network/open-repair-history.ts": "5f88d2335c552b4edb3eaa84f397ab6ce294261f1a98e97810800346914c7bf2", + "clients/typescript/network/open-repair-mailbox-authority.ts": "2098c3eaa3af6636751781c2f9129e8ab96b660437cc080742f68a98bf4dd1be", + "clients/typescript/network/open-repair-mailbox-feed.ts": "3996efc4e76fa199ae0ea109e31cc5702a65db282d54334d1777d1ef8d9d4ad6", + "clients/typescript/network/open-repair-mailbox-inbox.ts": "51f8c3274ebb603b729dd5dbf082fc3fd37a817c028d99de5ae48eef4fc349f0", + "clients/typescript/network/open-repair-mailbox-member.ts": "25d16b4437bed2c3984116487088e94b51a75259ab9a9bc36a0408e7274c7784", + "clients/typescript/network/open-repair-mailbox-range.ts": "c6d6b6078f8632784abc5ee8b52dd2ed7e531d1631b68491c568c9a5646c9527", + "clients/typescript/network/open-repair-mailbox-read.ts": "9f5fe5d369ae4db353c0a9c1332caabf7218bf37155376035777d1ead9eba091", + "clients/typescript/network/open-repair-occupied.ts": "9c43836e6957dfe7319dad7deffabe87730ec4190c3fc921b94b75c0239c6fb1", + "clients/typescript/network/open-repair-offer-client.ts": "8e0ec6d63b4c26274d709c76ca9f396d0c6863292324bb9c05b179c208265d21", + "clients/typescript/network/open-repair-original.ts": "d289733c05d0b6a0691fdfe7458aac4fb0b577a9297676d05cafe245eff4aa01", + "clients/typescript/network/open-repair-probe.ts": "3720d5a45b1354a4c0e7f6cb8cbb4e8a8d6ec998c1ac52f883fe90621420eb7b", + "clients/typescript/network/open-repair-proof.ts": "40ee00f74fd3147434bb54a7f978340d06da9ef448177b4d2d2b6aa288b88502", + "clients/typescript/network/open-repair-receipt.ts": "25e345006f8e5cd1f8a807c024c2319ef1aad9177582975ea2d91a05b7bd0a36", + "clients/typescript/network/open-repair-resource.ts": "b42c85fddd32672687cb3d2a613b11b223dcc04d9326b70967da43fc20484b43", + "clients/typescript/network/open-repair-status.ts": "3e98f1e2b9374730c18ae37ba6605c66948235f0c45d04965672b822a9f7fcdb", + "clients/typescript/network/open-repair-wire.ts": "82f92918da5ba4975e22a9021e5985ed7760ba9145ba359c6578e311f7c02862", + "clients/typescript/network/open-routing.ts": "ce59007a2b3703fab81d97a01d298ea825e34884ef0b694600f6af206a3e880b", + "clients/typescript/network/open-state.ts": "00d37b59b8db9b5d7749249d535e53ea2f2e0a0d0722ab9dae03579f9065801c", + "clients/typescript/network/open-transport.ts": "64b765bb76427375ad07e84d8ae2e3ce17d580593ef5e89ef2def6ab3fba949f", + "clients/typescript/network/peer.ts": "1e6e1614aa51584db194582f819af6cbc8d2179c3e803da8406c4dd58c4ac781", + "clients/typescript/network/privacy.ts": "b4a21f3d95f2fdb16518e881dc930be764f8cd52b96991e6eab41feb231d52af", + "clients/typescript/network/ranking_math.ts": "b509bec4360d23c39429d17d491766ed172334f10ca71667659eb9b7e5c1af6b", + "clients/typescript/network/records.ts": "6e0f2d66f128db7593e9ca4b53bba2b037ae682f32cc852660c1cfe5af76408c", + "clients/typescript/network/retrieval.ts": "ee5baa5fddbf6d780ce5a3ef27bb91cd98985693040852bf484b425e5b6cd47d", + "clients/typescript/network/retrieval_text.ts": "98e007279ad065aea7bdc29ebbb846a1709205016143cc75860b583feaca041f", + "clients/typescript/network/setup.ts": "5bce583aaf60f164f5fad2e694c0f88f6cfe4cc5be3c3e2402ed20754425da44", + "clients/typescript/network/topics.ts": "b1ae2e39490b3126f2b2090a6fbea3dcf3828192dab8545f88193f909e7e3fae", + "clients/typescript/network/transport-state.ts": "814ea3808a1242cba010df02272c4a12b836e515820b82c6b0af21663ef63323", + "clients/typescript/network/transport.ts": "ff986ed1378bcb531773257e5857798d58d3bf3fb80178a36fdcaf98d42430df", + "clients/typescript/network/vault.ts": "8a0465bbd6aa7dae73d80b3ccc0e6ee43ae1690587024e46c60c7f141be42adc" + }, + "result": { + "mode": "prior_first_approved_session", + "agent_runtime": "native", + "native_before": [ + { + "cpu": { + "user": 693131, + "system": 158666 + }, + "rss": 133808128, + "subprocessCalls": 0, + "distribution": false + }, + { + "cpu": { + "user": 458167, + "system": 26637 + }, + "rss": 124485632, + "subprocessCalls": 0, + "distribution": false + } + ], + "native_after": [ + { + "cpu": { + "user": 6338370, + "system": 628028 + }, + "rss": 153665536, + "subprocessCalls": 0, + "distribution": false + }, + { + "cpu": { + "user": 4263213, + "system": 339086 + }, + "rss": 171638784, + "subprocessCalls": 0, + "distribution": false + } + ], + "offered_messages": 48, + "offered_messages_per_second": 2, + "offer_window_seconds": 24.0, + "elapsed_including_drain": 24.002671291993465, + "successful_chains": 39, + "success_rate": 0.8125, + "successful_chains_per_second": 1.6248191514004182, + "statuses": { + "ok": 39, + "open_delivery_quota": 9 + }, + "successful_p95_ms": 345.6919579766691, + "successful_p99_ms": 403.5438749997411, + "all_p95_ms": 343.1104579940438, + "all_p99_ms": 403.5438749997411, + "offer_lag_p99_ms": 10.21012497949414, + "successful_by_node": { + "0": 15, + "1": 24 + }, + "recipient_saved_inbox": 39, + "recipient_vault_memories": 39, + "per_node": [ + { + "node": 0, + "counts": { + "http_requests": 255, + "second_1790947783": 0, + "admitted": 255, + "rpc_lease": 0, + "request_bytes": 744667, + "response_bytes": 716419, + "rpc_policy.put": 0, + "rpc_find": 14, + "rpc_put": 1, + "rpc_get": 2, + "rpc_policy.get": 0, + "rpc_challenge": 0, + "rpc_submit": 0, + "rpc_poll": 0, + "rpc_decide": 0, + "rpc_result": 0, + "second_1790947784": 12, + "rpc_prepare": 15, + "rpc_start": 15, + "blob_upload.chunk": 15, + "blob_input_bytes": 105990, + "blob_output_bytes": 105990, + "rpc_commit": 15, + "rpc_receipt.get": 30, + "rpc_list.prepare": 39, + "rpc_list": 39, + "rpc_read.prepare": 15, + "rpc_read.start": 15, + "blob_download.chunk": 15, + "rpc_receipt.put": 15, + "second_1790947785": 24, + "second_1790947786": 24, + "second_1790947787": 24, + "second_1790947788": 24, + "second_1790947789": 24, + "second_1790947790": 32, + "rpc_hello": 10, + "second_1790947791": 25, + "second_1790947792": 8, + "second_1790947793": 4, + "second_1790947794": 6, + "second_1790947795": 4, + "second_1790947796": 6, + "second_1790947797": 4, + "second_1790947798": 5, + "second_1790947799": 5, + "second_1790947800": 4, + "second_1790947801": 6, + "second_1790947802": 4, + "second_1790947803": 6, + "second_1790947805": 2, + "second_1790947807": 2 + }, + "peak_http_requests_per_second": 32, + "stored_items": 15, + "stored_ciphertext_bytes": 105990, + "saved_receipts": 15, + "delivery_leases": [ + { + "lease_id": "lease_94d8f931025e2e831c51ab8a6d719e265a7e2299efcc6c351bc4aef7bb8cd7cd", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 1.7946000000000002, + "system_cpu_seconds": 0.377117, + "max_rss_bytes": 45858816 + }, + { + "node": 1, + "counts": { + "http_requests": 342, + "second_1790947790": 4, + "admitted": 342, + "rpc_lease": 2, + "request_bytes": 1232447, + "response_bytes": 1001603, + "rpc_policy.put": 1, + "rpc_find": 12, + "rpc_put": 1, + "second_1790947791": 16, + "rpc_hello": 10, + "rpc_get": 1, + "rpc_policy.get": 1, + "rpc_challenge": 2, + "rpc_submit": 1, + "rpc_poll": 1, + "rpc_decide": 1, + "rpc_list.prepare": 25, + "rpc_list": 25, + "rpc_result": 1, + "second_1790947792": 24, + "rpc_prepare": 33, + "rpc_start": 33, + "blob_upload.chunk": 24, + "blob_input_bytes": 169632, + "blob_output_bytes": 169632, + "rpc_commit": 24, + "rpc_receipt.get": 48, + "rpc_read.prepare": 24, + "rpc_read.start": 24, + "blob_download.chunk": 24, + "rpc_receipt.put": 24, + "second_1790947793": 26, + "second_1790947794": 24, + "second_1790947795": 26, + "second_1790947796": 24, + "second_1790947797": 26, + "second_1790947798": 24, + "second_1790947799": 26, + "second_1790947800": 24, + "second_1790947801": 26, + "second_1790947802": 24, + "second_1790947803": 26, + "second_1790947804": 4, + "error_open_delivery_quota": 9, + "second_1790947805": 6, + "second_1790947806": 4, + "second_1790947807": 6, + "second_1790947808": 2 + }, + "peak_http_requests_per_second": 26, + "stored_items": 24, + "stored_ciphertext_bytes": 169632, + "saved_receipts": 24, + "delivery_leases": [ + { + "lease_id": "lease_c26b23bc12ba25c55edee7bcab86726c83db16bb9e59d40a80d46edecb2bdcd4", + "max_items": 24, + "max_bytes": 2097152 + } + ], + "user_cpu_seconds": 2.637645, + "system_cpu_seconds": 0.5067849999999999, + "max_rss_bytes": 46743552 + } + ], + "join": { + "started_seconds": 6.001860874996055, + "authorized_seconds": 7.375652207992971 + }, + "join_errors": [], + "setup_attempts": [ + { + "node": 0, + "attempt": 1, + "ok": true, + "code": null, + "time": 251337.198063125 + }, + { + "node": 1, + "attempt": 1, + "ok": false, + "code": "contact_unavailable", + "time": 251344.87656325 + }, + { + "node": 1, + "attempt": 2, + "ok": true, + "code": null, + "time": 251345.60706525 + } + ], + "requests": [ + { + "index": 0, + "offered_seconds": 0.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.20462498650886118, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_27df0135c4a68a719a062b4ced61dcfc2e55e59f95fdf4d7cd032b724a1b9d99", + "latency_ms": 339.797249995172, + "completed_seconds": 0.33979829199961387 + }, + { + "index": 1, + "offered_seconds": 0.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.086583002703264, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ffa70736e3608cfb681fb6a94b9054c6e9d0de5112bf401decc10a3ab4c82bf4", + "latency_ms": 320.2794169774279, + "completed_seconds": 0.8202804579923395 + }, + { + "index": 2, + "offered_seconds": 1.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.47133297752589, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f58a6bf9ad321ec92960dd4fe86c19248b9f96b59b6644a2d727254792d241bd", + "latency_ms": 319.5926249900367, + "completed_seconds": 1.3195935419935267 + }, + { + "index": 3, + "offered_seconds": 1.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.80662498739548, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a5217d85bbd2ff77dc81e85ad8ffecc59f37618ce702f83f71a4fa16a5bfe069", + "latency_ms": 319.14745797985233, + "completed_seconds": 1.8191484579874668 + }, + { + "index": 4, + "offered_seconds": 2.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.2973750019446015, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c57db38380cbf8727a01bd3d7b0a6f031d69b3a436f387afdf2b9d974500a7a3", + "latency_ms": 310.9174580022227, + "completed_seconds": 2.310918582981685 + }, + { + "index": 5, + "offered_seconds": 2.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.529583002906293, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ecadd2e92e06443b922a6c6a032c9e2a1630200bc063ad2fd0a63b93958bea3e", + "latency_ms": 306.01429200032726, + "completed_seconds": 2.806015166977886 + }, + { + "index": 6, + "offered_seconds": 3.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.65191697771661, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c2eb3c6c3a3200e208b44dab56d70899076f341f7868ff59ec3797ff0fa327e0", + "latency_ms": 316.6577919910196, + "completed_seconds": 3.3166586669976823 + }, + { + "index": 7, + "offered_seconds": 3.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 8.552249986678362, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3766e217ae94365a7032037389608af075d3033c8672bbfba2346a542c36331c", + "latency_ms": 322.04458297928795, + "completed_seconds": 3.822045499982778 + }, + { + "index": 8, + "offered_seconds": 4.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 3.5538749943953007, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8bef2ee3d725e507077c229f3ee8c264cf1796d222874e812e1d158e296e3c06", + "latency_ms": 319.44520797696896, + "completed_seconds": 4.3194462079845835 + }, + { + "index": 9, + "offered_seconds": 4.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.788541991729289, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_7835afcda13eb85aed8cdd08cb5c0076f90d62523f977a69d1c2f739f4b16217", + "latency_ms": 316.006249981001, + "completed_seconds": 4.816007207991788 + }, + { + "index": 10, + "offered_seconds": 5.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 4.770667001139373, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a644cd02d8a1fab9a9e755a4f1f227e5e556309869f7bc0d23285aeee2cbefc6", + "latency_ms": 309.7827919991687, + "completed_seconds": 5.3097836249799 + }, + { + "index": 11, + "offered_seconds": 5.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.776332985376939, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_2dadd8c9544712231f9ed07137953c34fe959b9fd49a9b866fda7510539bbddf", + "latency_ms": 305.1470420032274, + "completed_seconds": 5.805147957988083 + }, + { + "index": 12, + "offered_seconds": 6.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 2.1234579908195883, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_77d7b6feb44a2841df9e73dbfd33757501258c96a1f40e7f8637c811726721c5", + "latency_ms": 403.5438749997411, + "completed_seconds": 6.403544999979204 + }, + { + "index": 13, + "offered_seconds": 6.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.79362498037517, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_7fbe58a73e14529a05be4324d7eaa122ffa2d2b5bdb486a182532e21f0a972a9", + "latency_ms": 289.8156250012107, + "completed_seconds": 6.789816417003749 + }, + { + "index": 14, + "offered_seconds": 7.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.083916978444904, + "node": 0, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_4991843b15a58ef7a6dc83d698f3f498b30f4f81be52a528442bdbbb43aefa5f", + "latency_ms": 477.58395798155107, + "completed_seconds": 7.477584666979965 + }, + { + "index": 15, + "offered_seconds": 7.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.067249997518957, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_0a734ee4802ce5ea218ced84aa40b0bf93c2e60b8944a97cdfe6b847ee556f2b", + "latency_ms": 312.6083329843823, + "completed_seconds": 7.8126094169856515 + }, + { + "index": 16, + "offered_seconds": 8.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.357000001007691, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_ad023554aefb10e03b55f2b52f31d2e6820c3be5f5287b11cc9b09db0ad361d7", + "latency_ms": 336.31695798248984, + "completed_seconds": 8.336318041983759 + }, + { + "index": 17, + "offered_seconds": 8.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.172457987209782, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_86ae290d10d79650bcb34020b8eb7bc7416e9b5bb2d5038763e9482fc7c55373", + "latency_ms": 340.3323329985142, + "completed_seconds": 8.840333208005177 + }, + { + "index": 18, + "offered_seconds": 9.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.302750000031665, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a780bfbb49273276b12e8b62305d2e419cb1a809c09a519b5efebf440bba4eef", + "latency_ms": 338.7631249788683, + "completed_seconds": 9.33876416698331 + }, + { + "index": 19, + "offered_seconds": 9.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.850457983091474, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_68f45bdd93e96d0cfa67d480e77cbbaa26f02b71c3cd306b5aedbc21f16e0d36", + "latency_ms": 336.3378749927506, + "completed_seconds": 9.836338833003538 + }, + { + "index": 20, + "offered_seconds": 10.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 1.0559170041233301, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_2abaabf907992676747afc0d25bcbdad76f074edf9ed73c31924e8b2e28216f5", + "latency_ms": 332.36912498250604, + "completed_seconds": 10.332369791984092 + }, + { + "index": 21, + "offered_seconds": 10.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.241041989298537, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_0d2a1275d34c061f7be6f9d623a74a24095f72501261315b1994a3f877897762", + "latency_ms": 342.3760829900857, + "completed_seconds": 10.842377041990403 + }, + { + "index": 22, + "offered_seconds": 11.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.11741699767299, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_59cb3054b18d13111cb746e7220e145e4741a6bf50c5e3b3655bcc8cb8805e3b", + "latency_ms": 341.231832979247, + "completed_seconds": 11.34123270798591 + }, + { + "index": 23, + "offered_seconds": 11.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.11216698680073, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c3192d88307a2d2a432b1130eba64bda30b38456036a89adc38062b7e074c5d5", + "latency_ms": 341.6482500033453, + "completed_seconds": 11.841649166977732 + }, + { + "index": 24, + "offered_seconds": 12.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.14450000366196, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_f36a21866aeeaf29f0b6c70a65461617ff99bee1c6ab2d9d711556310da163df", + "latency_ms": 340.23412500391714, + "completed_seconds": 12.340234999981476 + }, + { + "index": 25, + "offered_seconds": 12.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.141416976694018, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_43067bd3e3487fa80490ca1db114dd2727565dd93a9c6e5ef950cbe3c9f3cb63", + "latency_ms": 326.22299998183735, + "completed_seconds": 12.8262238749885 + }, + { + "index": 26, + "offered_seconds": 13.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.301874993368983, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_3131c42d99f94713439e60b9c3871e968744add7ad826d0f63fe6102fe915015", + "latency_ms": 338.44145797775127, + "completed_seconds": 13.338442457985366 + }, + { + "index": 27, + "offered_seconds": 13.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 0.256957981036976, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_8622e70e0fb0a4a65133a8e76e0122dc90516518964fa3041ca72df917c16891", + "latency_ms": 327.36766699235886, + "completed_seconds": 13.82736841699807 + }, + { + "index": 28, + "offered_seconds": 14.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 3.043374978005886, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_40feb22168dbe4e2e7b6060cd781bc2c0d807a5afe9d24400cc836443f543e62", + "latency_ms": 332.9546669847332, + "completed_seconds": 14.332955457997741 + }, + { + "index": 29, + "offered_seconds": 14.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.117500001797453, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_65319104848cacd7343d4bc1f34c64eaea3298a9524554bb18a5beefb5536ff5", + "latency_ms": 310.37187497713603, + "completed_seconds": 14.810372624982847 + }, + { + "index": 30, + "offered_seconds": 15.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.176541982218623, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_cc3748ce14473c67c09053f27d1cef260d58782945761d4df97fa443ad984232", + "latency_ms": 343.1104579940438, + "completed_seconds": 15.343111374997534 + }, + { + "index": 31, + "offered_seconds": 15.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.896250005112961, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_45a152b134c8d65a4798c5cc77a6c53962d530503d6c4ba0893d2d4e73b4ed90", + "latency_ms": 342.4699169991072, + "completed_seconds": 15.842470791976666 + }, + { + "index": 32, + "offered_seconds": 16.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 9.048332984093577, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_dbf498d76e602ea9f47927e24429a1260d1f67a74a848abf72e3034cbfce7e50", + "latency_ms": 338.04800000507385, + "completed_seconds": 16.338048957986757 + }, + { + "index": 33, + "offered_seconds": 16.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.0802079832647, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_44f6486596a6c2053ebdbfeecdd2f4ff70fdf6994e2b0ceab6b5f7de7d22ce29", + "latency_ms": 314.3101669847965, + "completed_seconds": 16.81431104199146 + }, + { + "index": 34, + "offered_seconds": 17.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 6.945207976968959, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_83ac2f8885762f3e4311dbf16f35eff90f0c7c6fc2d6eabfd629d5bc539f1596", + "latency_ms": 329.1951249993872, + "completed_seconds": 17.32919595798012 + }, + { + "index": 35, + "offered_seconds": 17.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.249249986372888, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_e5c13de4d730243b331529cb0ff9e936495a11349b87626fec76288ec333105c", + "latency_ms": 345.6919579766691, + "completed_seconds": 17.84569279200514 + }, + { + "index": 36, + "offered_seconds": 18.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.744000002276152, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_c1c50e1ffd4f0789bc4edf1042b487d6f59d771f3b58b3fe12478d586ea5cd0a", + "latency_ms": 336.4784169825725, + "completed_seconds": 18.336479416990187 + }, + { + "index": 37, + "offered_seconds": 18.5, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 7.602208002936095, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_5b619bfa04109036e8921bf709f310b00dfbff8cfc44ac1e3a9c441bf3ff51b2", + "latency_ms": 340.51829198142514, + "completed_seconds": 18.84051929198904 + }, + { + "index": 38, + "offered_seconds": 19.0, + "status": "ok", + "stages": [ + { + "stage": "send", + "ok": true, + "code": null + }, + { + "stage": "receive", + "ok": true, + "errors": [] + }, + { + "stage": "receipt", + "ok": true, + "code": null + } + ], + "offer_lag_ms": 10.137124976608902, + "node": 1, + "storage_accepted": true, + "saved_receipt": true, + "local_read": true, + "memory_recalled": true, + "message_id": "msg_a315da76dce866f791c33e216ba539c22a907b901434a280941bb832746871b4", + "latency_ms": 315.02437498420477, + "completed_seconds": 19.31502520799404 + }, + { + "index": 39, + "offered_seconds": 19.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 3.6427919985726476, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 117.27504199370742, + "completed_seconds": 19.617276042001322 + }, + { + "index": 40, + "offered_seconds": 20.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 0.5479169776663184, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 115.80499997944571, + "completed_seconds": 20.115806124988012 + }, + { + "index": 41, + "offered_seconds": 20.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 6.151374982437119, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 122.4948329909239, + "completed_seconds": 20.622496041993145 + }, + { + "index": 42, + "offered_seconds": 21.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 8.526624995283782, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 87.92895800434053, + "completed_seconds": 21.08792995798285 + }, + { + "index": 43, + "offered_seconds": 21.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 6.70687499223277, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 121.67724999017082, + "completed_seconds": 21.62167891699937 + }, + { + "index": 44, + "offered_seconds": 22.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 10.189332999289036, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 126.61791700520553, + "completed_seconds": 22.126618791982764 + }, + { + "index": 45, + "offered_seconds": 22.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 8.597291976911947, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 125.30191699624993, + "completed_seconds": 22.62530320798396 + }, + { + "index": 46, + "offered_seconds": 23.0, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 9.995375003200024, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 126.90704199485481, + "completed_seconds": 23.12690812497749 + }, + { + "index": 47, + "offered_seconds": 23.5, + "status": "open_delivery_quota", + "stages": [ + { + "stage": "send", + "ok": false, + "code": "open_delivery_quota" + } + ], + "offer_lag_ms": 10.21012497949414, + "node": null, + "storage_accepted": false, + "saved_receipt": false, + "local_read": false, + "memory_recalled": false, + "latency_ms": 125.14387498958968, + "completed_seconds": 23.625145041994983 + } + ], + "driver": { + "user_cpu_seconds": 0.11804799999999999, + "system_cpu_seconds": 0.064217, + "max_rss_bytes": 60915712 + } + } +} diff --git a/docs/evidence/capacity-alpha40/native-replica.json b/docs/evidence/capacity-alpha40/native-replica.json new file mode 100644 index 0000000..3fdd52c --- /dev/null +++ b/docs/evidence/capacity-alpha40/native-replica.json @@ -0,0 +1,52 @@ +{ + "synthetic_replica_closure": { + "message_id": "msg_2ebb8cf80c810667c7b47075c6c7f7d98c8868f06d29a9b4920c427ab5c56a6d", + "envelope_sha256": "a5ef1acfee4e12d00904e9fe3dccdb18b832fc5b8b1aeb127a71e03681b558d1", + "copied_ciphertext_bytes": 7059, + "recipient_saved": true, + "native_offline_recall": true, + "retained_revocation_code": "repair_authority_revoked", + "revoked_repair_http_calls": 0, + "recipient_runtime": "native", + "cold_known_message_load": [ + { + "offered": 1, + "completed": 1, + "elapsed_ms": 17902.777874987805, + "repair_http_calls": 12, + "copy_work_before": 46, + "copy_work_after": 58, + "copied_ciphertext_bytes": 7059, + "subprocess_calls": 0 + }, + { + "offered": 1, + "completed": 0, + "elapsed_ms": 9330.418333003763, + "repair_http_calls": 7, + "copy_work_before": 58, + "copy_work_after": 64, + "copied_ciphertext_bytes": 0, + "subprocess_calls": 0, + "errors": [ + "open_request_rejected" + ] + }, + { + "offered": 1, + "completed": 0, + "elapsed_ms": 577.4701249902137, + "repair_http_calls": 1, + "copy_work_before": 64, + "copy_work_after": 64, + "copied_ciphertext_bytes": 0, + "subprocess_calls": 0, + "errors": [ + "open_request_rejected" + ] + } + ] + }, + "original_send_native_confirmed": true, + "frozen_send_replay_network_calls": 0 +} diff --git a/docs/evidence/capacity-alpha40/replica-profile.json b/docs/evidence/capacity-alpha40/replica-profile.json new file mode 100644 index 0000000..caff251 --- /dev/null +++ b/docs/evidence/capacity-alpha40/replica-profile.json @@ -0,0 +1,115 @@ +{ + "synthetic": true, + "real_models": 0, + "physical_hosts": 1, + "cpu": "Apple M1", + "scope": "additional known-message demand; full COPY and current READ preparation only; timing-only instrumentation; excludes HTTP/packing/Node client CPU", + "request_work_ceiling": 64, + "passed": true, + "requests": [ + { + "kind": "challenge", + "wall_seconds": 1.1301507920143194, + "thread_cpu_seconds": 1.1269313749999998 + }, + { + "kind": "proof", + "wall_seconds": 1.122289249993628, + "thread_cpu_seconds": 1.120069167 + }, + { + "kind": "child", + "wall_seconds": 1.1198066660144832, + "thread_cpu_seconds": 1.117508125 + }, + { + "kind": "child", + "wall_seconds": 1.1400847500190139, + "thread_cpu_seconds": 1.1351167920000003 + }, + { + "kind": "child", + "wall_seconds": 1.1360727910068817, + "thread_cpu_seconds": 1.1338473750000002 + }, + { + "kind": "child", + "wall_seconds": 1.1336871660023462, + "thread_cpu_seconds": 1.1317049999999993 + }, + { + "kind": "child", + "wall_seconds": 1.137808249972295, + "thread_cpu_seconds": 1.1355606670000005 + }, + { + "kind": "child", + "wall_seconds": 1.1339372500078753, + "thread_cpu_seconds": 1.1312616249999996 + }, + { + "kind": "child", + "wall_seconds": 1.132088250014931, + "thread_cpu_seconds": 1.1297155830000003 + }, + { + "kind": "child", + "wall_seconds": 1.1340743750042748, + "thread_cpu_seconds": 1.1320607919999999 + }, + { + "kind": "child", + "wall_seconds": 1.1349039579799864, + "thread_cpu_seconds": 1.1327826250000004 + }, + { + "kind": "child", + "wall_seconds": 1.1311838749970775, + "thread_cpu_seconds": 1.1274406659999998 + }, + { + "kind": "challenge", + "wall_seconds": 1.1351207080006134, + "thread_cpu_seconds": 1.132373167 + }, + { + "kind": "proof", + "wall_seconds": 1.127089750021696, + "thread_cpu_seconds": 1.1238766670000002 + }, + { + "kind": "child", + "wall_seconds": 1.128002541983733, + "thread_cpu_seconds": 1.1249620420000008 + }, + { + "kind": "child", + "wall_seconds": 1.127287166978931, + "thread_cpu_seconds": 1.1240482500000004 + }, + { + "kind": "child", + "wall_seconds": 1.12175745802233, + "thread_cpu_seconds": 1.1187367499999992 + }, + { + "kind": "child", + "wall_seconds": 1.1303392910049297, + "thread_cpu_seconds": 1.1272125830000004 + }, + { + "kind": "child", + "wall_seconds": 0.0043163750087842345, + "thread_cpu_seconds": 0.004315875000000524 + }, + { + "kind": "challenge", + "wall_seconds": 0.004289208009140566, + "thread_cpu_seconds": 0.004282499999999523 + } + ], + "total_server_thread_cpu_seconds": 20.313807626000003, + "total_server_request_wall_seconds": 20.36428987205727, + "first_successful_cold_read_server_preparation_thread_cpu_seconds": 13.553999792, + "first_successful_cold_read_server_preparation_wall_seconds": 13.586087373027112 +} diff --git a/llms.txt b/llms.txt index 364a487..7da1570 100644 --- a/llms.txt +++ b/llms.txt @@ -1,7 +1,7 @@ # Memory Vault Model-neutral encrypted agent communication and persistent original memories. -Published prerelease: v0.28.0-alpha.0.37. Check the published release record for exact +Current release target: v0.28.0-alpha.0.40. Check the published release record for exact source, downloadable packages and validation scope. Start with docs/OPEN_NETWORK_QUICKSTART.md and AI_START_HERE.md. A participant @@ -30,7 +30,7 @@ Mailbox consent preparation (Python): memory_vault_open_repair_mailbox_consent.p Ordinary replica reception (Python): connect/register_replica retains an explicitly selected authorized message replica; ordinary receive recovers it through the durable inbox after source loss and restart, with current destination checks and no duplicate import. -Native original-source ACK confirmation: connect/recover_receipt and unchanged original send can confirm independently retained receipts from an existing preparation. Original-root revocations survive restart, language changes and replacement bootstraps. Preparation and replica reception remain Python. +Native original-source ACK confirmation: connect/recover_receipt and unchanged original send can confirm independently retained receipts from an existing preparation. Original-root revocations survive restart, language changes and replacement bootstraps. Preparation and native ACK/root/feed replica reception remain Python. Directory-discovered ACK confirmation: Python and native TypeScript connect/recover_discovered_receipt use an explicit independent directory, then independently read the exact receipt. Missing facts and wrong custody cannot confirm a send. Directory replacement and publication still require explicit independent permission. @@ -38,15 +38,15 @@ Routed ACK recovery: Python and native TypeScript connect/recover_routed_receipt Native saved receipt return: connect/return_receipt authenticates the actual saved inbox and separate original return authority, shares the pending/completed journal with Python, and resumes the same upload after restart without a new grant. Native provisioning and full mailbox parity remain unfinished. -Native remote mailbox recovery: register_mailbox selects an explicitly authorized source; ordinary receive verifies current READ permission, full original feed and ciphertext, then imports through the durable inbox. Receiver configuration and refusal history are shared with Python. Native provisioning and replica inbox recovery remain unfinished. +Native remote mailbox recovery: register_mailbox selects an explicitly authorized source; ordinary receive verifies current READ permission, full original feed and ciphertext, then imports through the durable inbox. Receiver configuration and refusal history are shared with Python. Native provisioning and ACK/root/feed replica inbox recovery remain unfinished; native message replica recovery is available. -Native cold-mailbox receipt returns: register_mailbox_receipt_return retains an independently selected ACK destination. Ordinary receive returns the saved receipt with original permission, same-upload restart and cross-client history. Expired uncertain returns stop for reconciliation. Native provisioning and replica reception remain unfinished. +Native cold-mailbox receipt returns: register_mailbox_receipt_return retains an independently selected ACK destination. Ordinary receive returns the saved receipt with original permission, same-upload restart and cross-client history. Expired uncertain returns stop for reconciliation. Native provisioning and ACK/root/feed replica reception remain unfinished; native message replica recovery is available. Ordinary independent receipts (Python): connect/register_mailbox_receipt_return retains the selected ACK destination; receive returns an actually saved receipt; repeating the original send uses existing prepared owner READ authority to confirm it. Lost replies and restarts preserve the same requests. Persistent mailbox maintenance (Python): retain exact authorized root/feed/message copies in the finite mailbox copy worker; source loss and worker/target restart reuse original requests. ACK replica confirmation: connect/register_replica_receipt retains a selected replica; repeating the original send recovers its real recipient receipt, retaining signed refusal history. -Mailbox replicas (Python Agent): docs/OPEN_ACK_RECOVERY.md — explicit separately authorized directory/feed/message copies; original-key message and memory reception through the durable inbox after source loss; independent receipt return when the original message includes ACK authority. Automatic replacement and native replica reception remain unfinished. +Mailbox replicas (Python Agent): docs/OPEN_ACK_RECOVERY.md — explicit separately authorized directory/feed/message copies; original-key message and memory reception through the durable inbox after source loss; independent receipt return when the original message includes ACK authority. Automatic replacement and native ACK/root/feed replica reception remain unfinished. Receipt recovery: - docs/OPEN_ACK_PROVISIONING.md — independent remote source setup with only the sender's own config and the selected source's public origin/key; original unbound source precedes ciphertext. @@ -72,3 +72,5 @@ Reference: Use generic implementation and synthetic fixtures in public contributions. Never publish private memories, credentials or identity files. Reading this repository enables no relay, hook or background service. + +Bounded alpha40 capacity: docs/CAPACITY_HANDOFF.md. Native message replicas verify full historical COPY and current READ with shared protected receiver/status state. Same-host 48-message load completes 39 -> 48; independent-host/global capacity is unverified. Finite work grants and slow cold reads remain explicit limits. diff --git a/memory_vault.py b/memory_vault.py index 9554cbe..6949a29 100644 --- a/memory_vault.py +++ b/memory_vault.py @@ -60,7 +60,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence -VERSION = "0.28.0-alpha.0.39" +VERSION = "0.28.0-alpha.0.40" REQUEST_SCHEMA = "universal-agent-memory-request/v1" RESULT_SCHEMA = "universal-agent-memory-result/v1" RECORD_SCHEMA = "universal-memory-record/v1" diff --git a/memory_vault_network.py b/memory_vault_network.py index 6303f9e..fc2e8e8 100644 --- a/memory_vault_network.py +++ b/memory_vault_network.py @@ -287,22 +287,30 @@ def db(self): CREATE TABLE IF NOT EXISTS quarantine(message_id TEXT PRIMARY KEY,digest TEXT NOT NULL UNIQUE, sender TEXT NOT NULL,envelope BLOB NOT NULL,code TEXT NOT NULL); """) - with connection: - connection.execute("BEGIN IMMEDIATE") - if "recipients" not in {row[1] for row in connection.execute("PRAGMA table_info(outbox)")}: - # Older completely offline rows did not preserve routing. - # Keep them intact; only their original caller can supply - # missing recipients when no frozen envelope exists. - connection.execute("ALTER TABLE outbox ADD COLUMN recipients BLOB") - binding = connection.execute("SELECT value FROM state WHERE key='configuration_binding'").fetchone() - if binding is None: - occupied = any(connection.execute("SELECT 1 FROM " + table + " LIMIT 1").fetchone() - for table in ("state", "outbox", "inbox", "acknowledgements", "quarantine")) - if occupied: - raise MemoryError("network_state_binding_missing") - connection.execute("INSERT INTO state VALUES('configuration_binding',?)", (canonical_bytes(self._binding).decode(),)) - elif strict_json_loads(binding["value"]) != self._binding: - raise MemoryError("network_state_configuration_mismatch") + binding = connection.execute("SELECT value FROM state WHERE key='configuration_binding'").fetchone() + migrated = "recipients" in {row[1] for row in connection.execute("PRAGMA table_info(outbox)")} + if binding is not None and strict_json_loads(binding["value"]) != self._binding: + raise MemoryError("network_state_configuration_mismatch") + # Normal opens verify the current durable binding without becoming + # writers. Initialization/migration still rechecks under the writer + # lock so two first opens cannot admit different configurations. + if binding is None or not migrated: + with connection: + connection.execute("BEGIN IMMEDIATE") + if "recipients" not in {row[1] for row in connection.execute("PRAGMA table_info(outbox)")}: + # Older completely offline rows did not preserve routing. + # Keep them intact; only their original caller can supply + # missing recipients when no frozen envelope exists. + connection.execute("ALTER TABLE outbox ADD COLUMN recipients BLOB") + binding = connection.execute("SELECT value FROM state WHERE key='configuration_binding'").fetchone() + if binding is None: + occupied = any(connection.execute("SELECT 1 FROM " + table + " LIMIT 1").fetchone() + for table in ("state", "outbox", "inbox", "acknowledgements", "quarantine")) + if occupied: + raise MemoryError("network_state_binding_missing") + connection.execute("INSERT INTO state VALUES('configuration_binding',?)", (canonical_bytes(self._binding).decode(),)) + elif strict_json_loads(binding["value"]) != self._binding: + raise MemoryError("network_state_configuration_mismatch") with connection: yield connection finally: diff --git a/memory_vault_open_delivery_client.py b/memory_vault_open_delivery_client.py index 287c5c2..e525bb2 100644 --- a/memory_vault_open_delivery_client.py +++ b/memory_vault_open_delivery_client.py @@ -351,11 +351,12 @@ async def send(self, request_id, recipients, text="", memory_ids=None, control=N budget = _DeliveryBudget() if _deadline is not None:budget.deadline=min(budget.deadline,_deadline) if row["session"] is None: - session = await self._sending_session(recipients[0], budget) - row = self._encrypt_outbox(row, session) + session = await self._sending_session(recipients[0], budget, row=row) + row = self._outbox(request_id) else: session = document(bytes(row["session"]), maximum=MAX_SESSION_BYTES) - self._encrypt_outbox(row, session) + row = self._encrypt_outbox(row, session) + session = document(bytes(row["session"]), maximum=MAX_SESSION_BYTES) node, pending_code = None, None if row["result"] is None: # Recover a committed upload even if its final response was lost @@ -515,7 +516,7 @@ def _keys(session): "recipient_signing_key": policy["signing_key"], "recipient_encryption_key": policy["encryption_key"]} - async def _sending_session(self, recipient, budget): + async def _sending_session(self, recipient, budget, *, row=None): sessions = self._contact_sessions(outgoing=True, recipient=recipient) if not sessions: # Fetch only an existing explicit request's decision. This neither @@ -543,13 +544,101 @@ async def _sending_session(self, recipient, budget): budget.merge_routing(route_budget) break sessions = self._contact_sessions(outgoing=True, recipient=recipient) + eligible, unavailable = [], [] for session in sessions: try: - return await self._checked_session(session, budget) + checked = await self._checked_session(session, budget) + if row is None: + # Repair callers retain their original first valid choice. + return checked + eligible.append(checked) except ContactError as exc: - if exc.code not in {"contact_expired", "contact_policy_mismatch"}: + if exc.code in {"contact_expired", "contact_policy_mismatch"}: + continue + if row is not None and exc.code == "contact_resource_unresolved": + unavailable.append(exc) + continue + else: raise - raise MemoryError("open_contact_approval_required") + if not eligible: + if unavailable: + raise unavailable[0] + raise MemoryError("open_contact_approval_required") + frozen = self._freeze_dispatch(row, eligible) + return document(bytes(frozen["session"]), maximum=MAX_SESSION_BYTES) + + @staticmethod + def _session_scope(session): + """The exact signed resource lease; aliases cannot multiply its quota.""" + lease = session["decision"]["payload"]["grant"]["payload"]["resource_lease"] + return hashlib.sha256(canonical_bytes(lease)).hexdigest() + + def _freeze_dispatch(self, row, sessions): + """Atomically place a new immutable envelope in a finite approved slot. + + Count every local frozen send, including unfinished uploads, against its + exact grant. The server independently rechecks live authority and quota. + Existing frozen sends never move to a different node on retry or failure. + """ + keys = self._keys(sessions[0]) + # Preserve the formerly selected recipient key incarnation. Never pool + # approvals for different encryption identities under one ciphertext. + sessions = [session for session in sessions if self._keys(session) == keys] + envelope = create_envelope(bytes(row["body"]), signer=self.identity, + sender_encryption_key=self.encryption.public_descriptor(), + recipient_signing_key=keys["recipient_signing_key"], + recipient_encryption_key=keys["recipient_encryption_key"], + message_id=row["message_id"], object_key=secrets.token_hex(32), created_at=row["created_at"]) + encoded = canonical_bytes(document(envelope, maximum=MAX_ENVELOPE_BYTES)) + with self.participant.state.db() as db: + db.execute("BEGIN IMMEDIATE") + prior = db.execute("SELECT * FROM open_delivery_outbox WHERE request_id=?", (row["request_id"],)).fetchone() + if prior is None: + raise MemoryError("open_delivery_outbox_missing") + if prior["envelope"] is not None: + return dict(prior) + used = {} + # MAX_OUTBOX_RECORDS is enforced before admission; no new roster or + # unbounded historical archive is introduced by this local scan. + for saved in db.execute("SELECT session,length(envelope) AS bytes FROM open_delivery_outbox WHERE session IS NOT NULL LIMIT ?", (MAX_OUTBOX_RECORDS,)): + session = document(bytes(saved["session"]), maximum=MAX_SESSION_BYTES) + scope = self._session_scope(session) + items, size = used.get(scope, (0, 0)) + used[scope] = (items+1, size+(saved["bytes"] or MAX_ENVELOPE_BYTES)) + available = [] + for session in sessions: + lease = session["decision"]["payload"]["grant"]["payload"]["resource_lease"]["payload"] + scope = self._session_scope(session) + items, size = used.get(scope, (0, 0)) + if items < lease["max_items"] and size+len(encoded) <= lease["max_bytes"]: + available.append((items/lease["max_items"], size/lease["max_bytes"], scope, session)) + if not available: + if all(len(encoded) > session["decision"]["payload"]["grant"]["payload"]["resource_lease"]["payload"]["max_bytes"] for session in sessions): + raise MemoryError("open_delivery_approval_bytes_insufficient") + raise MemoryError("open_delivery_approval_capacity") + session = min(available, key=lambda x: x[:3])[3] + proof = canonical_bytes(document(session, maximum=MAX_SESSION_BYTES)) + _, size = self._local_size(db, "open_delivery_outbox") + if size-MAX_ENVELOPE_BYTES-MAX_SESSION_BYTES+len(encoded)+len(proof) > MAX_LOCAL_BYTES: + raise MemoryError("network_outbox_capacity") + db.execute("UPDATE open_delivery_outbox SET envelope=?,session=? WHERE request_id=?", (encoded,proof,row["request_id"])) + return dict(db.execute("SELECT * FROM open_delivery_outbox WHERE request_id=?", (row["request_id"],)).fetchone()) + + def _receiving_sessions(self): + sessions = self._contact_sessions(outgoing=False) + with self.participant.state.db() as db: + row = db.execute("SELECT body FROM open_delivery_client_state WHERE key='receive_session_cursor'").fetchone() + if row is not None: + previous = bytes(row[0]).decode("ascii") + for index, session in enumerate(sessions): + if self._session_scope(session) == previous: + return sessions[index+1:]+sessions[:index+1] + return sessions + + def _advance_receiving_session(self, session): + with self.participant.state.db() as db: + db.execute("INSERT INTO open_delivery_client_state VALUES('receive_session_cursor',?) ON CONFLICT(key) DO UPDATE SET body=excluded.body", + (self._session_scope(session).encode("ascii"),)) def _encrypt_outbox(self, row, session): if row["envelope"] is not None: @@ -955,7 +1044,7 @@ async def receive(self, limit=4, *, _pending_only=False, _skip_pending=False, _d self._advance_pending_receipt(pending_row) if _pending_only: return dict(messages=messages,errors=errors[:4],network_accessed=budget.requests>0) - for candidate in self._contact_sessions(outgoing=False): + for candidate in self._receiving_sessions(): if len(messages) >= limit: break try: @@ -994,6 +1083,8 @@ async def receive(self, limit=4, *, _pending_only=False, _skip_pending=False, _d errors.append({"code": exc.code, "retryable": exc.retryable}) if len(errors) >= 4: break + finally: + self._advance_receiving_session(candidate) return {"messages": messages, "errors": errors[:4], "network_accessed": budget.requests > 0} def read_message(self, message_id, offset=0): diff --git a/memory_vault_open_index.py b/memory_vault_open_index.py index 9073017..cd04892 100644 --- a/memory_vault_open_index.py +++ b/memory_vault_open_index.py @@ -84,23 +84,33 @@ def _prune(self, now: int) -> None: self.connection.execute("DELETE FROM open_contact_floors WHERE retain_until<=? AND owner NOT IN (SELECT owner FROM open_contacts)", (now,)) self.connection.execute("DELETE FROM open_index_replay WHERE retain_until<=?", (now,)) - def _capacity(self) -> None: + def _capacity(self, *, now: int | None = None) -> None: db = self.connection - records = db.execute("SELECT count(*) FROM open_contact_floors").fetchone()[0] - replays = db.execute("SELECT count(*) FROM open_index_replay").fetchone()[0] + # A read snapshot applies the same logical expiry as _prune without + # turning every directory lookup into a serialized garbage-collection + # write. Allocation still prunes and checks physical obligations. + floors_where = "" if now is None else " WHERE retain_until>? OR owner IN (SELECT owner FROM open_contacts WHERE expires_at>?)" + floor_args = () if now is None else (now, now) + replay_where = "" if now is None else " WHERE retain_until>?" + contact_where = "" if now is None else " WHERE expires_at>?" + live_args = () if now is None else (now,) + records = db.execute("SELECT count(*) FROM open_contact_floors" + floors_where, floor_args).fetchone()[0] + replays = db.execute("SELECT count(*) FROM open_index_replay" + replay_where, live_args).fetchone()[0] # Reserve two maximum descriptors per floor so observing a same-revision # fork can always invalidate a previously accepted contact, even when # all unreserved capacity is full. This is reserved capacity, not a # claim that padding or two records physically exist on disk. floors = records * (2 * 4096 + 512) - contacts = db.execute("SELECT coalesce(sum(length(record)+length(lease)+256),0) FROM open_contacts").fetchone()[0] - replay_bytes = db.execute("SELECT coalesce(sum(length(response)+256),0) FROM open_index_replay").fetchone()[0] + contacts = db.execute("SELECT coalesce(sum(length(record)+length(lease)+256),0) FROM open_contacts" + contact_where, live_args).fetchone()[0] + replay_bytes = db.execute("SELECT coalesce(sum(length(response)+256),0) FROM open_index_replay" + replay_where, live_args).fetchone()[0] if records > self.maximum_records or replays > self.maximum_replays or floors + contacts + replay_bytes > self.maximum_bytes: raise OpenControlError("open_index_capacity", retryable=True) def _get(self, key: str, now: int) -> dict[str, Any]: floor = self.connection.execute( - "SELECT owner,status FROM open_contact_floors WHERE lookup_key=?", (key,)).fetchone() + """SELECT owner,status FROM open_contact_floors WHERE lookup_key=? AND + (retain_until>? OR owner IN (SELECT owner FROM open_contacts WHERE expires_at>?))""", + (key, now, now)).fetchone() if floor is None: return {"state": "not_found"} if floor[1] in {"revoked", "conflict"}: @@ -141,17 +151,18 @@ def handle(self, request: Mapping[str, Any] | bytes, *, now: int | None = None) db = self.connection if db.in_transaction: raise OpenControlError("open_storage_transaction") - db.execute("BEGIN IMMEDIATE") + reading = checked["action"] == "get" + db.execute("BEGIN" if reading else "BEGIN IMMEDIATE") try: # A wait for the writer lock must not extend request authority. current = int(time.time()) if now is None else integer(now) checked = verify_request(signed_request, node=self.node, now=current) - self._prune(current) - if checked["action"] == "get": + if reading: result = self._get(checked["body"]["key"], current) else: + self._prune(current) result, failure = self._write(signed_request, checked, current) - self._capacity() + self._capacity(now=current if reading else None) db.commit() except BaseException: db.rollback() diff --git a/memory_vault_open_node.py b/memory_vault_open_node.py index 49144a4..18f2acd 100644 --- a/memory_vault_open_node.py +++ b/memory_vault_open_node.py @@ -15,6 +15,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import secrets +import queue import socket from socketserver import TCPServer import sqlite3 @@ -24,7 +25,7 @@ from memory_vault import MemoryError, canonical_bytes from memory_vault_network import NetworkClient, _read_private -from memory_vault_network_crypto import document, document_sha256, integer, object_fields +from memory_vault_network_crypto import document, document_sha256, integer, object_fields, digest from memory_vault_open_control import ( contact_key, coordinate, issue_node, sign_request, sign_response, verify_contact, verify_node, verify_response, verify_request, @@ -220,6 +221,9 @@ def __init__(self, identity: Identity, state_directory: Path, *, seeds: list, self._publication = None self._maintenance_cycle = 0 self._seed_refresh_after = 0.0 + self._neighbor_lock = threading.Lock() + self._neighbor_cursor = 0 + self._neighbor_probe_after = 0.0 with self.state.db() as db: OpenCheckpoints(db).initialize() db.execute("CREATE TABLE IF NOT EXISTS open_peer_cache (key_id TEXT PRIMARY KEY,node BLOB NOT NULL,seen_at INTEGER NOT NULL)") @@ -557,6 +561,73 @@ async def _lookup(self, target, view, budget): result["partial"] = result["partial"] or bool(refresh_errors) return result + async def find_neighbors(self, target: str, *, view="general", budget=None): + """One distributed routing step, returning introductions, not a contact. + + Repeated callers share a bounded, endpoint-proven router pool instead + of pinning every routing query to the initial seed. A reply can introduce + one new router for a fresh hello proof, at most once per five seconds. + This does not replace iterative lookup or its multi-shard contact checks. + No remote node receives the caller's authority or private message data. + """ + digest(target) + if view not in ("general", "directory"): + raise MemoryError("open_invalid_view") + budget = budget or LookupBudget() + errors = await self._refresh_seed_introductions(budget) + introductions = [node for lane in self._initial(target, "general") for node in lane] + proven = {node["payload"]["signing_key"]["key_id"]: node for node in introductions + if "router" in node["payload"]["roles"] and self.table.has_verified(node)} + nodes = sorted(proven.values(), key=lambda n: n["payload"]["signing_key"]["key_id"]) + with self._neighbor_lock: + offset = self._neighbor_cursor % len(nodes) if nodes else 0 + self._neighbor_cursor += 1 + nodes = nodes[offset:] + nodes[:offset] + # Cold starts and failed pools retain independently configured/cache + # introductions. They still need a valid response from the exact peer. + seen = set(proven) + for node in introductions: + key = node["payload"]["signing_key"]["key_id"] + if key not in seen and "router" in node["payload"]["roles"]: + nodes.append(node) + seen.add(key) + for node in nodes[:3]: + try: + body = await self._call(node, "find", {"target": target, "view": view}, budget) + except (MemoryError, OSError, TimeoutError) as exc: + self.table.mark_failed(node["payload"]["signing_key"]["key_id"]) + errors.append(getattr(exc, "code", "open_rpc_unreachable")) + continue + # The responder's proof never transfers to a returned descriptor. + # Use normal checkpoint and source/bucket limits on each new proof. + # An empty/fully proven reply must not consume the five-second + # probe allowance before a joining peer becomes discoverable. + candidates = [] + for introduced in sorted(body["nodes"], key=lambda n: + n["payload"]["signing_key"]["key_id"] in seen): + try: + if (introduced["payload"]["signing_key"]["key_id"] != self.identity.key_id + and "router" in introduced["payload"]["roles"] + and not self.table.has_verified(introduced)): + candidates.append(introduced) + except MemoryError: + continue + with self._neighbor_lock: + probe = bool(candidates) and time.monotonic() >= self._neighbor_probe_after + if probe: + self._neighbor_probe_after = time.monotonic() + 5 + if probe: + try: + await self._call(candidates[0], "hello", {"node": None}, budget) + except (MemoryError, OSError, TimeoutError) as exc: + errors.append(getattr(exc, "code", "open_rpc_unreachable")) + return {"state": "introduced", "responder": node, "nodes": body["nodes"], + "metrics": self._metrics(budget), "errors": errors, + "introductions_verified": False, "discovery_grants_access": False} + return {"state": "unreachable", "responder": None, "nodes": [], + "metrics": self._metrics(budget), "errors": errors, + "introductions_verified": False, "discovery_grants_access": False} + @staticmethod def _metrics(budget): return {"requests": budget.requests, "request_bytes": budget.request_bytes, @@ -829,7 +900,11 @@ def handle_blob(self, request, chunk): class OpenHTTPServer(ThreadingHTTPServer): - """Bounded HTTP admission. No plaintext, HTTP redirects or secret logging.""" + """Eight fixed workers and a finite queue; no per-connection worker spawn. + + Queue time is part of the existing input deadline. Overload is an explicit + retryable HTTP refusal, never a signed acceptance or a new protocol grant. + """ daemon_threads = True request_queue_size = 16 @@ -841,27 +916,85 @@ def server_bind(self): def __init__(self, address, participant): self.participant = participant - self._slots = threading.BoundedSemaphore(8) + self._requests = queue.Queue(maxsize=16) + self._closing = threading.Event() + self._deadlines = {} + self._deadlines_lock = threading.Lock() + self._workers = [] self._rate_lock = threading.Lock() self._rate = OrderedDict() self._global_rate = (0, 0) super().__init__(address, _Handler) + self._workers = [threading.Thread(target=self._work, daemon=True) for _ in range(8)] + for worker in self._workers: + worker.start() def process_request(self, request, client_address): - if not self._slots.acquire(blocking=False): - request.close() + if self._closing.is_set(): + self.shutdown_request(request) return try: - super().process_request(request, client_address) - except BaseException: - self._slots.release() - raise + self._requests.put_nowait((request, client_address, time.monotonic() + HTTP_REQUEST_SECONDS)) + except queue.Full: + self._busy(request) - def process_request_thread(self, request, client_address): + def _busy(self, request): + # A tiny fixed response, bounded send time, no input parsing or echo. try: - super().process_request_thread(request, client_address) + request.settimeout(.05) + request.sendall(b"HTTP/1.0 503 Service Unavailable\r\nContent-Length: 0\r\n" + b"Retry-After: 1\r\nConnection: close\r\n\r\n") + request.shutdown(socket.SHUT_WR) + # Closing with a normal client's unread upload can turn the 503 + # into a TCP reset. Drain only bytes already buffered, with a fixed + # ceiling; a peer cannot make admission wait for more input. + request.setblocking(False) + for _ in range(4): + if not request.recv(4096): + break + except OSError: + pass finally: - self._slots.release() + self.shutdown_request(request) + + def _work(self): + while not self._closing.is_set(): + try: + request, address, deadline = self._requests.get(timeout=.1) + except queue.Empty: + continue + try: + if self._closing.is_set() or time.monotonic() >= deadline: + self._busy(request) + continue + with self._deadlines_lock: + self._deadlines[request] = deadline + self.process_request_thread(request, address) + finally: + with self._deadlines_lock: + self._deadlines.pop(request, None) + self._requests.task_done() + + def input_deadline(self, request): + with self._deadlines_lock: + return self._deadlines[request] + + def server_close(self): + self._closing.set() + super().server_close() + while True: + try: + request, _, _ = self._requests.get_nowait() + except queue.Empty: + break + self.shutdown_request(request) + self._requests.task_done() + # Idle workers stop promptly. Active finite handlers retain the same + # daemon/timeout semantics as the previous server. + deadline = time.monotonic() + .2 + for worker in self._workers: + if worker is not threading.current_thread(): + worker.join(max(0, deadline-time.monotonic())) def admitted(self, address): now = int(time.monotonic()) @@ -886,13 +1019,17 @@ class _Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.0" def setup(self): - self.request.settimeout(HTTP_REQUEST_SECONDS) + deadline = self.server.input_deadline(self.request) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("open_request_timeout") + self.request.settimeout(remaining) super().setup() self.rfile = _HeaderBudget(self.rfile) # Bound the complete incoming headers/body, including drip-fed input. self._deadline_lock = threading.Lock() - self._deadline_at = time.monotonic() + HTTP_REQUEST_SECONDS - self._deadline = threading.Timer(HTTP_REQUEST_SECONDS, self._abort) + self._deadline_at = deadline + self._deadline = threading.Timer(remaining, self._abort) self._deadline.daemon = True self._deadline.start() diff --git a/plugins/memory-vault-client/.codex-plugin/plugin.json b/plugins/memory-vault-client/.codex-plugin/plugin.json index 46f92bb..8550a68 100644 --- a/plugins/memory-vault-client/.codex-plugin/plugin.json +++ b/plugins/memory-vault-client/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "memory-vault-client", - "version": "0.28.0-alpha.0.39", + "version": "0.28.0-alpha.0.40", "description": "Full authorized client for the open memory protocol: MCP, visible-turn capture, queued signed sync, host adapters and recovery.", "author": { "name": "Memory Vault contributors" diff --git a/scripts/benchmark_native_runtime.py b/scripts/benchmark_native_runtime.py new file mode 100644 index 0000000..90c6889 --- /dev/null +++ b/scripts/benchmark_native_runtime.py @@ -0,0 +1,92 @@ +"""Two persistent native Agents for the finite synthetic capacity harness only.""" +import json +import os +from pathlib import Path +import select +import shutil +import subprocess +import tempfile +import threading + +DRIVER = r''' +import child from 'node:child_process'; +import {syncBuiltinESMExports} from 'node:module'; +import {createInterface} from 'node:readline'; +let subprocessCalls=0; +const deny=()=>{subprocessCalls++;throw Error('native Agent must not delegate to a subprocess');}; +for(const name of ['spawn','spawnSync','exec','execSync','execFile','execFileSync','fork'])child[name]=deny; +syncBuiltinESMExports(); +const {Agent}=await import('./agent.ts'); +const {OpenDeliveryClient}=await import('./open-delivery-client.ts'); +let agent; +for await(const line of createInterface({input:process.stdin,crlfDelay:Infinity})){ + if(Buffer.byteLength(line)>1048576)throw Error('finite synthetic input limit'); + let result; + try{ + const request=JSON.parse(line); + if(!agent){agent=new Agent(request.client_config,request.network_config);result={ready:true};} + else if(request.capacity_metrics){result={cpu:process.cpuUsage(),rss:process.memoryUsage().rss,subprocessCalls, + distribution:typeof OpenDeliveryClient.prototype.freezeDispatch==='function'};} + else result=await agent.handle(request); + }catch(error){result={synthetic_driver_error:error.code??error.name};} + process.stdout.write(JSON.stringify(result)+'\n'); +} +''' + + +class NativeAgent: + def __init__(self,node,directory,agent): + self.lock=threading.Lock() + self.process=subprocess.Popen([node,'--experimental-strip-types',str(directory/'driver.mjs')], + cwd=directory,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + self.handle(dict(client_config=str(agent.client_config),network_config=str(agent.network_config))) + + def handle(self,request): + encoded=json.dumps(request,separators=(',',':')).encode()+b'\n' + if len(encoded)>1048576:raise RuntimeError('finite native input limit') + with self.lock: + self.process.stdin.write(encoded);self.process.stdin.flush() + if not select.select([self.process.stdout],[],[],60)[0]:raise TimeoutError('native synthetic reply deadline') + reply=self.process.stdout.readline(1048577) + if not reply or len(reply)>1048576:raise RuntimeError('native synthetic process/reply failure') + result=json.loads(reply) + if 'synthetic_driver_error' in result:raise RuntimeError(result['synthetic_driver_error']) + return result + + def close(self): + if self.process.poll() is None: + self.process.terminate() + try:self.process.wait(2) + except subprocess.TimeoutExpired:self.process.kill();self.process.wait(2) + for pipe in (self.process.stdin,self.process.stdout,self.process.stderr):pipe.close() + + +class NativeRuntime: + def __init__(self,fixture,root): + node=os.environ.get('MEMORY_VAULT_NODE') or shutil.which('node') + if not node:raise RuntimeError('existing Node >=22.19 required') + selected=os.environ.get('MEMORY_VAULT_JOSE_MODULE') + package=Path(selected).expanduser().resolve().parents[2] if selected else root/'clients/typescript/network/node_modules/jose' + metadata=json.loads((package/'package.json').read_text()) + if metadata.get('name')!='jose' or metadata.get('version')!='6.2.10':raise RuntimeError('locked existing jose 6.2.10 required') + self.temporary=tempfile.TemporaryDirectory(prefix='synthetic-native-capacity-') + directory=Path(self.temporary.name) + source=root/'clients/typescript/network' + for path in [*source.glob('*.ts'),source/'package.json']:shutil.copyfile(path,directory/path.name) + shutil.copyfile(root/'memory_vault_open_capacity_schema.json',directory/'memory_vault_open_capacity_schema.json') + (directory/'node_modules').mkdir();(directory/'node_modules/jose').symlink_to(package,target_is_directory=True) + (directory/'driver.mjs').write_text(DRIVER) + self.agents=[] + try: + for agent in (fixture.a,fixture.b): + native=NativeAgent(node,directory,agent);self.agents.append(native);agent.handle=native.handle + except BaseException:self.close();raise + + def usage(self): + result=[agent.handle(dict(capacity_metrics=True)) for agent in self.agents] + assert all(value['subprocessCalls']==0 for value in result) + return result + + def close(self): + for agent in self.agents:agent.close() + self.temporary.cleanup() diff --git a/scripts/benchmark_open_delivery.py b/scripts/benchmark_open_delivery.py new file mode 100644 index 0000000..069c57c --- /dev/null +++ b/scripts/benchmark_open_delivery.py @@ -0,0 +1,254 @@ +"""Finite real encrypted delivery -> saved receipt -> local read/Memory recall. + +Two loopback node processes, one sender and one recipient. Each node receives an +independent explicit finite resource approval; no grant is copied to another node. +Offers continue at a fixed rate while the second node starts and is authorized. +At most 48 offers, 48 seconds and one pipeline worker; all failures/lag are retained. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import platform +import resource +import signal +import sys +import tempfile +import threading +import time + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) + + +def usage(): + r=resource.getrusage(resource.RUSAGE_SELF) + return dict(user_cpu_seconds=r.ru_utime,system_cpu_seconds=r.ru_stime, + max_rss_bytes=r.ru_maxrss if sys.platform=='darwin' else r.ru_maxrss*1024) + + +def serve(config_path,metrics_path): + import memory_vault_open_node as node + from memory_vault import canonical_bytes + from memory_vault_storage import atomic_write + lock=threading.RLock();counts=Counter();serial=0 + OriginalParticipant=node.OpenParticipant;OriginalServer=node.OpenHTTPServer + class Participant(OriginalParticipant): + def handle(self,request): + response=super().handle(request) + with lock: + action=request['payload']['action'];counts['rpc_'+action]+=1 + counts['request_bytes']+=len(canonical_bytes(request)) + counts['response_bytes']+=len(canonical_bytes(response)) + error=response.get('payload',{}).get('body',{}).get('error') + if error:counts['error_'+error['code']]+=1 + return response + def handle_blob(self,request,chunk): + result=super().handle_blob(request,chunk) + with lock: + counts['blob_'+request['payload']['action']]+=1 + counts['blob_input_bytes']+=len(chunk) + counts['blob_output_bytes']+=len(result[1]) + return result + class Server(OriginalServer): + def admitted(self,address): + accepted=super().admitted(address) + with lock: + counts['http_requests']+=1;counts['second_'+str(int(time.time()))]+=1 + counts['admitted' if accepted else 'rate_refused']+=1 + return accepted + def _busy(self,request): + with lock:counts['queue_refused']+=1 + return super()._busy(request) + node.OpenParticipant=Participant;node.OpenHTTPServer=Server + def snapshot(*_): + nonlocal serial + with lock: + serial+=1;value=dict(serial=serial,counts=dict(counts),usage=usage(),time=time.monotonic()) + atomic_write(metrics_path,json.dumps(value).encode(),replace=True) + signal.signal(signal.SIGUSR1,snapshot) + with node._publication_lock(config_path): + node._run_node(config_path) + + +def percentile(values,p): + ordered=sorted(values) + return ordered[min(len(ordered)-1,int((len(ordered)-1)*p))] if ordered else None + + +def main_run(args): + from memory_vault_client import ClientConfig + from memory_vault_open_delivery_client import OpenDeliveryClient + from tests.test_open_delivery_distribution import DeliveryDistributionFixture + from tests.test_open_node import HTTPNodes + class InstrumentedNodes(HTTPNodes): + def command(self,index): + return [sys.executable,str(Path(__file__).resolve()),'--serve-config',str(self.configs[index]), + '--metrics',str(self.root/f'node_{index}.metrics.json')] + def snapshot(self,index): + if index not in self.processes: + return dict(counts={},usage=dict(user_cpu_seconds=0,system_cpu_seconds=0,max_rss_bytes=0)) + path=self.root/f'node_{index}.metrics.json' + previous=json.loads(path.read_bytes())['serial'] if path.exists() else 0 + os.kill(self.processes[index].pid,signal.SIGUSR1) + deadline=time.monotonic()+2 + while time.monotonic()previous:return value + time.sleep(.01) + raise RuntimeError('bounded synthetic snapshot deadline') + with tempfile.TemporaryDirectory(prefix='synthetic-delivery-capacity-') as temporary: + fixture=DeliveryDistributionFixture(temporary,host_class=InstrumentedNodes,late_node=True) + finished=threading.Event() + native=None + try: + items=args.messages//2 + fixture.approve(0,items=items,bytes_limit=2*1024*1024) + if args.runtime=='native': + from benchmark_native_runtime import NativeRuntime + native=NativeRuntime(fixture,ROOT) + memory=[] + for i in range(args.messages): + text=f'Synthetic observation {i}: independent evidence must be checked before reuse.' + value=fixture.call(fixture.a,op='remember',request_id=f'req_capacity_memory_{i}', + kind='observation',text=text) + memory.append(dict(memory_id=value['memory_id'],text=text)) + snapshots=[fixture.host.snapshot(i) for i in range(2)] + native_before=native.usage() if native else None + before=usage();start=time.perf_counter();offers=[];join={};join_error=[] + def hard_deadline(): + if not finished.wait(args.messages/args.rate+30): + # Terminate only this disposable harness and its two children, + # including queued work. Never let socket failure multiply + # the ordinary per-operation deadline into an unbounded run. + if native:native.close() + fixture.close() + os.write(2,b'synthetic benchmark exceeded its finite drain allowance\n') + os._exit(124) + threading.Thread(target=hard_deadline,daemon=True).start() + def join_node(): + join['started_seconds']=time.perf_counter()-start + try: + fixture.host.start(1) + fixture.approve(1,items=items,bytes_limit=2*1024*1024) + join['authorized_seconds']=time.perf_counter()-start + except Exception as exc: + join_error.append(str(exc));join['failed_seconds']=time.perf_counter()-start + def pipeline(i,offered): + row=dict(index=i,offered_seconds=offered-start,status='unknown',stages=[], + offer_lag_ms=(time.perf_counter()-offered)*1000,node=None,storage_accepted=False, + saved_receipt=False,local_read=False,memory_recalled=False) + try: + sent=fixture.send(i,memory_ids=[memory[i]['memory_id']]) + row['stages'].append(dict(stage='send',ok=sent['ok'],code=sent.get('error',{}).get('code'))) + if not sent['ok']: + row['status']=sent['error']['code'];return row + result=sent['result'];row['message_id']=result['message_id'];row['storage_accepted']=result['storage_accepted'] + with fixture.a._network() as network: + with network.participant.state.db() as db: + session=db.execute('SELECT session FROM open_delivery_outbox WHERE request_id=?', + (f'req_synthetic_distribution_send_{i}',)).fetchone()[0] + key=json.loads(session)['node']['payload']['signing_key']['key_id'] + row['node']=next(j for j,n in enumerate(fixture.host.nodes) if n['payload']['signing_key']['key_id']==key) + received=fixture.b.handle({'op':'receive','limit':4}) + row['stages'].append(dict(stage='receive',ok=received['ok'],errors=received.get('result',{}).get('errors',[]))) + if not received['ok'] or received['result']['errors']: + row['status']=received.get('error',{}).get('code','receive_errors');return row + ack=fixture.send(i,memory_ids=[memory[i]['memory_id']]) + row['stages'].append(dict(stage='receipt',ok=ack['ok'],code=ack.get('error',{}).get('code'))) + row['saved_receipt']=bool(ack['ok'] and ack['result']['endpoint_validated']) + read=fixture.b.handle({'op':'receive','message_id':result['message_id'],'offset':0}) + row['local_read']=bool(read['ok'] and f'payload {i}' in read['result']['text']) + recall=fixture.b.handle({'op':'recall','memory_id':memory[i]['memory_id']}) + row['memory_recalled']=bool(recall['ok'] and any(x['text']==memory[i]['text'] for x in recall['result']['hits'])) + row['status']='ok' if all(row[n] for n in ('storage_accepted','saved_receipt','local_read','memory_recalled')) else 'incomplete_chain' + return row + except Exception as exc: + row['status']=type(exc).__name__;row['detail']=str(exc)[:256];return row + finally: + row['latency_ms']=(time.perf_counter()-offered)*1000 + row['completed_seconds']=time.perf_counter()-start + worker=None + with ThreadPoolExecutor(max_workers=1) as pool: + for i in range(args.messages): + offered=start+i/args.rate + time.sleep(max(0,offered-time.perf_counter())) + if worker is None and i>=args.join_after: + worker=threading.Thread(target=join_node,daemon=True);worker.start() + offers.append(pool.submit(pipeline,i,offered)) + offer_window=args.messages/args.rate + time.sleep(max(0,start+offer_window-time.perf_counter())) + rows=[future.result() for future in offers] + if worker:worker.join(8) + elapsed=time.perf_counter()-start;after=usage();per_node=[] + for i in range(2): + snapshot=fixture.host.snapshot(i) + counts={k:v-snapshots[i]['counts'].get(k,0) for k,v in snapshot['counts'].items()} + with __import__('sqlite3').connect(fixture.root/f'node_{i}'/'transport/network.sqlite3') as db: + stored=db.execute('SELECT count(*),coalesce(sum(length(envelope)),0) FROM open_delivery_messages').fetchone() + receipts=db.execute('SELECT count(*) FROM open_delivery_receipts').fetchone()[0] + leases=[dict(lease_id=r[0],max_items=r[1],max_bytes=r[2]) for r in db.execute( + "SELECT lease_id,max_items,max_bytes FROM open_contact_resource_leases WHERE purpose='delivery'")] + per_node.append(dict(node=i,counts=counts,peak_http_requests_per_second=max( + [v for k,v in counts.items() if k.startswith('second_')] or [0]), + stored_items=stored[0],stored_ciphertext_bytes=stored[1],saved_receipts=receipts,delivery_leases=leases, + user_cpu_seconds=snapshot['usage']['user_cpu_seconds']-snapshots[i]['usage']['user_cpu_seconds'], + system_cpu_seconds=snapshot['usage']['system_cpu_seconds']-snapshots[i]['usage']['system_cpu_seconds'], + max_rss_bytes=snapshot['usage']['max_rss_bytes'])) + ok=[r for r in rows if r['status']=='ok'] + with fixture.b._network() as network: + with network.participant.state.db() as db: + inbox=db.execute("SELECT count(*) FROM open_delivery_inbox WHERE phase='saved' AND receipt_sent=1").fetchone()[0] + with __import__('sqlite3').connect(ClientConfig.load(fixture.b.client_config).vault_path) as db: + vault_memories=db.execute('SELECT count(*) FROM memories').fetchone()[0] + native_after=native.usage() if native else None + return dict(mode='approved_resource_distribution' if (native_after[0]['distribution'] if native else hasattr(OpenDeliveryClient,'_freeze_dispatch')) else 'prior_first_approved_session', + agent_runtime=args.runtime,native_before=native_before,native_after=native_after, + offered_messages=args.messages,offered_messages_per_second=args.rate,offer_window_seconds=offer_window, + elapsed_including_drain=elapsed,successful_chains=len(ok),success_rate=len(ok)/len(rows), + successful_chains_per_second=len(ok)/elapsed,statuses=dict(Counter(r['status'] for r in rows)), + successful_p95_ms=percentile([r['latency_ms'] for r in ok],.95),successful_p99_ms=percentile([r['latency_ms'] for r in ok],.99), + all_p95_ms=percentile([r['latency_ms'] for r in rows],.95),all_p99_ms=percentile([r['latency_ms'] for r in rows],.99), + offer_lag_p99_ms=percentile([r['offer_lag_ms'] for r in rows],.99),successful_by_node=dict(Counter(str(r['node']) for r in ok)), + recipient_saved_inbox=inbox,recipient_vault_memories=vault_memories,per_node=per_node,join=join,join_errors=join_error, + setup_attempts=fixture.setup_attempts,requests=rows,driver=dict( + user_cpu_seconds=after['user_cpu_seconds']-before['user_cpu_seconds'], + system_cpu_seconds=after['system_cpu_seconds']-before['system_cpu_seconds'],max_rss_bytes=after['max_rss_bytes'])) + finally: + finished.set() + if native:native.close() + fixture.close() + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--messages',type=int,default=48) + parser.add_argument('--rate',type=int,default=2) + parser.add_argument('--join-after',type=int,default=12,help='offer index at which second node starts') + parser.add_argument('--output',type=Path) + parser.add_argument('--runtime',choices=('python','native'),default='python') + parser.add_argument('--serve-config',type=Path) + parser.add_argument('--metrics',type=Path) + args=parser.parse_args() + if args.serve_config: + if not args.metrics:parser.error('metrics path required for synthetic child') + serve(args.serve_config,args.metrics);return + if args.messages not in (8,48) or args.rate not in (1,2) or not 1<=args.join_after= args.join_after: + for pipe in pipes[1:]: + pipe.send("join"); pipe.recv() + joined = True + futures.append(pool.submit(operation, i, offered, start)) + # Include the complete sustained offer window, without removing + # scheduling gaps. Drain is reported separately and in total rate. + time.sleep(max(0, start+args.seconds-time.perf_counter())) + rows = [future.result() for future in futures] + elapsed = time.perf_counter()-start + driver_after = usage() + per_node = [] + for i, pipe in enumerate(pipes): + pipe.send("snapshot"); end = pipe.recv() + per_node.append(dict(node=i, counts={key: value-snapshots[i]["counts"].get(key,0) + for key,value in end["counts"].items()}, routing=end["routing"], + user_cpu_seconds=end["usage"]["user_cpu_seconds"]-snapshots[i]["usage"]["user_cpu_seconds"], + system_cpu_seconds=end["usage"]["system_cpu_seconds"]-snapshots[i]["usage"]["system_cpu_seconds"], + max_rss_bytes=end["usage"]["max_rss_bytes"])) + client.close() + return dict(mode="automatic_neighbors" if adaptive else "prior_fixed_entry_find", + offered_rate=args.rate, offer_window_seconds=args.seconds, elapsed_including_drain=elapsed, + join_after_seconds=args.join_after, caller_initial_introductions=1, + overall=summary(rows, elapsed), + completed_during_offer_window=sum(r["status"] == "ok" and + r["offered_seconds"]+r["latency_ms"]/1000 <= args.seconds for r in rows), + completed_during_last_eight_seconds=sum(r["status"] == "ok" and + args.seconds-8 <= r["offered_seconds"]+r["latency_ms"]/1000 <= args.seconds for r in rows), + steady_last_eight_seconds=summary( + [r for r in rows if r["offered_seconds"] >= args.seconds-8],8), + windows=[summary([r for r in rows if second <= r["offered_seconds"] < second+4],4) + for second in range(0,args.seconds,4)], per_node=per_node, + driver=dict(user_cpu_seconds=driver_after["user_cpu_seconds"]-driver_before["user_cpu_seconds"], + system_cpu_seconds=driver_after["system_cpu_seconds"]-driver_before["system_cpu_seconds"], + max_rss_bytes=driver_after["max_rss_bytes"]), requests=rows) + finally: + for pipe in pipes: + try: pipe.send("stop") + except (OSError, BrokenPipeError): pass + for process in processes: + process.join(3) + if process.is_alive(): process.terminate(); process.join(2) + if process.is_alive(): process.kill(); process.join(2) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rate", type=int, default=80) + parser.add_argument("--seconds", type=int, default=24) + parser.add_argument("--join-after", type=int, default=6) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if not 1<=args.rate<=96 or args.seconds not in (16,20,24) or not 2<=args.join_after<=6: + parser.error("bounded load: rate <=96; seconds 16,20,24; join-after 2..6") + report=dict(synthetic=True, real_models=0, physical_hosts=1, cpu_count=multiprocessing.cpu_count(), + python=sys.version, platform=platform.platform(), source_hashes={name:hashlib.sha256((ROOT/name).read_bytes()).hexdigest() + for name in ("memory_vault_network.py","memory_vault_open_index.py","memory_vault_open_node.py","memory_vault_open_routing.py")}, + scope="signed public neighbor query, not full contact resolution or message I/O", result=run(args)) + args.output.parent.mkdir(parents=True,exist_ok=True) + args.output.write_text(json.dumps(report,indent=2)+"\n") + print(json.dumps({key:value for key,value in report["result"].items() if key!="requests"}),flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_client_plugin.py b/scripts/build_client_plugin.py index e689022..6018b6b 100644 --- a/scripts/build_client_plugin.py +++ b/scripts/build_client_plugin.py @@ -147,6 +147,10 @@ "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", + "clients/typescript/network/open-repair-mailbox-replica.ts", + "clients/typescript/network/open-mailbox-replica-client.ts", + "clients/typescript/network/open-repair-mailbox-replica-inbox.ts", + "clients/typescript/network/open-mailbox-replicas.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", diff --git a/scripts/build_release.py b/scripts/build_release.py index 7f1ce3a..56802f1 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -56,6 +56,9 @@ "docs/CONTINUATION_TRIAL.md", "docs/NATIVE_OPEN_PROVIDER.md", "docs/DEPENDENCIES_NETWORK.md", + "docs/HTTP_CAPACITY.md", "docs/ROUTING_CAPACITY.md", "docs/DELIVERY_CAPACITY.md", "docs/CAPACITY_HANDOFF.md", + "docs/evidence/capacity-alpha40/before.json", "docs/evidence/capacity-alpha40/after.json", + "docs/evidence/capacity-alpha40/native-replica.json", "docs/evidence/capacity-alpha40/replica-profile.json", "docs/NETWORK_RECOVERY.md", "docs/NETWORK_NODE_TRANSFER.md", "docs/NETWORK_TYPESCRIPT.md", ) # Each executable network review fixture is selected deliberately. A matching @@ -70,6 +73,7 @@ "tests/test_open_contact_directory_maintenance.py", "tests/test_open_contact_typescript.py", "tests/test_open_contact_typescript_http.py", "tests/test_open_delivery_http.py", + "tests/test_open_mailbox_replica_typescript.py", "tests/test_open_read_concurrency.py", "tests/test_open_neighbors.py", "tests/test_open_delivery_distribution.py", "tests/test_open_delivery_distribution_typescript.py", "tests/test_capacity_replica_closure.py", "tests/test_open_provider_typescript_http.py", "tests/test_open_provider_typescript.py", "tests/test_open_provider_status.py", "tests/test_open_provider_status_typescript.py", @@ -249,6 +253,10 @@ "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", + "clients/typescript/network/open-repair-mailbox-replica.ts", + "clients/typescript/network/open-mailbox-replica-client.ts", + "clients/typescript/network/open-repair-mailbox-replica-inbox.ts", + "clients/typescript/network/open-mailbox-replicas.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", @@ -326,6 +334,8 @@ def review_sources(material: list[Path], source_tree: ReleaseSource) -> list[Pat "scripts/build_client_plugin.py", "scripts/build_release.py", "scripts/release_source.py", "scripts/verify_client_package.py", "scripts/continuation_trial.py", + "scripts/benchmark_open_http.py", "scripts/benchmark_open_neighbors.py", + "scripts/benchmark_open_delivery.py", "scripts/benchmark_native_runtime.py", "docs/proposals/OPEN_DELIVERY_REPAIR_R3.md", "docs/OPEN_NETWORK_ARCHITECTURE.md", "docs/OPEN_NETWORK_LIFECYCLE.md", "docs/OPEN_NETWORK_ACCEPTANCE.md", "docs/OPEN_NETWORK_ROADMAP.md", diff --git a/scripts/release_source.py b/scripts/release_source.py index 1c8c0fa..bec6389 100644 --- a/scripts/release_source.py +++ b/scripts/release_source.py @@ -20,10 +20,10 @@ MAX_SOURCE_BYTES = 2 * 1024 * 1024 MAX_TREE_BYTES = 2 * 1024 * 1024 MAX_TREE_ENTRIES = 8192 -# The explicit native mailbox runtime/review inventory adds eleven files -# to the alpha35 inventory and now exceeds 624 selected source files. -# Keep a finite source-count ceiling independent of the unchanged byte caps. -MAX_SELECTED_FILES = 640 +# Alpha40's explicit native replica modules, tests, benchmark adapters and +# evidence bring the declared review inventory to 649 files. Keep a finite +# count ceiling independent of the unchanged per-file/tree/aggregate byte caps. +MAX_SELECTED_FILES = 664 MAX_SELECTED_BYTES = 32 * 1024 * 1024 GIT_TIMEOUT_SECONDS = 15 _SHA = re.compile(r"[0-9a-f]{40}") diff --git a/scripts/run_open_network_ci.py b/scripts/run_open_network_ci.py index 6295227..2a3f2aa 100644 --- a/scripts/run_open_network_ci.py +++ b/scripts/run_open_network_ci.py @@ -55,6 +55,7 @@ "tests.test_open_repair_provision", "tests.test_open_repair_onboarding", "tests.test_open_repair_bind_recovery", "tests.test_open_repair_remote_setup", "tests.test_open_repair_remote_provision", "tests.test_open_repair_status_observer", "tests.test_open_repair_mailbox_resources", "tests.test_open_repair_copy_resources", "tests.test_open_repair_copy_prepare", "tests.test_open_repair_copy_state", "tests.test_open_repair_copy_service", "tests.test_open_repair_copy_upload", "tests.test_open_repair_copy_empty", "tests.test_open_repair_copy_occupied", "tests.test_open_repair_mailbox_activation", "tests.test_open_repair_mailbox_range", "tests.test_open_repair_mailbox_authority_typescript", "tests.test_open_repair_mailbox_member_typescript", "tests.test_open_mailbox_client_typescript", "tests.test_open_mailbox_receipts_typescript", "tests.test_open_mailbox_destination_typescript", "tests.test_open_mailbox_send_typescript", "tests.test_open_repair_mailbox_root", "tests.test_open_repair_mailbox_status", "tests.test_open_repair_mailbox_source", "tests.test_open_repair_mailbox_snapshot", "tests.test_open_repair_mailbox_copy", "tests.test_open_repair_mailbox_copy_authority", "tests.test_open_repair_mailbox_copy_upload", "tests.test_open_repair_mailbox_reservation", "tests.test_open_repair_mailbox_consent", "tests.test_open_mailbox_replica_receive", "tests.test_open_mailbox_receipt_jobs", "tests.test_open_ack_replica_send", "tests.test_open_ack_typescript_agent", "tests.test_open_ack_discovery_typescript", "tests.test_open_ack_discovery_agent", "tests.test_open_mailbox_copy_jobs", "tests.test_open_repair_mailbox_copy_admin", "tests.test_open_repair_mailbox_feed_copy", "tests.test_open_repair_mailbox_message_copy", + "tests.test_open_mailbox_replica_typescript", "tests.test_open_read_concurrency", "tests.test_open_neighbors", "tests.test_open_delivery_distribution", "tests.test_open_delivery_distribution_typescript", "tests.test_capacity_replica_closure", "tests.test_open_provider_merge", "tests.test_continuation_trial", "tests.test_network_typescript_agent_network", "tests.test_network_packaging") EXPECTED = { @@ -244,7 +245,7 @@ def initialize(reports, mode, seed, partition_index=0, partition_count=1): "open-control.ts", "open-routing.ts", "open-state.ts", "client-config.ts", "transport-state.ts", "open-contact.ts", "open-contact-state.ts", "open-contact-client.ts", "open-provider.ts", "open-provider-client.ts", "open-blob.ts", "open-repair-wire.ts", "open-repair-history.ts", "open-repair-original.ts", "open-repair-resource.ts", - "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-mailbox-send.ts", "open-repair-ack.ts", "open-capacity.ts", + "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-mailbox-send.ts", "open-repair-mailbox-replica.ts", "open-mailbox-replica-client.ts", "open-repair-mailbox-replica-inbox.ts", "open-mailbox-replicas.ts", "open-repair-ack.ts", "open-capacity.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-delivery.ts", "open-delivery-control.ts", "open-delivery-client.ts", diff --git a/tests/test_capacity_replica_closure.py b/tests/test_capacity_replica_closure.py new file mode 100644 index 0000000..5a63f38 --- /dev/null +++ b/tests/test_capacity_replica_closure.py @@ -0,0 +1,169 @@ +"""Native frozen send, full replica verification and independent saved receipt. + +Python serves the independently approved replica. Native sender and recipient +execute their Agent operations without subprocess delegation. Finite replica +work limits are measured, retained and never increased for the load test. +""" +import hashlib +import json +import os +from pathlib import Path +import time +import unittest +from unittest.mock import patch + +from tests import test_open_delivery_http as delivery_fixtures +from tests import test_open_repair_mailbox_feed_copy as feed_fixtures +from tests import test_open_mailbox_replica_receive as registered_fixtures +from tests import test_open_ack_typescript_agent as native_ack +from tests import test_network_typescript_agent_network as native_runtime + + +class CrossLanguageReplicaClosureTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + native_runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(native_ack.DRIVER.replace('for(const name of', + "if(process.env.MEMORY_VAULT_FIXTURE_NOW)Date.now=()=>Number(process.env.MEMORY_VAULT_FIXTURE_NOW)*1000;\nfor(const name of",1)) + + def native(self, agent, requests, **options): + return native_ack.NativeAckAgentTests.native(self,agent,requests,**options) + + def run_fixture(self, inspect): + fixture=delivery_fixtures.MailboxStagingHTTPTests('test_ack_configuration_survives_mailbox_custody') + fixture.setUp();self.addCleanup(fixture.doCleanups);fixture.ack_cold_return=True + original=fixture.call + def call(agent,**request): + if (agent is fixture.a and request['op']=='send') or (getattr(self,'native_recipient',False) and agent is fixture.b): + response=self.native(agent,[request])['results'][0] + self.assertTrue(response['ok'],response) + return response['result'] + return original(agent,**request) + fixture.call=call;self.delivery_fixture=fixture + def committed(staging,slot,head): + h=feed_fixtures.FeedCopyFixture(self,fixture,staging,slot,head,message=True) + # Both runtimes use the same fixture clock. Real monotonic deadlines + # remain active; the protocol rejects future-dated probes strictly. + with patch('time.time',return_value=h.now),patch.dict(os.environ,{'MEMORY_VAULT_FIXTURE_NOW':str(h.now)}):inspect(h) + fixture.inspect_committed_mailbox=committed + fixture.test_ack_configuration_survives_mailbox_custody() + + def inspect_registration(self,h,registration,configured): + # Includes wrong-epoch refusal and bounded independent destination count. + registered_fixtures.RegisteredReplicaTests.inspect_registration(self,h,registration,configured) + self.original_envelope=h.snapshot.read(h.envelope_ref) + self.assertEqual(hashlib.sha256(self.original_envelope).hexdigest(),h.envelope_ref['raw_sha256']) + if getattr(self,'native_recipient',False): + from memory_vault_open_control import issue_node + changed=issue_node(h.identity,base_url=registration['base_url'],storage_epoch='synthetic_replaced_native_target', + roles=['directory','router'],revision=100,issued_at=h.now,expires_at=h.now+300) + wrong=self.native(h.recipient_agent,[dict(op='receive')],node_override=changed) + self.assertTrue(wrong['results'][0]['ok'],wrong) + self.assertTrue(any(e['code']=='open_invalid_mailbox_receiver' for e in wrong['results'][0]['result']['errors']),wrong) + self.assertFalse(any(c['kind']=='requestRepair' and c['base']==registration['base_url'] for c in wrong['calls'])) + + def cold_load(self,h,registration): + """Three additional empty Vaults reuse explicit existing permission. + + This measures a bounded known-message demand on the added replica. It + does not inject unknown future IDs into a pre-message frozen recipient, + or represent three independent owners/failure domains. + """ + from memory_vault_agent import Agent + from memory_vault import canonical_bytes + from memory_vault_trust import _write_new_private + client=json.loads(Path(h.recipient_agent.client_config).read_bytes()) + network=json.loads(Path(h.recipient_agent.network_config).read_bytes()) + self.cold_results=[] + for i in range(3): + folder=h.directory/('synthetic-native-cold-'+str(i));folder.mkdir(mode=0o700) + # A new private state binds to its own empty Vault. Copying another + # state's frozen Vault binding would correctly be refused. + state=folder/'transport';state.mkdir(mode=0o700) + vault=folder/'vault';vault.mkdir(mode=0o700) + copied_client=dict(client,vault_path=str(vault/'memory.sqlite3')) + client_path=folder/'client.json';config_path=folder/'open.json' + _write_new_private(client_path,canonical_bytes(copied_client)) + _write_new_private(config_path,canonical_bytes(dict(network,client_config_path=str(client_path),state_directory=str(state)))) + agent=Agent(client_path,config_path) + work_before=h.db.execute('SELECT requests FROM open_repair_copy_work WHERE resource_id=?',(h.rid,)).fetchone()[0] + started=time.perf_counter() + out=self.native(agent,[dict(op='connect',invitation=registration),dict(op='receive')]);elapsed=(time.perf_counter()-started)*1000 + self.assertTrue(out['results'][0]['ok'],out) + received=out['results'][1];self.assertTrue(received['ok'],received) + repair=[c for c in out['calls'] if c['kind']=='requestRepair'];self.assertTrue(repair) + self.assertTrue(all(c['base']==registration['base_url'] for c in repair)) + work_after=h.db.execute('SELECT requests FROM open_repair_copy_work WHERE resource_id=?',(h.rid,)).fetchone()[0] + self.assertLessEqual(work_after,64) + evidence=dict(offered=1,completed=len(received['result']['messages']),elapsed_ms=elapsed, + repair_http_calls=len(repair),copy_work_before=work_before,copy_work_after=work_after, + copied_ciphertext_bytes=h.envelope_ref['size'] if received['result']['messages'] else 0,subprocess_calls=0) + self.cold_results.append(evidence) + if not received['result']['messages']: + # Upload and complete reads use this resource's original finite + # work grant. Additional demand must fail closed, rather + # than reset the durable count or silently expand permission. + self.assertEqual(received['result']['messages'],[],received) + self.assertEqual(work_after,64) + self.assertTrue(received['result']['errors'],received) + evidence['errors']=[v['code'] for v in received['result']['errors']] + continue + self.assertEqual(len(received['result']['messages']),1,received) + message=received['result']['messages'][0];self.assertEqual(message['share']['records_added'],1) + offline=self.native(agent,[dict(op='receive',message_id=message['message_id']),dict(op='recall',query='Synthetic mailbox memory')],no_network=True) + self.assertTrue(all(row['ok'] for row in offline['results']),offline) + self.assertGreaterEqual(sum(v['completed'] for v in self.cold_results),1,self.cold_results) + self.assertTrue(any(not v['completed'] for v in self.cold_results),self.cold_results) + + def inspect_received_registration(self,h,registration,configured): + f=self.delivery_fixture + message=json.loads(self.original_envelope)['payload']['context']['message_id'] + with f.b._network() as network: + delivery=network._delivery();row=delivery._inbox(message) + self.assertEqual(bytes(row['envelope']),self.original_envelope) + self.assertEqual(row['phase'],'saved');self.assertIsNotNone(row['receipt']) + # Native reopens the imported Memory and saved message with zero network. + read=self.native(f.b,[dict(op='receive',message_id=message), + dict(op='recall',query='Synthetic mailbox memory')],no_network=True)['results'] + self.assertTrue(all(value['ok'] for value in read),read) + self.assertTrue(any(hit['text']=='Synthetic mailbox memory: consult current evidence before reuse.' + for hit in read[1]['result']['hits'])) + if getattr(self,'native_recipient',False):self.cold_load(h,registration) + # Fixture's existing explicit ACK authority carries the endpoint receipt + # independently; original send identity/envelope/session stay unchanged. + registered_fixtures.RegisteredReplicaTests.inspect_received_registration(self,h,registration,configured) + # Removing a selection cannot erase the authenticated denial. A direct + # retry after client restart must stop before any repair proof HTTP. + from memory_vault_open_transport import OpenHTTPTransport + with patch.object(OpenHTTPTransport,'request_repair',side_effect=AssertionError('retained revoked authority attempted repair HTTP')): + refused=f.b.handle(dict(op='connect',invitation=dict(registration,action='receive_replica'))) + self.assertFalse(refused['ok'],refused) + self.assertEqual(refused['error']['code'],'repair_authority_revoked') + if getattr(self,'native_recipient',False): + denied=self.native(f.b,[dict(op='connect',invitation=dict(registration,action='receive_replica'))],no_network=True)['results'][0] + self.assertFalse(denied['ok'],denied) + self.assertEqual(denied['error']['code'],'repair_authority_revoked') + self.closure_evidence=dict(message_id=message,envelope_sha256=h.envelope_ref['raw_sha256'], + copied_ciphertext_bytes=h.envelope_ref['size'],recipient_saved=True,native_offline_recall=True, + retained_revocation_code=refused['error']['code'],revoked_repair_http_calls=0) + self.closure_evidence['recipient_runtime']='native' if getattr(self,'native_recipient',False) else 'python' + if hasattr(self,'cold_results'):self.closure_evidence['cold_known_message_load']=self.cold_results + + def test_native_send_python_replica_receive_native_offline_reuse_and_independent_receipt(self): + self.with_independent_return=True + feed_fixtures.MailboxFeedCopyTests.http_roundtrip(self,message=True,commands=True,agent=True,registered=True) + f=self.delivery_fixture + confirmed=self.native(f.a,[f.ack_original_send_request],no_network=True)['results'][0] + self.assertTrue(confirmed['ok'],confirmed) + self.assertTrue(confirmed['result']['endpoint_validated']) + self.assertEqual(confirmed['result']['state'],'validated_saved') + self.assertEqual(confirmed['result']['message_id'],self.closure_evidence['message_id']) + print(json.dumps(dict(synthetic_replica_closure=self.closure_evidence, + original_send_native_confirmed=True,frozen_send_replay_network_calls=0))) + + def test_native_recipient_verifies_full_replica_proof_after_original_source_loss(self): + self.native_recipient=True + self.test_native_send_python_replica_receive_native_offline_reuse_and_independent_receipt() + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index e4dc618..6ae32a4 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -47,7 +47,7 @@ "open-repair-resource.ts", "open-repair-bootstrap.ts", "open-repair-status.ts", "open-repair-mailbox-range.ts", "open-repair-mailbox-authority.ts", "open-repair-mailbox-member.ts", "open-repair-mailbox-feed.ts", "open-repair-mailbox-read.ts", "open-repair-mailbox-inbox.ts", "open-mailbox-client.ts", "open-mailbox-journal.ts", "open-mailbox-receivers.ts", "open-mailbox-receipts.ts", "open-mailbox-destination.ts", "open-mailbox-send.ts", - "open-repair-ack.ts", + "open-repair-mailbox-replica.ts", "open-mailbox-replica-client.ts", "open-repair-mailbox-replica-inbox.ts", "open-mailbox-replicas.ts", "open-repair-ack.ts", "open-repair-probe.ts", "open-repair-proof.ts", "open-repair-client.ts", "open-ack-client.ts", "open-ack-status.ts", "open-ack-discovery.ts", "open-repair-admin.ts", "open-repair-bound.ts", "open-repair-empty.ts", "open-repair-occupied.ts", "open-repair-offer-client.ts", "open-repair-receipt.ts")} TS_ENDPOINT_TESTS = {"tests/test_network_typescript_" + name + ".py" for name in @@ -102,7 +102,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): self.assertEqual(len(documents), len(set(documents))) self.assertEqual(len(review), len(set(review))) self.assertGreaterEqual(len(review), 39) - self.assertEqual(len(TS_NETWORK), 70) + self.assertEqual(len(TS_NETWORK), 74) self.assertTrue(RUNTIME_DATA <= set(documents)) self.assertTrue(TS_NETWORK <= set(documents)) self.assertTrue(TS_ENDPOINT_TESTS <= set(review)) @@ -189,7 +189,7 @@ def test_new_sdk_docs_and_review_fixtures_are_explicit_public_paths(self): "clients/typescript/network/open-repair-original.ts", "clients/typescript/network/open-repair-resource.ts", "clients/typescript/network/open-repair-bootstrap.ts", "clients/typescript/network/open-repair-status.ts", "clients/typescript/network/open-repair-mailbox-range.ts", "clients/typescript/network/open-repair-mailbox-authority.ts", "clients/typescript/network/open-repair-mailbox-member.ts", "clients/typescript/network/open-repair-mailbox-feed.ts", "clients/typescript/network/open-repair-mailbox-read.ts", "clients/typescript/network/open-repair-mailbox-inbox.ts", "clients/typescript/network/open-mailbox-client.ts", "clients/typescript/network/open-mailbox-journal.ts", "clients/typescript/network/open-mailbox-receivers.ts", "clients/typescript/network/open-mailbox-receipts.ts", "clients/typescript/network/open-mailbox-destination.ts", "clients/typescript/network/open-mailbox-send.ts", - "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", + "clients/typescript/network/open-repair-mailbox-replica.ts", "clients/typescript/network/open-mailbox-replica-client.ts", "clients/typescript/network/open-repair-mailbox-replica-inbox.ts", "clients/typescript/network/open-mailbox-replicas.ts", "clients/typescript/network/open-repair-ack.ts", "clients/typescript/network/open-capacity.ts", "clients/typescript/network/open-repair-probe.ts", "clients/typescript/network/open-repair-proof.ts", "clients/typescript/network/open-repair-client.ts", "clients/typescript/network/open-ack-client.ts", "clients/typescript/network/open-ack-status.ts", "clients/typescript/network/open-ack-discovery.ts", "clients/typescript/network/open-repair-admin.ts", diff --git a/tests/test_open_delivery_distribution.py b/tests/test_open_delivery_distribution.py new file mode 100644 index 0000000..ee22ba0 --- /dev/null +++ b/tests/test_open_delivery_distribution.py @@ -0,0 +1,214 @@ +"""Explicit independent grants distribute real encrypted delivery and receipts.""" +import asyncio +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sqlite3 +import tempfile +import time +from types import SimpleNamespace +import unittest + +from memory_vault import canonical_bytes +from memory_vault_client import ClientConfig +from memory_vault_open_contact_client import CONNECT_SCHEMA, OpenContactClient +from memory_vault_open_delivery_client import OpenDeliveryClient +from memory_vault_open_routing import LookupBudget +from memory_vault_storage import atomic_write +from memory_vault_trust import TrustStore +from tests.test_open_agent import configured_agent +from tests.test_open_node import HTTPNodes + + +class DeliveryDistributionFixture: + """A and B retain their own keys/Vaults; R0/R1 get distinct owner grants.""" + def __init__(self, directory, *, host_class=HTTPNodes, late_node=False): + self.root = Path(directory).resolve() + self.host = host_class(self.root, 2) + for i in range(2): + self.host.stop(i) + config=json.loads(self.host.configs[i].read_bytes()) + config.update(contact_policy={"enabled":True},delivery_policy={"enabled":True}) + atomic_write(self.host.configs[i],canonical_bytes(config),replace=True) + if not late_node or i == 0: + self.host.start(i) + self.a,self.ai,*_=configured_agent(SimpleNamespace(root=self.root/"a",nodes=self.host.nodes[:1])) + self.b,self.bi,*_=configured_agent(SimpleNamespace(root=self.root/"b",nodes=self.host.nodes[:1])) + TrustStore(ClientConfig.load(self.b.client_config).trust_path).add(self.ai.public_descriptor()) + self.approvals=[] + self.setup_attempts=[] + + def close(self): + self.host.close() + + @staticmethod + def call(agent, **request): + value=agent.handle(request) + if not value["ok"]: raise AssertionError(value) + return value["result"] + + def contact(self, agent, action, **values): + request_id = values.pop("request_id") if action == "request" else None + return self.call(agent,op="connect",**({"request_id":request_id} if request_id else {}), + invitation={"schema_version":CONNECT_SCHEMA,"action":action,**values}) + + def approve(self, index, *, items=4, bytes_limit=2*1024*1024): + enabled=self.contact(self.b,"enable",node=self.host.nodes[index], + allocation_id=f"req_synthetic_distribution_knock_{index}",max_pending=2,lease_seconds=3600,revision=index+1) + request_id=f"req_synthetic_distribution_contact_{index}" + # A newly started R may not have been challenged by its first neighbor + # yet. These explicit fixture setup attempts are finite and retained. + for attempt in range(6): + value=self.a.handle({"op":"connect","request_id":request_id,"invitation":{ + "schema_version":CONNECT_SCHEMA,"action":"request","recipient_key_id":self.bi.key_id}}) + self.setup_attempts.append({"node":index,"attempt":attempt+1,"ok":value["ok"], + "code":value.get("error",{}).get("code"),"time":time.monotonic()}) + if value["ok"]: break + if not value["error"]["retryable"] or attempt==5: raise AssertionError(value) + time.sleep(.5) + pending=self.contact(self.b,"poll",lease_id=enabled["lease_id"]) + reference=next(row["request_ref"] for row in pending["requests"] if row["request_id"]==request_id) + self.contact(self.b,"decide",request_ref=reference,decision="approved",max_items=items,max_bytes=bytes_limit) + self.contact(self.a,"result",request_id=request_id) + self.approvals.append(enabled) + return enabled + + def send(self, i, *, memory_ids=None): + return self.a.handle({"op":"send","request_id":f"req_synthetic_distribution_send_{i}", + "recipients":[self.bi.key_id],"text":f"Synthetic distributed payload {i}","memory_ids":memory_ids or []}) + + def stored(self): + result=[] + for i in range(2): + with sqlite3.connect(self.root/f"node_{i}"/"transport/network.sqlite3") as db: + result.append(db.execute("SELECT count(*) FROM open_delivery_messages").fetchone()[0]) + return result + + +class DeliveryDistributionTests(unittest.TestCase): + def setUp(self): + temporary=tempfile.TemporaryDirectory(prefix="synthetic-delivery-distribution-") + self.addCleanup(temporary.cleanup) + self.fixture=DeliveryDistributionFixture(temporary.name) + self.addCleanup(self.fixture.close) + + def test_late_independent_approval_carries_new_messages_and_saved_memory_receipts(self): + f=self.fixture;f.approve(0,items=2) + memory=f.call(f.a,op="remember",request_id="req_synthetic_distribution_memory",kind="observation", + text="Synthetic shared observation retained through approved delivery.") + first=f.send(0,memory_ids=[memory["memory_id"]]);self.assertTrue(first["ok"],first) + f.approve(1,items=2) + messages=[first]+[f.send(i,memory_ids=[memory["memory_id"]]) for i in range(1,4)] + self.assertTrue(all(x["ok"] for x in messages),messages) + self.assertEqual(f.stored(),[2,2]) + received=[] + for _ in range(4): + value=f.call(f.b,op="receive",limit=1) + self.assertEqual(value["errors"],[]) + received+=value["messages"] + self.assertEqual(len(received),4) + self.assertEqual({x["state"] for x in received},{"validated_saved"}) + location={} + for index in range(2): + with sqlite3.connect(f.root/f"node_{index}"/"transport/network.sqlite3") as db: + location.update((row[0],index) for row in db.execute("SELECT message_id FROM open_delivery_messages")) + self.assertNotEqual(location[received[0]["message_id"]],location[received[1]["message_id"]]) + for i,result in enumerate(messages): + ack=f.send(i,memory_ids=[memory["memory_id"]]) + self.assertTrue(ack["ok"],ack) + self.assertTrue(ack["result"]["endpoint_validated"]) + read=f.call(f.b,op="receive",message_id=result["result"]["message_id"],offset=0) + self.assertIn(f"payload {i}",read["text"]) + recalled=f.call(f.b,op="recall",memory_id=memory["memory_id"]) + self.assertIn("Synthetic shared observation",recalled["hits"][0]["text"]) + self.assertEqual(f.stored(),[2,2]) + extra=f.send(4) + self.assertFalse(extra["ok"]) + self.assertEqual(extra["error"]["code"],"open_delivery_approval_capacity") + self.assertEqual(f.stored(),[2,2]) + + def test_atomic_new_send_placement_preserves_each_grant_and_frozen_retry(self): + f=self.fixture;f.approve(0,items=2);f.approve(1,items=2) + with f.a._network() as network: + delivery=OpenDeliveryClient(network.participant,network.encryption,network.client_config) + sessions=delivery._contact_sessions(outgoing=True,recipient=f.bi.key_id) + rows=[] + for i in range(4): + request_id,text=f"synthetic_freeze_{i}",f"Synthetic frozen {i}" + input_sha,_=delivery._send_input(request_id,[f.bi.key_id],text,[],None) + rows.append(delivery._prepare_outbox(request_id,f.bi.key_id,input_sha,text,[])) + with ThreadPoolExecutor(max_workers=4) as pool: + frozen=list(pool.map(lambda row:delivery._freeze_dispatch(row,sessions),rows)) + counts={} + for row in frozen: + session=json.loads(row["session"]) + scope=delivery._session_scope(session);counts[scope]=counts.get(scope,0)+1 + retry=delivery._freeze_dispatch(row,list(reversed(sessions))) + self.assertEqual(retry["session"],row["session"]) + self.assertEqual(retry["envelope"],row["envelope"]) + self.assertEqual(sorted(counts.values()),[2,2]) + + def test_revoked_exact_policy_does_not_store_on_another_node_without_approval(self): + f=self.fixture;enabled=f.approve(0,items=2) + with f.b._network() as network: + contact=OpenContactClient(network.participant,network.encryption) + session=contact._load("policy",enabled["lease_id"]) + payload={**session["policy"]["payload"],"revision":2,"status":"revoked"} + revoked={"payload":payload,"proof":f.bi.sign_message(payload)} + asyncio.run(contact.call(f.host.nodes[0],"policy.put",{"lease":session["lease"],"policy":revoked},LookupBudget())) + denied=f.send(0) + self.assertFalse(denied["ok"]) + self.assertEqual(denied["error"]["code"],"open_delivery_revoked") + self.assertEqual(f.stored(),[0,0]) + + def test_duplicate_session_aliases_do_not_multiply_one_signed_resource(self): + f=self.fixture;f.approve(0,items=1) + with f.a._network() as network: + delivery=OpenDeliveryClient(network.participant,network.encryption,network.client_config) + session=delivery._contact_sessions(outgoing=True,recipient=f.bi.key_id)[0] + rows=[] + for i in range(2): + request_id,text=f"synthetic_alias_{i}",f"Synthetic alias {i}" + sha,_=delivery._send_input(request_id,[f.bi.key_id],text,[],None) + rows.append(delivery._prepare_outbox(request_id,f.bi.key_id,sha,text,[])) + delivery._freeze_dispatch(rows[0],[session,dict(session)]) + from memory_vault import MemoryError + with self.assertRaises(MemoryError) as caught: + delivery._freeze_dispatch(rows[1],[session,dict(session)]) + self.assertEqual(caught.exception.code,"open_delivery_approval_capacity") + + def test_known_old_node_revocation_preserves_frozen_send_and_independent_new_grant(self): + from memory_vault import MemoryError + from memory_vault_open_control import issue_node + f=self.fixture;f.approve(0,items=2) + first=f.send(0);self.assertTrue(first["ok"],first) + f.approve(1,items=2) + raw=f.host.nodes[0]["payload"];now=int(time.time()) + revoked=issue_node(f.host.identities[0],base_url=raw["base_url"],storage_epoch=raw["storage_epoch"], + roles=raw["roles"],revision=2,status="revoked",issued_at=now,expires_at=now+600) + with f.a._network() as network: + with self.assertRaises(MemoryError):network.participant._accept(revoked) + independent=f.send(1);self.assertTrue(independent["ok"],independent) + self.assertEqual(f.stored(),[1,1]) + retry=f.send(0);self.assertTrue(retry["ok"],retry) + self.assertEqual(retry["result"]["message_id"],first["result"]["message_id"]) + self.assertFalse(retry["result"]["endpoint_validated"]) + self.assertEqual(f.stored(),[1,1]) + + def test_one_nodes_approved_grant_cannot_be_retargeted_to_another_node(self): + from memory_vault_open_delivery import sign_rpc, verify_response + f=self.fixture;f.approve(0,items=1) + self.assertTrue(f.send(0)["ok"]) + with f.a._network() as network: + with network.participant.state.db() as db: + row=db.execute("SELECT intent FROM open_delivery_outbox WHERE request_id='req_synthetic_distribution_send_0'").fetchone() + request=sign_rpc(f.ai,node=f.host.nodes[0],action="prepare",body={"intent":json.loads(row[0])}) + request["payload"]["node_key_id"]=f.host.nodes[1]["payload"]["signing_key"]["key_id"] + request["payload"]["storage_epoch"]=f.host.nodes[1]["payload"]["storage_epoch"] + request["proof"]=f.ai.sign_message(request["payload"]) + reply=network.participant.transport.request(f.host.nodes[1]["payload"]["base_url"],request,deadline=time.monotonic()+4) + body=verify_response(reply.response,request=request,node=f.host.nodes[1])["body"] + self.assertIn("error",body) + self.assertEqual(body["error"]["code"],"contact_wrong_node") + self.assertNotIn("challenge",body) + self.assertEqual(f.stored(),[1,0]) diff --git a/tests/test_open_delivery_distribution_typescript.py b/tests/test_open_delivery_distribution_typescript.py new file mode 100644 index 0000000..94b53a0 --- /dev/null +++ b/tests/test_open_delivery_distribution_typescript.py @@ -0,0 +1,115 @@ +"""Native Agent and Python share finite grants, placement and receive cursors.""" +import asyncio +import json +import sqlite3 +import tempfile +import unittest + +from memory_vault_open_contact_client import OpenContactClient +from memory_vault_open_routing import LookupBudget +from tests.test_open_delivery_distribution import DeliveryDistributionFixture +from tests import test_open_ack_typescript_agent as native_ack +from tests import test_network_typescript_agent_network as native_runtime + + +class NativeDistributionTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + native_runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(native_ack.DRIVER) + + def setUp(self): + temporary=tempfile.TemporaryDirectory(prefix='synthetic-native-distribution-') + self.addCleanup(temporary.cleanup) + self.f=DeliveryDistributionFixture(temporary.name) + self.addCleanup(self.f.close) + + def native(self, agent, requests, **options): + return native_ack.NativeAckAgentTests.native(self,agent,requests,**options) + + def send(self, i, memory_ids=()): + return dict(op='send',request_id=f'req_synthetic_distribution_send_{i}', + recipients=[self.f.bi.key_id],text=f'Synthetic distributed payload {i}',memory_ids=list(memory_ids)) + + def test_python_native_alternate_sender_receiver_and_reopen_saved_receipts(self): + f=self.f;f.approve(0,items=2) + memory=f.call(f.a,op='remember',request_id='req_native_distributed_memory',kind='observation', + text='Synthetic cross-language resource reuse observation.')['memory_id'] + first=self.native(f.a,[self.send(0,[memory])])['results'][0];self.assertTrue(first['ok'],first) + f.approve(1,items=2) + sent=[first,f.send(1,memory_ids=[memory]),self.native(f.a,[self.send(2,[memory])])['results'][0],f.send(3,memory_ids=[memory])] + self.assertTrue(all(row['ok'] for row in sent),sent);self.assertEqual(f.stored(),[2,2]) + received=[] + for index in range(4): + value=(self.native(f.b,[dict(op='receive',limit=1)])['results'][0]['result'] + if index%2==0 else f.call(f.b,op='receive',limit=1)) + self.assertEqual(value['errors'],[]);self.assertEqual(len(value['messages']),1,value) + received+=value['messages'] + locations={} + for index in range(2): + with sqlite3.connect(f.root/f'node_{index}'/'transport/network.sqlite3') as db: + locations.update((row[0],index) for row in db.execute('SELECT message_id FROM open_delivery_messages')) + self.assertNotEqual(locations[received[0]['message_id']],locations[received[1]['message_id']]) + for index,row in enumerate(sent): + result=self.native(f.a,[self.send(index,[memory])])['results'][0] + self.assertTrue(result['ok'],result);self.assertTrue(result['result']['endpoint_validated']) + read=self.native(f.b,[dict(op='receive',message_id=row['result']['message_id']), + dict(op='recall',memory_id=memory)],no_network=True)['results'] + self.assertTrue(all(value['ok'] for value in read),read) + self.assertIn('Synthetic cross-language',read[1]['result']['hits'][0]['text']) + full=self.native(f.a,[self.send(4)])['results'][0] + self.assertFalse(full['ok']);self.assertEqual(full['error']['code'],'open_delivery_approval_capacity') + self.assertEqual(f.stored(),[2,2]) + + def test_native_obeys_live_policy_revocation_without_unapproved_retarget(self): + f=self.f;enabled=f.approve(0,items=2) + with f.b._network() as network: + contact=OpenContactClient(network.participant,network.encryption) + session=contact._load('policy',enabled['lease_id']) + payload={**session['policy']['payload'],'revision':2,'status':'revoked'} + revoked={'payload':payload,'proof':f.bi.sign_message(payload)} + asyncio.run(contact.call(f.host.nodes[0],'policy.put',{'lease':session['lease'],'policy':revoked},LookupBudget())) + refused=self.native(f.a,[self.send(0)])['results'][0] + self.assertFalse(refused['ok']);self.assertEqual(refused['error']['code'],'open_delivery_revoked') + self.assertEqual(f.stored(),[0,0]) + + def test_native_freeze_survives_source_failure_and_restarts_without_duplicate_store(self): + f=self.f;f.approve(0,items=2) + first=self.native(f.a,[self.send(0)])['results'][0];self.assertTrue(first['ok'],first) + with f.a._network() as network: + with network.participant.state.db() as db: + frozen=tuple(bytes(x) for x in db.execute('SELECT envelope,session,intent FROM open_delivery_outbox WHERE request_id=?',(self.send(0)['request_id'],)).fetchone()) + f.approve(1,items=2);f.host.stop(0) + retried=self.native(f.a,[self.send(0)])['results'][0] + self.assertTrue(retried['ok'],retried);self.assertFalse(retried['result']['endpoint_validated']) + self.assertEqual(retried['result']['message_id'],first['result']['message_id']) + f.host.start(0) + received=f.call(f.b,op='receive',limit=1);self.assertEqual(len(received['messages']),1,received) + confirmed=self.native(f.a,[self.send(0)])['results'][0] + self.assertTrue(confirmed['ok'],confirmed);self.assertTrue(confirmed['result']['endpoint_validated']) + with f.a._network() as network: + with network.participant.state.db() as db: + again=tuple(bytes(x) for x in db.execute('SELECT envelope,session,intent FROM open_delivery_outbox WHERE request_id=?',(self.send(0)['request_id'],)).fetchone()) + self.assertEqual(again,frozen);self.assertEqual(f.stored(),[1,0]) + + def test_native_new_send_uses_independent_grant_after_known_old_node_revocation(self): + import time + from memory_vault import MemoryError + from memory_vault_open_control import issue_node + f=self.f;f.approve(0,items=2) + first=self.native(f.a,[self.send(0)])['results'][0];self.assertTrue(first['ok'],first) + f.approve(1,items=2) + p=f.host.nodes[0]['payload'];now=int(time.time()) + revoked=issue_node(f.host.identities[0],base_url=p['base_url'],storage_epoch=p['storage_epoch'], + roles=p['roles'],revision=2,status='revoked',issued_at=now,expires_at=now+600) + with f.a._network() as network: + with self.assertRaises(MemoryError):network.participant._accept(revoked) + next_send=self.native(f.a,[self.send(1)])['results'][0] + self.assertTrue(next_send['ok'],next_send);self.assertEqual(f.stored(),[1,1]) + old=self.native(f.a,[self.send(0)])['results'][0] + self.assertTrue(old['ok'],old);self.assertFalse(old['result']['endpoint_validated']) + self.assertEqual(old['result']['message_id'],first['result']['message_id']) + self.assertEqual(f.stored(),[1,1]) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_open_mailbox_replica_typescript.py b/tests/test_open_mailbox_replica_typescript.py new file mode 100644 index 0000000..ad176e4 --- /dev/null +++ b/tests/test_open_mailbox_replica_typescript.py @@ -0,0 +1,54 @@ +"""Native reconstruction of exact historical COPY, without Python delegation.""" +import json +import subprocess +import unittest +from tests import test_network_typescript_agent_network as runtime +from tests import test_open_repair_mailbox_message_copy as fixtures + +DRIVER=runtime.DRIVER.split('const {Agent}',1)[0]+r''' +import {RepairBudget,LocalRawResolver} from './open-repair-wire.ts'; +import {DEFAULT_REPAIR_CLIENT_POLICY} from './open-repair-client.ts'; +import {verifyMailboxMessageReplica} from './open-repair-mailbox-replica.ts'; +const parts=[];let size=0;for await(const p of process.stdin){size+=p.length;if(size>1048576)throw Error('finite fixture input');parts.push(p);} +const v=JSON.parse(Buffer.concat(parts)),decode=e=>({ref:e.ref,raw:Buffer.from(e.raw_base64url,'base64url')}),results=[]; +for(const change of v.contexts){try{const policy={...DEFAULT_REPAIR_CLIENT_POLICY,max_signature_checks:512},budget=new RepairBudget(policy),resolver=new LocalRawResolver(policy,budget); + for(const item of v.originals){const e=decode(item);resolver.put(e.ref.namespace,e.ref.key,e.raw);} + const result=verifyMailboxMessageReplica(decode(v.manifest),resolver,decode(v.custody),{...change,policy,budget}); + results.push({ok:true,envelope:result.source.message_scope.envelope_ref,originals:result.entries.size,read_until:result.authority.read_until}); +}catch(error){results.push({ok:false,code:error.code??error.name});}} +process.stdout.write(JSON.stringify({results,subprocessCalls})); +''' + +class NativeMessageReplicaProofTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runtime.TypeScriptAgentNetworkTests.setUpClass.__func__(cls) + (cls.fixture/'driver.mjs').write_text(DRIVER) + + def run_fixture(self,inspect): + return fixtures.MailboxMessageCopyTests.run_fixture(self,inspect) + + def test_full_original_closure_and_independent_expected_bindings(self): + def inspect(h): + custody=h.commit();replica=h.store.restore_message(h.rid,**h.context()) + manifest_ref=json.loads(custody['raw'])['payload']['replica_manifest_ref'] + manifest=h.wrap(h.store.read_local_original(h.rid,manifest_ref)) + originals={json.dumps(e['ref'],sort_keys=True):e for values in replica['entries'].values() for e in values} + context=dict(expectedSlot=h.slot,expectedOwner=h.owner,expectedSender=h.sender,expectedSource=h.source.target, + sourceStorageEpoch=h.slot['writer_storage_epoch'],expectedMaintainer=h.source.target,expectedTarget=h.target, + targetStorageEpoch=h.node['payload']['storage_epoch'],expectedEnvelopeRef=h.envelope_ref,limitPolicy=h.limits) + changes=[context] + for name,value in (('sourceStorageEpoch','synthetic_replaced_source'),('targetStorageEpoch','synthetic_replaced_target'), + ('expectedSource',h.target),('expectedSender',h.owner),('expectedMaintainer',h.owner)): + changes.append(dict(context,**{name:value})) + bad=json.loads(json.dumps(context));bad['expectedEnvelopeRef']['raw_sha256']='f'*64;changes.append(bad) + result=subprocess.run([self.node,'--experimental-strip-types',str(self.fixture/'driver.mjs')],cwd=self.fixture, + input=json.dumps(dict(manifest=h.encode(manifest),custody=h.encode(custody),originals=[h.encode(e) for e in originals.values()],contexts=changes)).encode(), + stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30) + self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace')[-2000:]);out=json.loads(result.stdout) + self.assertEqual(out['subprocessCalls'],0);self.assertTrue(out['results'][0]['ok'],out) + self.assertEqual(out['results'][0]['envelope'],h.envelope_ref) + self.assertTrue(all(not row['ok'] for row in out['results'][1:]),out) + self.run_fixture(inspect) + +if __name__=='__main__':unittest.main() diff --git a/tests/test_open_neighbors.py b/tests/test_open_neighbors.py new file mode 100644 index 0000000..96cf539 --- /dev/null +++ b/tests/test_open_neighbors.py @@ -0,0 +1,130 @@ +"""Bounded synthetic routing hints; exact signatures and checkpoints stay live.""" +import asyncio +import copy +import tempfile +import time +import unittest +from pathlib import Path +from types import SimpleNamespace + +from memory_vault import MemoryError, canonical_bytes +from memory_vault_open_control import coordinate, issue_node, sign_response, verify_request +from memory_vault_open_node import OpenParticipant +from memory_vault_open_routing import LookupBudget +from memory_vault_trust import Identity + + +class NeighborTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.identities = [Identity.generate(Path(self.tmp.name).resolve() / str(i) / "identity.json") for i in range(4)] + now = int(time.time()) + self.nodes = [issue_node(identity, base_url=f"https://synthetic-{i}.invalid", + storage_epoch=f"synthetic_epoch_{i}", roles=["directory", "router"], + revision=1, issued_at=now, expires_at=now+600) for i, identity in enumerate(self.identities[:3])] + self.client = OpenParticipant(self.identities[3], Path(self.tmp.name).resolve()/"client", seeds=[self.nodes[0]]) + self.addCleanup(self.client.close) + self.client.transport.close() + self.offline = set() + self.bad_proofs = set() + self.calls = [] + self.reply_nodes = None + self.client.transport = SimpleNamespace(request=self.request, close=lambda: None) + self.target = coordinate(self.identities[3].key_id) + + def request(self, url, request, *, deadline): + index = next(i for i, node in enumerate(self.nodes) if node["payload"]["base_url"] == url) + self.calls.append((index, request["payload"]["action"])) + if index in self.offline: + raise OSError("synthetic offline") + checked = verify_request(request, node=self.nodes[index]) + if checked["action"] == "hello": + body = {"node": self.nodes[index]} + else: + body = {"nodes": self.reply_nodes if self.reply_nodes is not None else [self.nodes[(index+1) % 3]]} + response = sign_response(self.identities[index], request=request, node=self.nodes[index], body=body, + issued_at=int(time.time()), expires_at=int(time.time())+60) + if index in self.bad_proofs: + response = copy.deepcopy(response) + response["payload"]["expires_at"] -= 1 + return SimpleNamespace(response=response, observed_address=f"10.{index}.0.1", wire_bytes=len(canonical_bytes(response))) + + def step(self, **kwargs): + return asyncio.run(self.client.find_neighbors(self.target, **kwargs)) + + def test_new_neighbor_is_challenged_then_used_without_injecting_members(self): + first = self.step(view="directory") + self.assertEqual(first["state"], "introduced") + self.assertFalse(first["introductions_verified"]) + self.assertFalse(first["discovery_grants_access"]) + self.assertEqual(self.calls, [(0, "find"), (1, "hello")]) + self.assertTrue(self.client.table.has_verified(self.nodes[1])) + before = len(self.calls) + for _ in range(6): + self.step() + reads = [index for index, action in self.calls[before:] if action == "find"] + self.assertEqual(reads.count(0), 3) + self.assertEqual(reads.count(1), 3) + self.assertNotIn((2, "find"), self.calls) + self.client._neighbor_probe_after = 0 # advance the bounded probe gate + for _ in range(2): + self.step() + self.assertIn((2, "hello"), self.calls) + for _ in range(3): + self.step() + self.assertIn((2, "find"), self.calls) + + def test_bad_introduction_proof_never_enters_dispatch_pool(self): + self.bad_proofs.add(1) + result = self.step() + self.assertEqual(result["state"], "introduced") + self.assertTrue(result["errors"]) + self.assertFalse(self.client.table.has_verified(self.nodes[1])) + for _ in range(4): + self.step() + self.assertNotIn((1, "find"), self.calls) + + def test_offline_peer_fails_over_but_observed_revocation_is_not_usable(self): + self.step() + self.offline.add(0) + for _ in range(3): + self.assertEqual(self.step()["state"], "introduced") + now = int(time.time()) + revoked = issue_node(self.identities[1], base_url=self.nodes[1]["payload"]["base_url"], + storage_epoch="synthetic_epoch_1", roles=["directory", "router"], revision=2, + status="revoked", issued_at=now, expires_at=now+600) + with self.assertRaises(MemoryError): + self.client._accept(revoked) + self.assertEqual(self.step()["state"], "unreachable") + + def test_shared_budget_bounds_probe_and_invalid_target_sends_nothing(self): + result = self.step(budget=LookupBudget(maximum_requests=1)) + self.assertEqual(result["state"], "introduced") + self.assertEqual(result["metrics"]["requests"], 1) + self.assertEqual(self.calls, [(0, "find")]) + self.assertFalse(self.client.table.has_verified(self.nodes[1])) + before = len(self.calls) + with self.assertRaises(MemoryError): + asyncio.run(self.client.find_neighbors("not-a-digest")) + with self.assertRaises(MemoryError): + self.step(view="private") + self.assertEqual(len(self.calls), before) + + def test_failed_old_peer_does_not_starve_a_new_introduction(self): + self.step() + self.client.table.mark_failed(self.nodes[1]["payload"]["signing_key"]["key_id"]) + self.reply_nodes = [self.nodes[1], self.nodes[2]] + self.client._neighbor_probe_after = 0 + self.step() + self.assertIn((2, "hello"), self.calls) + self.assertTrue(self.client.table.has_verified(self.nodes[2])) + + def test_empty_reply_does_not_delay_the_first_actual_new_peer_probe(self): + self.reply_nodes=[] + self.step() + self.assertEqual(self.client._neighbor_probe_after,0) + self.reply_nodes=[self.nodes[1]] + self.step() + self.assertIn((1,"hello"),self.calls) + self.assertGreater(self.client._neighbor_probe_after,0) diff --git a/tests/test_open_read_concurrency.py b/tests/test_open_read_concurrency.py new file mode 100644 index 0000000..6581551 --- /dev/null +++ b/tests/test_open_read_concurrency.py @@ -0,0 +1,144 @@ +"""Read admission under real WAL writers and bounded real socket overload.""" +import socket +import sqlite3 +import tempfile +import threading +import time +import unittest +from pathlib import Path + +from memory_vault import MemoryError +from memory_vault_open_node import OpenHTTPServer, _TransportState +from memory_vault_trust import Identity +from tests import test_open_index as index_fixture + + +class OpenReadConcurrencyTests(unittest.TestCase): + def fixture(self): + fixture = index_fixture.OpenIndexTests() + fixture.setUp() + self.addCleanup(fixture.doCleanups) + fixture.db.execute("PRAGMA journal_mode=WAL") + fixture.db.execute("PRAGMA busy_timeout=10") + fixture.put() + writer = sqlite3.connect(fixture.db_path, timeout=.01) + self.addCleanup(writer.close) + return fixture, writer + + def test_reader_observes_committed_revocation_without_waiting_for_uncommitted_writer(self): + fixture, writer = self.fixture() + writer.execute("BEGIN IMMEDIATE") + writer.execute("UPDATE open_contact_floors SET status='revoked'") + self.assertEqual(fixture.get()["state"], "found") + writer.commit() + self.assertEqual(fixture.get(), {"state": "revoked"}) + + def test_expired_rows_are_invisible_without_mutation_and_reclaimed_on_next_write(self): + fixture, writer = self.fixture() + writer.execute("UPDATE open_contacts SET expires_at=?", (fixture.now,)) + writer.execute("UPDATE open_contact_floors SET retain_until=?", (fixture.now,)) + writer.execute("UPDATE open_index_replay SET retain_until=?", (fixture.now,)) + writer.commit() + changes = fixture.db.total_changes + self.assertEqual(fixture.get(), {"state": "not_found"}) + self.assertEqual(fixture.db.total_changes, changes) + self.assertEqual(fixture.db.execute("SELECT count(*) FROM open_contact_floors").fetchone()[0], 1) + fixture.put(request_id="synthetic_after_expiry") + self.assertEqual(fixture.get()["state"], "found") + self.assertEqual(fixture.db.execute("SELECT count(*) FROM open_index_replay").fetchone()[0], 1) + + def test_transport_open_remains_readable_under_writer_and_checks_binding_every_time(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + identity = Identity.generate(root/"synthetic_identity.json") + state = _TransportState(root/"transport", identity, "synthetic_epoch") + with state.db(): + pass + writer = sqlite3.connect(root/"transport"/"network.sqlite3") + try: + writer.execute("BEGIN IMMEDIATE") + writer.execute("INSERT INTO state VALUES('synthetic_writer','uncommitted')") + with state.db() as reader: + self.assertIsNone(reader.execute("SELECT value FROM state WHERE key='synthetic_writer'").fetchone()) + writer.rollback() + writer.execute("UPDATE state SET value='{}' WHERE key='configuration_binding'") + writer.commit() + with self.assertRaisesRegex(MemoryError, "network_state_configuration_mismatch"): + with state.db(): + pass + finally: + writer.close() + + +class OpenHTTPAdmissionTests(unittest.TestCase): + def test_occupied_listener_preserves_bind_error_without_starting_workers(self): + with socket.socket() as reservation: + reservation.bind(("127.0.0.1",0)) + reservation.listen() + with self.assertRaises(OSError): + OpenHTTPServer(reservation.getsockname(),None) + + def start(self, participant): + server = OpenHTTPServer(("127.0.0.1", 0), participant) + worker = threading.Thread(target=server.serve_forever, kwargs={"poll_interval":.01}, daemon=True) + worker.start() + def cleanup(): + server.shutdown();server.server_close();worker.join(1) + self.addCleanup(cleanup) + return server + + def connect(self, server, request=b"GET /open/v1/node HTTP/1.0\r\n\r\n"): + connection=socket.create_connection(server.server_address,timeout=4) + self.addCleanup(connection.close) + connection.sendall(request) + return connection + + def wait(self, check): + deadline=time.monotonic()+2 + while time.monotonic() Date: Fri, 2 Oct 2026 22:04:05 +0800 Subject: [PATCH 22/24] Bound complete runtime inventories independently from source modules --- plugins/memory-vault-client/scripts/launcher.py | 5 ++++- tests/test_network_packaging.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/memory-vault-client/scripts/launcher.py b/plugins/memory-vault-client/scripts/launcher.py index 90ef8fa..1277095 100644 --- a/plugins/memory-vault-client/scripts/launcher.py +++ b/plugins/memory-vault-client/scripts/launcher.py @@ -98,6 +98,9 @@ } REQUIRED_MODULES = ALLOWED_MODULES MAX_MODULE_BYTES = 1024 * 1024 +# The closed 153-module inventory exceeds 16 KiB with ordinary indentation. +# Bound metadata independently; module names, file set and every hash stay strict. +MAX_INVENTORY_BYTES = 32 * 1024 def _regular(path: Path) -> bool: @@ -140,7 +143,7 @@ def main() -> int: try: if not _regular(Path(__file__).absolute()): raise ValueError("runtime_not_built") - inventory = json.loads(_read(manifest, 16 * 1024).decode("utf-8"), object_pairs_hook=_unique) + inventory = json.loads(_read(manifest, MAX_INVENTORY_BYTES).decode("utf-8"), object_pairs_hook=_unique) if not isinstance(inventory, dict) or set(inventory) != {"schema_version", "modules"} or inventory["schema_version"] != "memory-vault-client-runtime/v1": raise ValueError("invalid_runtime_inventory") modules = inventory["modules"] diff --git a/tests/test_network_packaging.py b/tests/test_network_packaging.py index 6ae32a4..6ea5887 100644 --- a/tests/test_network_packaging.py +++ b/tests/test_network_packaging.py @@ -340,7 +340,10 @@ def test_isolated_source_runtime_launcher_and_strict_inventory(self): data = (ROOT / name).read_bytes() (runtime / name).write_bytes(data) hashes[name] = hashlib.sha256(data).hexdigest() - (runtime / "MANIFEST.json").write_text(json.dumps({"schema_version": "memory-vault-client-runtime/v1", "modules": hashes})) + manifest = runtime / "MANIFEST.json" + manifest.write_text(json.dumps({"schema_version": "memory-vault-client-runtime/v1", "modules": hashes}, indent=2)) + inventory_bytes = manifest.read_bytes() + self.assertGreater(len(inventory_bytes), 16 * 1024) config = root / "must-not-be-created.json" command = [sys.executable, "-I", "-S", "-B", str(launcher), "--config", str(config), "agent", "request"] def launch(): @@ -353,6 +356,11 @@ def launch(): self.assertTrue(answer["ok"], answer) self.assertFalse(answer["result"]["network_accessed"]) self.assertFalse(config.exists()) + # The full ordinary builder inventory must launch, while an + # oversized metadata document still fails before application code. + manifest.write_bytes(inventory_bytes + b" " * (32 * 1024 + 1 - len(inventory_bytes))) + self.assertEqual(launch().returncode, 1) + manifest.write_bytes(inventory_bytes) # The packaged client's new recovery command must be reachable # without reading a config or installing network/server extras. help_result = subprocess.run(command[:-2] + ["network-recovery", "--help"], From 1f4aea4ed4eca5e8df16558f4905b8eb08a5e635 Mon Sep 17 00:00:00 2001 From: smartyue Date: Fri, 2 Oct 2026 22:29:42 +0800 Subject: [PATCH 23/24] Keep synthetic neighbor replies within their request deadline --- tests/test_open_neighbors.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_open_neighbors.py b/tests/test_open_neighbors.py index 96cf539..bf82170 100644 --- a/tests/test_open_neighbors.py +++ b/tests/test_open_neighbors.py @@ -6,9 +6,10 @@ import unittest from pathlib import Path from types import SimpleNamespace +from unittest.mock import patch from memory_vault import MemoryError, canonical_bytes -from memory_vault_open_control import coordinate, issue_node, sign_response, verify_request +from memory_vault_open_control import coordinate, issue_node, sign_response, verify_request, verify_response from memory_vault_open_node import OpenParticipant from memory_vault_open_routing import LookupBudget from memory_vault_trust import Identity @@ -43,8 +44,9 @@ def request(self, url, request, *, deadline): body = {"node": self.nodes[index]} else: body = {"nodes": self.reply_nodes if self.reply_nodes is not None else [self.nodes[(index+1) % 3]]} + now = int(time.time()) response = sign_response(self.identities[index], request=request, node=self.nodes[index], body=body, - issued_at=int(time.time()), expires_at=int(time.time())+60) + issued_at=now, expires_at=min(now+60, checked["expires_at"])) if index in self.bad_proofs: response = copy.deepcopy(response) response["payload"]["expires_at"] -= 1 @@ -53,6 +55,16 @@ def request(self, url, request, *, deadline): def step(self, **kwargs): return asyncio.run(self.client.find_neighbors(self.target, **kwargs)) + def test_response_after_second_boundary_retains_original_request_deadline(self): + request = self.client._request(self.nodes[0], "find", {"target": self.target, "view": "general"}) + now = request["payload"]["issued_at"] + 1 + # Only the synthetic responder clock moves; protocol verification stays live. + with patch("tests.test_open_neighbors.time", SimpleNamespace(time=lambda: now)): + reply = self.request(self.nodes[0]["payload"]["base_url"], request, deadline=float("inf")) + response = verify_response(reply.response, request=request, node=self.nodes[0], now=now) + self.assertEqual(response["issued_at"], now) + self.assertEqual(response["expires_at"], request["payload"]["expires_at"]) + def test_new_neighbor_is_challenged_then_used_without_injecting_members(self): first = self.step(view="directory") self.assertEqual(first["state"], "introduced") From 36562e894da83855708ec4c99a68a7b726dd6737 Mon Sep 17 00:00:00 2001 From: qh-work <309895166+qh-work@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:05:37 +0800 Subject: [PATCH 24/24] Align packaged alpha40 notes with shipped capabilities and limits --- packaging/CLIENT_README.md | 25 +++++++++++++++++++++---- packaging/PROTOCOL_README.md | 25 +++++++++++++++++++++---- packaging/RELEASE_NOTES.md | 27 ++++++++++++++++++++++----- packaging/REVIEW_README.md | 23 ++++++++++++++++++++--- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/packaging/CLIENT_README.md b/packaging/CLIENT_README.md index b110254..0a32271 100644 --- a/packaging/CLIENT_README.md +++ b/packaging/CLIENT_README.md @@ -1,4 +1,21 @@ -# Memory Vault v0.28.0-alpha.0.39 — authorized full client +# Memory Vault v0.28.0-alpha.0.40 — authorized full client + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. Native TypeScript senders can prepare and retain an existing outgoing message in the recipient-authorized mailbox through Agent `connect`. The sender keeps @@ -10,7 +27,7 @@ this does not claim the recipient saved the message or acknowledged it. An optional `ack_request_id` selects an already prepared independent receipt permission. Complete signed controls and statuses travel with the message, including bounded compressed transport for the larger draft. No new permission -or storage allocation is inferred. Native source provisioning and replica inbox +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox recovery still use Python. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. @@ -41,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory @@ -180,7 +197,7 @@ remain unfinished. There is no project-operated public seed or verified thousand-agent/global-availability result. Use the source-bound release record for actual validation and compare archive bytes with published checksums. -This full-client package targets **v0.28.0-alpha.0.39 open-delivery source**, +This full-client package targets **v0.28.0-alpha.0.40 open-delivery source**, not a stable-release or complete runtime-certification claim. Existing published versions remain immutable. Match the artifact's source and hashes to its manifest; this README does not establish installation or publication. The plugin is under diff --git a/packaging/PROTOCOL_README.md b/packaging/PROTOCOL_README.md index b9d7953..2968508 100644 --- a/packaging/PROTOCOL_README.md +++ b/packaging/PROTOCOL_README.md @@ -1,4 +1,21 @@ -# Memory Vault v0.28.0-alpha.0.39 — independent protocol +# Memory Vault v0.28.0-alpha.0.40 — independent protocol + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. Native TypeScript senders can prepare and retain an existing outgoing message in the recipient-authorized mailbox through Agent `connect`. The sender keeps @@ -10,7 +27,7 @@ this does not claim the recipient saved the message or acknowledged it. An optional `ack_request_id` selects an already prepared independent receipt permission. Complete signed controls and statuses travel with the message, including bounded compressed transport for the larger draft. No new permission -or storage allocation is inferred. Native source provisioning and replica inbox +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox recovery still use Python. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. @@ -41,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory @@ -197,7 +214,7 @@ require independently configured providers; reading metadata cannot grant authority or enroll keys. The complete Python client and executable synthetic review kit are separate -artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.39; +artifacts described in `docs/RELEASE.md`. This package targets v0.28.0-alpha.0.40; previous published versions remain immutable. The optional native network adds communication around existing records without changing canonical record/v1 or share-v1. It has no MCP, A2A, Matrix, Nostr or Graphiti adapter or compatibility diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index 86301ad..d5aa5cb 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -1,4 +1,21 @@ -# Memory Vault v0.28.0-alpha.0.39 — native sender mailbox admission +# Memory Vault v0.28.0-alpha.0.40 — bounded delivery distribution and native message replicas + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. Native TypeScript senders can prepare and retain an existing outgoing message in the recipient-authorized mailbox through Agent `connect`. The sender keeps @@ -10,7 +27,7 @@ this does not claim the recipient saved the message or acknowledged it. An optional `ack_request_id` selects an already prepared independent receipt permission. Complete signed controls and statuses travel with the message, including bounded compressed transport for the larger draft. No new permission -or storage allocation is inferred. Native source provisioning and replica inbox +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox recovery still use Python. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. @@ -41,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory @@ -206,7 +223,7 @@ TypeScript recognizes the mailbox replica proof and status profiles. Participant operate their own authorized nodes; no central authority or project-operated public seed is required or provided. -Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.39/docs/OPEN_NETWORK_QUICKSTART.md) -and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.39/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). +Use the [quickstart](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.40/docs/OPEN_NETWORK_QUICKSTART.md) +and [mailbox replica guide](https://github.com/qh-work/memory-vault-sync/blob/v0.28.0-alpha.0.40/docs/OPEN_ACK_RECOVERY.md#receive-messages-and-shared-memories-from-a-replica-development-after-alpha025). Preserve existing identity and state files when installing. The archives contain implementation, public documentation and wholly synthetic fixtures only. diff --git a/packaging/REVIEW_README.md b/packaging/REVIEW_README.md index 7418368..431cdbb 100644 --- a/packaging/REVIEW_README.md +++ b/packaging/REVIEW_README.md @@ -1,4 +1,21 @@ -# Memory Vault v0.28.0-alpha.0.39 independent review kit +# Memory Vault v0.28.0-alpha.0.40 independent review kit + +Alpha.0.40 distributes new messages across separately approved item and byte +resources in Python and native TypeScript. Existing frozen ciphertext is never +retargeted. Receive polling rotates across approved sessions; bounded HTTP +admission and concurrent SQLite reads preserve the existing execution limits. + +Native recipients can recover an explicitly selected message replica after its +original source disappears, with complete historical COPY and current READ +verification, retained revocation and independent receipt confirmation. +Source provisioning and native ACK/root/feed replica reception still use Python. + +In the synthetic same-host workload, 48 selected Memories offered at 2/second +complete 39 to 48 chains; each approved node completes 24. This measures use of +authorized capacity, not maximum throughput. Cold replica recovery remains +about 17.9 seconds and the existing 64-work allowance remains finite. Independent +hosts, WAN capacity and automatic universal repair are unverified. See +`docs/CAPACITY_HANDOFF.md` for the workload and explicit limits. Native TypeScript senders can prepare and retain an existing outgoing message in the recipient-authorized mailbox through Agent `connect`. The sender keeps @@ -10,7 +27,7 @@ this does not claim the recipient saved the message or acknowledged it. An optional `ack_request_id` selects an already prepared independent receipt permission. Complete signed controls and statuses travel with the message, including bounded compressed transport for the larger draft. No new permission -or storage allocation is inferred. Native source provisioning and replica inbox +or storage allocation is inferred. Native source provisioning and ACK/root/feed replica inbox recovery still use Python. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md` for the existing invitations. @@ -41,7 +58,7 @@ inbox. Retained revocations survive restart. Python and TypeScript share receive configuration, status history and saved inbox evidence; either can resume an interrupted import. The native path performs its own HTTP and cryptography. Saved receipts remain available through independently authorized return requests. -Native provisioning and replica inbox recovery remain unfinished. See +Native provisioning and ACK/root/feed replica inbox recovery remain unfinished. See `clients/typescript/network/README.md` and `docs/OPEN_NETWORK_CONTACT.md`. Native TypeScript recipients can return an actually saved message or memory