diff --git a/crates/jcode-tui/src/tui/session_picker/cloud_preview/projection_tests.rs b/crates/jcode-tui/src/tui/session_picker/cloud_preview/projection_tests.rs index bc046ba405a..b2411445fc4 100644 --- a/crates/jcode-tui/src/tui/session_picker/cloud_preview/projection_tests.rs +++ b/crates/jcode-tui/src/tui/session_picker/cloud_preview/projection_tests.rs @@ -320,3 +320,316 @@ fn remote_projection_ignores_stale_sidecar_and_respects_away_lease() { let result: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); assert!(result["sessions"].as_array().unwrap().is_empty()); } + +#[test] +fn remote_journal_dedupe_matches_rust_checkpoint_replay() { + use serde_json::{Value, json}; + fn message(id: usize) -> Value { + json!({"id":id.to_string(),"role":"user","content":[{"type":"text","text":id.to_string()}]}) + } + fn entry(snapshot: &Value, ids: &[usize], title: &str, stamp: &str) -> String { + let mut meta = snapshot.clone(); + meta["title"] = json!(title); + meta["model"] = json!(title); + meta["working_dir"] = json!(format!("/{title}")); + meta["updated_at"] = json!(stamp); + // Match the canonical struct field order used by Rust glued salvage. + format!( + "{{\"meta\":{},\"append_messages\":{}}}", + meta, + serde_json::to_string(&ids.iter().copied().map(message).collect::>()).unwrap() + ) + } + let id = "dedupe_parity"; + let session = Session::create_with_id(id.into(), None, Some("snapshot-new".into())); + let mut snapshot = serde_json::to_value(session).unwrap(); + snapshot["updated_at"] = json!("2026-01-01T00:00:10.123456789Z"); + snapshot["model"] = json!("snapshot-new"); + snapshot["working_dir"] = json!("/snapshot-new"); + snapshot["messages"] = json!((1..=100).map(message).collect::>()); + let old = "2026-01-01T00:00:01Z"; + let fresh = "2026-01-01T00:00:11Z"; + let stale = entry(&snapshot, &(1..=50).collect::>(), "old", old); + let overlap = entry( + &snapshot, + &[95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 101, 105], + "fresh", + fresh, + ); + let metadata = entry(&snapshot, &[], "metadata", "2026-01-01T00:00:10.123456790Z"); + let cases = [ + ("stale", format!("{stale}\n"), "snapshot-new", 100), + ( + "mixed-overlap", + format!("{stale}\n{overlap}\n"), + "fresh", + 105, + ), + ( + "stale-ids-new-time", + entry(&snapshot, &[1, 2], "old", "2099-01-01T00:00:00Z"), + "snapshot-new", + 100, + ), + ( + "metadata-equal-offset", + entry(&snapshot, &[], "old", "2026-01-01T01:00:10.123456789+01:00"), + "snapshot-new", + 100, + ), + ( + "metadata-nanosecond", + format!("{stale}\n{metadata}\n"), + "metadata", + 100, + ), + ( + "metadata-after-fresh", + format!("{overlap}\n{}\n", entry(&snapshot, &[], "after", old)), + "after", + 105, + ), + ( + "glued-torn-prefix", + format!("{{\"meta\":{{{overlap}\n"), + "fresh", + 105, + ), + ( + "malformed-and-torn-tail", + format!("{stale}\nnot-json\n{overlap}\n{{\"meta\":"), + "fresh", + 105, + ), + ]; + let mut cases: Vec<_> = cases + .into_iter() + .map(|(name, journal, title, count)| (name, journal, title, count, 100)) + .collect(); + let new = entry(&snapshot, &(101..=110).collect::>(), "fresh", fresh); + cases.extend([ + ("no-journal", String::new(), "snapshot-new", 100, 100), + ("empty-no-journal", String::new(), "snapshot-new", 0, 0), + ( + "empty-old-metadata", + entry(&snapshot, &[], "old", old), + "snapshot-new", + 0, + 0, + ), + ("empty-new-metadata", metadata.clone(), "metadata", 0, 0), + ( + "empty-unseen-ids-old-time", + entry(&snapshot, &[1, 2], "unseen", old), + "unseen", + 2, + 0, + ), + ( + "new-after-stale", + format!("{stale}\n{new}\n"), + "fresh", + 110, + 100, + ), + ( + "subset-noncontiguous", + entry(&snapshot, &[99, 1, 50], "old", fresh), + "snapshot-new", + 100, + 100, + ), + ( + "duplicates-after-fresh", + format!("{overlap}\n{}\n", entry(&snapshot, &[1, 101], "after", old)), + "after", + 105, + 100, + ), + ( + "salvaged-stale-prefix", + format!("{{\"meta\":{{{stale}{new}\n"), + "fresh", + 110, + 100, + ), + ]); + for (case, journal, title, count, initial_count) in cases { + let home = tempfile::tempdir().unwrap(); + let sessions = home.path().join("sessions"); + std::fs::create_dir(&sessions).unwrap(); + let path = sessions.join(format!("{id}.json")); + let journal_path = path.with_extension("journal.jsonl"); + let mut snapshot = snapshot.clone(); + if initial_count == 0 { + snapshot["messages"] = json!([]); + } + let bytes = serde_json::to_vec(&snapshot).unwrap(); + std::fs::write(&path, &bytes).unwrap(); + if !journal.is_empty() { + std::fs::write(&journal_path, journal.as_bytes()).unwrap(); + } + let rust = Session::load_from_path(&path).unwrap(); + assert_eq!(rust.messages.len(), count, "Rust oracle must replay {case}"); + assert_eq!( + rust.title.as_deref(), + Some(title), + "Rust oracle metadata {case}" + ); + let (projected, _) = + parse_preview(&run_projection(home.path(), "preview", id).stdout).unwrap(); + let expected: Vec<_> = rust + .messages + .iter() + .rev() + .take(80) + .rev() + .map(|m| m.id.as_str()) + .collect(); + let actual: Vec<_> = projected.messages.iter().map(|m| m.id.as_str()).collect(); + assert_eq!(actual, expected, "message order for {case}"); + assert_eq!(projected.title, rust.title, "preview title for {case}"); + assert_eq!(projected.model, rust.model, "preview model for {case}"); + assert_eq!( + projected.updated_at, rust.updated_at, + "preview timestamp for {case}" + ); + let output = run_projection(home.path(), "inventory", id); + let inventory: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + inventory["sessions"][0]["title"], + json!(rust.title), + "inventory {case}" + ); + assert_eq!( + inventory["sessions"][0]["cwd"], + json!(rust.working_dir), + "inventory cwd {case}" + ); + let stamp: chrono::DateTime = + serde_json::from_value(inventory["sessions"][0]["updated_at"].clone()).unwrap(); + assert_eq!(stamp, rust.updated_at, "inventory timestamp {case}"); + assert_eq!( + std::fs::read(&path).unwrap(), + bytes, + "snapshot changed for {case}" + ); + if journal.is_empty() { + assert!(!journal_path.exists(), "journal created for {case}"); + } else { + assert_eq!( + std::fs::read(&journal_path).unwrap(), + journal.as_bytes(), + "journal changed for {case}" + ); + } + } +} + +fn run_projector_checks(checks: &str) { + let mut child = std::process::Command::new("python3") + .arg("-") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let definitions = REMOTE_PROJECTOR.split("if __name__ ==").next().unwrap(); + let mut input = child.stdin.take().unwrap(); + write!(input, "{definitions}\n{checks}").unwrap(); + drop(input); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn remote_timestamp_uses_microsecond_parser_without_losing_nanoseconds() { + run_projector_checks( + r#" +import tempfile +real_datetime = datetime +class MicrosecondParser: + # Python 3.9/3.10's fromisoformat accepts only 3/6 fractional digits. + @staticmethod + def fromisoformat(value): + fraction = re.search(r'\.(\d+)', value) + if fraction and len(fraction.group(1)) not in (3, 6): + raise ValueError('unsupported fractional precision') + return real_datetime.fromisoformat(value) +datetime = MicrosecondParser +try: + for fraction, nanos in [('1',100000000), ('123',123000000), ('123456',123456000), ('123456789',123456789)]: + value = timestamp('2026-01-01T00:00:10.' + fraction + 'Z') + assert value is not None and value[1] == nanos, (fraction, value) + assert timestamp('2026-01-01T00:00:10.123456789Z') == timestamp('2026-01-01T01:00:10.123456789+01:00') + assert timestamp('2026-01-01T00:00:10.123456788Z') < timestamp('2026-01-01T00:00:10.123456789Z') + with tempfile.TemporaryDirectory() as directory: + path = Path(directory)/'compat.json' + path.write_text(json.dumps({'id':'compat','title':'snapshot','updated_at':'2026-01-01T00:00:10.123456789Z','messages':[]})) + path.with_suffix('.journal.jsonl').write_text(json.dumps({'meta':{'title':'stale','updated_at':'2026-01-01T00:00:10.123456788Z'},'append_messages':[]})+'\n') + for preview in (True,False): + assert project(path,preview)['title'] == 'snapshot' +finally: + datetime = real_datetime +"#, + ); +} + +#[test] +fn remote_journal_candidates_and_replay_share_checkpoint_view() { + run_projector_checks( + r#" +import tempfile +original_collect = journal_message_ids +original_open = Path.open +message = lambda i: {'id':str(i),'role':'user','content':[{'type':'text','text':str(i)}]} +for mutation in ('append','replace'): + for preview in (True,False): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory)/'race.json' + journal = path.with_suffix('.journal.jsonl') + before = {'id':'race','title':'snapshot','updated_at':'2026-01-01T00:00:10Z','messages':[message(i) for i in range(1,101)]} + old_entry = json.dumps({'meta':{'title':'old','updated_at':'2026-01-01T00:00:01Z'},'append_messages':[message(1)]})+'\n' + new_entry = json.dumps({'meta':{'title':'late','updated_at':'2026-01-01T00:00:11Z'},'append_messages':[message(101)]})+'\n' + path.write_text(json.dumps(before)); journal.write_text(old_entry) + opens = [] + def tracked_open(target, *args, **kwargs): + if target == journal and args and args[0] == 'rb': + opens.append(target) + return original_open(target, *args, **kwargs) + def checkpoint_after_candidates(source): + result = original_collect(source) + # Deterministic writer barrier: publish a new snapshot containing + # a message appended after the candidate mapping was captured. + before['messages'].append(message(101)) + before['title'] = 'checkpoint' + before['updated_at'] = '2026-01-01T00:00:12Z' + path.write_text(json.dumps(before)) + if mutation == 'append': + with journal.open('a') as writer: writer.write(new_entry) + else: + replacement = journal.with_suffix('.replacement') + replacement.write_text(old_entry + new_entry) + replacement.replace(journal) + return result + Path.open = tracked_open + journal_message_ids = checkpoint_after_candidates + try: + result = project(path,preview) + finally: + Path.open = original_open + journal_message_ids = original_collect + assert result['title'] == 'checkpoint', (mutation,preview,result['title']) + assert result['updated_at'] == '2026-01-01T00:00:12Z' + if preview: + assert [m['id'] for m in result['messages']] == [str(i) for i in range(22,102)] + assert len(opens) == 1, opens + assert json.loads(path.read_text()) == before + assert journal.read_text() == old_entry + new_entry +"#, + ); +} diff --git a/crates/jcode-tui/src/tui/session_picker/cloud_preview/remote.py b/crates/jcode-tui/src/tui/session_picker/cloud_preview/remote.py index 6b645a0f6d6..a230e65ea93 100644 --- a/crates/jcode-tui/src/tui/session_picker/cloud_preview/remote.py +++ b/crates/jcode-tui/src/tui/session_picker/cloud_preview/remote.py @@ -1,5 +1,6 @@ # Read-only remote projection. No sidecars or canonical session writes. import collections +from datetime import datetime import json import mmap import os @@ -151,6 +152,63 @@ def journal_fields(data): yield from record +def array_items(data, start, end): + if data[start] != 91: + raise ValueError('expected message array') + pos = skip_space(data, start + 1) + while pos < end and data[pos] != 93: + stop = value_end(data, pos) + if stop <= pos or stop > end: + raise ValueError('invalid message value') + yield pos, stop + pos = skip_space(data, stop) + if data[pos] == 44: + pos = skip_space(data, pos + 1) + elif data[pos] != 93: + raise ValueError('expected message separator') + + +def message_id(data, start): + if data[start] == 123: + for name, a, b in fields(data, start): + if name == 'id': + value = json.loads(data[a:b]) + return value if isinstance(value, str) else None + return None + + +def journal_message_ids(data): + # Index only journal candidates, not every message in a huge snapshot. + ids = set() + if data is not None: + for record in journal_records(data): + try: + for key, start, end in record: + if key == 'append_messages': + for a, _ in array_items(data, start, end): + mid = message_id(data, a) + if mid is not None: + ids.add(mid) + except (ValueError, IndexError, TypeError): + continue + return ids + + +def timestamp(value): + try: + # Python 3.9/3.10 accept microseconds, while Chrono can emit nanos. + # Normalize only the parser input; compare the original precision below. + normalized = re.sub(r'\.(\d+)', lambda m: '.' + (m.group(1) + '000000')[:6], value) + stamp = datetime.fromisoformat(normalized.replace('Z', '+00:00')) + if stamp.tzinfo is None: + return None + fraction = re.search(r'\.(\d+)', value) + nanos = int((fraction.group(1) + '000000000')[:9]) if fraction else 0 + return int(stamp.replace(microsecond=0).timestamp()), nanos + except (ValueError, TypeError, AttributeError, OverflowError): + return None + + def recent_messages(data, start, end, messages, ranges=None): # Locate the tail without deserializing old message bodies. The range deque # is bounded even for snapshots with millions of messages. @@ -192,41 +250,90 @@ def recent_messages(data, start, end, messages, ranges=None): @timed('projection') def project(path, preview): + journal = path.with_suffix('.journal.jsonl') + try: + source = journal.open('rb') + except FileNotFoundError: + return project_with_journal(path, preview, None) + with source: + size = os.fstat(source.fileno()).st_size + if not size: + return project_with_journal(path, preview, None) + # Pin both inode and length before scanning the checkpoint. Appends or + # atomic journal replacement must not change the candidate/replay view. + with mmap.mmap(source.fileno(), size, access=mmap.ACCESS_READ) as data: + return project_with_journal(path, preview, data) + + +def project_with_journal(path, preview, journal_data): session = {} messages = collections.deque(maxlen=MAX_MESSAGES) + candidates = journal_message_ids(journal_data) + seen = set() with path.open('rb') as source, mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ) as data: ranges = collections.deque(maxlen=MAX_MESSAGES) if preview else None for key, start, end in fields(data, message_ranges=ranges): if key == 'messages': + if candidates: + for a, _ in array_items(data, start, end): + mid = message_id(data, a) + if mid in candidates: + seen.add(mid) if preview: recent_messages(data, start, end, messages, ranges) elif key not in BULKY and (preview or key in INVENTORY_FIELDS) and end - start <= 256 * 1024: session[key] = json.loads(data[start:end]) - journal = path.with_suffix('.journal.jsonl') - if journal.exists() and journal.stat().st_size: - with journal.open('rb') as source, mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ) as data: - for record in journal_records(data): - pending_meta = {} - pending_messages = collections.deque(maxlen=MAX_MESSAGES) - try: - for key, start, stop in record: - if key == 'meta': - meta = {} - route_present = False - for name, a, b in fields(data, start): - if name == 'route_api_method_present': - route_present = data[a:b] == b'true' - if name in META_FIELDS and (preview or name in INVENTORY_FIELDS) and b - a <= 256 * 1024: - meta[name] = json.loads(data[a:b]) - if not route_present: - meta.pop('route_api_method', None) - pending_meta.update(meta) - elif key == 'append_messages' and preview: - recent_messages(data, start, stop, pending_messages) - except (ValueError, IndexError, TypeError): - continue - session.update(pending_meta) - messages.extend(pending_messages) + del candidates + snapshot_updated_at = timestamp(session.get('updated_at')) + in_stale_prefix = True + if journal_data is not None: + data = journal_data + for record in journal_records(data): + pending_meta = {} + pending_messages = collections.deque(maxlen=MAX_MESSAGES) + new_ids = set() + has_messages = False + all_seen = True + try: + for key, start, stop in record: + if key == 'meta': + meta = {} + route_present = False + for name, a, b in fields(data, start): + if name == 'route_api_method_present': + route_present = data[a:b] == b'true' + if name in META_FIELDS and (preview or name in INVENTORY_FIELDS) and b - a <= 256 * 1024: + meta[name] = json.loads(data[a:b]) + if not route_present: + meta.pop('route_api_method', None) + pending_meta.update(meta) + elif key == 'append_messages': + fresh = collections.deque(maxlen=MAX_MESSAGES) + for a, b in array_items(data, start, stop): + mid = message_id(data, a) + has_messages = True + all_seen = all_seen and mid is not None and mid in seen + if mid is None or (mid not in seen and mid not in new_ids): + fresh.append((a, b)) + if mid is not None: + new_ids.add(mid) + if preview: + recent_messages(data, start, stop, pending_messages, fresh) + except (ValueError, IndexError, TypeError): + continue + entry_updated_at = timestamp(pending_meta.get('updated_at')) + stale = all_seen if has_messages else ( + snapshot_updated_at is not None and entry_updated_at is not None + and entry_updated_at <= snapshot_updated_at + ) + # Match JournalReplayDedupe: only a folded prefix is stale; + # later entries still apply metadata while filtering messages. + if in_stale_prefix and stale: + continue + in_stale_prefix = False + seen.update(new_ids) + session.update(pending_meta) + messages.extend(pending_messages) if preview: session['messages'] = list(messages) for key in ('env_snapshots', 'memory_injections', 'replay_events'):