diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 792f79e..4c1550c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,6 +123,7 @@ jobs: - filter_e2e # include/exclude filters - hardlink_e2e # hardlinks: preserve dedups across a multi-agent fleet - hardlink_resilience_e2e # agent dies mid-LINKFIX; oversized group falls back + - on_dest_newer_e2e # copy.on_dest_newer: skip (default) vs overwrite conflict resolution - probe_e2e # per-agent mount probe gates pass start - scale_e2e # pathological shapes: huge dir, huge file - temp_reclaim_e2e # sweep reclaims crash residue, spares live temps diff --git a/agent/Makefile b/agent/Makefile index 2b8cba6..7397ded 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -6,7 +6,7 @@ SRC := src/pb.c src/msgs.c src/wire.c src/tls.c src/state.c src/uring.c src/copy OBJ := $(SRC:.c=.o) BIN := bin/drsync-agent -.PHONY: all clean test filter-test fidelity-test ucopy-test tempname-test xattr-copy-test poolsize-test state-test opts-test estat-test link-test +.PHONY: all clean test filter-test fidelity-test ucopy-test tempname-test xattr-copy-test poolsize-test state-test opts-test estat-test link-test on-dest-newer-test all: $(BIN) @@ -18,7 +18,7 @@ $(BIN): $(OBJ) $(CC) $(CFLAGS) -c -o $@ $< # Unit tests that need no coordinator/fleet. -test: filter-test fidelity-test xattr-copy-test ucopy-test tempname-test poolsize-test state-test opts-test estat-test link-test +test: filter-test fidelity-test xattr-copy-test ucopy-test tempname-test poolsize-test state-test opts-test estat-test link-test on-dest-newer-test # Standalone unit test for the glob matcher. filter-test: bin/filter_test @@ -110,5 +110,15 @@ bin/link_test: test/link_test.c src/link.o src/delete.o src/split.o src/jrn.o sr @mkdir -p bin $(CC) $(CFLAGS) -o $@ test/link_test.c src/link.o src/delete.o src/split.o src/jrn.o src/msgs.o src/pb.o src/state.o src/uring.o -pthread -lzstd +# copy.on_dest_newer wire decode: absent field / CONFLICT_UNSPECIFIED / _SKIP / +# _OVERWRITE all resolve job_options.on_dest_newer_skip correctly, driven +# through the real dec_work_grant -> dec_job_options -> dec_copy_opts path. +on-dest-newer-test: bin/on_dest_newer_test + ./bin/on_dest_newer_test + +bin/on_dest_newer_test: test/on_dest_newer_test.c src/msgs.o src/pb.o + @mkdir -p bin + $(CC) $(CFLAGS) -o $@ test/on_dest_newer_test.c src/msgs.o src/pb.o -pthread -lzstd + clean: - rm -f $(OBJ) $(BIN) bin/filter_test bin/fidelity_test bin/xattr_copy_test bin/ucopy_test bin/tempname_test bin/poolsize_test bin/state_test bin/opts_test bin/estat_test bin/link_test + rm -f $(OBJ) $(BIN) bin/filter_test bin/fidelity_test bin/xattr_copy_test bin/ucopy_test bin/tempname_test bin/poolsize_test bin/state_test bin/opts_test bin/estat_test bin/link_test bin/on_dest_newer_test diff --git a/agent/src/msgs.c b/agent/src/msgs.c index 73f6eb8..4734d33 100644 --- a/agent/src/msgs.c +++ b/agent/src/msgs.c @@ -213,6 +213,7 @@ static void enc_counters(pb_buf *b, uint32_t field, const struct shard_counters pb_put_u64(&sub, 17, c->links_created); pb_put_u64(&sub, 18, c->link_anchor_races); pb_put_u64(&sub, 19, c->link_fallback); + pb_put_u64(&sub, 20, c->skipped_newer); pb_put_msg(b, field, &sub); pb_free(&sub); } @@ -428,6 +429,11 @@ static bool dec_shard(const uint8_t *p, size_t n, struct shard_item *it) static bool dec_copy_opts(const uint8_t *p, size_t n, struct job_options *o) { + /* Pre-set before the field loop so an old coordinator that never sends + * field 10 at all (and, on the wire, a present-but-UNSPECIFIED value — + * proto's own zero default) both land on the safer skip behavior, not + * silently on the pre-this-field overwrite-always behavior. See the + * on_dest_newer_skip comment in msgs.h. */ pb_cur c; pb_cur_init(&c, p, n); uint32_t f; @@ -442,6 +448,14 @@ static bool dec_copy_opts(const uint8_t *p, size_t n, struct job_options *o) case 6: pb_get_strn(&c, o->temp_prefix, sizeof o->temp_prefix); break; case 7: o->fsync_per_file = pb_get_varint(&c) == 1; /* FSYNC_PER_FILE */ break; case 9: o->direct_write = pb_get_varint(&c) != 0; break; + case 10: + /* CONFLICT_UNSPECIFIED (0, proto's own zero value — should never + * be sent deliberately, but a corrupt/misbehaving coordinator is + * not a reason to fall back to the destructive choice) and + * CONFLICT_SKIP_IF_DEST_NEWER (1) both mean skip; only an + * explicit CONFLICT_OVERWRITE (2) turns it off. */ + o->on_dest_newer_skip = pb_get_varint(&c) != 2; + break; default: pb_skip(&c, wt); } } @@ -559,6 +573,20 @@ static bool dec_filter_rule(const uint8_t *p, size_t n, struct filter_rule *fr) static bool dec_job_options(const uint8_t *p, size_t n, struct job_options *o) { memset(o, 0, sizeof(*o)); + /* Pre-set here, not just in dec_copy_opts: pb_put_msg (Go encoder side is + * equivalent) omits an entirely empty CopyOptions submessage from the + * wire rather than sending a zero-length one, so a JobOptions whose + * CopyOptions has every field at proto's zero value never reaches + * dec_copy_opts's own field loop at all — case 6 below is simply never + * taken. Setting it only inside dec_copy_opts left exactly that case + * (which is also what an old coordinator's CopyOptions predating this + * field would produce whenever its other fields also happened to be + * zero) resolving to false/overwrite, silently defeating the mixed-fleet + * safety this field exists for. dec_copy_opts still sets it again when + * that submessage IS present, which is harmless and keeps the two + * decode functions independently correct if either is ever called on + * its own. */ + o->on_dest_newer_skip = true; pb_cur c; pb_cur_init(&c, p, n); uint32_t f; diff --git a/agent/src/msgs.h b/agent/src/msgs.h index 7e0c271..1d0f4c4 100644 --- a/agent/src/msgs.h +++ b/agent/src/msgs.h @@ -77,6 +77,7 @@ enum { JR_SRC_CHANGED = 14, JR_LINK_CREATED = 15, /* hardlink member linked to its group's anchor */ JR_LINK_FALLBACK = 16, /* hardlink group fell back to independent copy */ + JR_SKIPPED_NEWER = 17, /* on_dest_newer_skip: dest mtime > src, left alone */ }; /* Control.Command */ @@ -132,6 +133,16 @@ struct job_options { char temp_prefix[64]; bool fsync_per_file; bool direct_write; /* copy a new file straight to its final name */ + /* CONFLICT_* (proto CopyOptions.ConflictPolicy). A destination file + * strictly newer than the source (beyond mtime_slop_ns) is skipped + * rather than overwritten unless the job set on_dest_newer: overwrite + * (the pre-this-field behavior — source always wins regardless of + * direction). dec_copy_opts (msgs.c) pre-sets this true before decoding + * field 10, specifically so a coordinator old enough to never send that + * field at all — struct starts memset(0) and the field is then simply + * never written — still resolves to the newer, safer skip behavior + * rather than silently falling back to overwrite-always. */ + bool on_dest_newer_skip; /* metadata */ bool meta_owner, meta_mode, meta_times, meta_xattrs, meta_specials; bool acl_posix, acl_nfs4; @@ -299,6 +310,7 @@ struct shard_counters { uint64_t links_created; /* member links created via linkat (space saved) */ uint64_t link_anchor_races; /* redundant speculative copies (§3.4) */ uint64_t link_fallback; /* groups that fell back to independent-copy */ + uint64_t skipped_newer; /* on_dest_newer_skip: dest mtime > src, left alone */ }; struct cached_opts { diff --git a/agent/src/walker.c b/agent/src/walker.c index acaf7c6..5d6c0eb 100644 --- a/agent/src/walker.c +++ b/agent/src/walker.c @@ -277,6 +277,19 @@ static bool times_equal(const struct walk_ctx *ctx, const struct timespec *a, return ts_diff_ns(a, b) <= ctx->oe->o.mtime_slop_ns; } +/* Signed counterpart to times_equal, for the on_dest_newer_skip conflict + * check (§ handle_entry S_IFREG): unlike times_equal/ts_diff_ns, direction + * matters here — a destination strictly newer than the source by more than + * mtime_slop_ns is the one case this policy treats differently from a plain + * "mtime differs" copy trigger. */ +static bool dest_is_newer(const struct walk_ctx *ctx, const struct timespec *src, + const struct timespec *dst) +{ + int64_t d = ((int64_t)dst->tv_sec - src->tv_sec) * 1000000000 + + (dst->tv_nsec - src->tv_nsec); + return d > ctx->oe->o.mtime_slop_ns; +} + static void apply_meta_dirfd(struct walk_ctx *ctx, int fd, const struct estat *ss, const char *path) { @@ -776,6 +789,22 @@ static void handle_entry(struct walk_ctx *ctx, struct dpend *dp, const char *rel } bool need = !type_match || ds->size != ss->size || !times_equal(ctx, &ss->mtim, &ds->mtim); + /* Conflict check (copy.on_dest_newer, default skip): a destination + * regular file strictly newer than the source is left entirely + * alone — no copy, no metadata fix, not even the owner/mode/xattr + * lazy fixups below — rather than treated as drift to reconcile. + * Only applies once both sides agree it's the same file (type_match + * — ds is non-NULL whenever that's true) and only for the "dest is + * newer" direction: a source that's newer, or equal within slop, + * proceeds through the normal diff exactly as before. */ + if (o->on_dest_newer_skip && type_match && + dest_is_newer(ctx, &ss->mtim, &ds->mtim)) { + CTR_ADD(ctx->c.skipped_newer, 1); + char srel[PATH_MAX]; + snprintf(srel, sizeof srel, "%s%s%s", rel, rel[0] ? "/" : "", name); + jrn_emit(ctx, JR_SKIPPED_NEWER, srel, ss, ds, 0, NULL); + return; + } if (!need) { /* diff predicate steps 5–6: owner/mode, then lazy xattr digest — * paid only by files that are otherwise clean (design §2.1) */ diff --git a/agent/test/on_dest_newer_test.c b/agent/test/on_dest_newer_test.c new file mode 100644 index 0000000..f150a6c --- /dev/null +++ b/agent/test/on_dest_newer_test.c @@ -0,0 +1,161 @@ +/* Unit test for the copy.on_dest_newer wire decode (msgs.c dec_copy_opts, + * field 10 of CopyOptions inside JobOptions inside WorkGrant). Covers the + * mixed-fleet safety property this field was specifically designed for: an + * old coordinator that never sends the field at all (or a value that + * decodes to CONFLICT_UNSPECIFIED, proto's own zero default) must still + * resolve to job_options.on_dest_newer_skip == true — the newer, safer + * default — not silently fall back to the pre-this-field overwrite-always + * behavior. Only an explicit CONFLICT_OVERWRITE (2) turns it off. + * + * Hand-builds each WorkGrant's wire bytes directly with the same pb_put_* + * helpers the coordinator's Go encoder ultimately produces equivalent bytes + * for, so this exercises the real decode path (dec_work_grant -> + * dec_job_options -> dec_copy_opts) with no socket, no coordinator, no fleet. + * Build+run: `make on-dest-newer-test` in agent/. Exits non-zero on failure. */ +#include "../src/agent.h" +#include "../src/pb.h" + +#include +#include + +/* ---- stubs for symbols on paths this test does not exercise ---- */ +void log_line(const char *level, const char *fmt, ...) +{ + (void)level; + (void)fmt; +} +void out_push(uint16_t type, pb_buf *b) +{ + (void)type; + (void)b; +} +void walk_err(struct walk_ctx *ctx, const char *what, const char *path) +{ + (void)ctx; + (void)what; + (void)path; +} + +static int failures; +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL: " __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + failures++; \ + } \ + } while (0) + +/* Builds a minimal WorkGrant containing one JobOptions (job_id=1, minimal + * valid src/dst roots) whose CopyOptions carries field 10 (on_dest_newer) + * IF have_conflict_policy is set, to the given enum value. Returns the + * marshaled WorkGrant bytes in *out (caller frees via pb_free semantics — + * out itself is a plain pb_buf owned by the caller). */ +static void build_grant(pb_buf *out, bool have_conflict_policy, uint64_t conflict_policy) +{ + pb_buf copy; + pb_init(©); + if (have_conflict_policy) + pb_put_u64(©, 10, conflict_policy); + + pb_buf jo; + pb_init(&jo); + pb_put_u64(&jo, 1, 1); /* job_id */ + pb_put_str(&jo, 3, "/tmp/src"); /* src_root */ + pb_put_str(&jo, 4, "/tmp/dst"); /* dst_root */ + pb_put_msg(&jo, 6, ©); /* copy (CopyOptions) */ + pb_free(©); + + pb_init(out); + pb_put_msg(out, 2, &jo); /* WorkGrant.options[] (repeated field 2) */ + pb_free(&jo); +} + +/* An old coordinator that predates this field never sends CopyOptions field + * 10 at all — the mixed-fleet case on_dest_newer_skip's pre-set-true default + * (in dec_copy_opts, before the field loop runs) exists for. */ +static void test_field_absent_defaults_skip(void) +{ + pb_buf grant; + build_grant(&grant, false, 0); + + struct work_grant g; + bool ok = dec_work_grant(grant.p, grant.len, &g); + pb_free(&grant); + + CHECK(ok, "dec_work_grant failed to decode a grant with no on_dest_newer field"); + CHECK(g.n_options == 1, "n_options = %zu, want 1", g.n_options); + if (ok && g.n_options == 1) + CHECK(g.options[0].on_dest_newer_skip, + "on_dest_newer_skip = false with the field entirely absent from the wire, want true"); + work_grant_free(&g); +} + +/* CONFLICT_UNSPECIFIED (0) on the wire — proto's own zero value, should never + * be sent deliberately, but a misbehaving coordinator sending it explicitly + * must not be treated as an instruction to overwrite (the destructive + * choice); it means the same as the field being absent. */ +static void test_explicit_unspecified_defaults_skip(void) +{ + pb_buf grant; + build_grant(&grant, true, 0 /* CONFLICT_UNSPECIFIED */); + + struct work_grant g; + bool ok = dec_work_grant(grant.p, grant.len, &g); + pb_free(&grant); + + CHECK(ok, "dec_work_grant failed to decode CONFLICT_UNSPECIFIED"); + if (ok && g.n_options == 1) + CHECK(g.options[0].on_dest_newer_skip, + "on_dest_newer_skip = false for explicit CONFLICT_UNSPECIFIED, want true"); + work_grant_free(&g); +} + +/* CONFLICT_SKIP_IF_DEST_NEWER (1), the explicit form of the default. */ +static void test_explicit_skip(void) +{ + pb_buf grant; + build_grant(&grant, true, 1 /* CONFLICT_SKIP_IF_DEST_NEWER */); + + struct work_grant g; + bool ok = dec_work_grant(grant.p, grant.len, &g); + pb_free(&grant); + + CHECK(ok, "dec_work_grant failed to decode CONFLICT_SKIP_IF_DEST_NEWER"); + if (ok && g.n_options == 1) + CHECK(g.options[0].on_dest_newer_skip, + "on_dest_newer_skip = false for explicit CONFLICT_SKIP_IF_DEST_NEWER, want true"); + work_grant_free(&g); +} + +/* CONFLICT_OVERWRITE (2) — the only value that turns skip off. */ +static void test_explicit_overwrite(void) +{ + pb_buf grant; + build_grant(&grant, true, 2 /* CONFLICT_OVERWRITE */); + + struct work_grant g; + bool ok = dec_work_grant(grant.p, grant.len, &g); + pb_free(&grant); + + CHECK(ok, "dec_work_grant failed to decode CONFLICT_OVERWRITE"); + if (ok && g.n_options == 1) + CHECK(!g.options[0].on_dest_newer_skip, + "on_dest_newer_skip = true for explicit CONFLICT_OVERWRITE, want false"); + work_grant_free(&g); +} + +int main(void) +{ + test_field_absent_defaults_skip(); + test_explicit_unspecified_defaults_skip(); + test_explicit_skip(); + test_explicit_overwrite(); + + if (failures) { + fprintf(stderr, "%d failure(s)\n", failures); + return 1; + } + printf("on_dest_newer_test: OK\n"); + return 0; +} diff --git a/cli/drsync/commands.go b/cli/drsync/commands.go index 2471d25..236361a 100644 --- a/cli/drsync/commands.go +++ b/cli/drsync/commands.go @@ -799,6 +799,7 @@ var journalCats = []struct{ key, label, color string }{ {"ORPHAN", "orphan", ansiYellow}, {"SRC_CHANGED", "src_changed", ansiYellow}, {"LINK_FALLBACK", "link_fallback", ansiYellow}, + {"SKIPPED_NEWER", "skipped_newer", ansiYellow}, {"ERROR", "error", ansiRed}, {"FIDELITY_EXCEPTION", "fidelity_exception", ansiRed}, {"VERIFY_FAIL", "verify_fail", ansiRed}, @@ -901,6 +902,9 @@ func cmdReport(args []string) error { if lc, lr, lf := i64(t["links_created"]), i64(t["link_anchor_races"]), i64(t["link_fallback"]); lc+lr+lf > 0 { fmt.Printf("hardlinks: %d linked, %d anchor races, %d fell back to independent copy\n", lc, lr, lf) } + if sn := i64(t["skipped_newer"]); sn > 0 { + fmt.Printf("skipped: %d files left untouched (destination newer than source)\n", sn) + } fmt.Printf("converged: %v orphans remaining: %d delete pass ran: %v\n", rep["converged"], i64(rep["orphans_remaining"]), rep["delete_pass_ran"]) if n := i64(rep["parked_shard_count"]); n > 0 { diff --git a/coordinator/internal/api/api.go b/coordinator/internal/api/api.go index df807f9..ced1f73 100644 --- a/coordinator/internal/api/api.go +++ b/coordinator/internal/api/api.go @@ -321,8 +321,11 @@ type passView struct { LinksCreated int64 `json:"links_created"` LinkAnchorRaces int64 `json:"link_anchor_races"` LinkFallback int64 `json:"link_fallback"` - StartedAtMs int64 `json:"started_at_ms,omitempty"` - FinishedAtMs int64 `json:"finished_at_ms,omitempty"` + // copy.on_dest_newer = skip (default): files left untouched because the + // destination's mtime was newer than the source's. + SkippedNewer int64 `json:"skipped_newer"` + StartedAtMs int64 `json:"started_at_ms,omitempty"` + FinishedAtMs int64 `json:"finished_at_ms,omitempty"` // DurationMs is finished-started for a completed pass, or elapsed-so-far // (now-started) for one still running. Zero if the pass never started. DurationMs int64 `json:"duration_ms"` @@ -336,7 +339,7 @@ func passViewOf(p *store.Pass) passView { FidelityExc: p.FidelityExceptions, VerifyOK: p.VerifyOK, VerifyFail: p.VerifyFail, LinksCreated: p.LinksCreated, LinkAnchorRaces: p.LinkAnchorRaces, - LinkFallback: p.LinkFallback, + LinkFallback: p.LinkFallback, SkippedNewer: p.SkippedNewer, } if p.Started.Valid { v.StartedAtMs = p.Started.Int64 diff --git a/coordinator/internal/api/query.go b/coordinator/internal/api/query.go index 23ad371..1b93475 100644 --- a/coordinator/internal/api/query.go +++ b/coordinator/internal/api/query.go @@ -381,6 +381,7 @@ func (s *Server) getReport(w http.ResponseWriter, r *http.Request) { totals.LinksCreated += p.LinksCreated totals.LinkAnchorRaces += p.LinkAnchorRaces totals.LinkFallback += p.LinkFallback + totals.SkippedNewer += p.SkippedNewer if p.EntriesWalked > 0 { // scan pass: orphan census supersedes previous orphans = p.Orphans } else if p.Orphans > 0 { // delete pass reports removals here @@ -432,6 +433,7 @@ func (s *Server) getReport(w http.ResponseWriter, r *http.Request) { "links_created": totals.LinksCreated, "link_anchor_races": totals.LinkAnchorRaces, "link_fallback": totals.LinkFallback, + "skipped_newer": totals.SkippedNewer, }, "converged": converged, "orphans_remaining": orphans, diff --git a/coordinator/internal/model/spec.go b/coordinator/internal/model/spec.go index 3d846b5..4d84d6f 100644 --- a/coordinator/internal/model/spec.go +++ b/coordinator/internal/model/spec.go @@ -74,6 +74,16 @@ type JobSpec struct { TempNaming string `yaml:"temp_naming"` Fsync string `yaml:"fsync"` DirectWrite *bool `yaml:"direct_write"` + // OnDestNewer resolves a conflict where the destination's mtime is + // strictly newer than the source's (beyond tuning.mtime_slop_ns): + // skip (default) leaves that file alone, recorded as + // JR_SKIPPED_NEWER; overwrite is the behavior before this field + // existed — source always wins regardless of direction. skip is + // the safer default for a dataset merge (an unexpected skip loses + // nothing; an unexpected overwrite is destructive and permanent). + // A true one-directional mirror, where the destination must never + // diverge from the source, should set this to overwrite explicitly. + OnDestNewer string `yaml:"on_dest_newer"` } `yaml:"copy"` Metadata struct { Owner *bool `yaml:"owner"` @@ -232,6 +242,9 @@ func (s *JobSpec) ApplyDefaults() { sp.Copy.Fsync = "batched" } boolDefault(&sp.Copy.DirectWrite, true) + if sp.Copy.OnDestNewer == "" { + sp.Copy.OnDestNewer = "skip" + } boolDefault(&sp.Metadata.Owner, true) boolDefault(&sp.Metadata.Mode, true) boolDefault(&sp.Metadata.Times, true) @@ -399,6 +412,11 @@ func (s *JobSpec) Validate() error { default: return fmt.Errorf("deletes.mode must be report|mirror") } + switch s.Spec.Copy.OnDestNewer { + case "skip", "overwrite": + default: + return fmt.Errorf("copy.on_dest_newer must be skip|overwrite") + } switch s.Spec.Verify.Mode { case "on", "off": default: @@ -492,6 +510,14 @@ func (s *JobSpec) ToJobOptions(jobID uint64, dryRun bool) (*drsyncpb.JobOptions, default: return nil, fmt.Errorf("copy.server_side_copy must be auto|off|require") } + switch sp.Copy.OnDestNewer { + case "skip": + o.Copy.OnDestNewer = drsyncpb.CopyOptions_CONFLICT_SKIP_IF_DEST_NEWER + case "overwrite": + o.Copy.OnDestNewer = drsyncpb.CopyOptions_CONFLICT_OVERWRITE + default: + return nil, fmt.Errorf("copy.on_dest_newer must be skip|overwrite") + } if sp.Copy.Fsync == "per_file" { o.Copy.FsyncMode = drsyncpb.CopyOptions_FSYNC_PER_FILE } else { diff --git a/coordinator/internal/model/spec_defaults_test.go b/coordinator/internal/model/spec_defaults_test.go index 9decc58..63999f8 100644 --- a/coordinator/internal/model/spec_defaults_test.go +++ b/coordinator/internal/model/spec_defaults_test.go @@ -1,6 +1,10 @@ package model -import "testing" +import ( + "testing" + + drsyncpb "drsync/proto/gen/drsyncpb" +) // TestDefaultsAppliedToMinimalSpec locks down ApplyDefaults' resolved values // for a spec that specifies nothing beyond the required fields, so the @@ -84,3 +88,58 @@ func TestDirectWriteExplicitFalseIsRespected(t *testing.T) { t.Errorf("copy.direct_write = %v, want explicit false to stick", s.Spec.Copy.DirectWrite) } } + +// TestOnDestNewerDefaultsToSkip: a spec that doesn't set copy.on_dest_newer +// must resolve to "skip" — the deliberate behavior change from the +// pre-this-field always-overwrite default (source always won regardless of +// mtime direction). Checked at both the YAML-resolved level and the wire +// level (ToJobOptions), since a mismatch between the two would mean the +// coordinator's own idea of the default diverges from what agents receive. +func TestOnDestNewerDefaultsToSkip(t *testing.T) { + s, err := ParseSpec([]byte(filterBase)) + if err != nil { + t.Fatal(err) + } + if s.Spec.Copy.OnDestNewer != "skip" { + t.Errorf("copy.on_dest_newer = %q, want skip", s.Spec.Copy.OnDestNewer) + } + o, err := s.ToJobOptions(1, false) + if err != nil { + t.Fatal(err) + } + if o.Copy.OnDestNewer != drsyncpb.CopyOptions_CONFLICT_SKIP_IF_DEST_NEWER { + t.Errorf("resolved JobOptions.Copy.OnDestNewer = %v, want CONFLICT_SKIP_IF_DEST_NEWER", + o.Copy.OnDestNewer) + } +} + +// TestOnDestNewerOverwriteIsRespected: an operator who explicitly opts into +// the old always-overwrite behavior must have that honored end to end, not +// silently defaulted back to skip. +func TestOnDestNewerOverwriteIsRespected(t *testing.T) { + spec := filterBase + " copy:\n on_dest_newer: overwrite\n" + s, err := ParseSpec([]byte(spec)) + if err != nil { + t.Fatal(err) + } + if s.Spec.Copy.OnDestNewer != "overwrite" { + t.Errorf("copy.on_dest_newer = %q, want explicit overwrite to stick", s.Spec.Copy.OnDestNewer) + } + o, err := s.ToJobOptions(1, false) + if err != nil { + t.Fatal(err) + } + if o.Copy.OnDestNewer != drsyncpb.CopyOptions_CONFLICT_OVERWRITE { + t.Errorf("resolved JobOptions.Copy.OnDestNewer = %v, want CONFLICT_OVERWRITE", o.Copy.OnDestNewer) + } +} + +// TestOnDestNewerRejectsUnknownValue guards the validation switch: a typo'd +// value must fail fast at submit time, not silently fall through to +// whatever the zero enum value happens to mean. +func TestOnDestNewerRejectsUnknownValue(t *testing.T) { + spec := filterBase + " copy:\n on_dest_newer: sometimes\n" + if _, err := ParseSpec([]byte(spec)); err == nil { + t.Fatal("expected an error for copy.on_dest_newer: sometimes, got nil") + } +} diff --git a/coordinator/internal/store/store.go b/coordinator/internal/store/store.go index e273aa8..6401f73 100644 --- a/coordinator/internal/store/store.go +++ b/coordinator/internal/store/store.go @@ -186,6 +186,9 @@ CREATE TABLE IF NOT EXISTS passes ( links_created INTEGER NOT NULL DEFAULT 0, link_anchor_races INTEGER NOT NULL DEFAULT 0, link_fallback INTEGER NOT NULL DEFAULT 0, + -- copy.on_dest_newer = skip (default): files left untouched because the + -- destination's mtime was newer than the source's. + skipped_newer INTEGER NOT NULL DEFAULT 0, -- Historical DONE-shard total the Shard Reaper has deleted for this pass. -- ShardStateCounts adds this to the live DONE count from shard_counts, so a -- reaped pass still reports the DONE total it actually had — reaping frees @@ -548,6 +551,7 @@ var migrations = []string{ // such column" on its very first delete pass without these. `ALTER TABLE delete_groups ADD COLUMN done_streaming INTEGER NOT NULL DEFAULT 0`, `ALTER TABLE delete_groups ADD COLUMN pending_children INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE passes ADD COLUMN skipped_newer INTEGER NOT NULL DEFAULT 0`, } func (s *Store) Close() error { @@ -933,12 +937,15 @@ type Pass struct { FidelityExceptions, VerifyOK, VerifyFail int64 // Hardlink preservation (docs/DESIGN-hardlinks.md), all pass-scoped. LinksCreated, LinkAnchorRaces, LinkFallback int64 + // copy.on_dest_newer = skip (default): files left untouched because the + // destination's mtime was newer than the source's. + SkippedNewer int64 } const passCols = `id, job_id, pass_no, state, started_at, finished_at, entries_walked, files_copied, bytes_copied, meta_fixed, orphans, errors, nlink_dup_files, nlink_dup_bytes, fidelity_exceptions, verify_ok, verify_fail, - links_created, link_anchor_races, link_fallback` + links_created, link_anchor_races, link_fallback, skipped_newer` func scanPass(row interface{ Scan(...any) error }) (*Pass, error) { var p Pass @@ -946,7 +953,7 @@ func scanPass(row interface{ Scan(...any) error }) (*Pass, error) { &p.EntriesWalked, &p.FilesCopied, &p.BytesCopied, &p.MetaFixed, &p.Orphans, &p.Errors, &p.NlinkDupFiles, &p.NlinkDupBytes, &p.FidelityExceptions, &p.VerifyOK, &p.VerifyFail, - &p.LinksCreated, &p.LinkAnchorRaces, &p.LinkFallback); err != nil { + &p.LinksCreated, &p.LinkAnchorRaces, &p.LinkFallback, &p.SkippedNewer); err != nil { return nil, err } return &p, nil @@ -1058,12 +1065,13 @@ func accumulatePassCountersTx(x dbOrTx, passID int64, c *drsyncpb.ShardCounters) verify_fail = verify_fail + ?, links_created = links_created + ?, link_anchor_races = link_anchor_races + ?, - link_fallback = link_fallback + ? + link_fallback = link_fallback + ?, + skipped_newer = skipped_newer + ? WHERE id = ?`, c.EntriesWalked, c.FilesCopied, c.BytesCopied, c.MetaFixed, c.Orphans, c.Errors, c.NlinkDupFiles, c.NlinkDupBytes, c.FidelityExceptions, c.VerifyOk, c.VerifyFail, - c.LinksCreated, c.LinkAnchorRaces, c.LinkFallback, passID) + c.LinksCreated, c.LinkAnchorRaces, c.LinkFallback, c.SkippedNewer, passID) return err } diff --git a/docs/DESIGN-agent.md b/docs/DESIGN-agent.md index 8685950..1205cc9 100644 --- a/docs/DESIGN-agent.md +++ b/docs/DESIGN-agent.md @@ -271,6 +271,74 @@ the stale subtree is picked up and removed by the next explicit DELETE pass through the same fan-out machinery any other orphan uses, no coordinator-side awareness of "type change" required. +### 2.1b Conflict resolution: which side wins on a timestamp disagreement + +Steps 1–3 of the diff predicate above decide **whether** to copy; they say +nothing about **direction** — `times_equal` (`agent/src/walker.c`) is an +unsigned `|src.mtime - dst.mtime|` comparison, so before `copy.on_dest_newer` +existed, any mtime difference beyond `mtime_slop_ns` meant "copy," full stop, +regardless of which side was actually newer. drsync's normal operating +mode is one-directional (source is authoritative, destination is a +migration target), so this was never wrong for that case — but it also meant +a destination edited out-of-band (a dataset **merge**, not a pure migration: +an operator or another process wrote directly into the destination tree) +would be silently clobbered on the very next pass, with no way to protect it +short of excluding the path entirely. + +`copy.on_dest_newer` (proto `CopyOptions.ConflictPolicy`, job-spec +`copy.on_dest_newer: skip|overwrite`) adds a direction-aware check ahead of +the normal diff, using a second, *signed* comparison (`dest_is_newer`, +alongside `times_equal` — direction matters here in a way it deliberately +doesn't for the plain equality check the rest of the predicate uses): + +- **`skip` (default):** when the destination's mtime is strictly newer than + the source's (beyond `mtime_slop_ns`) and both sides agree on type, the + entry is left completely untouched — no copy, no owner/mode/xattr fixup + either, since those are normally applied as a cheap side-channel on an + otherwise-clean file (predicate steps 5–6) and would still be clobbering + something the operator may have deliberately changed. Recorded as + `JR_SKIPPED_NEWER` (own counter, `shard_counters.skipped_newer`) — every + occurrence, not sampled, since (unlike `JR_SKIPPED_CLEAN`) this is a real + conflict an operator needs to be able to audit in full, not routine + "nothing to do" volume. +- **`overwrite`:** the pre-this-field behavior — source always wins, + direction never considered. The correct choice for a strict + one-directional mirror where the destination must never diverge from the + source, which is most drsync jobs; an operator opts into it explicitly. + +Two things keep this narrowly scoped rather than reshaping the predicate: + +- It only fires when `type_match` is true (same `d_type` on both sides) — + a type change (§2.1a) is not a "which side is newer" question, and still + goes through `remove_dst`/replace exactly as before regardless of + `on_dest_newer`. +- A file the VERIFY phase never learns about cannot be "un-skipped" by + verification's own re-check: `passctrl.seedVerify` seeds VERIFY entries + only from `JR_COPIED`/`JR_META_FIXED` journal records (docs/DESIGN- + coordinator.md §5), and a skip emits neither — so a skipped file is + invisible to VERIFY by construction, not by an extra check inside + `verify.c`. This matters because VERIFY's own mismatch handling + (`recopy`, on ANY field disagreement including mtime) would otherwise + silently re-copy exactly the file this option exists to protect. + +**Mixed-fleet default safety:** the field defaults to `skip`, and that +default has to survive an agent that cannot decode it at all — either +because it predates this field (proto skips unknown field 10 on decode, +`job_options.on_dest_newer_skip` never gets written) or because the +coordinator's `CopyOptions` submessage happened to be empty on the wire +(`pb_put_msg` omits an entirely zero-valued submessage rather than sending +an empty one — a real, not merely theoretical, case: this is exactly the +shape an old coordinator's `CopyOptions` produces). `dec_job_options` +(`agent/src/msgs.c`) sets `on_dest_newer_skip = true` immediately after its +`memset`, before any field parsing — not only inside `dec_copy_opts`'s own +field loop, which a completely absent `copy` submessage never reaches. An +agent that cannot see this field at all must land on the newer, safer +default, never silently regress to overwrite-always; caught by +`agent/test/on_dest_newer_test.c` before it shipped (a first version that +only set the default inside `dec_copy_opts` passed the "explicit value" +cases but failed exactly the "field absent" and "explicit +`CONFLICT_UNSPECIFIED`" cases this note describes). + ### 2.2 statx batching — the NFS scan multiplier Serial `stat` over NFS at 0.5 ms RTT = 2k entries/s/thread — hopeless. The walker diff --git a/docs/DESIGN-jobspec.md b/docs/DESIGN-jobspec.md index 22d8a7a..a9b1621 100644 --- a/docs/DESIGN-jobspec.md +++ b/docs/DESIGN-jobspec.md @@ -65,6 +65,16 @@ spec: # Trades atomicity: a crash mid-write leaves a # partial file, re-copied next pass. Only new # files; updates keep the atomic temp+rename. + on_dest_newer: skip # skip (default) | overwrite. A destination file + # whose mtime is strictly newer than the source's + # (beyond tuning.mtime_slop_ns) is left untouched + # (JR_SKIPPED_NEWER) instead of overwritten — the + # safer default for a dataset merge, where an + # out-of-band destination edit shouldn't be + # silently clobbered. overwrite restores the + # behavior from before this field existed: source + # always wins regardless of mtime direction, for a + # strict one-directional mirror. metadata: owner: true # uid/gid (needs root) diff --git a/proto/drsync.proto b/proto/drsync.proto index 90eb594..2cc9245 100644 --- a/proto/drsync.proto +++ b/proto/drsync.proto @@ -257,6 +257,30 @@ message CopyOptions { // applied when the destination file does not already exist; updates keep the // atomic temp+rename. Default off. bool direct_write = 9; + enum ConflictPolicy { + CONFLICT_UNSPECIFIED = 0; + // A destination file whose mtime is strictly newer than the source's + // (beyond tuning.mtime_slop_ns) is left alone: not copied, not + // metadata-fixed, no VERIFY_FAIL — recorded as JR_SKIPPED_NEWER instead. + // Default: drsync is a one-directional migration/consolidation tool + // (source is normally authoritative), but an out-of-band destination + // edit or a dataset merge scenario means "newer" isn't always "source's + // copy". This is the safer default for a merge — an unexpected skip + // loses nothing (the file just isn't touched), where an unexpected + // overwrite is destructive and cannot be undone by a later pass. + CONFLICT_SKIP_IF_DEST_NEWER = 1; + // The behavior before this field existed: mtime difference in EITHER + // direction means copy, source always wins. Opt in explicitly for a true + // one-directional mirror where the destination must never diverge. + CONFLICT_OVERWRITE = 2; + } + // Default CONFLICT_SKIP_IF_DEST_NEWER — see the enum's own comment. An old + // agent that does not understand this field (pre-dates it, decodes as 0/ + // UNSPECIFIED via proto default-on-absence) falls back to its own built-in + // default at the agent/src/msgs.c decode site, not to whatever the + // coordinator resolved — see docs/DESIGN-protocol.md §5.1's "old agent, + // new coordinator field" mixed-fleet caveat. + ConflictPolicy on_dest_newer = 10; } message LimitOptions { @@ -620,6 +644,11 @@ message ShardCounters { // at most one per group, tracked for visibility uint64 link_fallback = 19; // groups that fell back to independent-copy // (anchor failure or hardlinks_max_group_scan cap) + // copy.on_dest_newer = CONFLICT_SKIP_IF_DEST_NEWER (the default): files + // left untouched because the destination's mtime was strictly newer than + // the source's. Each one also gets its own JR_SKIPPED_NEWER journal + // record (not sampled) — this counter is the pass-summary rollup of that. + uint64 skipped_newer = 20; } message ShardResult { @@ -689,6 +718,12 @@ message JournalRecord { // Hardlink preservation (docs/DESIGN-hardlinks.md): JR_LINK_CREATED = 15; // member linked to its group's anchor JR_LINK_FALLBACK = 16; // group fell back to independent copy + // copy.on_dest_newer = CONFLICT_SKIP_IF_DEST_NEWER (the default): the + // destination's mtime was strictly newer than the source's, so the + // otherwise-due copy/meta-fix was skipped. Not sampled — every skip is + // recorded, since (unlike JR_SKIPPED_CLEAN) this is a conflict an + // operator needs to be able to audit in full. + JR_SKIPPED_NEWER = 17; } Type type = 1; bytes rel_path = 2; diff --git a/proto/gen/drsyncpb/drsync.pb.go b/proto/gen/drsyncpb/drsync.pb.go index 704bc9e..055b746 100644 --- a/proto/gen/drsyncpb/drsync.pb.go +++ b/proto/gen/drsyncpb/drsync.pb.go @@ -27,7 +27,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.12 // protoc (unknown) // source: drsync.proto @@ -525,6 +525,67 @@ func (CopyOptions_FsyncMode) EnumDescriptor() ([]byte, []int) { return file_drsync_proto_rawDescGZIP(), []int{14, 1} } +type CopyOptions_ConflictPolicy int32 + +const ( + CopyOptions_CONFLICT_UNSPECIFIED CopyOptions_ConflictPolicy = 0 + // A destination file whose mtime is strictly newer than the source's + // (beyond tuning.mtime_slop_ns) is left alone: not copied, not + // metadata-fixed, no VERIFY_FAIL — recorded as JR_SKIPPED_NEWER instead. + // Default: drsync is a one-directional migration/consolidation tool + // (source is normally authoritative), but an out-of-band destination + // edit or a dataset merge scenario means "newer" isn't always "source's + // copy". This is the safer default for a merge — an unexpected skip + // loses nothing (the file just isn't touched), where an unexpected + // overwrite is destructive and cannot be undone by a later pass. + CopyOptions_CONFLICT_SKIP_IF_DEST_NEWER CopyOptions_ConflictPolicy = 1 + // The behavior before this field existed: mtime difference in EITHER + // direction means copy, source always wins. Opt in explicitly for a true + // one-directional mirror where the destination must never diverge. + CopyOptions_CONFLICT_OVERWRITE CopyOptions_ConflictPolicy = 2 +) + +// Enum value maps for CopyOptions_ConflictPolicy. +var ( + CopyOptions_ConflictPolicy_name = map[int32]string{ + 0: "CONFLICT_UNSPECIFIED", + 1: "CONFLICT_SKIP_IF_DEST_NEWER", + 2: "CONFLICT_OVERWRITE", + } + CopyOptions_ConflictPolicy_value = map[string]int32{ + "CONFLICT_UNSPECIFIED": 0, + "CONFLICT_SKIP_IF_DEST_NEWER": 1, + "CONFLICT_OVERWRITE": 2, + } +) + +func (x CopyOptions_ConflictPolicy) Enum() *CopyOptions_ConflictPolicy { + p := new(CopyOptions_ConflictPolicy) + *p = x + return p +} + +func (x CopyOptions_ConflictPolicy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CopyOptions_ConflictPolicy) Descriptor() protoreflect.EnumDescriptor { + return file_drsync_proto_enumTypes[8].Descriptor() +} + +func (CopyOptions_ConflictPolicy) Type() protoreflect.EnumType { + return &file_drsync_proto_enumTypes[8] +} + +func (x CopyOptions_ConflictPolicy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CopyOptions_ConflictPolicy.Descriptor instead. +func (CopyOptions_ConflictPolicy) EnumDescriptor() ([]byte, []int) { + return file_drsync_proto_rawDescGZIP(), []int{14, 2} +} + type JournalRecord_Type int32 const ( @@ -546,6 +607,12 @@ const ( // Hardlink preservation (docs/DESIGN-hardlinks.md): JournalRecord_JR_LINK_CREATED JournalRecord_Type = 15 // member linked to its group's anchor JournalRecord_JR_LINK_FALLBACK JournalRecord_Type = 16 // group fell back to independent copy + // copy.on_dest_newer = CONFLICT_SKIP_IF_DEST_NEWER (the default): the + // destination's mtime was strictly newer than the source's, so the + // otherwise-due copy/meta-fix was skipped. Not sampled — every skip is + // recorded, since (unlike JR_SKIPPED_CLEAN) this is a conflict an + // operator needs to be able to audit in full. + JournalRecord_JR_SKIPPED_NEWER JournalRecord_Type = 17 ) // Enum value maps for JournalRecord_Type. @@ -568,6 +635,7 @@ var ( 14: "JR_SRC_CHANGED", 15: "JR_LINK_CREATED", 16: "JR_LINK_FALLBACK", + 17: "JR_SKIPPED_NEWER", } JournalRecord_Type_value = map[string]int32{ "JR_UNSPECIFIED": 0, @@ -587,6 +655,7 @@ var ( "JR_SRC_CHANGED": 14, "JR_LINK_CREATED": 15, "JR_LINK_FALLBACK": 16, + "JR_SKIPPED_NEWER": 17, } ) @@ -601,11 +670,11 @@ func (x JournalRecord_Type) String() string { } func (JournalRecord_Type) Descriptor() protoreflect.EnumDescriptor { - return file_drsync_proto_enumTypes[8].Descriptor() + return file_drsync_proto_enumTypes[9].Descriptor() } func (JournalRecord_Type) Type() protoreflect.EnumType { - return &file_drsync_proto_enumTypes[8] + return &file_drsync_proto_enumTypes[9] } func (x JournalRecord_Type) Number() protoreflect.EnumNumber { @@ -1783,7 +1852,14 @@ type CopyOptions struct { // a partial file at its real name, which the next pass re-copies. Only ever // applied when the destination file does not already exist; updates keep the // atomic temp+rename. Default off. - DirectWrite bool `protobuf:"varint,9,opt,name=direct_write,json=directWrite,proto3" json:"direct_write,omitempty"` + DirectWrite bool `protobuf:"varint,9,opt,name=direct_write,json=directWrite,proto3" json:"direct_write,omitempty"` + // Default CONFLICT_SKIP_IF_DEST_NEWER — see the enum's own comment. An old + // agent that does not understand this field (pre-dates it, decodes as 0/ + // UNSPECIFIED via proto default-on-absence) falls back to its own built-in + // default at the agent/src/msgs.c decode site, not to whatever the + // coordinator resolved — see docs/DESIGN-protocol.md §5.1's "old agent, + // new coordinator field" mixed-fleet caveat. + OnDestNewer CopyOptions_ConflictPolicy `protobuf:"varint,10,opt,name=on_dest_newer,json=onDestNewer,proto3,enum=drsync.v1.CopyOptions_ConflictPolicy" json:"on_dest_newer,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1881,6 +1957,13 @@ func (x *CopyOptions) GetDirectWrite() bool { return false } +func (x *CopyOptions) GetOnDestNewer() CopyOptions_ConflictPolicy { + if x != nil { + return x.OnDestNewer + } + return CopyOptions_CONFLICT_UNSPECIFIED +} + type LimitOptions struct { state protoimpl.MessageState `protogen:"open.v1"` BandwidthPerAgent uint64 `protobuf:"varint,1,opt,name=bandwidth_per_agent,json=bandwidthPerAgent,proto3" json:"bandwidth_per_agent,omitempty"` // bytes/s, 0 = unlimited @@ -3752,7 +3835,13 @@ type ShardCounters struct { LinksCreated uint64 `protobuf:"varint,17,opt,name=links_created,json=linksCreated,proto3" json:"links_created,omitempty"` // member links created via linkat (space saved) LinkAnchorRaces uint64 `protobuf:"varint,18,opt,name=link_anchor_races,json=linkAnchorRaces,proto3" json:"link_anchor_races,omitempty"` // redundant speculative copies (§3.4) — bounded, // at most one per group, tracked for visibility - LinkFallback uint64 `protobuf:"varint,19,opt,name=link_fallback,json=linkFallback,proto3" json:"link_fallback,omitempty"` // groups that fell back to independent-copy + LinkFallback uint64 `protobuf:"varint,19,opt,name=link_fallback,json=linkFallback,proto3" json:"link_fallback,omitempty"` // groups that fell back to independent-copy + // (anchor failure or hardlinks_max_group_scan cap) + // copy.on_dest_newer = CONFLICT_SKIP_IF_DEST_NEWER (the default): files + // left untouched because the destination's mtime was strictly newer than + // the source's. Each one also gets its own JR_SKIPPED_NEWER journal + // record (not sampled) — this counter is the pass-summary rollup of that. + SkippedNewer uint64 `protobuf:"varint,20,opt,name=skipped_newer,json=skippedNewer,proto3" json:"skipped_newer,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3920,6 +4009,13 @@ func (x *ShardCounters) GetLinkFallback() uint64 { return 0 } +func (x *ShardCounters) GetSkippedNewer() uint64 { + if x != nil { + return x.SkippedNewer + } + return 0 +} + type ShardResult struct { state protoimpl.MessageState `protogen:"open.v1"` ShardId uint64 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` @@ -5102,7 +5198,7 @@ const file_drsync_proto_rawDesc = "" + "OnMismatch\x12\x1b\n" + "\x17ON_MISMATCH_UNSPECIFIED\x10\x00\x12\x16\n" + "\x12ON_MISMATCH_RECOPY\x10\x01\x12\x14\n" + - "\x10ON_MISMATCH_FAIL\x10\x02\"\xb4\x04\n" + + "\x10ON_MISMATCH_FAIL\x10\x02\"\xe4\x05\n" + "\vCopyOptions\x12'\n" + "\x0fchunk_threshold\x18\x01 \x01(\x04R\x0echunkThreshold\x12\x1d\n" + "\n" + @@ -5117,7 +5213,9 @@ const file_drsync_proto_rawDesc = "" + "fsync_mode\x18\a \x01(\x0e2 .drsync.v1.CopyOptions.FsyncModeR\tfsyncMode\x12\x1f\n" + "\vfsync_batch\x18\b \x01(\rR\n" + "fsyncBatch\x12!\n" + - "\fdirect_write\x18\t \x01(\bR\vdirectWrite\"Q\n" + + "\fdirect_write\x18\t \x01(\bR\vdirectWrite\x12I\n" + + "\ron_dest_newer\x18\n" + + " \x01(\x0e2%.drsync.v1.CopyOptions.ConflictPolicyR\vonDestNewer\"Q\n" + "\x0eServerSideCopy\x12\x13\n" + "\x0fSSC_UNSPECIFIED\x10\x00\x12\f\n" + "\bSSC_AUTO\x10\x01\x12\v\n" + @@ -5126,7 +5224,11 @@ const file_drsync_proto_rawDesc = "" + "\tFsyncMode\x12\x15\n" + "\x11FSYNC_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eFSYNC_PER_FILE\x10\x01\x12\x11\n" + - "\rFSYNC_BATCHED\x10\x02\"d\n" + + "\rFSYNC_BATCHED\x10\x02\"c\n" + + "\x0eConflictPolicy\x12\x18\n" + + "\x14CONFLICT_UNSPECIFIED\x10\x00\x12\x1f\n" + + "\x1bCONFLICT_SKIP_IF_DEST_NEWER\x10\x01\x12\x16\n" + + "\x12CONFLICT_OVERWRITE\x10\x02\"d\n" + "\fLimitOptions\x12.\n" + "\x13bandwidth_per_agent\x18\x01 \x01(\x04R\x11bandwidthPerAgent\x12$\n" + "\x0eiops_per_agent\x18\x02 \x01(\x04R\fiopsPerAgent\"\x88\x03\n" + @@ -5303,7 +5405,7 @@ const file_drsync_proto_rawDesc = "" + "\rShardSplitAck\x12&\n" + "\x0fparent_shard_id\x18\x01 \x01(\x04R\rparentShardId\x12\x10\n" + "\x03seq\x18\x02 \x01(\x04R\x03seq\x12,\n" + - "\x12assigned_shard_ids\x18\x03 \x03(\x04R\x10assignedShardIds\"\xfd\x04\n" + + "\x12assigned_shard_ids\x18\x03 \x03(\x04R\x10assignedShardIds\"\xa2\x05\n" + "\rShardCounters\x12%\n" + "\x0eentries_walked\x18\x01 \x01(\x04R\rentriesWalked\x12!\n" + "\ffiles_copied\x18\x02 \x01(\x04R\vfilesCopied\x12!\n" + @@ -5326,7 +5428,8 @@ const file_drsync_proto_rawDesc = "" + "verifyFail\x12#\n" + "\rlinks_created\x18\x11 \x01(\x04R\flinksCreated\x12*\n" + "\x11link_anchor_races\x18\x12 \x01(\x04R\x0flinkAnchorRaces\x12#\n" + - "\rlink_fallback\x18\x13 \x01(\x04R\flinkFallback\"\xe9\x01\n" + + "\rlink_fallback\x18\x13 \x01(\x04R\flinkFallback\x12#\n" + + "\rskipped_newer\x18\x14 \x01(\x04R\fskippedNewer\"\xe9\x01\n" + "\vShardResult\x12\x19\n" + "\bshard_id\x18\x01 \x01(\x04R\ashardId\x12\x19\n" + "\blease_id\x18\x02 \x01(\x04R\aleaseId\x12/\n" + @@ -5347,7 +5450,7 @@ const file_drsync_proto_rawDesc = "" + "\vverify_fail\x18\t \x01(\x04R\n" + "verifyFail\"B\n" + "\x0fTaskResultBatch\x12/\n" + - "\aresults\x18\x01 \x03(\v2\x15.drsync.v1.TaskResultR\aresults\"\xed\x04\n" + + "\aresults\x18\x01 \x03(\v2\x15.drsync.v1.TaskResultR\aresults\"\x83\x05\n" + "\rJournalRecord\x121\n" + "\x04type\x18\x01 \x01(\x0e2\x1d.drsync.v1.JournalRecord.TypeR\x04type\x12\x19\n" + "\brel_path\x18\x02 \x01(\fR\arelPath\x12\x13\n" + @@ -5357,7 +5460,7 @@ const file_drsync_proto_rawDesc = "" + "\axxh3_lo\x18\x06 \x01(\x04R\x06xxh3Lo\x12\x17\n" + "\axxh3_hi\x18\a \x01(\x04R\x06xxh3Hi\x12\x14\n" + "\x05errno\x18\b \x01(\x05R\x05errno\x12\x16\n" + - "\x06detail\x18\t \x01(\tR\x06detail\"\xca\x02\n" + + "\x06detail\x18\t \x01(\tR\x06detail\"\xe0\x02\n" + "\x04Type\x12\x12\n" + "\x0eJR_UNSPECIFIED\x10\x00\x12\r\n" + "\tJR_COPIED\x10\x01\x12\x11\n" + @@ -5377,7 +5480,8 @@ const file_drsync_proto_rawDesc = "" + "JR_DELETED\x10\r\x12\x12\n" + "\x0eJR_SRC_CHANGED\x10\x0e\x12\x13\n" + "\x0fJR_LINK_CREATED\x10\x0f\x12\x14\n" + - "\x10JR_LINK_FALLBACK\x10\x10\"\x96\x01\n" + + "\x10JR_LINK_FALLBACK\x10\x10\x12\x14\n" + + "\x10JR_SKIPPED_NEWER\x10\x11\"\x96\x01\n" + "\fJournalBatch\x12\x10\n" + "\x03seq\x18\x01 \x01(\x04R\x03seq\x12\x15\n" + "\x06job_id\x18\x02 \x01(\x04R\x05jobId\x12\x17\n" + @@ -5455,7 +5559,7 @@ func file_drsync_proto_rawDescGZIP() []byte { return file_drsync_proto_rawDescData } -var file_drsync_proto_enumTypes = make([]protoimpl.EnumInfo, 9) +var file_drsync_proto_enumTypes = make([]protoimpl.EnumInfo, 10) var file_drsync_proto_msgTypes = make([]protoimpl.MessageInfo, 52) var file_drsync_proto_goTypes = []any{ (FrameType)(0), // 0: drsync.v1.FrameType @@ -5466,117 +5570,119 @@ var file_drsync_proto_goTypes = []any{ (VerifyOptions_OnMismatch)(0), // 5: drsync.v1.VerifyOptions.OnMismatch (CopyOptions_ServerSideCopy)(0), // 6: drsync.v1.CopyOptions.ServerSideCopy (CopyOptions_FsyncMode)(0), // 7: drsync.v1.CopyOptions.FsyncMode - (JournalRecord_Type)(0), // 8: drsync.v1.JournalRecord.Type - (*StatInfo)(nil), // 9: drsync.v1.StatInfo - (*MountCaps)(nil), // 10: drsync.v1.MountCaps - (*Hello)(nil), // 11: drsync.v1.Hello - (*HelloAck)(nil), // 12: drsync.v1.HelloAck - (*MountHealth)(nil), // 13: drsync.v1.MountHealth - (*InflightItem)(nil), // 14: drsync.v1.InflightItem - (*Heartbeat)(nil), // 15: drsync.v1.Heartbeat - (*HeartbeatAck)(nil), // 16: drsync.v1.HeartbeatAck - (*Control)(nil), // 17: drsync.v1.Control - (*ProtocolError)(nil), // 18: drsync.v1.ProtocolError - (*FilterRule)(nil), // 19: drsync.v1.FilterRule - (*AclOptions)(nil), // 20: drsync.v1.AclOptions - (*MetadataOptions)(nil), // 21: drsync.v1.MetadataOptions - (*VerifyOptions)(nil), // 22: drsync.v1.VerifyOptions - (*CopyOptions)(nil), // 23: drsync.v1.CopyOptions - (*LimitOptions)(nil), // 24: drsync.v1.LimitOptions - (*TuningOptions)(nil), // 25: drsync.v1.TuningOptions - (*JobOptions)(nil), // 26: drsync.v1.JobOptions - (*WorkRequest)(nil), // 27: drsync.v1.WorkRequest - (*FileGen)(nil), // 28: drsync.v1.FileGen - (*WalkOverrides)(nil), // 29: drsync.v1.WalkOverrides - (*Shard)(nil), // 30: drsync.v1.Shard - (*EntryListShard)(nil), // 31: drsync.v1.EntryListShard - (*ChunkTask)(nil), // 32: drsync.v1.ChunkTask - (*DirMeta)(nil), // 33: drsync.v1.DirMeta - (*DirFixBatch)(nil), // 34: drsync.v1.DirFixBatch - (*VerifyEntry)(nil), // 35: drsync.v1.VerifyEntry - (*VerifyBatch)(nil), // 36: drsync.v1.VerifyBatch - (*DeleteBatch)(nil), // 37: drsync.v1.DeleteBatch - (*ProbeTask)(nil), // 38: drsync.v1.ProbeTask - (*LinkTask)(nil), // 39: drsync.v1.LinkTask - (*LinkEntry)(nil), // 40: drsync.v1.LinkEntry - (*LinkTaskBatch)(nil), // 41: drsync.v1.LinkTaskBatch - (*WorkItem)(nil), // 42: drsync.v1.WorkItem - (*WorkGrant)(nil), // 43: drsync.v1.WorkGrant - (*ShardSplit)(nil), // 44: drsync.v1.ShardSplit - (*ShardSplitAck)(nil), // 45: drsync.v1.ShardSplitAck - (*ShardCounters)(nil), // 46: drsync.v1.ShardCounters - (*ShardResult)(nil), // 47: drsync.v1.ShardResult - (*TaskResult)(nil), // 48: drsync.v1.TaskResult - (*TaskResultBatch)(nil), // 49: drsync.v1.TaskResultBatch - (*JournalRecord)(nil), // 50: drsync.v1.JournalRecord - (*JournalBatch)(nil), // 51: drsync.v1.JournalBatch - (*JournalAck)(nil), // 52: drsync.v1.JournalAck - (*LatencyHistogram)(nil), // 53: drsync.v1.LatencyHistogram - (*StatsReport)(nil), // 54: drsync.v1.StatsReport - (*WorkRequest_CachedOptions)(nil), // 55: drsync.v1.WorkRequest.CachedOptions - (*ShardSplit_NewShard)(nil), // 56: drsync.v1.ShardSplit.NewShard - (*ShardSplit_NewEntryList)(nil), // 57: drsync.v1.ShardSplit.NewEntryList - (*ShardSplit_BigFile)(nil), // 58: drsync.v1.ShardSplit.BigFile - (*ShardSplit_LinkSighting)(nil), // 59: drsync.v1.ShardSplit.LinkSighting - (*ShardSplit_DeleteRemainder)(nil), // 60: drsync.v1.ShardSplit.DeleteRemainder + (CopyOptions_ConflictPolicy)(0), // 8: drsync.v1.CopyOptions.ConflictPolicy + (JournalRecord_Type)(0), // 9: drsync.v1.JournalRecord.Type + (*StatInfo)(nil), // 10: drsync.v1.StatInfo + (*MountCaps)(nil), // 11: drsync.v1.MountCaps + (*Hello)(nil), // 12: drsync.v1.Hello + (*HelloAck)(nil), // 13: drsync.v1.HelloAck + (*MountHealth)(nil), // 14: drsync.v1.MountHealth + (*InflightItem)(nil), // 15: drsync.v1.InflightItem + (*Heartbeat)(nil), // 16: drsync.v1.Heartbeat + (*HeartbeatAck)(nil), // 17: drsync.v1.HeartbeatAck + (*Control)(nil), // 18: drsync.v1.Control + (*ProtocolError)(nil), // 19: drsync.v1.ProtocolError + (*FilterRule)(nil), // 20: drsync.v1.FilterRule + (*AclOptions)(nil), // 21: drsync.v1.AclOptions + (*MetadataOptions)(nil), // 22: drsync.v1.MetadataOptions + (*VerifyOptions)(nil), // 23: drsync.v1.VerifyOptions + (*CopyOptions)(nil), // 24: drsync.v1.CopyOptions + (*LimitOptions)(nil), // 25: drsync.v1.LimitOptions + (*TuningOptions)(nil), // 26: drsync.v1.TuningOptions + (*JobOptions)(nil), // 27: drsync.v1.JobOptions + (*WorkRequest)(nil), // 28: drsync.v1.WorkRequest + (*FileGen)(nil), // 29: drsync.v1.FileGen + (*WalkOverrides)(nil), // 30: drsync.v1.WalkOverrides + (*Shard)(nil), // 31: drsync.v1.Shard + (*EntryListShard)(nil), // 32: drsync.v1.EntryListShard + (*ChunkTask)(nil), // 33: drsync.v1.ChunkTask + (*DirMeta)(nil), // 34: drsync.v1.DirMeta + (*DirFixBatch)(nil), // 35: drsync.v1.DirFixBatch + (*VerifyEntry)(nil), // 36: drsync.v1.VerifyEntry + (*VerifyBatch)(nil), // 37: drsync.v1.VerifyBatch + (*DeleteBatch)(nil), // 38: drsync.v1.DeleteBatch + (*ProbeTask)(nil), // 39: drsync.v1.ProbeTask + (*LinkTask)(nil), // 40: drsync.v1.LinkTask + (*LinkEntry)(nil), // 41: drsync.v1.LinkEntry + (*LinkTaskBatch)(nil), // 42: drsync.v1.LinkTaskBatch + (*WorkItem)(nil), // 43: drsync.v1.WorkItem + (*WorkGrant)(nil), // 44: drsync.v1.WorkGrant + (*ShardSplit)(nil), // 45: drsync.v1.ShardSplit + (*ShardSplitAck)(nil), // 46: drsync.v1.ShardSplitAck + (*ShardCounters)(nil), // 47: drsync.v1.ShardCounters + (*ShardResult)(nil), // 48: drsync.v1.ShardResult + (*TaskResult)(nil), // 49: drsync.v1.TaskResult + (*TaskResultBatch)(nil), // 50: drsync.v1.TaskResultBatch + (*JournalRecord)(nil), // 51: drsync.v1.JournalRecord + (*JournalBatch)(nil), // 52: drsync.v1.JournalBatch + (*JournalAck)(nil), // 53: drsync.v1.JournalAck + (*LatencyHistogram)(nil), // 54: drsync.v1.LatencyHistogram + (*StatsReport)(nil), // 55: drsync.v1.StatsReport + (*WorkRequest_CachedOptions)(nil), // 56: drsync.v1.WorkRequest.CachedOptions + (*ShardSplit_NewShard)(nil), // 57: drsync.v1.ShardSplit.NewShard + (*ShardSplit_NewEntryList)(nil), // 58: drsync.v1.ShardSplit.NewEntryList + (*ShardSplit_BigFile)(nil), // 59: drsync.v1.ShardSplit.BigFile + (*ShardSplit_LinkSighting)(nil), // 60: drsync.v1.ShardSplit.LinkSighting + (*ShardSplit_DeleteRemainder)(nil), // 61: drsync.v1.ShardSplit.DeleteRemainder } var file_drsync_proto_depIdxs = []int32{ 1, // 0: drsync.v1.StatInfo.type:type_name -> drsync.v1.EntryType - 13, // 1: drsync.v1.Heartbeat.mounts:type_name -> drsync.v1.MountHealth - 14, // 2: drsync.v1.Heartbeat.inflight:type_name -> drsync.v1.InflightItem + 14, // 1: drsync.v1.Heartbeat.mounts:type_name -> drsync.v1.MountHealth + 15, // 2: drsync.v1.Heartbeat.inflight:type_name -> drsync.v1.InflightItem 3, // 3: drsync.v1.Control.command:type_name -> drsync.v1.Control.Command 4, // 4: drsync.v1.AclOptions.untranslatable:type_name -> drsync.v1.AclOptions.Untranslatable - 20, // 5: drsync.v1.MetadataOptions.acls:type_name -> drsync.v1.AclOptions + 21, // 5: drsync.v1.MetadataOptions.acls:type_name -> drsync.v1.AclOptions 5, // 6: drsync.v1.VerifyOptions.on_mismatch:type_name -> drsync.v1.VerifyOptions.OnMismatch 6, // 7: drsync.v1.CopyOptions.server_side_copy:type_name -> drsync.v1.CopyOptions.ServerSideCopy 7, // 8: drsync.v1.CopyOptions.fsync_mode:type_name -> drsync.v1.CopyOptions.FsyncMode - 19, // 9: drsync.v1.JobOptions.filters:type_name -> drsync.v1.FilterRule - 23, // 10: drsync.v1.JobOptions.copy:type_name -> drsync.v1.CopyOptions - 21, // 11: drsync.v1.JobOptions.metadata:type_name -> drsync.v1.MetadataOptions - 22, // 12: drsync.v1.JobOptions.verify:type_name -> drsync.v1.VerifyOptions - 24, // 13: drsync.v1.JobOptions.limits:type_name -> drsync.v1.LimitOptions - 25, // 14: drsync.v1.JobOptions.tuning:type_name -> drsync.v1.TuningOptions - 55, // 15: drsync.v1.WorkRequest.cached:type_name -> drsync.v1.WorkRequest.CachedOptions - 29, // 16: drsync.v1.Shard.overrides:type_name -> drsync.v1.WalkOverrides - 29, // 17: drsync.v1.EntryListShard.overrides:type_name -> drsync.v1.WalkOverrides - 28, // 18: drsync.v1.ChunkTask.gen:type_name -> drsync.v1.FileGen - 33, // 19: drsync.v1.DirFixBatch.dirs:type_name -> drsync.v1.DirMeta - 35, // 20: drsync.v1.VerifyBatch.entries:type_name -> drsync.v1.VerifyEntry - 28, // 21: drsync.v1.LinkTask.anchor_gen:type_name -> drsync.v1.FileGen - 28, // 22: drsync.v1.LinkEntry.anchor_gen:type_name -> drsync.v1.FileGen - 40, // 23: drsync.v1.LinkTaskBatch.links:type_name -> drsync.v1.LinkEntry - 30, // 24: drsync.v1.WorkItem.shard:type_name -> drsync.v1.Shard - 31, // 25: drsync.v1.WorkItem.entry_list:type_name -> drsync.v1.EntryListShard - 32, // 26: drsync.v1.WorkItem.chunk:type_name -> drsync.v1.ChunkTask - 34, // 27: drsync.v1.WorkItem.dirfix:type_name -> drsync.v1.DirFixBatch - 36, // 28: drsync.v1.WorkItem.verify:type_name -> drsync.v1.VerifyBatch - 37, // 29: drsync.v1.WorkItem.delete:type_name -> drsync.v1.DeleteBatch - 38, // 30: drsync.v1.WorkItem.probe:type_name -> drsync.v1.ProbeTask - 39, // 31: drsync.v1.WorkItem.link:type_name -> drsync.v1.LinkTask - 41, // 32: drsync.v1.WorkItem.link_batch:type_name -> drsync.v1.LinkTaskBatch - 42, // 33: drsync.v1.WorkGrant.items:type_name -> drsync.v1.WorkItem - 26, // 34: drsync.v1.WorkGrant.options:type_name -> drsync.v1.JobOptions - 56, // 35: drsync.v1.ShardSplit.subdirs:type_name -> drsync.v1.ShardSplit.NewShard - 57, // 36: drsync.v1.ShardSplit.entry_lists:type_name -> drsync.v1.ShardSplit.NewEntryList - 58, // 37: drsync.v1.ShardSplit.big_files:type_name -> drsync.v1.ShardSplit.BigFile - 59, // 38: drsync.v1.ShardSplit.link_sightings:type_name -> drsync.v1.ShardSplit.LinkSighting - 60, // 39: drsync.v1.ShardSplit.delete_remainders:type_name -> drsync.v1.ShardSplit.DeleteRemainder - 56, // 40: drsync.v1.ShardSplit.delete_subdirs:type_name -> drsync.v1.ShardSplit.NewShard - 2, // 41: drsync.v1.ShardResult.status:type_name -> drsync.v1.ResultStatus - 46, // 42: drsync.v1.ShardResult.counters:type_name -> drsync.v1.ShardCounters - 2, // 43: drsync.v1.TaskResult.status:type_name -> drsync.v1.ResultStatus - 10, // 44: drsync.v1.TaskResult.src_caps:type_name -> drsync.v1.MountCaps - 10, // 45: drsync.v1.TaskResult.dst_caps:type_name -> drsync.v1.MountCaps - 48, // 46: drsync.v1.TaskResultBatch.results:type_name -> drsync.v1.TaskResult - 8, // 47: drsync.v1.JournalRecord.type:type_name -> drsync.v1.JournalRecord.Type - 9, // 48: drsync.v1.JournalRecord.src:type_name -> drsync.v1.StatInfo - 9, // 49: drsync.v1.JournalRecord.dst:type_name -> drsync.v1.StatInfo - 53, // 50: drsync.v1.StatsReport.latencies:type_name -> drsync.v1.LatencyHistogram - 51, // [51:51] is the sub-list for method output_type - 51, // [51:51] is the sub-list for method input_type - 51, // [51:51] is the sub-list for extension type_name - 51, // [51:51] is the sub-list for extension extendee - 0, // [0:51] is the sub-list for field type_name + 8, // 9: drsync.v1.CopyOptions.on_dest_newer:type_name -> drsync.v1.CopyOptions.ConflictPolicy + 20, // 10: drsync.v1.JobOptions.filters:type_name -> drsync.v1.FilterRule + 24, // 11: drsync.v1.JobOptions.copy:type_name -> drsync.v1.CopyOptions + 22, // 12: drsync.v1.JobOptions.metadata:type_name -> drsync.v1.MetadataOptions + 23, // 13: drsync.v1.JobOptions.verify:type_name -> drsync.v1.VerifyOptions + 25, // 14: drsync.v1.JobOptions.limits:type_name -> drsync.v1.LimitOptions + 26, // 15: drsync.v1.JobOptions.tuning:type_name -> drsync.v1.TuningOptions + 56, // 16: drsync.v1.WorkRequest.cached:type_name -> drsync.v1.WorkRequest.CachedOptions + 30, // 17: drsync.v1.Shard.overrides:type_name -> drsync.v1.WalkOverrides + 30, // 18: drsync.v1.EntryListShard.overrides:type_name -> drsync.v1.WalkOverrides + 29, // 19: drsync.v1.ChunkTask.gen:type_name -> drsync.v1.FileGen + 34, // 20: drsync.v1.DirFixBatch.dirs:type_name -> drsync.v1.DirMeta + 36, // 21: drsync.v1.VerifyBatch.entries:type_name -> drsync.v1.VerifyEntry + 29, // 22: drsync.v1.LinkTask.anchor_gen:type_name -> drsync.v1.FileGen + 29, // 23: drsync.v1.LinkEntry.anchor_gen:type_name -> drsync.v1.FileGen + 41, // 24: drsync.v1.LinkTaskBatch.links:type_name -> drsync.v1.LinkEntry + 31, // 25: drsync.v1.WorkItem.shard:type_name -> drsync.v1.Shard + 32, // 26: drsync.v1.WorkItem.entry_list:type_name -> drsync.v1.EntryListShard + 33, // 27: drsync.v1.WorkItem.chunk:type_name -> drsync.v1.ChunkTask + 35, // 28: drsync.v1.WorkItem.dirfix:type_name -> drsync.v1.DirFixBatch + 37, // 29: drsync.v1.WorkItem.verify:type_name -> drsync.v1.VerifyBatch + 38, // 30: drsync.v1.WorkItem.delete:type_name -> drsync.v1.DeleteBatch + 39, // 31: drsync.v1.WorkItem.probe:type_name -> drsync.v1.ProbeTask + 40, // 32: drsync.v1.WorkItem.link:type_name -> drsync.v1.LinkTask + 42, // 33: drsync.v1.WorkItem.link_batch:type_name -> drsync.v1.LinkTaskBatch + 43, // 34: drsync.v1.WorkGrant.items:type_name -> drsync.v1.WorkItem + 27, // 35: drsync.v1.WorkGrant.options:type_name -> drsync.v1.JobOptions + 57, // 36: drsync.v1.ShardSplit.subdirs:type_name -> drsync.v1.ShardSplit.NewShard + 58, // 37: drsync.v1.ShardSplit.entry_lists:type_name -> drsync.v1.ShardSplit.NewEntryList + 59, // 38: drsync.v1.ShardSplit.big_files:type_name -> drsync.v1.ShardSplit.BigFile + 60, // 39: drsync.v1.ShardSplit.link_sightings:type_name -> drsync.v1.ShardSplit.LinkSighting + 61, // 40: drsync.v1.ShardSplit.delete_remainders:type_name -> drsync.v1.ShardSplit.DeleteRemainder + 57, // 41: drsync.v1.ShardSplit.delete_subdirs:type_name -> drsync.v1.ShardSplit.NewShard + 2, // 42: drsync.v1.ShardResult.status:type_name -> drsync.v1.ResultStatus + 47, // 43: drsync.v1.ShardResult.counters:type_name -> drsync.v1.ShardCounters + 2, // 44: drsync.v1.TaskResult.status:type_name -> drsync.v1.ResultStatus + 11, // 45: drsync.v1.TaskResult.src_caps:type_name -> drsync.v1.MountCaps + 11, // 46: drsync.v1.TaskResult.dst_caps:type_name -> drsync.v1.MountCaps + 49, // 47: drsync.v1.TaskResultBatch.results:type_name -> drsync.v1.TaskResult + 9, // 48: drsync.v1.JournalRecord.type:type_name -> drsync.v1.JournalRecord.Type + 10, // 49: drsync.v1.JournalRecord.src:type_name -> drsync.v1.StatInfo + 10, // 50: drsync.v1.JournalRecord.dst:type_name -> drsync.v1.StatInfo + 54, // 51: drsync.v1.StatsReport.latencies:type_name -> drsync.v1.LatencyHistogram + 52, // [52:52] is the sub-list for method output_type + 52, // [52:52] is the sub-list for method input_type + 52, // [52:52] is the sub-list for extension type_name + 52, // [52:52] is the sub-list for extension extendee + 0, // [0:52] is the sub-list for field type_name } func init() { file_drsync_proto_init() } @@ -5601,7 +5707,7 @@ func file_drsync_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_drsync_proto_rawDesc), len(file_drsync_proto_rawDesc)), - NumEnums: 9, + NumEnums: 10, NumMessages: 52, NumExtensions: 0, NumServices: 0, diff --git a/template.yaml b/template.yaml index 5e28e8c..afd05d6 100644 --- a/template.yaml +++ b/template.yaml @@ -49,6 +49,11 @@ spec: direct_write: true # write NEW files straight to their final name (faster # on GPFS/Weka; a crash leaves a partial, re-copied # next pass). Updates always use the atomic temp+rename. + on_dest_newer: skip # skip (default) | overwrite — a destination file whose + # mtime is newer than the source's is left untouched + # (recorded as JR_SKIPPED_NEWER) rather than overwritten. + # Set overwrite for a strict one-directional mirror where + # source must always win, regardless of destination mtime. metadata: owner: true # uid/gid (needs root on the agents) diff --git a/test/on_dest_newer_e2e.sh b/test/on_dest_newer_e2e.sh new file mode 100755 index 0000000..d39d513 --- /dev/null +++ b/test/on_dest_newer_e2e.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# drsync copy.on_dest_newer e2e: a merge-dataset scenario where the +# destination has been edited out-of-band and is now newer than the source +# for one file. Default (skip) must leave that file's content and mtime +# completely untouched and record JR_SKIPPED_NEWER — not silently overwrite +# it, which was the only behavior before this option existed (source always +# won regardless of mtime direction, agent/src/walker.c's times_equal is an +# unsigned |diff| check). Also covers the ordinary "source is newer" case +# still copying normally, and the explicit on_dest_newer: overwrite opt-out +# restoring the old always-overwrite behavior. +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +. "$ROOT/test/lib.sh" +WORK=$(mktemp -d "${TMPDIR:-/tmp}/drsync-destnewer.XXXXXX") +read -r _CP _HP < <(pick_ports) +CP=${CP:-$_CP}; HP=${HP:-$_HP} +API="http://127.0.0.1:${HP}"; AUTH="Authorization: Bearer destnewertok" +PASS=0 +cleanup() { + for p in "${APID:-}" "${CPID:-}"; do [[ -n "$p" ]] && kill "$p" 2>/dev/null || true; done + wait 2>/dev/null || true + if [[ $PASS -eq 1 ]]; then rm -rf "$WORK"; else echo "work dir kept: $WORK"; fi +} +trap cleanup EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } +export DRSYNC_SERVER="$API" DRSYNC_TOKEN=destnewertok + +API_TOKEN_FILE="$WORK/api-token" +echo -n destnewertok >"$API_TOKEN_FILE" +chmod 600 "$API_TOKEN_FILE" +DRSYNC="$ROOT/bin/drsync" + +wait_completed() { + local job=$1 i + for i in $(seq 1 60); do + curl -sf -H "$AUTH" "$API/api/v1/jobs/$job" | grep -q '"state":"COMPLETED"' && return 0 + sleep 0.25 + done + tail -n 8 "$WORK"/agent.log "$WORK"/coord.log + fail "$job did not converge" +} + +# --- build ------------------------------------------------------------------- +make -C "$ROOT/agent" -s +( cd "$ROOT" && go build -o bin/drsyncd ./coordinator/cmd/drsyncd \ + && go build -o bin/drsync ./cli/drsync ) + +SRC="$WORK/src"; DST="$WORK/dst" +mkdir -p "$SRC" +echo "source v1" > "$SRC/a.txt" +echo "source v1" > "$SRC/b.txt" +echo "source v1" > "$SRC/c.txt" + +# --- services ---------------------------------------------------------------- +"$ROOT/bin/drsyncd" -data-dir "$WORK/coord" -listen-agent 127.0.0.1:$CP \ + -listen-http 127.0.0.1:$HP -api-token-file "$API_TOKEN_FILE" -log-level warn \ + >"$WORK/coord.log" 2>&1 & +CPID=$! +wait_coordinator "$API" "$AUTH" || exit 1 +"$ROOT/agent/bin/drsync-agent" -c 127.0.0.1:$CP -i destnewer-agent -w 4 -C 4 \ + >"$WORK/agent.log" 2>&1 & +APID=$! +sleep 1 + +cat > "$WORK/job.yaml" < "$DST/a.txt" +touch -d "+1 hour" "$DST/a.txt" + +# b.txt: source edited normally (source newer, ordinary case) — must still +# copy exactly as before this option existed. +echo "source v2 (should win)" > "$SRC/b.txt" + +# c.txt: left alone on both sides — must stay "clean", not spuriously +# skipped or recopied. + +# --- pass 2: default on_dest_newer (skip) -------------------------------------- +has() { local pat=$1; shift; "$@" | grep -q -- "$pat"; } +has "pass triggered" "$DRSYNC" pass trigger destnewer || fail "pass 2 trigger failed" +wait_completed destnewer + +[[ "$(cat "$DST/a.txt")" == "dest edited (should survive)" ]] \ + || fail "pass 2: a.txt was overwritten despite being newer at the destination (default should be skip)" +[[ "$(cat "$DST/b.txt")" == "source v2 (should win)" ]] \ + || fail "pass 2: b.txt (source newer, ordinary case) was not copied" +[[ "$(cat "$DST/c.txt")" == "source v1" ]] || fail "pass 2: c.txt (unchanged) diverged unexpectedly" + +"$DRSYNC" report destnewer --json > "$WORK/report2.json" +python3 - "$WORK/report2.json" <<'EOF' || fail "pass 2 report: unexpected totals" +import json, sys +r = json.load(open(sys.argv[1])) +t = r["totals"] +assert t["errors"] == 0, t["errors"] +assert t["skipped_newer"] == 1, t["skipped_newer"] +EOF + +SKIPPED=$(curl -sf -H "$AUTH" "$API/api/v1/jobs/destnewer/journal?type=SKIPPED_NEWER&pass=2" | python3 -c ' +import sys, json +recs = json.load(sys.stdin).get("records", []) +print("\n".join(r.get("rel_path", "") for r in recs)) +') +grep -qF "a.txt" <<<"$SKIPPED" \ + || fail "a.txt not recorded as JR_SKIPPED_NEWER in pass 2's journal (got: $SKIPPED)" + +echo "default on_dest_newer (skip): dest-newer file preserved, source-newer file still copied, clean file untouched" + +# --- pass 3: on_dest_newer: overwrite must restore the old always-wins behavior +cat > "$WORK/job2.yaml" < "$SRC/bigdir/f$(printf %04d "$i").txt"; done echo nested > "$SRC/bigdir/sub/deep.txt" # subdir inside the split dir echo stale > "$DST/bigdir/f0001.txt" # must be replaced +touch -d '2000-01-01' "$DST/bigdir/f0001.txt" # older than the source it stands in + # for — copy.on_dest_newer defaults to + # skip, so an unbackdated destination + # write here (which lands at "now", + # after the source's own write above) + # would read as newer and be left + # alone instead of replaced, which is + # not what this fixture is testing echo orphan > "$DST/bigdir/zzz-orphan.txt" # dst-only → orphan (report-only) head -c 629145600 /dev/urandom > "$SRC/huge.bin" # 600 MiB → 3 ranges HUGE_SUM=$(sha256sum "$SRC/huge.bin" | cut -d' ' -f1) diff --git a/webui/console.html b/webui/console.html index da00191..fea6b0b 100644 --- a/webui/console.html +++ b/webui/console.html @@ -1011,7 +1011,7 @@

New job

const kindLabel = k => k === "entrylist" ? "large-dir" : k; // ---------- job template (embedded verbatim from ../template.yaml) ---------- - const JOB_TEMPLATE = "# drsync job template \u2014 copy this, edit source/destination, and submit with:\n# drsync job submit template.yaml --start\n#\n# Every value below is the shipped default unless noted, so a minimal job only\n# needs apiVersion, kind, metadata.name and the two paths \u2014 the rest can be\n# deleted to inherit defaults. Sizes accept KiB/MiB/GiB/TiB suffixes (binary);\n# plain integers are bytes or counts. Only the fields shown here are recognised;\n# unknown keys are rejected at submit time. Full reference: docs/DESIGN-jobspec.md\n# and docs/ADMIN.md.\n\napiVersion: drsync/v1\nkind: Job\n\nmetadata:\n name: example-sync # REQUIRED, unique; also the journal directory name\n description: \"example drsync job\" # optional, free text\n\nspec:\n # REQUIRED. Absolute paths, identical on every agent host, and disjoint from\n # each other. These are the roots on the source and destination mounts.\n source:\n path: /mnt/src/data\n destination:\n path: /mnt/dst/data\n\n # Optional include/exclude rules, evaluated in order \u2014 first match wins; no\n # match keeps the entry (implicit `include: \"**\"`). Globs: ? and * stop at /,\n # ** crosses /. Max 64 rules, each pattern <= 255 bytes. Delete this block to\n # copy everything.\n filters:\n - exclude: \"**/.snapshot/**\" # snapshot dirs of either filesystem\n - exclude: \"**/*.tmp\"\n\n passes:\n max: 5 # hard ceiling; convergence usually stops sooner\n schedule: continuous # continuous | manual (operator triggers each pass)\n converge_when: # stop once a pass's delta is under EITHER (OR-combined)\n delta_files_below: 0 # 0 = only the zero-delta fixpoint stops the job\n delta_bytes_below: 0 # e.g. 50GiB to stop while a small delta remains\n\n copy:\n chunk_threshold: 24GiB # files >= this are eligible to split into chunk tasks\n chunk_size: 8GiB # bytes per chunk; a file > this fans out across agents\n buffer_size: 1MiB # copy buffer unit\n preserve_sparse: true # SEEK_HOLE/DATA, with zero-detect fallback\n server_side_copy: auto # auto | off | require (copy_file_range / reflink)\n temp_naming: \".drsync.tmp.\" # prefix for in-progress destination names\n fsync: batched # per_file | batched\n direct_write: true # write NEW files straight to their final name (faster\n # on GPFS/Weka; a crash leaves a partial, re-copied\n # next pass). Updates always use the atomic temp+rename.\n\n metadata:\n owner: true # uid/gid (needs root on the agents)\n mode: true # permission bits\n times: true # atime + mtime, ns precision\n xattrs: true # all readable xattr namespaces\n specials: true # device nodes, FIFOs, sockets (needs root)\n acls:\n posix: true\n nfs4: true\n untranslatable: warn # warn | fail | skip \u2014 how to treat an untranslatable ACL\n hardlinks: preserve # preserve (default) | report \u2014 preserve links\n # nlink>1 files to a shared destination copy;\n # report copies them independently (D3 behavior;\n # set this to opt out) \u2014 docs/DESIGN-hardlinks.md\n hardlinks_max_group_scan: 0 # cap a link group's member count before giving up and\n # falling back to independent copies; 0 = unlimited\n\n # Mount probe: at pass start each agent verifies its source and destination\n # roots before any bulk work runs, gating the whole pass until all agents pass.\n probe:\n require_mount: true # require each root to sit on a real mounted\n # filesystem, so an unmounted volume's leftover\n # stub directory parks the pass instead of syncing\n # into the underlying rootfs. Set false only when a\n # root legitimately lives on the host root filesystem.\n\n verify:\n mode: \"on\" # on | off \u2014 off skips the verify phase entirely\n checksum:\n sample_rate: 0.01 # deterministic fraction of copied files re-checksummed\n on_mismatch: recopy # recopy | fail\n\n deletes:\n mode: mirror # report | mirror \u2014 mirror deletes destination orphans\n # and requires a second explicit gate at pass-trigger time\n\n limits:\n bandwidth_per_agent: 0 # 0 = unlimited; else bytes/s per agent\n iops_per_agent: 0 # 0 = unlimited\n\n # Rarely changed; defaults are sized for a small fleet of fast agents.\n tuning:\n shard_budget: 2000 # entries a walker processes before pushing subdirs back\n dir_split_threshold: 50000 # single-directory size that triggers entry-list sharding\n entrylist_batch: 4000 # names per entry-list shard (sets a huge dir's fan-out)\n delete_split_threshold: 200000 # single orphan-directory size that triggers delete sharding\n delete_split_batch: 20000 # names per split-produced DELETE shard\n delete_shard_budget: 250000 # objects one DELETE shard removes before pushing subdirs back\n statx_batch: 256 # in-flight statx per walker = io_uring ring depth (1\u20134096)\n mtime_slop_ns: 1000000 # 1ms slop for cross-filesystem timestamp granularity\n spread_mode: auto # auto | off | always \u2014 coordinator-side walk fan-out\n spread_target_per_agent: 32 # walk shards per agent to aim for while spreading\n\n # Optional email; inert unless the coordinator has an SMTP config (-smtp-config).\n # Delete this block for no notifications.\n notifications:\n recipients: # required if either flag below is set\n - ops@example.com\n on_pass_complete: false # email as each pass finishes (the convergence trace)\n on_job_complete: false # one summary email when the job reaches COMPLETED\n # (per-pass table includes each pass's duration)\n # Parked-shard alerts are NOT a flag here: as soon as any shard hits its\n # retry ceiling, `recipients` above gets an email automatically (batched\n # per job per check), independent of the two flags \u2014 see DESIGN-jobspec.md \u00a71.2.\n"; + const JOB_TEMPLATE = "# drsync job template \u2014 copy this, edit source/destination, and submit with:\n# drsync job submit template.yaml --start\n#\n# Every value below is the shipped default unless noted, so a minimal job only\n# needs apiVersion, kind, metadata.name and the two paths \u2014 the rest can be\n# deleted to inherit defaults. Sizes accept KiB/MiB/GiB/TiB suffixes (binary);\n# plain integers are bytes or counts. Only the fields shown here are recognised;\n# unknown keys are rejected at submit time. Full reference: docs/DESIGN-jobspec.md\n# and docs/ADMIN.md.\n\napiVersion: drsync/v1\nkind: Job\n\nmetadata:\n name: example-sync # REQUIRED, unique; also the journal directory name\n description: \"example drsync job\" # optional, free text\n\nspec:\n # REQUIRED. Absolute paths, identical on every agent host, and disjoint from\n # each other. These are the roots on the source and destination mounts.\n source:\n path: /mnt/src/data\n destination:\n path: /mnt/dst/data\n\n # Optional include/exclude rules, evaluated in order \u2014 first match wins; no\n # match keeps the entry (implicit `include: \"**\"`). Globs: ? and * stop at /,\n # ** crosses /. Max 64 rules, each pattern <= 255 bytes. Delete this block to\n # copy everything.\n filters:\n - exclude: \"**/.snapshot/**\" # snapshot dirs of either filesystem\n - exclude: \"**/*.tmp\"\n\n passes:\n max: 5 # hard ceiling; convergence usually stops sooner\n schedule: continuous # continuous | manual (operator triggers each pass)\n converge_when: # stop once a pass's delta is under EITHER (OR-combined)\n delta_files_below: 0 # 0 = only the zero-delta fixpoint stops the job\n delta_bytes_below: 0 # e.g. 50GiB to stop while a small delta remains\n\n copy:\n chunk_threshold: 24GiB # files >= this are eligible to split into chunk tasks\n chunk_size: 8GiB # bytes per chunk; a file > this fans out across agents\n buffer_size: 1MiB # copy buffer unit\n preserve_sparse: true # SEEK_HOLE/DATA, with zero-detect fallback\n server_side_copy: auto # auto | off | require (copy_file_range / reflink)\n temp_naming: \".drsync.tmp.\" # prefix for in-progress destination names\n fsync: batched # per_file | batched\n direct_write: true # write NEW files straight to their final name (faster\n # on GPFS/Weka; a crash leaves a partial, re-copied\n # next pass). Updates always use the atomic temp+rename.\n on_dest_newer: skip # skip (default) | overwrite — a destination file whose\n # mtime is newer than the source's is left untouched\n # (recorded as JR_SKIPPED_NEWER) rather than overwritten.\n # Set overwrite for a strict one-directional mirror where\n # source must always win, regardless of destination mtime.\n\n metadata:\n owner: true # uid/gid (needs root on the agents)\n mode: true # permission bits\n times: true # atime + mtime, ns precision\n xattrs: true # all readable xattr namespaces\n specials: true # device nodes, FIFOs, sockets (needs root)\n acls:\n posix: true\n nfs4: true\n untranslatable: warn # warn | fail | skip \u2014 how to treat an untranslatable ACL\n hardlinks: preserve # preserve (default) | report \u2014 preserve links\n # nlink>1 files to a shared destination copy;\n # report copies them independently (D3 behavior;\n # set this to opt out) \u2014 docs/DESIGN-hardlinks.md\n hardlinks_max_group_scan: 0 # cap a link group's member count before giving up and\n # falling back to independent copies; 0 = unlimited\n\n # Mount probe: at pass start each agent verifies its source and destination\n # roots before any bulk work runs, gating the whole pass until all agents pass.\n probe:\n require_mount: true # require each root to sit on a real mounted\n # filesystem, so an unmounted volume's leftover\n # stub directory parks the pass instead of syncing\n # into the underlying rootfs. Set false only when a\n # root legitimately lives on the host root filesystem.\n\n verify:\n mode: \"on\" # on | off \u2014 off skips the verify phase entirely\n checksum:\n sample_rate: 0.01 # deterministic fraction of copied files re-checksummed\n on_mismatch: recopy # recopy | fail\n\n deletes:\n mode: mirror # report | mirror \u2014 mirror deletes destination orphans\n # and requires a second explicit gate at pass-trigger time\n\n limits:\n bandwidth_per_agent: 0 # 0 = unlimited; else bytes/s per agent\n iops_per_agent: 0 # 0 = unlimited\n\n # Rarely changed; defaults are sized for a small fleet of fast agents.\n tuning:\n shard_budget: 2000 # entries a walker processes before pushing subdirs back\n dir_split_threshold: 50000 # single-directory size that triggers entry-list sharding\n entrylist_batch: 4000 # names per entry-list shard (sets a huge dir's fan-out)\n delete_split_threshold: 200000 # single orphan-directory size that triggers delete sharding\n delete_split_batch: 20000 # names per split-produced DELETE shard\n delete_shard_budget: 250000 # objects one DELETE shard removes before pushing subdirs back\n statx_batch: 256 # in-flight statx per walker = io_uring ring depth (1\u20134096)\n mtime_slop_ns: 1000000 # 1ms slop for cross-filesystem timestamp granularity\n spread_mode: auto # auto | off | always \u2014 coordinator-side walk fan-out\n spread_target_per_agent: 32 # walk shards per agent to aim for while spreading\n\n # Optional email; inert unless the coordinator has an SMTP config (-smtp-config).\n # Delete this block for no notifications.\n notifications:\n recipients: # required if either flag below is set\n - ops@example.com\n on_pass_complete: false # email as each pass finishes (the convergence trace)\n on_job_complete: false # one summary email when the job reaches COMPLETED\n # (per-pass table includes each pass's duration)\n # Parked-shard alerts are NOT a flag here: as soon as any shard hits its\n # retry ceiling, `recipients` above gets an email automatically (batched\n # per job per check), independent of the two flags \u2014 see DESIGN-jobspec.md \u00a71.2.\n"; // ---------- live state ---------- const S = { jobs:[], queue:{depth:[],parked:[]}, agents:[], rates:{}, glob:{}, @@ -1574,6 +1574,7 @@

New job

["WOULD_COPY", "would copy (dry-run)", "warn"], ["WOULD_DELETE", "would delete (dry-run)", "warn"], ["NLINK_DUP", "hardlink duplicates", "warn"], ["ORPHAN", "orphans observed", "warn"], ["SRC_CHANGED", "source changed mid-copy", "warn"], + ["SKIPPED_NEWER", "skipped (dest newer)", "warn"], ["ERROR", "errors", "crit"], ["FIDELITY_EXCEPTION", "fidelity exceptions", "crit"], ["VERIFY_FAIL", "verify failed", "crit"], ];