From be375fb3122caa64f4fdadb4eed40a88b9fa9cc2 Mon Sep 17 00:00:00 2001 From: Bo Lorentsen Date: Sat, 24 Oct 2026 16:36:40 +0200 Subject: [PATCH] homehub: implement + validate sync, tombstones, manifest, alerts Tested end-to-end with two throwaway homes against a scratch hub root: deletion propagation, new-machine join safety, edit-wins-over-delete, reaping, forget, alert timing. Fixes found by those tests: - harvest deletions BEFORE the pull (pull used to undo every deletion) - gate attribution on $STATE/seen (a new machine would have tombstoned every hub file its peers had just uploaded) - active_path probed '+ /dir***' instead of '+ /dir/***' -> every tombstone looked out-of-scope, so deletions never applied - tombstone excludes must precede includes in the filter (first match wins) - rsync must get -e with the deploy key; harvest stderr must not be dropped - guard hub sync root that already holds unrelated data (178 G here) - scrub credentials out of repo remote URLs before publishing the manifest --- DESIGN.md | 27 ++++++++- README.md | 2 +- bin/homehub | 154 ++++++++++++++++++++++++++++++++++++++++------------ config | 12 ++-- 4 files changed, 150 insertions(+), 45 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index d8f52b3..843fd18 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,6 +1,29 @@ # homehub — lean home-drive sync across 3–4 machines via Synology -Status: **design, no code yet**. Open decisions in §13. +Status: **implemented and tested on g9; no personal file data uploaded yet.** Decisions in §13, +operations in `README.md`. + +### Verification status (2026-10-24) + +| Behaviour | Method | Result | +|---|---|---| +| default-deny filter, shadow warnings | `homehub filter` on real `$HOME` | 35 include / 95 deny rules, no shadowing | +| pending set is genuinely personal | dry run on real `$HOME` | 571 files, 470 MB (docs, photos, dotfiles, pi sessions) | +| class B manifest, no byte-sync of repos | `git inventory` | 114 repos in 11 s, credentials scrubbed from 3 remotes | +| class C union merge + idempotency | synthetic 2-host export | 53 urls, re-merge stable | +| alert timing (silent → 24 h → once) | backdated `broken.auth` | silent at 0 m, single alert at 2 d | +| new machine must not tombstone the hub | 2 fake homes + scratch hub root | 5 never-seen hub files correctly ignored | +| deletion propagates, never resurrects | same | harvested pre-pull; peer applied it; no resurrection | +| **edit wins over a later delete** | same | re-pulled peer's newer copy, classified `EDITED-ELSEWHERE` | +| reaper preserves the last copy | `GRACE_DAYS=0` | hub copy moved to `.hub/prev/reaped/2026-10/…`, tombstone pruned | +| `forget` escape hatch | same | file travels again after forgetting it | + +Not yet exercised: 3+ machine convergence, suspend/resume with `Persistent=true`, +real mtime-conflict resolution across skewed clocks, and the bookmark write-back +(v1 deliberately stops before overwriting the live file). + +**Blocking step before any real sync:** `ssh-copy-id -i ~/.ssh/homehub_ed25519.pub nas.lue.dk` +— see §0 row on the passphrase/agent problem. ## 0. Ground truth this design was built against @@ -256,7 +279,7 @@ freshness monitor, not an error notifier. - Transport: `sendmail` → your existing `mail.lue.dk`. Works with no GUI session, which `notify-send` does not. -## 11. Files to be created (this repo = the policy; ~200 lines of shell total) +## 11. Files to be created (~750 lines of bash in total — the comments are half of it) ``` ~/homehub/ # small git repo, hub-backed via a bare repo on the NAS diff --git a/README.md b/README.md index cc4c590..31226d4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ is only the operational half. ``` ~/homehub/ - bin/homehub the tool (~450 lines of bash + one python merge) + bin/homehub the tool (~750 lines of bash + one python merge) config machine-specific settings (per-machine, not synced) rules/denylist DEFAULT DENY, emitted first, wins over everything rules/sets.d/*.list named path-lists ("sets") diff --git a/bin/homehub b/bin/homehub index f03aa26..cc98c2e 100755 --- a/bin/homehub +++ b/bin/homehub @@ -19,7 +19,9 @@ HH_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) # shellcheck disable=SC1091 . "$HH_ROOT/config" -HOST=$(hostname -s) +# HH_HOST lets the deletion/convergence semantics be tested with two throwaway +# "machines" on one box; leave unset in normal use. +HOST=${HH_HOST:-$(hostname -s)} HUB_URL="$HUB_HOST:$HUB_PATH" RULES=$HH_ROOT/rules STATE=${STATE_ROOT:?config: STATE_ROOT unset} @@ -115,33 +117,36 @@ check_shadowing() { # $1 set name done < <(awk '!/^[[:space:]]*#/ && NF' "$RULES/denylist" 2>/dev/null) done < <(awk '!/^[[:space:]]*#/ && NF' "$f") } -# base filter: deny first, then includes (+ ancestors), NO trailing default-deny +# Filter composition. Order is load-bearing: rsync applies FIRST MATCH WINS, so +# 1. denylist (default-deny trees, wins over everything) +# 2. tombstone excludes (pull only — must precede the includes or they never fire) +# 3. includes (this host's sets + their traversable ancestors) +# 4. "- *" (default deny) +# Two variants: the push filter must NOT exclude tombstones, or a file you re-create +# after deleting it could never travel again (edits win, §3). gen_filter() { local sets s sets=$(sets_for_host) || return 1 - { - echo '- /.hub/***' # hub metadata can never leak into a home + + { echo '- /.hub/***' # hub metadata can never leak into a home [ -r "$RULES/denylist" ] && rules_from "$RULES/denylist" '-' - for s in $sets; do + } | awk '!seen[$0]++' > "$FILTER.deny" || return 1 + + { for s in $sets; do [ -r "$RULES/sets.d/$s.list" ] || { warn "unknown set '$s' (no rules/sets.d/$s.list)"; continue; } { rules_from "$RULES/sets.d/$s.list" '+' awk '!/^[[:space:]]*#/ && NF' "$RULES/sets.d/$s.list" | while read -r l; do ancestors_of "$l"; done; } check_shadowing "$s" done - } | awk '!seen[$0]++' > "$FILTER.base.tmp" || return 1 - mv "$FILTER.base.tmp" "$FILTER.base" - # push filter: default deny only - { cat "$FILTER.base"; echo '- *'; } > "$FILTER.tmp" && mv "$FILTER.tmp" "$FILTER" - # pull filter: default deny, PLUS still-tombstoned paths excluded so a deleted - # file is not re-downloaded every cycle for 30 days only to be re-deleted. - # Tombstones never enter the push filter: a file re-created after the delete - # must still be able to travel (edits win — DESIGN.md §3). - { - cat "$FILTER.base" - awk -F'\t' 'NF>=2 { p=$2; sub(/^\//, "", p); if (p != "" && p !~ /^\.hub\// && p !~/\.\./) { - print "- /" p; print "- /" p "/***" } }' "$STATE/tombstones.all" 2>/dev/null - echo '- *' - } | awk '!seen[$0]++' > "$FILTER.pull.tmp" && mv "$FILTER.pull.tmp" "$FILTER.pull" + } | awk '!seen[$0]++' > "$FILTER.incl" || return 1 + + awk -F'\t' 'NF>=2 { p=$2; sub(/^\//, "", p) + if (p != "" && p !~ /^\.hub\// && p !~ /\.\./) { print "- /" p; print "- /" p "/***" } }' \ + "$STATE/tombstones.all" 2>/dev/null | awk '!seen[$0]++' > "$FILTER.tomb" + + cat "$FILTER.deny" "$FILTER.tomb" "$FILTER.incl" > "$FILTER.base" + { cat "$FILTER.deny" "$FILTER.incl"; echo '- *'; } > "$FILTER.tmp" && mv "$FILTER.tmp" "$FILTER" + { cat "$FILTER.deny" "$FILTER.tomb" "$FILTER.incl"; echo '- *'; } > "$FILTER.tmp.pull" && mv "$FILTER.tmp.pull" "$FILTER.pull" } # --------------------------------------------------------------- rsync bits -- @@ -172,6 +177,20 @@ count_items() { grep -cE '^[<>]' "$1" 2>/dev/null || true; } show_head() { grep -E '^[<>]' "$1" 2>/dev/null | head -n "${2:-10}"; } # ----------------------------------------------------- deletions (tombstones) -- +# $STATE/seen = every path this machine has actually transferred (pushed or pulled). +# It is what makes "on the hub but not local" decidable: a file we transferred and +# then removed is OUR deletion and may propagate; a file only a peer ever uploaded is +# not. Without it, a newly added machine harvests every existing hub file as its own +# deletion and tombstones the whole hub on join. Built from rsync's own itemized +# output (no extra round trip); its failure mode is the safe one — a file that is not +# recognised is never deleted. +seen_update() { + { [ -s "$STATE/seen" ] && cat "$STATE/seen" + awk '/^[<>]/ { sub(/^[^|]*\|/, ""); if (length($0)) print }' \ + "$STATE/pull.out" "$STATE/push.out" 2>/dev/null + } | sed '/^\.$/d' | sort -u > "$STATE/seen.new" && mv "$STATE/seen.new" "$STATE/seen" +} + # Format on the hub: .hub/tombstones/ lines = epoch \t home-rel-path hub_tombstones_all() { hub_script "'$HUB_PATH'" <<'EOS' @@ -188,8 +207,11 @@ EOS active_path() { # $1 = path without leading slash local p="/$1" while :; do - grep -qxF "+ $p***" "$FILTER" && return 0 # inside an included subtree - grep -qxF "+ $p" "$FILTER" && return 0 # is itself an included file + # NOTE the slash before ***: the include rules are "+ /dir/***", so probing + # "+ /dir***" would never match and every tombstone would look out-of-scope. + grep -qxF "+ $p/***" "$FILTER" && return 0 # inside an included subtree + grep -qxF "+ $p" "$FILTER" && return 0 # is itself an included file + grep -qxF "+ $p/" "$FILTER" && return 0 # an included ancestor dir [ "${p%/*}" = "$p" ] && return 1 p=${p%/*} [ -z "$p" ] && return 1 @@ -218,20 +240,73 @@ apply_tombstones() { # deletions recorded by any machine, edit-wins (§3) done < "$all" v "tombstones: applied=$n edited-since-so-kept=$kept outside-sets=$out malformed=$bad" } -harvest_tombstones() { # record what this machine deleted; runs only after a real pull - local new=$STATE/tomb.new merged=$STATE/tomb.merged n ep - run_rsync 1 --delete "$HOME/" "$HUB_URL/" 2>/dev/null | - awk -F'|' '$1 ~ /^\*/ { sub(/^\//, "", $2); print $2 }' | sort -u > "$new" +harvest_tombstones() { # record what this machine deleted — BEFORE any pull, else + # the pull re-downloads the file and the deletion is silently lost forever. + # A hub copy newer than our last successful sync was edited by someone else + # since we last looked, so it is an edit, not a deletion: leave it alone (§3). + local new=$STATE/tomb.new merged=$STATE/tomb.merged n ep last rc=0 + # stderr is kept, never sent to /dev/null: an ssh / known_hosts / transport error + # here would otherwise look exactly like "nothing was deleted" and silently stop + # deletions propagating anywhere, forever. + run_rsync 1 "$FILTER" --delete "$HOME/" "$HUB_URL/" > "$STATE/harvest.out" 2> "$STATE/harvest.err" || rc=$? + if [ "$rc" != 0 ]; then + warn "deletion harvest failed (rsync rc=$rc): $(tr '\n' ' ' < "$STATE/harvest.err" | tail -c 200)" + return 1 + fi + awk -F'|' '$1 ~ /^\*/ { sub(/^\//, "", $2); print $2 }' "$STATE/harvest.out" | sort -u > "$new.raw" + # intersect with what we saw at the hub on the previous sync (first sync: no cache, + # so nothing can be claimed as a deletion — correct, nothing was ever ours to delete) + if [ -s "$STATE/seen" ]; then + comm -12 "$new.raw" "$STATE/seen" > "$new" + else + : > "$new" + v "nothing transferred by this machine yet: no deletions are attributable to it" + fi + local skipped; skipped=$(($(grep -c . "$new.raw" 2>/dev/null || true) - $(grep -c . "$new" 2>/dev/null || true))) + [ "${skipped:-0}" -gt 0 ] && v "ignored $skipped hub file(s) this machine never had (not our deletions)" n=$(grep -c . "$new" 2>/dev/null || true) - [ "${n:-0}" = 0 ] && { v "no local deletions to record"; return 0; } + if [ "${n:-0}" = 0 ]; then v "no local deletions to record"; return 0; fi + + last=$(hub "date -r '$HUB_PATH/.hub/meta/markers/$HOST' +%s 2>/dev/null || echo 0") + local kept=$STATE/tomb.kept edited=$STATE/tomb.edited + # Candidates exist on the hub by definition (hub has it, we do not), so spool the + # list there and let the remote loop stat it. A heredoc cannot share stdin with a + # pipe, which is why this is two ssh calls rather than one. + hub "mkdir -p '$HUB_PATH/.hub/tmp' && cat > '$HUB_PATH/.hub/tmp/cand.$HOST'" < "$new" || return 1 + hub_script "'$HUB_PATH' '${last:-0}' '$HOST'" > "$kept" <<'EOS' +root=$1; last=$2; me=$3 +cf="$root/.hub/tmp/cand.$me" +[ -f "$cf" ] || exit 0 +while IFS= read -r p; do + [ -n "$p" ] || continue + m=$(date -r "$root/$p" +%s 2>/dev/null || echo 0) + if [ "$last" != 0 ] && [ "$m" -gt "$last" ]; then + printf '%s\tEDITED-ELSEWHERE\n' "$p" + else + printf '%s\n' "$p" + fi +done < "$cf" +rm -f "$cf" +exit 0 +EOS + grep -c 'EDITED-ELSEWHERE' "$kept" 2>/dev/null | tr -d ' ' > "$edited" || echo 0 > "$edited" + grep -v 'EDITED-ELSEWHERE' "$kept" > "$kept.paths" || true + if [ "$(grep -c . "$kept.paths" 2>/dev/null || true)" = 0 ]; then + v "no deletions recorded ($(cat "$edited") path(s) were edited elsewhere -> kept as edits)"; return 0 + fi + ep=$(now) { hub "cat '$HUB_PATH/.hub/tombstones/$HOST' 2>/dev/null" - awk -v e="$ep" '{ print e "\t" $0 }' "$new" + awk -v e="$ep" 'NF { print e "\t" $0 }' "$kept.paths" } | awk -F'\t' 'NF>=2 { p=$2; if (!(p in e) || $1+0 < e[p]) e[p]=$1+0 } END { for (p in e) printf "%d\t%s\n", e[p], p }' | sort -t$'\t' -k2 > "$merged" hub "mkdir -p '$HUB_PATH/.hub/tombstones' && cat > '$HUB_PATH/.hub/tombstones/$HOST'" < "$merged" || { warn "could not write tombstones"; return 1; } - v "recorded $n deletion(s) as tombstones on the hub" + # fold into the local cache and refresh the pull filter so the pull cannot undo us + cat "$merged" >> "$STATE/tombstones.all" + sort -u -t$'\t' -k2,2 "$STATE/tombstones.all" -o "$STATE/tombstones.all" + gen_filter >/dev/null 2>&1 || true + v "recorded $(grep -c . "$kept.paths") deletion(s) as tombstones ($(cat "$edited") edited elsewhere, kept)" } reap() { # stop resurrecting files deleted more than GRACE_DAYS ago (§5) hub_script "'$HUB_PATH' '${GRACE_DAYS:-30}'" <<'EOS' || true @@ -261,7 +336,7 @@ EOS # $HOME on the first pull (and could be reaped by a tombstone). This guard exists # because /volume1/homes/bl really did hold 178 G of unrelated legacy data. check_hub_stray() { - [ -r "$FILTER.base" ] || return 0 + [ -r "$FILTER.base" ] || return 1 # cannot judge -> do not block the sync local allowed stray allowed=$(printf '.hub\n@eaDir\n'; grep '^+ /' "$FILTER.base" | sed -E 's#^\+ /([^/]+).*#\1#; s#/$##' | sort -u) stray=$(hub "cd '$HUB_PATH' 2>/dev/null && ls -A" | grep -vxF -f <(printf '%s\n' "$allowed") || true) @@ -290,18 +365,26 @@ cmd_sync() { esac clear_broken auth hub_setup_dirs - if check_hub_stray; then - [ $exec = 1 ] && [ "${HH_FORCE:-0}" != 1 ] && - die "refusing to sync into a hub root that is not ours. Fix HUB_PATH in config, or move that data away, or re-run with HH_FORCE=1 if you really mean it" - fi # tombstones must be known BEFORE the filter is built: they shape the pull filter hub_tombstones_all > "$STATE/tombstones.all" || true gen_filter || return 1 + # ...and the filter must exist before the hub root can be checked against it + if check_hub_stray; then + [ $exec = 1 ] && [ "${HH_FORCE:-0}" != 1 ] && + die "refusing to sync into a hub root that is not ours. Fix HUB_PATH in config, move that data away, or re-run with HH_FORCE=1 if you really mean it" + fi hr "homehub sync ($([ $exec = 1 ] && echo EXECUTE || echo 'dry run')) on $HOST" v "sets: $sets" v "filter: $(grep -c . "$FILTER" || true) push, $(grep -c . "$FILTER.pull" || true) pull rules, $(grep -c . "$STATE/tombstones.all" 2>/dev/null || true) tombstoned" [ $exec = 1 ] && touch_marker alive + # 0. harvest BEFORE the pull: a deletion the pull would otherwise undo + if [ $exec = 1 ]; then + hr "0. harvest this machine's deletions (before the pull can undo them)" + harvest_tombstones || rc=1 + gen_filter || return 1 # pull filter must know the fresh tombstones + fi + hr "1. pull $HUB_URL -> $HOME/" pull_rsync "$dry" > "$STATE/pull.out" 2> "$STATE/pull.err" || rc=$? if [ $rc != 0 ]; then @@ -314,10 +397,8 @@ cmd_sync() { if [ $exec = 1 ]; then hr "2. apply deletions recorded by any machine (edit-wins)" apply_tombstones - hr "3. harvest this machine's deletions" - harvest_tombstones || rc=1 else - hr "2./3. deletion handling skipped in dry run" + hr "2. deletion handling skipped in dry run (harvest only runs with --exec)" fi hr "4. push $HOME/ -> $HUB_URL" @@ -334,6 +415,7 @@ cmd_sync() { touch_marker markers hub "rm -f '$HUB_PATH/.hub/meta/alerted/$HOST'" || true clear_broken sync + seen_update elif [ $exec = 1 ]; then mark_broken sync "last sync failed with rsync rc=$rc (see $STATE/pull.err, $STATE/push.err)" fi diff --git a/config b/config index 0c6cd85..a1dff6b 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ HUB_HOST=nas.lue.dk # years of NAS files into ~/, and a tombstone could reap unrelated files by name # accident. The sync root is a dedicated subtree; .hub/ metadata lives inside it. HUB_BASE=/volume1/homes/bl -HUB_PATH=$HUB_BASE/homehub +HUB_PATH=${HUB_PATH:-$HUB_BASE/homehub} # Dedicated, PASSPHRASE-LESS deploy key. Your id_rsa is passphrase-protected and a # systemd *user* unit has no SSH_AUTH_SOCK, so using ~/.ssh/id_rsa here would make @@ -30,11 +30,11 @@ STATE_ROOT=$HOME/.local/state/homehub SHARED_ROOT=$HOME/.local/state/shared # --- policy knobs ------------------------------------------------------------- -GRACE_DAYS=30 # tombstone age before the hub copy is reaped (§5) -FRESH_MAX_AGE=86400 # alert if a live machine has not synced in 24 h -ALIVE_FRESH=10800 # "machine is on" = heartbeat younger than 3 h -REALERT_AGE=604800 # at most one alert per host per week -TOMB_LIMIT=100 # refuse to apply more deletions than this in one run +GRACE_DAYS=${GRACE_DAYS:-30} # tombstone age before the hub copy is reaped (§5) +FRESH_MAX_AGE=${FRESH_MAX_AGE:-86400} # alert if a live machine has not synced in 24 h +ALIVE_FRESH=${ALIVE_FRESH:-10800} # "machine is on" = heartbeat younger than 3 h +REALERT_AGE=${REALERT_AGE:-604800} # at most one alert per host per week +TOMB_LIMIT=${TOMB_LIMIT:-100} # refuse to apply more deletions than this in one run WARN_FILES=20000 # dry run warns above this many pending changes # Where to look for git repos for the class B manifest (DESIGN.md §6).