homehub: rsync-based home-drive sync via Synology hub
DESIGN.md is the rationale; README.md the operations half. Four data classes: files (mtime-arbitrated two-way), git repos (manifest + clone, never byte-synced), the browser link bar (union merge), and everything else denied by default.
This commit is contained in:
druppy committed
commit
78496330f3
16 files changed
+1576
No files matched your search
Executable
+664
@@ -0,0 +1,664 @@
|
||||
#!/usr/bin/env bash
|
||||
# homehub — lean home-drive sync: rsync against a Synology hub, mtime-arbitrated,
|
||||
# silent unless something has been broken for a day.
|
||||
#
|
||||
# sync [--exec] one pull+push cycle (DRY unless --exec)
|
||||
# status what would move, and who synced when
|
||||
# fresh [--dry] the only command allowed to complain (24 h rule)
|
||||
# git <sub> class B: repos.tsv manifest, clone on the far side
|
||||
# bookmarks [--write] class C: union-merge the link bar
|
||||
# forget <path>... drop tombstones so a path can come back
|
||||
# filter dump the generated rsync filter
|
||||
# setup | setup-key | install one-time plumbing
|
||||
#
|
||||
# Rationale for every odd choice: ../DESIGN.md
|
||||
set -uo pipefail
|
||||
|
||||
HH_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
[ -r "$HH_ROOT/config" ] || { printf 'homehub: no config at %s\n' "$HH_ROOT/config" >&2; exit 1; }
|
||||
# shellcheck disable=SC1091
|
||||
. "$HH_ROOT/config"
|
||||
|
||||
HOST=$(hostname -s)
|
||||
HUB_URL="$HUB_HOST:$HUB_PATH"
|
||||
RULES=$HH_ROOT/rules
|
||||
STATE=${STATE_ROOT:?config: STATE_ROOT unset}
|
||||
FILTER=$STATE/effective.filter
|
||||
mkdir -p "$STATE/tmp"
|
||||
|
||||
# systemd sets HH_QUIET=1: silence on success is a feature, not an omission (§10)
|
||||
v() { [ "${HH_QUIET:-0}" = 1 ] || printf ' %s\n' "$*"; }
|
||||
warn() { printf 'homehub: %s\n' "$*" >&2; }
|
||||
die() { printf 'homehub: %s\n' "$*" >&2; exit 1; }
|
||||
now() { date +%s; }
|
||||
hr() { [ "${HH_QUIET:-0}" = 1 ] || printf '\n== %s ==\n' "$*"; }
|
||||
|
||||
# ------------------------------------------------------------- ssh plumbing --
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT:-10}" -o LogLevel=ERROR)
|
||||
[ -f "$HH_KEY" ] && SSH_OPTS+=(-i "$HH_KEY" -o IdentitiesOnly=yes)
|
||||
hub() { ssh "${SSH_OPTS[@]}" "$HUB_HOST" "$@"; }
|
||||
hub_ok() { hub true >/dev/null 2>&1; }
|
||||
|
||||
# Telling "off the LAN" apart from "our own config is broken" is the whole point of
|
||||
# §10: the first must stay silent for weeks, the second must become loud after a
|
||||
# day. 0 = usable, 1 = network/unreachable (stays silent), 2 = auth/config (loud).
|
||||
hub_probe() {
|
||||
local out
|
||||
if ! out=$(ssh "${SSH_OPTS[@]}" "$HUB_HOST" true 2>&1); then
|
||||
case $out in
|
||||
*"Permission denied"*|*"Host key verification"*|*"no matching host key"*|*"invalid format"*|*"no such identity"*)
|
||||
return 2 ;;
|
||||
esac
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Local breakage markers — readable with the hub down, so a machine whose own key
|
||||
# stopped working can still alert about itself. First-failure epoch is preserved.
|
||||
mark_broken() { # kind message
|
||||
local f=$STATE/broken.$1 e
|
||||
e=$(awk -F'\t' 'NF{print $1; exit}' "$f" 2>/dev/null)
|
||||
[ -n "${e:-}" ] || e=$(now)
|
||||
printf '%s\t%s\n' "$e" "$2" > "$f"
|
||||
}
|
||||
clear_broken() { rm -f "$STATE/broken.$1" 2>/dev/null || true; }
|
||||
# Remote script with positional args, no quoting hell:
|
||||
# hub_script "'$HUB_PATH' '$HOST'" <<'EOS' ... EOS (POSIX sh on DSM!)
|
||||
hub_script() { ssh "${SSH_OPTS[@]}" "$HUB_HOST" "sh -s -- $1"; }
|
||||
|
||||
touch_marker() {
|
||||
hub "mkdir -p '$HUB_PATH/.hub/meta/$1' && touch '$HUB_PATH/.hub/meta/$1/$HOST'" || true
|
||||
}
|
||||
log_line() { # rc note
|
||||
printf '%s\t%s\t%s\t%s\n' "$(now)" "$HOST" "$1" "$2" |
|
||||
hub "mkdir -p '$HUB_PATH/.hub' && cat >> '$HUB_PATH/.hub/log.tsv'" 2>/dev/null || true
|
||||
}
|
||||
hub_setup_dirs() {
|
||||
hub "mkdir -p '$HUB_PATH/.hub/meta/markers' '$HUB_PATH/.hub/meta/alive' \
|
||||
'$HUB_PATH/.hub/meta/alerted' '$HUB_PATH/.hub/tombstones' '$HUB_PATH/.hub/prev' \
|
||||
'$HUB_PATH/.hub/gitman' '$HUB_PATH/.hub/bookmarks' && chmod 700 '$HUB_PATH'" || true
|
||||
}
|
||||
|
||||
# ---------------------------------------------------- filter (default deny) --
|
||||
sets_for_host() {
|
||||
local f=$RULES/hosts/$HOST
|
||||
[ -r "$f" ] || { warn "no rules/hosts/$HOST — this host syncs nothing (see rules/hosts/_laptop-template)"; return 1; }
|
||||
awk '!/^[[:space:]]*#/ && NF' "$f"
|
||||
}
|
||||
# one path per line -> rsync rule; trailing slash means "this dir and its subtree"
|
||||
rules_from() { # file sign
|
||||
awk -v s="$2" '!/^[[:space:]]*#/ && NF {
|
||||
if ($0 ~ /\/$/) printf "%s %s***\n", s, $0; else printf "%s %s\n", s, $0 }' "$1"
|
||||
}
|
||||
# ancestors must be explicitly traversable, otherwise a blanket "- *" can prune
|
||||
# the branch before rsync ever reaches the include below it
|
||||
ancestors_of() { # /a/b/c/ -> "+ /a/", "+ /a/b/"
|
||||
local p=${1%/}
|
||||
while [ -n "$p" ] && [ "${p#*/}" != "$p" ]; do
|
||||
p=${p%/*}
|
||||
[ -n "$p" ] && printf '+ %s/\n' "$p"
|
||||
done
|
||||
}
|
||||
check_shadowing() { # $1 set name
|
||||
local f=$RULES/sets.d/$1.list d inc nm
|
||||
[ -r "$f" ] || return 0
|
||||
while read -r inc; do
|
||||
while read -r d; do
|
||||
case $d in
|
||||
"$inc") warn "shadowed: include '$inc' is also denied verbatim (set $1)"; continue ;;
|
||||
esac
|
||||
case $d in
|
||||
/*/) case $inc in "$d"*|"${d%/}") warn "shadowed: include '$inc' sits under deny rule '$d' (set $1)" ;; esac ;;
|
||||
*/) nm=${d%/}; case "/$inc" in *"/$nm"/*|*"/$nm") warn "shadowed: include '$inc' matches unanchored deny '$d' (set $1)" ;; esac ;;
|
||||
esac
|
||||
done < <(awk '!/^[[:space:]]*#/ && NF' "$RULES/denylist" 2>/dev/null)
|
||||
done < <(awk '!/^[[:space:]]*#/ && NF' "$f")
|
||||
}
|
||||
# base filter: deny first, then includes (+ ancestors), NO trailing default-deny
|
||||
gen_filter() {
|
||||
local sets s
|
||||
sets=$(sets_for_host) || return 1
|
||||
{
|
||||
echo '- /.hub/***' # hub metadata can never leak into a home
|
||||
[ -r "$RULES/denylist" ] && rules_from "$RULES/denylist" '-'
|
||||
for s in $sets; do
|
||||
[ -r "$RULES/sets.d/$s.list" ] || { warn "unknown set '$s' (no rules/sets.d/$s.list)"; continue; }
|
||||
{ rules_from "$RULES/sets.d/$s.list" '+'
|
||||
awk '!/^[[:space:]]*#/ && NF' "$RULES/sets.d/$s.list" | while read -r l; do ancestors_of "$l"; done; }
|
||||
check_shadowing "$s"
|
||||
done
|
||||
} | awk '!seen[$0]++' > "$FILTER.base.tmp" || return 1
|
||||
mv "$FILTER.base.tmp" "$FILTER.base"
|
||||
# push filter: default deny only
|
||||
{ cat "$FILTER.base"; echo '- *'; } > "$FILTER.tmp" && mv "$FILTER.tmp" "$FILTER"
|
||||
# pull filter: default deny, PLUS still-tombstoned paths excluded so a deleted
|
||||
# file is not re-downloaded every cycle for 30 days only to be re-deleted.
|
||||
# Tombstones never enter the push filter: a file re-created after the delete
|
||||
# must still be able to travel (edits win — DESIGN.md §3).
|
||||
{
|
||||
cat "$FILTER.base"
|
||||
awk -F'\t' 'NF>=2 { p=$2; sub(/^\//, "", p); if (p != "" && p !~ /^\.hub\// && p !~/\.\./) {
|
||||
print "- /" p; print "- /" p "/***" } }' "$STATE/tombstones.all" 2>/dev/null
|
||||
echo '- *'
|
||||
} | awk '!seen[$0]++' > "$FILTER.pull.tmp" && mv "$FILTER.pull.tmp" "$FILTER.pull"
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------- rsync bits --
|
||||
# rsync must authenticate with the SAME identity as the rest of homehub, or it
|
||||
# silently picks another key/agent and unreachable comes back from the wrong door.
|
||||
RSH="ssh -o BatchMode=yes -o ConnectTimeout=${SSH_TIMEOUT:-10} -o LogLevel=ERROR"
|
||||
[ -f "${HH_KEY:-}" ] && RSH="$RSH -i $HH_KEY -o IdentitiesOnly=yes"
|
||||
RS=(-rltDz --partial --timeout=90 --out-format='%i|%n' -e "$RSH")
|
||||
run_rsync() { # dry filter extra args...
|
||||
local dry=$1 filt=$2; shift 2
|
||||
if [ "$dry" = 1 ]; then
|
||||
"$RSYNC_BIN" "${RS[@]}" --filter="merge $filt" -n "$@"
|
||||
else
|
||||
"$RSYNC_BIN" "${RS[@]}" --filter="merge $filt" "$@"
|
||||
fi
|
||||
}
|
||||
pull_rsync() { # dry
|
||||
run_rsync "$1" "$FILTER.pull" --update "$HUB_URL/" "$HOME/"
|
||||
}
|
||||
push_rsync() { # dry
|
||||
if [ "$1" = 1 ]; then
|
||||
run_rsync 1 "$FILTER" --update "$HOME/" "$HUB_URL/"
|
||||
else
|
||||
run_rsync 0 "$FILTER" --update --backup --backup-dir=".hub/prev/$(date +%Y-%m)" "$HOME/" "$HUB_URL/"
|
||||
fi
|
||||
}
|
||||
count_items() { grep -cE '^[<>]' "$1" 2>/dev/null || true; }
|
||||
show_head() { grep -E '^[<>]' "$1" 2>/dev/null | head -n "${2:-10}"; }
|
||||
|
||||
# ----------------------------------------------------- deletions (tombstones) --
|
||||
# Format on the hub: .hub/tombstones/<host> lines = epoch \t home-rel-path
|
||||
hub_tombstones_all() {
|
||||
hub_script "'$HUB_PATH'" <<'EOS'
|
||||
root=$1
|
||||
cd "$root/.hub/tombstones" 2>/dev/null || exit 0
|
||||
for f in *; do
|
||||
[ -f "$f" ] || continue
|
||||
while IFS= read -r l; do printf '%s\t%s\n' "${f%%:*}" "$l"; done < "$f"
|
||||
done
|
||||
exit 0
|
||||
EOS
|
||||
}
|
||||
# is this home-relative path inside one of this host's sets?
|
||||
active_path() { # $1 = path without leading slash
|
||||
local p="/$1"
|
||||
while :; do
|
||||
grep -qxF "+ $p***" "$FILTER" && return 0 # inside an included subtree
|
||||
grep -qxF "+ $p" "$FILTER" && return 0 # is itself an included file
|
||||
[ "${p%/*}" = "$p" ] && return 1
|
||||
p=${p%/*}
|
||||
[ -z "$p" ] && return 1
|
||||
done
|
||||
}
|
||||
apply_tombstones() { # deletions recorded by any machine, edit-wins (§3)
|
||||
local all=$STATE/tombstones.all n=0 kept=0 out=0 bad=0
|
||||
[ -s "$all" ] || { v "tombstones: none recorded"; return 0; }
|
||||
while IFS=$'\t' read -r src ep path; do
|
||||
[ -n "${path:-}" ] || continue
|
||||
path=${path#/}
|
||||
case $path in
|
||||
.hub/*|*..*|/*) warn "ignoring suspicious tombstone '$path'"; bad=$((bad+1)); continue ;;
|
||||
esac
|
||||
active_path "$path" || { out=$((out+1)); continue; }
|
||||
[ -e "$HOME/$path" ] || continue
|
||||
if [ -n "$ep" ] && find "$HOME/$path" -newermt "@$ep" -print -quit 2>/dev/null | grep -q .; then
|
||||
kept=$((kept+1)); continue # touched after the delete: edits win, keep it
|
||||
fi
|
||||
n=$((n+1))
|
||||
if [ "$n" -gt "${TOMB_LIMIT:-100}" ] && [ "${HH_FORCE:-0}" != 1 ]; then
|
||||
die "tombstones ask for more than $TOMB_LIMIT deletions — refusing. Check $HUB_HOST:$HUB_PATH/.hub/tombstones/ then re-run with HH_FORCE=1"
|
||||
fi
|
||||
v "deleted (tombstone from $src): $path"
|
||||
rm -rf --one-file-system -- "$HOME/$path"
|
||||
done < "$all"
|
||||
v "tombstones: applied=$n edited-since-so-kept=$kept outside-sets=$out malformed=$bad"
|
||||
}
|
||||
harvest_tombstones() { # record what this machine deleted; runs only after a real pull
|
||||
local new=$STATE/tomb.new merged=$STATE/tomb.merged n ep
|
||||
run_rsync 1 --delete "$HOME/" "$HUB_URL/" 2>/dev/null |
|
||||
awk -F'|' '$1 ~ /^\*/ { sub(/^\//, "", $2); print $2 }' | sort -u > "$new"
|
||||
n=$(grep -c . "$new" 2>/dev/null || true)
|
||||
[ "${n:-0}" = 0 ] && { v "no local deletions to record"; return 0; }
|
||||
ep=$(now)
|
||||
{ hub "cat '$HUB_PATH/.hub/tombstones/$HOST' 2>/dev/null"
|
||||
awk -v e="$ep" '{ print e "\t" $0 }' "$new"
|
||||
} | awk -F'\t' 'NF>=2 { p=$2; if (!(p in e) || $1+0 < e[p]) e[p]=$1+0 }
|
||||
END { for (p in e) printf "%d\t%s\n", e[p], p }' | sort -t$'\t' -k2 > "$merged"
|
||||
hub "mkdir -p '$HUB_PATH/.hub/tombstones' && cat > '$HUB_PATH/.hub/tombstones/$HOST'" < "$merged" ||
|
||||
{ warn "could not write tombstones"; return 1; }
|
||||
v "recorded $n deletion(s) as tombstones on the hub"
|
||||
}
|
||||
reap() { # stop resurrecting files deleted more than GRACE_DAYS ago (§5)
|
||||
hub_script "'$HUB_PATH' '${GRACE_DAYS:-30}'" <<'EOS' || true
|
||||
root=$1; grace=$2
|
||||
cd "$root/.hub/tombstones" 2>/dev/null || exit 0
|
||||
cutoff=$(( $(date +%s) - grace * 86400 ))
|
||||
for f in *; do
|
||||
[ -f "$f" ] || continue
|
||||
awk -F'\t' -v c="$cutoff" '$1+0 < c { print }' "$f" > "$f.gone"
|
||||
awk -F'\t' -v c="$cutoff" '$1+0 >= c { print }' "$f" > "$f.keep"
|
||||
while IFS=$'\t' read -r e p; do
|
||||
[ -n "$p" ] || continue
|
||||
t="$root/$p"
|
||||
[ -e "$t" ] || continue
|
||||
d="$root/.hub/prev/reaped/$(date -d "@$e" +%Y-%m 2>/dev/null || echo unknown)"
|
||||
mkdir -p "$d/$(dirname "$p")" 2>/dev/null && mv "$t" "$d/$p" 2>/dev/null && echo "reaped $p"
|
||||
done < "$f.gone"
|
||||
if [ -s "$f.keep" ]; then mv "$f.keep" "$f"; else rm -f "$f.keep" "$f"; fi
|
||||
rm -f "$f.gone"
|
||||
done
|
||||
exit 0
|
||||
EOS
|
||||
}
|
||||
|
||||
# The hub sync root must contain ONLY what homehub put there. Anything else means
|
||||
# HUB_PATH points somewhere that already has data — which would be merged into
|
||||
# $HOME on the first pull (and could be reaped by a tombstone). This guard exists
|
||||
# because /volume1/homes/bl really did hold 178 G of unrelated legacy data.
|
||||
check_hub_stray() {
|
||||
[ -r "$FILTER.base" ] || return 0
|
||||
local allowed stray
|
||||
allowed=$(printf '.hub\n@eaDir\n'; grep '^+ /' "$FILTER.base" | sed -E 's#^\+ /([^/]+).*#\1#; s#/$##' | sort -u)
|
||||
stray=$(hub "cd '$HUB_PATH' 2>/dev/null && ls -A" | grep -vxF -f <(printf '%s\n' "$allowed") || true)
|
||||
[ -z "$stray" ] && return 1
|
||||
warn "unexpected content in hub sync root $HUB_PATH (pull would merge it into $HOME):"
|
||||
printf ' %s\n' $stray
|
||||
return 0
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ sync ----
|
||||
cmd_sync() {
|
||||
local exec=0; [ "${1:-}" = "--exec" ] && exec=1
|
||||
local dry=1; [ $exec = 1 ] && dry=0
|
||||
local sets rc=0
|
||||
|
||||
sets=$(sets_for_host) || return 1
|
||||
|
||||
case "$(hub_probe; echo $?)" in
|
||||
1) # Off the LAN. Silent on purpose: the laptop-in-another-country case (§10).
|
||||
v "hub $HUB_HOST unreachable — nothing done (silent by design; a live machine that stays broken alerts through 'fresh' after ${FRESH_MAX_AGE}s)"
|
||||
return 0 ;;
|
||||
2) warn "cannot authenticate to $HUB_HOST (key: ${HH_KEY:-default agent}). This is a CONFIGURATION failure, not an offline machine."
|
||||
warn " fix: ssh-copy-id -i ${HH_KEY:-~/.ssh/id_ed25519}.pub $HUB_HOST"
|
||||
mark_broken auth "ssh auth to $HUB_HOST failed (key ${HH_KEY:-default})"
|
||||
return 64 ;;
|
||||
esac
|
||||
clear_broken auth
|
||||
hub_setup_dirs
|
||||
if check_hub_stray; then
|
||||
[ $exec = 1 ] && [ "${HH_FORCE:-0}" != 1 ] &&
|
||||
die "refusing to sync into a hub root that is not ours. Fix HUB_PATH in config, or move that data away, or re-run with HH_FORCE=1 if you really mean it"
|
||||
fi
|
||||
# tombstones must be known BEFORE the filter is built: they shape the pull filter
|
||||
hub_tombstones_all > "$STATE/tombstones.all" || true
|
||||
gen_filter || return 1
|
||||
hr "homehub sync ($([ $exec = 1 ] && echo EXECUTE || echo 'dry run')) on $HOST"
|
||||
v "sets: $sets"
|
||||
v "filter: $(grep -c . "$FILTER" || true) push, $(grep -c . "$FILTER.pull" || true) pull rules, $(grep -c . "$STATE/tombstones.all" 2>/dev/null || true) tombstoned"
|
||||
[ $exec = 1 ] && touch_marker alive
|
||||
|
||||
hr "1. pull $HUB_URL -> $HOME/"
|
||||
pull_rsync "$dry" > "$STATE/pull.out" 2> "$STATE/pull.err" || rc=$?
|
||||
if [ $rc != 0 ]; then
|
||||
warn "pull failed (rsync rc=$rc): $(tr '\n' ' ' < "$STATE/pull.err" | tail -c 300)"
|
||||
else
|
||||
v "$(count_items "$STATE/pull.out") change(s) inbound"
|
||||
[ $dry = 1 ] && show_head "$STATE/pull.out"
|
||||
fi
|
||||
|
||||
if [ $exec = 1 ]; then
|
||||
hr "2. apply deletions recorded by any machine (edit-wins)"
|
||||
apply_tombstones
|
||||
hr "3. harvest this machine's deletions"
|
||||
harvest_tombstones || rc=1
|
||||
else
|
||||
hr "2./3. deletion handling skipped in dry run"
|
||||
fi
|
||||
|
||||
hr "4. push $HOME/ -> $HUB_URL"
|
||||
push_rsync "$dry" > "$STATE/push.out" 2> "$STATE/push.err" || rc=$?
|
||||
if [ $rc != 0 ]; then
|
||||
warn "push failed (rsync rc=$rc): $(tr '\n' ' ' < "$STATE/push.err" | tail -c 300)"
|
||||
else
|
||||
v "$(count_items "$STATE/push.out") change(s) outbound"
|
||||
[ $dry = 1 ] && show_head "$STATE/push.out"
|
||||
fi
|
||||
|
||||
if [ $exec = 1 ] && [ $rc = 0 ]; then
|
||||
reap | sed 's/^/ /'
|
||||
touch_marker markers
|
||||
hub "rm -f '$HUB_PATH/.hub/meta/alerted/$HOST'" || true
|
||||
clear_broken sync
|
||||
elif [ $exec = 1 ]; then
|
||||
mark_broken sync "last sync failed with rsync rc=$rc (see $STATE/pull.err, $STATE/push.err)"
|
||||
fi
|
||||
[ $exec = 1 ] && log_line "$rc" "sync"
|
||||
|
||||
if [ $dry = 1 ]; then
|
||||
local tot; tot=$(count_items "$STATE/pull.out"); tot=$((tot + $(count_items "$STATE/push.out")))
|
||||
hr "summary"
|
||||
v "$tot change(s) pending — nothing was written. Run: homehub sync --exec"
|
||||
[ "${tot:-0}" -gt "${WARN_FILES:-20000}" ] &&
|
||||
warn "that is a lot of files for a first run; review $STATE/{pull,push}.out first"
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- status ----
|
||||
hub_meta() {
|
||||
hub_script "'$HUB_PATH'" <<'EOS'
|
||||
root=$1
|
||||
cd "$root/.hub" 2>/dev/null || exit 0
|
||||
for d in meta/markers meta/alive meta/alerted; do
|
||||
for f in "$d"/*; do
|
||||
[ -f "$f" ] || continue
|
||||
printf '%s\t%s\t%s\n' "${d##*/}" "$(basename "$f")" "$(date -r "$f" +%s 2>/dev/null || echo 0)"
|
||||
done
|
||||
done
|
||||
if [ -f log.tsv ]; then
|
||||
tail -n 400 log.tsv | awk -F'\t' '{ r[$2] = $3 "\t" $4 } END { for (h in r) printf "lastlog\t%s\t%s\n", h, r[h] }'
|
||||
fi
|
||||
for f in tombstones/*; do [ -f "$f" ] && printf 'tomb\t%s\t%s\n' "$(basename "$f")" "$(grep -c . "$f")"; done
|
||||
for f in gitman/repos.*; do [ -f "$f" ] && printf 'repos\t%s\t%s\n' "$(basename "$f" 2>/dev/null)" "$(grep -c . "$f")"; done
|
||||
for f in bookmarks/*; do [ -f "$f" ] && printf 'bmarks\t%s\t%s\n' "$(basename "$f" .json)" "$(date -r "$f" +%s)"; done
|
||||
exit 0
|
||||
EOS
|
||||
}
|
||||
age() { local n=$(( $(now) - ${1:-0} )); [ "${1:-0}" = 0 ] && { printf 'never'; return; }
|
||||
if [ $n -lt 3600 ]; then printf '%dm' $((n/60))
|
||||
elif [ $n -lt 172800 ]; then printf '%.1fh' "$(awk -v n=$n 'BEGIN{print n/3600}')"
|
||||
else printf '%.0fd' "$(awk -v n=$n 'BEGIN{print n/86400}')"; fi; }
|
||||
|
||||
cmd_status() {
|
||||
hub_ok || die "hub $HUB_HOST unreachable"
|
||||
sets_for_host >/dev/null || true
|
||||
gen_filter 2>/dev/null || true
|
||||
hub_meta > "$STATE/meta.tsv" || die "could not read hub metadata"
|
||||
|
||||
hr "machines"
|
||||
printf ' %-14s %-9s %-9s %-6s %s\n' HOST synced heartbeat last-rc notes
|
||||
local h
|
||||
for h in $(awk -F'\t' '$1=="markers"||$1=="alive"{print $2}' "$STATE/meta.tsv" | sort -u); do
|
||||
local m a rc note
|
||||
m=$(awk -F'\t' -v h="$h" '$1=="markers"&&$2==h{print $3}' "$STATE/meta.tsv")
|
||||
a=$(awk -F'\t' -v h="$h" '$1=="alive"&&$2==h{print $3}' "$STATE/meta.tsv")
|
||||
rc=$(awk -F'\t' -v h="$h" '$1=="lastlog"{split($0,x,"\t"); if (x[2]==h) print x[3]}' "$STATE/meta.tsv")
|
||||
note=$(awk -F'\t' -v h="$h" '$1=="lastlog"{split($0,x,"\t"); if (x[2]==h) print x[4]}' "$STATE/meta.tsv")
|
||||
printf ' %-14s %-9s %-9s %-6s %s\n' "$h" "$(age "$m")" "$(age "$a")" "${rc:-?}" "${note:-}"
|
||||
done
|
||||
|
||||
hr "this host ($HOST)"
|
||||
v "sets: $(sets_for_host | tr '\n' ' ')"
|
||||
v "included: $(grep -c '^+ ' "$FILTER" 2>/dev/null || echo 0) rules, denied: $(grep -c '^- ' "$FILTER" 2>/dev/null || echo 0)"
|
||||
v "state dir: $STATE"
|
||||
awk -F'\t' -v h="$HOST" '$1=="tomb"&&$2==h{print " tombstones: " $3 " paths"}' "$STATE/meta.tsv"
|
||||
v "pending dry-run lists: $STATE/pull.out $STATE/push.out (run 'homehub sync' to refresh)"
|
||||
|
||||
hr "class B — repo manifest"
|
||||
awk -F'\t' '$1=="repos"{printf " %-14s %s repos\n", $2, $3}' "$STATE/meta.tsv"
|
||||
cmd_git missing 2>/dev/null | sed 's/^/ /' || true
|
||||
|
||||
hr "class C — bookmark bar"
|
||||
awk -F'\t' '$1=="bmarks"{printf " %-14s exported %s\n", $2, ""}' "$STATE/meta.tsv"
|
||||
[ -r "$BOOKMARK_SRC" ] && v "live file: $BOOKMARK_SRC"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------- fresh ----
|
||||
alert_cmd() {
|
||||
if [ -n "${ALERT_CMD:-}" ]; then printf '%s' "$ALERT_CMD"; return; fi
|
||||
if command -v msmtp >/dev/null 2>&1; then printf 'msmtp -a homehub %s' "$ALERT_TO"
|
||||
elif command -v sendmail >/dev/null 2>&1; then printf 'sendmail -t'
|
||||
else printf 'cat >> "%s/alerts.out"' "$STATE"; fi
|
||||
}
|
||||
alert_send() { # subject body
|
||||
{ printf 'From: %s\nTo: %s\nSubject: %s\n\n%s\n' "${ALERT_FROM:-homehub}" "${ALERT_TO:-root}" "$1" "$2" |
|
||||
sh -c "$(alert_cmd)"; } 2>>"$STATE/alert.err" || warn "alert delivery failed (see $STATE/alert.err)"
|
||||
if [ "${HH_QUIET:-0}" != 1 ] && [ -n "${DISPLAY:-}" ] && command -v notify-send >/dev/null 2>&1; then
|
||||
notify-send -u critical "$1" "$2" || true
|
||||
fi
|
||||
}
|
||||
cmd_fresh() { # the ONLY place that may complain (§10)
|
||||
local dry=0; [ "${1:-}" = "--dry" ] && dry=1
|
||||
local n=$(now) h msgs="" f
|
||||
|
||||
# 1) Self-checks that work with the hub down. Closes the hole where a dead key
|
||||
# means no marker and no heartbeat, i.e. a machine that can never alert.
|
||||
for f in "$STATE"/broken.*; do
|
||||
[ -e "$f" ] || continue
|
||||
local kind=${f##*.} ep msg age already=0
|
||||
ep=$(awk -F'\t' 'NF{print $1; exit}' "$f"); msg=$(cut -f2- "$f")
|
||||
age=$(( n - ${ep:-0} ))
|
||||
[ -e "$STATE/alerted-local.$kind" ] &&
|
||||
[ $(( n - $(date -r "$STATE/alerted-local.$kind" +%s) )) -lt "${REALERT_AGE:-604800}" ] && already=1
|
||||
if [ "$age" -gt "${FRESH_MAX_AGE:-86400}" ] && [ $already = 0 ]; then
|
||||
msgs="$msgs$HOST, local check '$kind': $msg (failing for $(age "$ep"))"$'\n'
|
||||
touch "$STATE/alerted-local.$kind"
|
||||
else
|
||||
v "ok(local): $kind has been failing for $(age "$ep")$([ $already = 1 ] && echo ', already alerted')"
|
||||
fi
|
||||
done
|
||||
|
||||
# 2) Hub-side freshness of every machine
|
||||
if ! hub_ok; then
|
||||
v "hub unreachable — cannot judge other machines (the local checks above still ran)"
|
||||
else
|
||||
hub_meta > "$STATE/meta.tsv" || die "cannot read hub metadata"
|
||||
for h in $(awk -F'\t' '$1=="markers"||$1=="alive"{print $2}' "$STATE/meta.tsv" | sort -u); do
|
||||
local m a al rc mka maa
|
||||
m=$(awk -F'\t' -v h="$h" '$1=="markers"&&$2==h{print $3}' "$STATE/meta.tsv")
|
||||
a=$(awk -F'\t' -v h="$h" '$1=="alive"&&$2==h{print $3}' "$STATE/meta.tsv")
|
||||
al=$(awk -F'\t' -v h="$h" '$1=="alerted"&&$2==h{print $3}' "$STATE/meta.tsv")
|
||||
rc=$(awk -F'\t' -v h="$h" '$1=="lastlog"{split($0,x,"\t"); if (x[2]==h) print x[3]}' "$STATE/meta.tsv")
|
||||
mka=$(( n - ${m:-0} )); maa=$(( n - ${a:-$n} ))
|
||||
local alive_now=0; [ "${a:-0}" != 0 ] && [ $maa -lt "${ALIVE_FRESH:-10800}" ] && alive_now=1
|
||||
local netrc=0 x
|
||||
for x in ${NET_RC:-5 255}; do [ "$rc" = "$x" ] && netrc=1; done
|
||||
local realert_ok=1
|
||||
[ "${al:-0}" != 0 ] && [ $(( n - al )) -lt "${REALERT_AGE:-604800}" ] && realert_ok=0
|
||||
|
||||
if [ $alive_now = 1 ] && [ $mka -gt "${FRESH_MAX_AGE:-86400}" ] && [ $netrc = 0 ] && [ $realert_ok = 1 ]; then
|
||||
msgs="$msgs$h (last success $(age "$m"), last rc=${rc:-?})"$'\n'
|
||||
v "STALE: $h — on, but not synced for $(age "$m")"
|
||||
hub "mkdir -p '$HUB_PATH/.hub/meta/alerted' && touch '$HUB_PATH/.hub/meta/alerted/$h'" || true
|
||||
else
|
||||
v "ok: $h (sync $(age "$m"), heartbeat $(age "$a"), rc=${rc:--}$([ $netrc = 1 ] && echo ', off-LAN: silent'))"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
[ -n "$msgs" ] || return 0
|
||||
if [ $dry = 1 ]; then
|
||||
printf 'homehub: would alert for:\n%s' "$msgs" >&2
|
||||
return 0
|
||||
fi
|
||||
alert_send "homehub: sync stale on $(hostname -f 2>/dev/null || hostname)" \
|
||||
"These machines are running but have not completed a homehub sync in over $(( ${FRESH_MAX_AGE:-86400} / 3600 )) h:
|
||||
|
||||
$msgs
|
||||
Check with: homehub status
|
||||
Hub: $HUB_HOST:$HUB_PATH/.hub/log.tsv"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------- class B: the repos --
|
||||
git_root_p() { [ -e "$1/.git" ]; }
|
||||
git_row() { # relpath remote branch sha dirty unpushed (last two are ADVISORY ONLY)
|
||||
local d=$1 rel=${1#"$HOME"/} url br sha dirty unpush cred=ok
|
||||
url=$(git -C "$d" remote get-url origin 2>/dev/null || true); [ -n "$url" ] || url=-
|
||||
# SECURITY: remotes can carry credentials (https://user:TOKEN@host/...). They
|
||||
# must never be published to the hub. Strip userinfo and mark the row so
|
||||
# `git clone` can tell you the URL needs re-adding on the far side.
|
||||
case $url in
|
||||
*://*:*@*) url=$(printf '%s' "$url" | sed -E 's#(://)[^/@]*@#\1#'); cred=stripped ;;
|
||||
*://*@*) url=$(printf '%s' "$url" | sed -E 's#(://)[^/@]*@#\1#'); cred=stripped ;;
|
||||
esac
|
||||
br=$(git -C "$d" symbolic-ref --short HEAD 2>/dev/null || echo -)
|
||||
sha=$(git -C "$d" rev-parse --short HEAD 2>/dev/null || echo -)
|
||||
dirty=$(git -C "$d" status --porcelain 2>/dev/null | wc -l | tr -d ' ')
|
||||
unpush=$(git -C "$d" rev-list --count '@{u}..HEAD' 2>/dev/null | tail -1); [ -n "$unpush" ] || unpush=0
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$rel" "$url" "$br" "$sha" "${dirty:-0}" "${unpush:-0}" "$cred"
|
||||
}
|
||||
git_walk() { # dir depth — stops at the first .git, so submodules are not double counted
|
||||
local d=$1 depth=$2 c
|
||||
if git_root_p "$d"; then git_row "$d"; return; fi
|
||||
[ "$depth" -le 0 ] && return
|
||||
for c in "$d"/*/; do [ -d "$c" ] || continue; git_walk "${c%/}" $((depth - 1)); done
|
||||
}
|
||||
cmd_git() {
|
||||
local sub=${1:-inventory}; shift || true
|
||||
local lazy=0 exec=0 a
|
||||
for a in "$@"; do case $a in --lazy) lazy=1;; --exec) exec=1;; esac; done
|
||||
case $sub in
|
||||
inventory)
|
||||
local out=$STATE/repos.$HOST.tsv
|
||||
: > "$out"
|
||||
local r; for r in ${GIT_DIRS:-git src}; do [ -d "$HOME/$r" ] && git_walk "$HOME/$r" 3 >> "$out"; done
|
||||
v "$(grep -c . "$out") repo(s) under ${GIT_DIRS:-git src}"
|
||||
awk -F'\t' '$5+0>0 || $6+0>0 { printf " %-34s %s modified, %s unpushed\n", $1, $5, $6 }' "$out"
|
||||
awk -F'\t' '$7=="stripped" { printf " CREDENTIALS STRIPPED from remote of %s — the far side must supply them\n", $1 }' "$out"
|
||||
v "advisory only — neither the working tree nor unpushed commits ever leave this machine"
|
||||
if [ $exec = 1 ] || [ "${HH_PUSH_INVENTORY:-1}" = 1 ]; then
|
||||
hub "mkdir -p '$HUB_PATH/.hub/gitman' && cat > '$HUB_PATH/.hub/gitman/repos.$HOST'" < "$out" &&
|
||||
v "manifest published to hub"
|
||||
fi
|
||||
;;
|
||||
list)
|
||||
hub "cat '$HUB_PATH/.hub/gitman/repos.'* 2>/dev/null" | sort -u -t$'\t' -k1,1
|
||||
;;
|
||||
missing)
|
||||
hub "cat '$HUB_PATH/.hub/gitman/repos.'* 2>/dev/null" | sort -u -t$'\t' -k1,1 |
|
||||
while IFS=$'\t' read -r rel url br sha dirty unpush cred; do
|
||||
[ -n "${rel:-}" ] || continue
|
||||
[ -e "$HOME/$rel/.git" ] && continue
|
||||
printf '%-34s %s (%s%s%s)\n' "$rel" "$url" "$br" \
|
||||
"$( [ "${dirty:-0}" != 0 ] && printf ', %s modified elsewhere' "$dirty")" \
|
||||
"$( [ "${cred:-ok}" = stripped ] && printf ', creds stripped')"
|
||||
done
|
||||
;;
|
||||
clone)
|
||||
local n=0
|
||||
hub "cat '$HUB_PATH/.hub/gitman/repos.'* 2>/dev/null" | sort -u -t$'\t' -k1,1 |
|
||||
while IFS=$'\t' read -r rel url br sha dirty unpush cred; do
|
||||
[ -n "${rel:-}" ] || continue
|
||||
[ "$url" = - ] && continue
|
||||
[ -e "$HOME/$rel/.git" ] && continue
|
||||
if [ $exec != 1 ]; then
|
||||
printf ' would clone %s <- %s%s%s\n' "$rel" "$url" \
|
||||
"$( [ $lazy = 1 ] && echo ' (blobless)')" "$( [ "${cred:-ok}" = stripped ] && echo ' [creds stripped: set a credential helper or fix the remote]')"
|
||||
else
|
||||
printf ' cloning %s <- %s\n' "$rel" "$url"
|
||||
mkdir -p "$(dirname "$HOME/$rel")"
|
||||
local opts=(--origin origin)
|
||||
[ $lazy = 1 ] && opts+=(--filter=blob:none)
|
||||
[ "$br" != - ] && git clone "${opts[@]}" -b "$br" "$url" "$HOME/$rel" >/dev/null 2>&1 ||
|
||||
git clone "${opts[@]}" "$url" "$HOME/$rel" >/dev/null 2>&1 ||
|
||||
warn "clone failed: $rel ($url)"
|
||||
fi
|
||||
n=$((n+1))
|
||||
done
|
||||
;;
|
||||
*) die "git: unknown subcommand '$sub' (inventory|list|missing|clone)";;
|
||||
esac
|
||||
}
|
||||
|
||||
# ------------------------------------------------- class C: the link bar -----
|
||||
cmd_bookmarks() {
|
||||
local write=0 a
|
||||
for a in "$@"; do [ "$a" = --write ] && write=1; done
|
||||
[ -r "$BOOKMARK_SRC" ] || die "no bookmark file at $BOOKMARK_SRC"
|
||||
cp -- "$BOOKMARK_SRC" "$STATE/bookmarks.$HOST.json" || die "cannot read $BOOKMARK_SRC"
|
||||
local nu; nu=$(jq '[.. | objects | select(.type=="url")] | length' "$STATE/bookmarks.$HOST.json" 2>/dev/null || echo '?')
|
||||
hr "class C: bookmark bar ($HOST)"
|
||||
v "exported $nu url(s) from $BOOKMARK_SRC (the union-merge below covers the bar only)"
|
||||
if [ $write != 1 ]; then
|
||||
v "dry: nothing pushed, nothing merged, live file untouched. Use --write."
|
||||
return 0
|
||||
fi
|
||||
hub "mkdir -p '$HUB_PATH/.hub/bookmarks' && cat > '$HUB_PATH/.hub/bookmarks/$HOST.json'" < "$STATE/bookmarks.$HOST.json" ||
|
||||
die "could not publish bookmark export"
|
||||
hub "cd '$HUB_PATH/.hub/bookmarks' 2>/dev/null && for f in *.json; do printf '### %s\\n' \"\$f\"; cat \"\$f\"; done" > "$STATE/bookmarks.all.json" ||
|
||||
die "could not fetch exports"
|
||||
"$PY" "$HH_ROOT/lib/merge_bookmarks.py" "$STATE/bookmarks.all.json" > "$STATE/bookmarks.merged.json" ||
|
||||
die "merge failed"
|
||||
local mu; mu=$(jq '.urls' "$STATE/bookmarks.merged.json" 2>/dev/null || echo '?')
|
||||
v "merged union: $mu urls (from $(jq -r '.hosts | join(",")' "$STATE/bookmarks.merged.json" 2>/dev/null))"
|
||||
v "union result: $STATE/bookmarks.merged.json"
|
||||
if pgrep -f "$BOOKMARK_PGREP" >/dev/null 2>&1; then
|
||||
warn "browser is running — merged bar NOT written (it would be overwritten in memory). Close it and re-run."
|
||||
v "merged result is ready at $STATE/bookmarks.merged.json"
|
||||
return 0
|
||||
fi
|
||||
warn "writing the merged bar to $BOOKMARK_SRC is not enabled in v1 — inspect $STATE/bookmarks.merged.json first (see README)"
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------- housekeeping -
|
||||
cmd_forget() {
|
||||
[ $# -ge 1 ] || die "usage: homehub forget <home-relative-path> ..."
|
||||
hub_ok || die "hub unreachable"
|
||||
local p
|
||||
for p in "$@"; do
|
||||
p=${p#/}
|
||||
hub_script "'$HUB_PATH' '$p'" <<'EOS'
|
||||
root=$1; pat=$2
|
||||
cd "$root/.hub/tombstones" 2>/dev/null || exit 0
|
||||
for f in *; do
|
||||
[ -f "$f" ] || continue
|
||||
awk -F'\t' -v p="$pat" '$2 != p && $2 !~ ("^" p "/")' "$f" > "$f.t" && mv "$f.t" "$f"
|
||||
done
|
||||
exit 0
|
||||
EOS
|
||||
v "forget: $p"
|
||||
done
|
||||
}
|
||||
cmd_filter() { gen_filter || return 1; cat "$FILTER"; }
|
||||
|
||||
cmd_setup_key() {
|
||||
[ -f "$HH_KEY" ] && { warn "$HH_KEY already exists"; return 0; }
|
||||
mkdir -p "$(dirname "$HH_KEY")"
|
||||
ssh-keygen -q -t ed25519 -N '' -C "homehub@$HOST" -f "$HH_KEY" || die "keygen failed"
|
||||
hr "deploy key created"
|
||||
v "$HH_KEY (passphrase-less, by necessity: a systemd user unit has no agent)"
|
||||
v "Install it on the hub, then make ~/.ssh 700 there:"
|
||||
printf '\n ssh-copy-id -i %s.pub %s\n\n' "$HH_KEY" "$HUB_HOST"
|
||||
warn "Until that is done, background sync cannot authenticate — and it fails silently by design."
|
||||
}
|
||||
cmd_setup() {
|
||||
hub_ok || die "cannot reach $HUB_HOST with the current ssh setup"
|
||||
hub_setup_dirs
|
||||
mkdir -p "$SHARED_ROOT"
|
||||
v "shared state root ready: $SHARED_ROOT (put app state here, symlink apps to it)"
|
||||
hub "chmod 700 '$HUB_PATH' 2>/dev/null; ls -d '$HUB_PATH/.hub'/*" | sed 's/^/ /'
|
||||
hr "published rules"
|
||||
git -C "$HH_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1 &&
|
||||
v "rules live in git: $HH_ROOT — clone that repo on the other machines"
|
||||
[ -f "$HH_KEY" ] || cmd_setup_key
|
||||
}
|
||||
cmd_install() {
|
||||
local d=${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user
|
||||
mkdir -p "$d"
|
||||
cp -v "$HH_ROOT"/systemd/homehub-*.service "$HH_ROOT"/systemd/homehub-*.timer "$d/" || return 1
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now homehub-sync.timer homehub-fresh.timer
|
||||
systemctl --user --no-pager --full status homehub-sync.timer homehub-fresh.timer 2>&1 | grep -E 'Loaded:|Active:|→' || true
|
||||
}
|
||||
|
||||
usage() { sed -n '2,14p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }
|
||||
|
||||
case ${1:-help} in
|
||||
sync) shift; cmd_sync "$@";;
|
||||
status) shift; cmd_status "$@";;
|
||||
fresh) shift; cmd_fresh "$@";;
|
||||
git) shift; cmd_git "$@";;
|
||||
bookmarks) shift; cmd_bookmarks "$@";;
|
||||
forget) shift; cmd_forget "$@";;
|
||||
filter) shift; cmd_filter "$@";;
|
||||
setup) shift; cmd_setup "$@";;
|
||||
setup-key) shift; cmd_setup_key "$@";;
|
||||
install) shift; cmd_install "$@";;
|
||||
help|-h|--help) usage;;
|
||||
*) warn "unknown command '${1:-}'; try 'homehub help'"; exit 2;;
|
||||
esac
|
||||
Reference in new issue
Block a user