#!/bin/bash
# downpour.sh -- back up your iCloud-stored data to a local folder.
# https://downpour.sh
# Copyright (c) 2026 downpour.sh. MIT licensed: https://opensource.org/licenses/MIT
#
# Usage: bash [output_folder] (default output base: ~/Desktop)
# The dated downpour--YYYY-MM-DD-HHMMSS/ subfolder is created
# INSIDE output_folder. At most one positional path; positional and
# flag order don't matter.
#
# Flags — each sets the env var shown in [brackets]; exporting that env var
# directly is equivalent (e.g. PHOTOS_SYS_COPY=no bash ):
# Photos, system library (default ON)
# --no-system-originals skip copying system-library originals [PHOTOS_SYS_COPY=no]
# --no-system-report skip the system-library report [PHOTOS_SYS_REPORT=no]
# Photos, other libraries (default OFF)
# --other-libraries also process non-system libraries (copy + report)
# --no-other-originals other libraries: skip originals copy [PHOTOS_OTHER_COPY=no]
# --no-other-report other libraries: skip the report [PHOTOS_OTHER_REPORT=no]
# Photos, shared albums (default ON)
# --no-shared-originals skip the per-album asset copy [SHARED_COPY=no]
# --no-shared-report skip the shared-albums report rows [SHARED_REPORT=no]
# --no-verify-shared-matches skip czkawka match verification + fetch [PHOTOS_VERIFY_SHARED_MATCHES=0]
# (alias: --disable-verify-shared-matches)
# Auto-fetch components
# --disable-protoc Notes: skip protoc fetch, use strings [NOTES_FETCH_PROTOC=0]
# (alias: --no-protoc)
# --disable-imessage-exporter Messages: skip binary fetch, use bash [MESSAGES_USE_IMESSAGE_EXPORTER=0]
# (alias: --no-imessage-exporter)
# --disable-czkawka Photos: skip czkawka fetch (shared-album [PHOTOS_VERIFY_SHARED_MATCHES=0]
# matches stay unchecked) (alias: --no-czkawka)
# --disable-convert-docs skip iWork->ODF conversion + LibreOffice [CONVERT_DOCS=0]
# (alias: --no-convert-docs)
# -h / --help print this help and exit
#
# Console + GUI-menu only. SSH / headless / no-TTY / cron runs are refused
# (there is no unattended mode). Use the interactive menu to deselect leaves and to enable
# the off-by-default categories (Books, Freeform, Maps, Music Playlists,
# Safari Bookmarks, Stickies, Text Replacements, Voice Memos, Convert iWork
# Files) — the flags above cover only the Photos scopes and the auto-fetches.
if (return 0 2>/dev/null); then printf 'ERROR: this script must be executed, not sourced.\n' >&2; return 1; fi
: "${SKIP_ALIASES:=1}"
CLEAN_DOWNLOADS="${CLEAN_DOWNLOADS:-0}"
EXPORT_OVERHEAD_GB="${EXPORT_OVERHEAD_GB:-10}"
CONFIRM_AUTO_NO_SEC="${CONFIRM_AUTO_NO_SEC:-10}"
MENU_AUTO_CANCEL_SEC="${MENU_AUTO_CANCEL_SEC:-60}"
FILES_INCLUSIONS=(
Documents # wholesale mirror
Downloads # wholesale mirror
Desktop # wholesale mirror — prior downpour-/export- timestamped subfolders are pruned
Public # wholesale mirror (the shared / Drop Box folder)
Pictures # full minus *.photoslibrary (see FILES_INCLUDE_PHOTOSLIBRARY_BUNDLES)
Movies # full minus *.imovielibrary (see FILES_INCLUDE_IMOVIE_BUNDLES)
Music # full minus GarageBand/ (see FILES_INCLUDE_GARAGEBAND_BUNDLES)
)
FILES_EXCLUSIONS=(
"Movies/iMovie Theater.theater" # iMovie's shared-theater bundle
"Movies/iMovie Theater" # pre-bundle theater store (older macOS)
"Movies/TV" # the TV.app media library
"Music/Audio Music Apps" # Audio Units / plug-in scratch
"Music/Music" # the Music.app (iTunes) library
)
FILES_INCLUDE_IMOVIE_BUNDLES=1
FILES_INCLUDE_GARAGEBAND_BUNDLES=1
FILES_INCLUDE_PHOTOSLIBRARY_BUNDLES=0
FILES_INCLUDE_OTHER_HOME=1
: "${FREEFORM_GUI_WAIT_SEC:=20}"
: "${EXPORT_PASSWORDS_CONFIRM:=1}"
: "${PHOTOS_SYS_COPY:=yes}"
: "${PHOTOS_SYS_REPORT:=yes}"
: "${PHOTOS_OTHER_COPY:=no}"
: "${PHOTOS_OTHER_REPORT:=yes}"
: "${PHOTOS_INCLUDE_SHARED_SOURCE:=yes}"
: "${PHOTOS_PERLIB_HEADROOM_MB:=500}"
: "${PHOTO_LIBRARY_DOWNLOAD_MISSING:=1}"
: "${PHOTO_LIBRARY_DOWNLOAD_TIMEOUT_SEC:=60}"
SHARED_COPY="${SHARED_COPY:-yes}"
SHARED_REPORT="${SHARED_REPORT:-yes}"
: "${SHARED_ALBUMS_PER_ASSET_SEC:=60}"
: "${STICKIES_RTF_MAX_MB:=50}"
if (return 0 2>/dev/null); then
printf 'ERROR: this script must be executed, not sourced.\n' >&2
return 1
fi
trap '' USR1 2>/dev/null || true
[ -z "${BASH_VERSION:-}" ] && { printf 'ERROR: requires bash\n' >&2; exit 1; }
case "$BASH_VERSION" in [12].*|3.0*|3.1*)
printf 'ERROR: bash 3.2+ required (found %s)\n' "$BASH_VERSION" >&2; exit 1 ;;
esac
_macos_pv="$(sw_vers -productVersion 2>/dev/null)"
if [ -z "$_macos_pv" ]; then
printf 'ERROR: requires macOS (Tahoe 26+); could not read sw_vers -productVersion.\n' >&2
exit 1
fi
_macos_major="${_macos_pv%%.*}"
case "$_macos_major" in
''|*[!0-9]*)
printf 'ERROR: unexpected sw_vers -productVersion output: %s\n' "$_macos_pv" >&2
exit 1 ;;
esac
_macos_major=$((10#$_macos_major))
if [ "$_macos_major" -lt 26 ]; then
printf 'ERROR: macOS Tahoe (26) or newer required. This Mac reports %s.\n' "$_macos_pv" >&2
printf ' Earlier macOS versions are not supported.\n' >&2
exit 1
fi
unset _macos_pv _macos_major
if [ -z "${HOME:-}" ] || [ ! -d "$HOME" ]; then
printf 'ERROR: HOME is unset or not a directory (got %q)\n' "${HOME:-}" >&2
exit 1
fi
set -uo pipefail
IFS=$' \t\n'
export COPYFILE_DISABLE=1
WARN_COUNT=0
ERR_COUNT=0
: "${_LEAF_COUNTS_ID:=$$}"
SCRATCH_DIR=""
OUTPUT_PATHS_TO_CLEAN=()
CHILD_PIDS=()
_TEE_PIDS=()
_PRE_PREPARE_WARNS=()
_PRE_PREPARE_ERRS=()
SUCCESS_FLAG="no"
_CLEANUP_RUNNING=0
DOWNPOUR_TRACE_FILE=""
if [ -n "${DOWNPOUR_TRACE:-}" ]; then
DOWNPOUR_TRACE_FILE="${TMPDIR:-/tmp}/downpour-trace.$$.log"
: > "$DOWNPOUR_TRACE_FILE" 2>/dev/null || DOWNPOUR_TRACE_FILE=""
if [ -n "$DOWNPOUR_TRACE_FILE" ]; then
exec 2> >(tee -a "$DOWNPOUR_TRACE_FILE" >&2)
set -x
else
printf 'warn: DOWNPOUR_TRACE set but cannot write %s — tracing disabled\n' \
"${TMPDIR:-/tmp}/downpour-trace.$$.log" >&2
fi
fi
_TM_SUSPENDED=0
_LAST_BLANK=0
[ "${_INVOKED_BY_WRAPPER:-0}" = "1" ] && _LAST_BLANK=1
_BANNER_SHOWN=0
: "${TCC_POLL_TIMEOUT:=20}"
case "$TCC_POLL_TIMEOUT" in
''|*[!0-9]*) TCC_POLL_TIMEOUT=20 ;;
0|[1-9]*) ;;
*) TCC_POLL_TIMEOUT=$((10#$TCC_POLL_TIMEOUT)) ;;
esac
: "${TCC_POLL_INTERVAL_SEC:=2}"
case "$TCC_POLL_INTERVAL_SEC" in
''|*[!0-9]*) TCC_POLL_INTERVAL_SEC=2 ;;
0) TCC_POLL_INTERVAL_SEC=1 ;;
[1-9]*) ;;
*) TCC_POLL_INTERVAL_SEC=$((10#$TCC_POLL_INTERVAL_SEC)) ;;
esac
if [ -t 1 ] || [ "${_DOWNPOUR_FORCE_TTY_OUT:-0}" = "1" ]; then
_B=$'\033[1m' # bold on
_D=$'\033[2m' # dim on
_N=$'\033[22m' # bold + dim off (back to regular)
else
_B=""; _D=""; _N=""
fi
_TAG_OK='ok:'; _TAG_WARN='warn:'; _TAG_ERR='error:'
_TAG_OK_W=${#_TAG_OK}; _TAG_WARN_W=${#_TAG_WARN}; _TAG_ERR_W=${#_TAG_ERR}
_C_OK=''; _C_WARN=''; _C_ERR=''; _C_RST1=''; _C_RST2=''
_C_BRAND=''; _C_BRAND_RST=''
_status_color=0
_TPUT_COLORS=0
if [ -z "${NO_COLOR:-}" ] && command -v tput >/dev/null 2>&1; then
_sc_n="$(tput colors 2>/dev/null || echo 0)"
case "$_sc_n" in ''|*[!0-9]*) _sc_n=0 ;; esac
_TPUT_COLORS="$_sc_n"
[ "$_sc_n" -ge 8 ] && _status_color=1
unset _sc_n
fi
if [ -t 1 ] || [ "${_DOWNPOUR_FORCE_TTY_OUT:-0}" = "1" ]; then
_TAG_OK='✓'; _TAG_OK_W=1
if [ "$_status_color" = "1" ]; then
_C_OK=$'\033[32m'; _C_RST1=$'\033[0m'
if [ "${_TPUT_COLORS:-0}" -ge 256 ]; then _C_BRAND=$'\033[38;5;75m'; else _C_BRAND=$'\033[34m'; fi
_C_BRAND_RST=$'\033[0m'
fi
fi
if [ -t 2 ] || [ "${_DOWNPOUR_FORCE_TTY_ERR:-0}" = "1" ]; then
_TAG_WARN='!'; _TAG_WARN_W=1; _TAG_ERR='✗'; _TAG_ERR_W=1
[ "$_status_color" = "1" ] && { _C_WARN=$'\033[33m'; _C_ERR=$'\033[31m'; _C_RST2=$'\033[0m'; }
fi
unset _status_color
_blank() {
[ "$_LAST_BLANK" = "1" ] && return
printf '\n'
_LAST_BLANK=1
}
_WRAP_WIDTH="${DOWNPOUR_WRAP_WIDTH:-120}"
case "$_WRAP_WIDTH" in ''|*[!0-9]*) _WRAP_WIDTH=120 ;; esac
{ [ "$_WRAP_WIDTH" -ge 40 ] && [ "$_WRAP_WIDTH" -le 200 ]; } || _WRAP_WIDTH=120
_REPORT_WIDTH="${DOWNPOUR_FILE_WRAP_WIDTH:-120}"
case "$_REPORT_WIDTH" in ''|*[!0-9]*) _REPORT_WIDTH=120 ;; esac
[ "$_REPORT_WIDTH" -gt 2000 ] && _REPORT_WIDTH=2000
[ "$_REPORT_WIDTH" -lt 40 ] && _REPORT_WIDTH=40
_log_fold() {
_LF_P1="$1" _LF_W1="$2" _LF_CONT="$3" _LF_W="${_WRAP_WIDTH:-120}" \
LC_ALL=C awk '
BEGIN {
p1 = ENVIRON["_LF_P1"]; w1 = ENVIRON["_LF_W1"] + 0
cont = ENVIRON["_LF_CONT"]; W = ENVIRON["_LF_W"] + 0; if (W <= 0) W = 80
a1 = W - w1; if (a1 < 8) a1 = 8
ac = W - length(cont); if (ac < 8) ac = 8
}
{
n = split($0, word, " "); line = ""; first = 1
for (i = 1; i <= n; i++) {
if (word[i] == "") continue
lim = (first ? a1 : ac)
if (line == "") line = word[i]
else if (length(line) + 1 + length(word[i]) > lim) {
if (first) { printf "%s%s\n", p1, line; first = 0 }
else printf "%s%s\n", cont, line
line = word[i]
}
else line = line " " word[i]
}
if (first) printf "%s%s\n", p1, line
else printf "%s%s\n", cont, line
}'
}
log_info() {
local _m="$*"
case "$_m" in
*": /"*|*": ~/"*) printf ' %s\n' "$_m"; _LAST_BLANK=0; return 0 ;;
esac
if [ "$(( ${#_m} + 2 ))" -le "${_WRAP_WIDTH:-120}" ]; then
printf ' %s\n' "$_m"
else
local _lead="${_m%%[! ]*}" _txt
_txt="${_m#"$_lead"}"
printf '%s\n' "$_txt" | _log_fold " $_lead" "$(( 2 + ${#_lead} ))" " $_lead"
fi
_LAST_BLANK=0
}
log_ok() {
local _m="$*" _pw=$(( 2 + _TAG_OK_W + 1 ))
if [ "$(( ${#_m} + _pw ))" -le "${_WRAP_WIDTH:-120}" ]; then
printf ' %s%s%s %s\n' "$_C_OK" "$_TAG_OK" "$_C_RST1" "$_m"
else
printf '%s\n' "$_m" | _log_fold " ${_C_OK}${_TAG_OK}${_C_RST1} " "$_pw" " "
fi
_LAST_BLANK=0
}
log_warn() {
local _m="$*" _pw=$(( 2 + _TAG_WARN_W + 1 ))
if [ "$(( ${#_m} + _pw ))" -le "${_WRAP_WIDTH:-120}" ]; then
printf ' %s%s%s %s\n' "$_C_WARN" "$_TAG_WARN" "$_C_RST2" "$_m" >&2
else
printf '%s\n' "$_m" | _log_fold " ${_C_WARN}${_TAG_WARN}${_C_RST2} " "$_pw" " " >&2
fi
WARN_COUNT=$((WARN_COUNT+1))
local _msg="${*//$'\n'/ }"
if [ -n "${_LEAF_COUNTS_DIR:-}" ] && [ -d "${_LEAF_COUNTS_DIR:-}" ] \
&& [ -w "${_LEAF_COUNTS_DIR:-}" ]; then
printf '%s\n' "$_msg" >> "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.warns" 2>/dev/null || true
else
_PRE_PREPARE_WARNS+=( "$_msg" )
fi
_LAST_BLANK=0
}
log_err() {
local _m="$*" _pw=$(( 2 + _TAG_ERR_W + 1 ))
if [ "$(( ${#_m} + _pw ))" -le "${_WRAP_WIDTH:-120}" ]; then
printf ' %s%s%s %s\n' "$_C_ERR" "$_TAG_ERR" "$_C_RST2" "$_m" >&2
else
printf '%s\n' "$_m" | _log_fold " ${_C_ERR}${_TAG_ERR}${_C_RST2} " "$_pw" " " >&2
fi
ERR_COUNT=$((ERR_COUNT+1))
local _msg="${*//$'\n'/ }"
if [ -n "${_LEAF_COUNTS_DIR:-}" ] && [ -d "${_LEAF_COUNTS_DIR:-}" ] \
&& [ -w "${_LEAF_COUNTS_DIR:-}" ]; then
printf '%s\n' "$_msg" >> "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.errs" 2>/dev/null || true
else
_PRE_PREPARE_ERRS+=( "$_msg" )
fi
_LAST_BLANK=0
}
log_step() { _blank; printf '%s%s── %s%s%s\n' "$_C_BRAND" "$_B" "$*" "$_N" "$_C_BRAND_RST"; _LAST_BLANK=0; }
die() { log_err "$*"; exit 1; }
cleanup_handler() {
local rc="${1:-$?}"
trap '' INT TERM HUP QUIT USR1
[ "$_CLEANUP_RUNNING" -eq 1 ] && return
_CLEANUP_RUNNING=1
local p _any_alive=0
if declare -F _heartbeat_stop >/dev/null 2>&1; then
_heartbeat_stop 2>/dev/null || true
fi
if declare -F leaf_cleanup_hook >/dev/null 2>&1; then
( leaf_cleanup_hook ) || true
fi
if [ -n "${_LEAF_COUNTS_DIR:-}" ]; then
local _post_w _post_e
if [ -s "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.warns" ]; then
_post_w="$(wc -l < "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.warns" 2>/dev/null | tr -d ' ')"
case "$_post_w" in ''|*[!0-9]*) _post_w=0 ;; esac
[ "$_post_w" -gt "$WARN_COUNT" ] && WARN_COUNT="$_post_w"
fi
if [ -s "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.errs" ]; then
_post_e="$(wc -l < "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.errs" 2>/dev/null | tr -d ' ')"
case "$_post_e" in ''|*[!0-9]*) _post_e=0 ;; esac
[ "$_post_e" -gt "$ERR_COUNT" ] && ERR_COUNT="$_post_e"
fi
fi
_write_leaf_counts
if [ "${#_TEE_PIDS[@]}" -gt 0 ]; then
local _t _n
for _t in "${_TEE_PIDS[@]+"${_TEE_PIDS[@]}"}"; do
disown "$_t" 2>/dev/null || true
done
exec 1>/dev/null 2>/dev/null
local _alive
_n=0
while [ "$_n" -lt 20 ]; do
_alive=0
for _t in "${_TEE_PIDS[@]+"${_TEE_PIDS[@]}"}"; do
kill -0 "$_t" 2>/dev/null && { _alive=1; break; }
done
[ "$_alive" = "0" ] && break
sleep 0.1 2>/dev/null || true
_n=$((_n + 1))
done
for _t in "${_TEE_PIDS[@]+"${_TEE_PIDS[@]}"}"; do
kill -KILL "$_t" 2>/dev/null || true
pkill -KILL -P "$_t" 2>/dev/null || true
done
local _new_cp=() _cp _tp _skip_cp
for _cp in "${CHILD_PIDS[@]+"${CHILD_PIDS[@]}"}"; do
_skip_cp=0
for _tp in "${_TEE_PIDS[@]+"${_TEE_PIDS[@]}"}"; do
[ "$_cp" = "$_tp" ] && { _skip_cp=1; break; }
done
[ "$_skip_cp" = "1" ] && continue
_new_cp+=( "$_cp" )
done
CHILD_PIDS=( "${_new_cp[@]+"${_new_cp[@]}"}" )
fi
for p in "${CHILD_PIDS[@]+"${CHILD_PIDS[@]}"}"; do
[ -z "$p" ] && continue
if kill -0 "$p" 2>/dev/null; then _any_alive=1; break; fi
done
if [ "$_any_alive" = "1" ]; then
for p in "${CHILD_PIDS[@]+"${CHILD_PIDS[@]}"}"; do
[ -z "$p" ] && continue
kill -TERM "$p" 2>/dev/null || true
done
sleep 1 2>/dev/null || true
for p in "${CHILD_PIDS[@]+"${CHILD_PIDS[@]}"}"; do
[ -z "$p" ] && continue
kill -KILL "$p" 2>/dev/null || true
done
fi
[ "${#CHILD_PIDS[@]}" -gt 0 ] && CHILD_PIDS=()
[ "${#_TEE_PIDS[@]}" -gt 0 ] && _TEE_PIDS=()
if [ "$SUCCESS_FLAG" != "yes" ] && [ "${#OUTPUT_PATHS_TO_CLEAN[@]}" -gt 0 ]; then
if [ "${_INVOKED_BY_WRAPPER:-0}" != "1" ]; then
local _kept_root="${_OUT_DIR:-}"
if [ -z "$_kept_root" ]; then
local _cand
for _cand in "${OUTPUT_PATHS_TO_CLEAN[@]+"${OUTPUT_PATHS_TO_CLEAN[@]}"}"; do
[ -d "$_cand" ] && { _kept_root="$_cand"; break; }
done
fi
printf ' Keeping the partial (INCOMPLETE) export%s — scratch/temp removed.\n' "${_kept_root:+ at $_kept_root}" 2>/dev/null >/dev/tty \
|| printf ' Keeping the partial (INCOMPLETE) export — scratch/temp removed.\n' 2>/dev/null \
|| true
if [ -n "$_kept_root" ] && [ -d "$_kept_root" ]; then
{
printf '\357\273\277' # UTF-8 BOM, as above
_report_frame "ABORTED — export interrupted"
printf ' Result: INCOMPLETE \342\200\224 the run was interrupted before it finished.\n'
printf '\n'; _report_band "WHAT THIS MEANS"; printf '\n'
printf ' \342\200\242 The partial output was KEPT; only scratch / temporary files were removed.\n'
printf ' \342\200\242 Some categories may be missing, partial, or mid-write.\n'
printf ' \342\200\242 Completeness was NOT verified: the run never reached its summary step,\n'
printf ' so nothing here is a ledger of what you got. Do NOT treat this folder as\n'
printf ' a complete or trustworthy backup.\n'
printf ' \342\200\242 Re-run downpour.sh to produce a complete, verified export.\n'
} > "$_kept_root/_downpour_report.txt" 2>/dev/null || true
fi
fi
fi
if [ -z "${_INVOKED_BY_WRAPPER:-}" ] && [ -n "${_LEAF_COUNTS_DIR:-}" ] \
&& [ -f "$_LEAF_COUNTS_DIR/leaf_scratch_dirs" ]; then
local _lsd
while IFS= read -r _lsd; do
[ -n "$_lsd" ] && [ -d "$_lsd" ] || continue
case "$_lsd" in */downpour.*) ;; *) continue ;; esac
chmod -R u+w "$_lsd" 2>/dev/null || true
rm -rf -- "$_lsd" 2>/dev/null || true
done < "$_LEAF_COUNTS_DIR/leaf_scratch_dirs"
fi
if [ -n "$SCRATCH_DIR" ] && [ -d "$SCRATCH_DIR" ]; then
chmod -R u+w "$SCRATCH_DIR" 2>/dev/null || true
rm -rf -- "$SCRATCH_DIR" 2>/dev/null || true
fi
if [ -n "$DOWNPOUR_TRACE_FILE" ]; then
if [ "$rc" -eq 0 ] && [ "$SUCCESS_FLAG" = "yes" ]; then
rm -f -- "$DOWNPOUR_TRACE_FILE" 2>/dev/null || true
else
printf ' trace: %s\n' "$DOWNPOUR_TRACE_FILE" 2>/dev/null >/dev/tty \
|| printf ' trace: %s\n' "$DOWNPOUR_TRACE_FILE" >&2 \
|| true
fi
fi
if [ -z "${_INVOKED_BY_WRAPPER:-}" ] && [ -z "${_LEAF_COUNTS_DIR:-}" ]; then
_blank
fi
trap - INT TERM HUP QUIT EXIT
exit "$rc"
}
resolve_output_dir() {
local out="${1:-$HOME/Desktop}"
case "$out" in
[~]) out="$HOME" ;;
[~]/*) out="$HOME/${out#~/}" ;;
esac
[ -d "$out" ] || die "Output folder does not exist: $out"
[ -w "$out" ] || die "Output folder is not writable: $out"
case "$out" in
*$'\n'*) die "Output folder path contains a newline (unsupported): $out" ;;
esac
local abs
abs="$(cd "$out" 2>/dev/null && pwd -P)" || die "Cannot resolve output folder: $out"
[ -n "$abs" ] || die "Cannot resolve output folder: $out"
[ "$abs" = "/" ] && die "Refusing to use / as the output folder"
case "$abs" in
*$'\n'*) die "Output folder path contains a newline (unsupported): $out" ;;
esac
local prefix
for prefix in "${SOURCE_PROTECTED_PREFIXES[@]+"${SOURCE_PROTECTED_PREFIXES[@]}"}"; do
case "$abs/" in
"$prefix"/*|"$prefix"/)
die "Output folder is inside protected source-data path: $prefix
Choose a destination outside ~/Library." ;;
esac
done
printf '%s' "$abs"
}
_parse_output_folder_arg() {
local out_arg="" arg end_opts=0 _seen=0
while [ "$#" -gt 0 ]; do
arg="$1"; shift
if [ "$end_opts" -eq 0 ]; then
case "$arg" in
--) end_opts=1; continue ;;
-*) die "Unknown flag: $arg" ;;
esac
fi
if [ "$_seen" -eq 1 ]; then
die "Too many positional arguments: '$arg' (already saw '$out_arg'). Pass at most one output folder."
fi
out_arg="$arg"; _seen=1
done
printf '%s' "$out_arg"
}
require_no_collision() {
local p
for p in "$@"; do
if [ -e "$p" ] || [ -L "$p" ]; then
die "Output exists, refusing to clobber: '$p'"
fi
done
}
_fda_grant_hint() {
log_info " Grant it in: System Settings > Privacy & Security > Full Disk Access"
log_info " Add your terminal (Terminal, iTerm, Ghostty, etc.) and enable the toggle."
}
check_full_disk_access() {
local tcc="$HOME/Library/Application Support/com.apple.TCC/TCC.db"
[ -n "$(head -c1 -- "$tcc" 2>/dev/null)" ] && return 0
local _fda_path
for _fda_path in \
"$HOME/Library/Application Support/AddressBook" \
"$HOME/Library/Mail" \
"$HOME/Library/Messages" \
"$HOME/Library/Group Containers/group.com.apple.calendar" \
"$HOME/Library/Group Containers/group.com.apple.notes"; do
[ -d "$_fda_path" ] || continue
ls "$_fda_path" >/dev/null 2>&1 && return 0
done
return 1
}
require_full_disk_access() {
case "${_FDA_STATE:-unknown}" in
granted)
return 0
;;
denied)
if check_full_disk_access; then
log_ok "Full Disk Access: granted (after wrapper pre-flight)"
return 0
fi
log_err "Full Disk Access not granted (wrapper pre-flight already polled and timed out)."
_fda_grant_hint
log_info " Then re-run."
exit 1
;;
unknown|'')
;;
*)
log_warn "_FDA_STATE='$_FDA_STATE' unrecognized; treating as unset (full probe-and-poll path)."
;;
esac
check_full_disk_access && return 0
log_info "Full Disk Access not yet granted — opening System Settings."
log_info " Add your terminal (Terminal, iTerm, Ghostty, etc.) to the"
log_info " Full Disk Access list and toggle it ON."
log_info " Polling for up to ${TCC_POLL_TIMEOUT} seconds; the run will proceed when you grant."
open 'x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles' \
>/dev/null 2>&1 || true
if wait_for_grant check_full_disk_access "Full Disk Access" "$TCC_POLL_TIMEOUT"; then
log_ok "Full Disk Access: granted"
return 0
fi
log_err "Full Disk Access was not granted within ${TCC_POLL_TIMEOUT} s."
_fda_grant_hint
log_info " If polling didn't pick up your grant, quit and re-launch the"
log_info " terminal, then re-run this script."
exit 1
}
require_accessibility() {
case "${_ACCESSIBILITY_STATE:-unknown}" in
granted)
return 0
;;
denied|na)
if check_accessibility; then
return 0
fi
log_err "Accessibility not granted (wrapper pre-flight already probed)."
log_info " Grant it in: System Settings > Privacy & Security > Accessibility"
log_info " Add your terminal binary and enable the toggle, then re-run."
return 1
;;
esac
log_info "Probing Accessibility..."
if check_accessibility; then
log_ok "Accessibility: granted"
return 0
fi
log_info "Accessibility permission required — opening System Settings."
log_info " Add your terminal (Terminal, iTerm, Ghostty, etc.) to the"
log_info " Privacy & Security > Accessibility list and toggle it ON."
log_info " Polling for up to ${TCC_POLL_TIMEOUT} seconds; the run will proceed when you grant."
open 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility' \
>/dev/null 2>&1 || true
if wait_for_grant check_accessibility "Accessibility" "$TCC_POLL_TIMEOUT"; then
log_ok "Accessibility: granted"
return 0
fi
log_err "Accessibility was not granted within ${TCC_POLL_TIMEOUT} s."
log_info " Grant it in: System Settings > Privacy & Security > Accessibility"
log_info " Add your terminal binary and enable the toggle, then re-run."
return 1
}
print_summary() {
_probe_disclose
if [ "${_INVOKED_BY_WRAPPER:-0}" = "1" ]; then
_write_leaf_counts
return 0
fi
_blank
if [ "$ERR_COUNT" -eq 0 ] && [ "$WARN_COUNT" -eq 0 ]; then
printf '%sDone. No warnings or errors.%s\n' "$_D" "$_N"
elif [ "$ERR_COUNT" -eq 0 ]; then
printf '%sDone with %d %s.%s\n' \
"$_D" "$WARN_COUNT" "$(_pluralize "$WARN_COUNT" warning)" "$_N"
else
printf '%sDone with %d %s and %d %s.%s\n' \
"$_D" "$ERR_COUNT" "$(_pluralize "$ERR_COUNT" error)" \
"$WARN_COUNT" "$(_pluralize "$WARN_COUNT" warning)" "$_N"
fi
_LAST_BLANK=0
_blank
_write_leaf_counts
if [ -n "${_LEAF_REPORT_CATEGORY:-}" ] && [ -n "${_LEAF_REPORT_DIR:-}" ]; then
if [ ! -d "$_LEAF_REPORT_DIR" ] && [ -s "$(_report_store)" ]; then
mkdir -p "$_LEAF_REPORT_DIR" 2>/dev/null || true
fi
[ -d "$_LEAF_REPORT_DIR" ] && \
finalize_leaf_report "$_LEAF_REPORT_CATEGORY" "$_LEAF_REPORT_DIR"
fi
}
_write_leaf_counts() {
[ "${_INVOKED_BY_WRAPPER:-0}" = "1" ] || return 0
[ -n "${_LEAF_COUNTS_DIR:-}" ] || return 0
[ -d "${_LEAF_COUNTS_DIR:-}" ] && [ -w "${_LEAF_COUNTS_DIR:-}" ] || return 0
{
printf 'WARN=%d\n' "$WARN_COUNT"
printf 'ERR=%d\n' "$ERR_COUNT"
} > "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.counts" 2>/dev/null || true
}
maybe_show_help() {
local arg
for arg in "$@"; do
case "$arg" in
-h|--help|-\?|--help=*)
local src="${BASH_SOURCE[0]:-}"
if [ -n "$src" ] && [ -f "$src" ] && [ -r "$src" ]; then
awk 'NR==1 { next }
/^$/ { exit }
{ sub(/^# ?/, ""); print }' "$src"
else
printf 'Help text lives in this script'\''s header comment (the spec is the script).\n'
printf 'View it with: curl -fsSL | head -60\n'
fi
exit 0
;;
esac
done
}
_countdown_key_prompt() {
local _ckp_sec="$1" _ckp_var="$2" _ckp_text="$3" _ckp_s _ckp_stty=""
printf -v "$_ckp_var" '%s' '' 2>/dev/null || true
case "$_ckp_sec" in ''|*[!0-9]*|0) _ckp_sec=10 ;; esac
( : /dev/null || return 0
local _ckp_kf; _ckp_kf="$(mktemp "${SCRATCH_DIR:-${TMPDIR:-/tmp}}/.ckp.XXXXXX" 2>/dev/null)" || return 0
_ckp_stty="$(stty -g /dev/null || true)"
( _ckp_k=""; IFS= read -r -n 1 _ckp_k /dev/null \
&& printf '%s' "$_ckp_k" > "$_ckp_kf"; ) &
local _ckp_rpid=$!
for _ckp_s in $(LC_ALL=C seq "$_ckp_sec" -1 1); do
printf '\r %s (auto-no in %ds) ' "$_ckp_text" "$_ckp_s" >&2
[ -s "$_ckp_kf" ] && break
sleep 1 2>/dev/null || true
[ -s "$_ckp_kf" ] && break
done
kill -KILL "$_ckp_rpid" 2>/dev/null || true
wait "$_ckp_rpid" 2>/dev/null || true
[ -n "$_ckp_stty" ] && stty "$_ckp_stty" /dev/null || true
printf -v "$_ckp_var" '%s' "$(cat "$_ckp_kf" 2>/dev/null)" 2>/dev/null || true
rm -f "$_ckp_kf" 2>/dev/null
}
ensure_scratch() {
[ -n "$SCRATCH_DIR" ] && [ -d "$SCRATCH_DIR" ] && return 0
SCRATCH_DIR="$(mktemp -d -t downpour)" || die "Cannot create scratch directory"
[ -n "$SCRATCH_DIR" ] && [ -d "$SCRATCH_DIR" ] || die "Cannot create scratch directory (mktemp produced no usable path)."
case "$SCRATCH_DIR" in
*[!A-Za-z0-9./_-]*)
local _orphan="$SCRATCH_DIR"
SCRATCH_DIR=""
rm -rf -- "$_orphan" 2>/dev/null || true
die "Unsafe scratch path '$_orphan' (contains characters outside [A-Za-z0-9./_-]).
TMPDIR='$TMPDIR' produced this path; set TMPDIR to a path of safe characters and re-run."
;;
esac
if [ "${_INVOKED_BY_WRAPPER:-}" = "1" ] && [ -n "${_LEAF_COUNTS_DIR:-}" ] \
&& [ -d "$_LEAF_COUNTS_DIR" ]; then
printf '%s\n' "$SCRATCH_DIR" >> "$_LEAF_COUNTS_DIR/leaf_scratch_dirs" 2>/dev/null || true
fi
}
track_output_path() { OUTPUT_PATHS_TO_CLEAN+=( "$1" ); }
_untrack_output_path() {
local _target="$1" _cp _new=() _removed=0
for _cp in "${OUTPUT_PATHS_TO_CLEAN[@]+"${OUTPUT_PATHS_TO_CLEAN[@]}"}"; do
if [ "$_removed" -eq 0 ] && [ "$_cp" = "$_target" ]; then
_removed=1
continue
fi
_new+=( "$_cp" )
done
OUTPUT_PATHS_TO_CLEAN=( "${_new[@]+"${_new[@]}"}" )
}
mark_success() { SUCCESS_FLAG="yes"; }
_md5_hex() { md5 -q -- "$1" 2>/dev/null; }
_create_output_dir() {
local _path="$1" _ctx="${2:-}"
track_output_path "$_path"
mkdir -p "$_path" || die "Cannot create $_path${_ctx:+ ($_ctx)}"
}
is_macos_junk_path() {
case "$1" in
.DS_Store|*/.DS_Store) return 0 ;;
.localized|*/.localized) return 0 ;;
._*|*/._*) return 0 ;;
.AppleDouble|.AppleDouble/*|*/.AppleDouble|*/.AppleDouble/*) return 0 ;;
.Spotlight-V100|.Spotlight-V100/*|*/.Spotlight-V100|*/.Spotlight-V100/*) return 0 ;;
.fseventsd|.fseventsd/*|*/.fseventsd|*/.fseventsd/*) return 0 ;;
.Trashes|.Trashes/*|*/.Trashes|*/.Trashes/*) return 0 ;;
.TemporaryItems|.TemporaryItems/*|*/.TemporaryItems|*/.TemporaryItems/*) return 0 ;;
.DocumentRevisions-V100|.DocumentRevisions-V100/*|*/.DocumentRevisions-V100|*/.DocumentRevisions-V100/*) return 0 ;;
__MACOSX|__MACOSX/*|*/__MACOSX|*/__MACOSX/*) return 0 ;;
.VolumeIcon.icns|*/.VolumeIcon.icns) return 0 ;;
.com.apple.timemachine|*/.com.apple.timemachine) return 0 ;;
.com.apple.timemachine.*|*/.com.apple.timemachine.*) return 0 ;;
.apdisk|*/.apdisk) return 0 ;;
esac
return 1
}
is_macos_alias() {
local f="$1" size magic hex
_is_network_mount_for "$f" && return 1
size="$(stat -f %z -- "$f" 2>/dev/null)"
case "$size" in
''|*[!0-9]*) return 1 ;;
esac
[ "$size" -lt 200 ] && return 1
[ "$size" -gt 32768 ] && return 1
magic="$(LC_ALL=C od -An -tx1 -N 4 -- "$f" 2>/dev/null)"
magic="${magic// /}"
magic="${magic//$'\n'/}"
case "$magic" in
626f6f6b) return 0 ;;
esac
hex="$(xattr -px com.apple.FinderInfo -- "$f" 2>/dev/null | head -1)"
hex="${hex// /}"
hex="${hex:0:8}"
hex="${hex//A/a}"
hex="${hex//B/b}"
hex="${hex//C/c}"
hex="${hex//D/d}"
hex="${hex//E/e}"
hex="${hex//F/f}"
case "$hex" in
616c6973*|616c7973*) return 0 ;;
esac
return 1
}
should_skip_copy() {
local f="$1" rel="${2:-$1}"
[ -L "$f" ] && return 0
is_macos_junk_path "$rel" && return 0
[ "${SKIP_ALIASES:-0}" = "1" ] && is_macos_alias "$f" && return 0
return 1
}
is_icloud_stub() {
case "${1##*/}" in
.?*.icloud) return 0 ;;
esac
return 1
}
_strip_icloud_stub_name() {
local s="$1"
s="${s%.icloud}"
s="${s#.}"
if [ -n "${2:-}" ]; then printf -v "$2" '%s' "$s"; return 0; fi
printf '%s' "$s"
}
_expand_tilde() {
local p="$1"
case "$p" in
[~]|[~]/*) printf '%s' "$HOME${p#~}" ;;
*) printf '%s' "$p" ;;
esac
}
_is_gzipped() {
local mb
mb="$(LC_ALL=C head -c 2 -- "$1" 2>/dev/null | xxd -p 2>/dev/null)"
[ "$mb" = "1f8b" ]
}
_icloud_stub_sibling_for() {
local p="${1:-}"
[ -z "$p" ] && return 1
local _bn="${p##*/}"
[ -z "$_bn" ] && return 1
local _dir=""
case "$p" in
*/*) _dir="${p%/*}/" ;;
esac
local _stub="${_dir}.${_bn}.icloud"
[ -f "$_stub" ] && { printf '%s' "$_stub"; return 0; }
return 1
}
is_dataless() {
local _fl
_fl="$(stat -f '%f' -- "${1:-}" 2>/dev/null)" || return 1
case "$_fl" in ''|*[!0-9]*) return 1 ;; esac
[ "$(( 10#$_fl & 1073741824 ))" -ne 0 ]
}
_materialized_copy_incomplete() {
local _exp_size="$1" _dst="$2" _dst_size
_dst_size="$(stat -f '%z' -- "$_dst" 2>/dev/null)"
[ -n "$_dst_size" ] || return 0
if [ -n "$_exp_size" ] && [ "$_exp_size" != "$_dst_size" ]; then
return 0
fi
is_dataless "$_dst" && return 0
return 1
}
_emit_html_index() {
local _out="$1" _label="$2"
local _tmp _rows_rc=1 _group_rc
_tmp="$(mktemp "$_out.tmp.XXXXXX" 2>/dev/null)" || {
log_warn "Could not stage $_label — index not written."
return 1
}
track_output_path "$_tmp"
{
printf '\n\n\n'
printf '\n'
printf '%s\n' "${_idx_title:-Index}"
printf '\n\n\n'
printf '%s
\n' "${_idx_h1:-${_idx_title:-Index}}"
printf '\n%s\n\n' "${_idx_thead:-}"
"$_idx_rows_fn"
_rows_rc=$?
printf '\n
\n'
printf '\n' "$(_utc_now_iso8601)"
printf '\n\n'
} > "$_tmp"
_group_rc=$?
{ [ "$_rows_rc" -eq 0 ] && [ "$_group_rc" -eq 0 ]; }
_atomic_commit_tmp $? "$_tmp" "$_out" "$_label"
_untrack_output_path "$_tmp"
}
_epoch_to_utc_iso_into() {
local _e="${1:-}" _out="${2:-}"
[ -n "$_out" ] || return 0
case "$_e" in ''|*[!0-9]*) printf -v "$_out" '%s' ''; return 0 ;; esac
_e=$((10#$_e))
local _days=$((_e / 86400)) _sod=$((_e % 86400))
local _z=$((_days + 719468))
local _era=$((_z / 146097))
local _doe=$((_z - _era * 146097))
local _yoe=$(( (_doe - _doe/1460 + _doe/36524 - _doe/146096) / 365 ))
local _y=$((_yoe + _era * 400))
local _doy=$(( _doe - (365*_yoe + _yoe/4 - _yoe/100) ))
local _mp=$(( (5*_doy + 2) / 153 ))
local _d=$(( _doy - (153*_mp + 2)/5 + 1 ))
local _m
if [ "$_mp" -lt 10 ]; then _m=$((_mp + 3)); else _m=$((_mp - 9)); fi
[ "$_m" -le 2 ] && _y=$((_y + 1))
printf -v "$_out" '%04d-%02d-%02dT%02d:%02d:%02dZ' \
"$_y" "$_m" "$_d" "$((_sod / 3600))" "$(((_sod % 3600) / 60))" "$((_sod % 60))"
return 0
}
_var_key_into() {
local _s="${1:-}" _out="${2:-}" _i=0 _len=${#1} _c _o _h=2166136261
[ -n "$_out" ] || return 0
while [ "$_i" -lt "$_len" ]; do
_c="${_s:$_i:1}"
printf -v _o '%d' "'$_c" 2>/dev/null || _o=0
_h=$(( (_h ^ (_o & 255)) & 4294967295 ))
_h=$(( (_h * 16777619) & 4294967295 ))
_i=$((_i + 1))
done
printf -v "$_out" '%08x' "$_h"
return 0
}
_record_icloud_stub() {
local _cf="${1:-}" _label="${2:-}" _rel="${3:-}" _orig
[ -n "$_cf" ] || return 0
local _dir
_strip_icloud_stub_name "${_rel##*/}" _orig
tsv_escape "$_orig" _orig
tsv_escape "$_label" _label
case "$_rel" in
*/*) tsv_escape "${_rel%/*}" _dir
printf '%s\t%s/%s\n' "$_label" "$_dir" "$_orig" >> "$_cf" ;;
*) printf '%s\t%s\n' "$_label" "$_orig" >> "$_cf" ;;
esac
return 0
}
_ditto_dl_retry() {
local _s="$1" _d="$2" _wdl="${3:-0}" _e _r _try=0 _max
case "${ICLOUD_DOWNLOAD_RETRIES:-}" in
''|*[!0-9]*) _max=2 ;;
*) _max=$((10#${ICLOUD_DOWNLOAD_RETRIES})); [ "$_max" -gt 10 ] && _max=2 ;;
esac
while :; do
_e="$(ditto -- "$_s" "$_d" 2>&1 >/dev/null)"; _r=$?
[ "$_r" -eq 0 ] && break
[ "$_wdl" = "1" ] || break
case "$_e" in *"Permission denied"*|*"Operation not permitted"*) break ;; esac
[ "$_try" -ge "$_max" ] && break
_try=$((_try + 1))
rm -f -- "$_d" 2>/dev/null
sleep "$(( _try * 2 ))" 2>/dev/null || true
done
printf '%s' "$_e"
return "$_r"
}
_ditto_verified() {
local _s="$1" _d="$2" _rt="${3:-0}" _ev="${4:-}" _st _sz _fl _wdl=0 _e _r
_st="$(stat -f '%z %f' -- "$_s" 2>/dev/null)"
_sz="${_st%% *}"; _fl="${_st##* }"
case "$_sz" in ''|*[!0-9]*) _sz="" ;; esac
case "$_fl" in
''|*[!0-9]*) ;;
*) [ "$(( 10#$_fl & 1073741824 ))" -ne 0 ] && _wdl=1 ;;
esac
if [ "$_rt" = "1" ]; then
_e="$(_ditto_dl_retry "$_s" "$_d" "$_wdl")"; _r=$?
else
_e="$(ditto -- "$_s" "$_d" 2>&1 >/dev/null)"; _r=$?
fi
[ -n "$_ev" ] && printf -v "$_ev" '%s' "$_e"
if [ "$_r" -ne 0 ]; then
rm -f -- "$_d" 2>/dev/null
return 1
fi
if [ "$_wdl" = "1" ] && _materialized_copy_incomplete "$_sz" "$_d"; then
rm -f -- "$_d" 2>/dev/null
return 2
fi
return 0
}
_pluralize() {
local n="$1" singular="$2" plural="${3:-${2}s}"
n="${n#"${n%%[![:space:]]*}"}"
case "$n" in ''|*[!0-9]*) n=0 ;; esac
n=$((10#${n:-0}))
if [ "$n" = "1" ]; then printf '%s' "$singular"; else printf '%s' "$plural"; fi
}
_sql_cd_to_unix() {
local _col="$1"
printf '(CASE WHEN %s > 4000000000 THEN %s/1e9 + 978307200 ELSE %s + 978307200 END)' \
"$_col" "$_col" "$_col"
}
_count_lines() {
local _cl_file="$1" _cl_default="${2:-0}" _cl_n
_cl_n="$(LC_ALL=C awk 'END{print NR}' "$_cl_file" 2>/dev/null)"
case "$_cl_n" in
''|*[!0-9]*) printf '%s\n' "$_cl_default" ;;
*) printf '%s\n' "$_cl_n" ;;
esac
}
_count_csv_rows() {
local _ccr_file="$1" _ccr_default="${2:-0}" _ccr_n
_ccr_n="$(LC_ALL=C awk 'END{print NR-1}' "$_ccr_file" 2>/dev/null)"
case "$_ccr_n" in
''|*[!0-9]*) printf '%s\n' "$_ccr_default" ;;
*) printf '%s\n' "$_ccr_n" ;;
esac
}
_to_int() {
local _ti_v="$1" _ti_default="${2:-0}"
case "$_ti_v" in
''|*[!0-9]*) printf '%s\n' "$_ti_default" ;;
*) printf '%s\n' "$_ti_v" ;;
esac
}
_clamp_env_int() {
local _name="$1" _default="$2" _max="${3:-}"
local _v="${!_name:-}"
local _clamped_new="$_default"
case "$_v" in
''|*[!0-9]*) _clamped_new="$_default" ;;
0|0[0-9]*) _clamped_new="$_default" ;;
*)
if ! _is_int64_safe "$_v"; then
_clamped_new="$_default"
elif [ -n "$_max" ] && [ "$_v" -gt "$_max" ]; then
_clamped_new="$_default"
else
_clamped_new="$_v"
fi
;;
esac
printf -v "$_name" '%s' "$_clamped_new"
printf '%s' "$_clamped_new"
}
_dir_size_human() {
local path="$1" _s
[ -d "$path" ] || { printf '0B'; return; }
_s="$(LC_ALL=C du -sh "$path" 2>/dev/null | LC_ALL=C awk 'NR==1 { print $1; exit }')"
case "$_s" in
''|*[!0-9.BKMGTP]*|[!0-9]*) _s=0B ;;
esac
case "$_s" in
*[BKMGTP]) ;;
*) _s="${_s}B" ;;
esac
printf '%s' "${_s:-0B}"
}
_df_mountpoint_of() {
df -P "${1:-}" 2>/dev/null | LC_ALL=C awk '
NR==2 && NF>=6 { m=$6; for (i=7;i<=NF;i++) m=m" "$i; print m }'
}
_mount_fstype_of() {
LC_ALL=C mount 2>/dev/null | mp="$1" LC_ALL=C awk '
BEGIN { mp = ENVIRON["mp"] }
{
needle = " on " mp " ("
i = index($0, needle)
if (i == 0) next
fs = substr($0, i + length(needle))
k = index(fs, ","); if (k > 0) fs = substr(fs, 1, k - 1)
l = index(fs, ")"); if (l > 0) fs = substr(fs, 1, l - 1)
print tolower(fs); exit
}'
}
_target_filesystem_type() {
local _path="${1:-}" _mp _fs
[ -e "$_path" ] || return 0
_mp="$(_df_mountpoint_of "$_path")"
[ -z "$_mp" ] && return 0
_fs="$(_mount_fstype_of "$_mp")"
printf '%s' "$_fs"
}
_is_network_mount_for() {
local _p="${1:-}" _mp _key _cached _cached_mp _fs
[ -n "$_p" ] || return 1
_mp="$(_df_mountpoint_of "$_p")"
[ -n "$_mp" ] || return 1
_mp="${_mp%/}"
[ -n "$_mp" ] || _mp="/"
_key="$(printf '%s' "$_mp" | LC_ALL=C tr -c 'A-Za-z0-9_' '_')"
[ -n "$_key" ] || return 1
local _mp_ref="_NETMOUNT_CACHE_MP_$_key"
local _fs_ref="_NETMOUNT_CACHE_FS_$_key"
_cached_mp="${!_mp_ref:-}"
_cached="${!_fs_ref:-}"
if [ -z "$_cached" ] || [ "$_cached_mp" != "$_mp" ]; then
_fs="$(_mount_fstype_of "$_mp")"
case "$_fs" in
nfs|smbfs|afpfs|webdav|webdavfs|ftp|cifs) _cached=n ;;
*) _cached=l ;;
esac
printf -v "$_mp_ref" '%s' "$_mp"
printf -v "$_fs_ref" '%s' "$_cached"
fi
[ "$_cached" = "n" ]
}
warn_if_fragile_filesystem() {
local _fs
_fs="$(_target_filesystem_type "${1:-}")"
case "$_fs" in
exfat|msdos|ntfs)
log_warn "Output volume is $_fs — case-insensitive (filenames differing only in case collide), no POSIX permissions, no extended attributes; some exported metadata cannot be preserved on this volume."
;;
macfuse*|osxfuse*|tuxera_*|paragon_*|fuse|fuse-*|fuse_*|fusefs*|fuse[0-9]*)
log_warn "Output volume is a FUSE / third-party driver ($_fs) — case-insensitivity and extended-attribute support depend on the driver. Some exported metadata may not preserve."
;;
smbfs|nfs|webdav|webdavfs|cifs|afpfs|ftp)
log_warn "Output volume is a network filesystem ($_fs) — permission and metadata fidelity depend on the server."
;;
esac
return 0
}
_fmt_duration() {
local s="$1"
case "$s" in ''|*[!0-9]*) s=0 ;; esac
if [ "$s" -lt 60 ]; then
printf '%ds' "$s"
elif [ "$s" -lt 3600 ]; then
printf '%dm %ds' "$((s / 60))" "$((s % 60))"
else
printf '%dh %dm' "$((s / 3600))" "$(((s % 3600) / 60))"
fi
}
_fmt_relative_mtime() {
local _p="${1:-}"
[ -e "$_p" ] || { printf ''; return; }
local _mtime _now _delta
_mtime="$(stat -f '%m' -- "$_p" 2>/dev/null)"
case "$_mtime" in
''|*[!0-9]*) printf ''; return ;;
esac
_now="$(date +%s 2>/dev/null)"
case "$_now" in
''|*[!0-9]*) printf ''; return ;;
esac
_delta=$((_now - _mtime))
[ "$_delta" -lt 0 ] && _delta=0
if [ "$_delta" -lt 60 ]; then printf 'just now'
elif [ "$_delta" -lt 3600 ]; then printf '%dm ago' "$((_delta / 60))"
elif [ "$_delta" -lt 86400 ]; then printf '%dh ago' "$((_delta / 3600))"
elif [ "$_delta" -lt 2592000 ]; then printf '%dd ago' "$((_delta / 86400))"
elif [ "$_delta" -lt 31536000 ];then printf '%dmo ago' "$((_delta / 2592000))"
else printf '%dy ago' "$((_delta / 31536000))"
fi
}
_progress_tick() {
if [ -z "${_pt_last+x}" ] || [ -z "${_pt_inline+x}" ]; then
if [ -z "${_PROGRESS_TICK_WARNED:-}" ]; then
log_warn "_progress_tick called without caller-side 'local _pt_last=0 _pt_inline=0' — progress display may misbehave; this is a leaf-code bug."
_PROGRESS_TICK_WARNED=1
fi
_pt_last="${_pt_last:-0}"
_pt_inline="${_pt_inline:-0}"
fi
if [ "${_pt_last:-0}" -eq 0 ]; then
_pt_last=$((SECONDS + 1))
return 0
fi
[ "$(( SECONDS - _pt_last ))" -ge 5 ] || return 0
_pt_last=$SECONDS
mark_activity
local _d="${1:-0}" _t="${2:-0}" _n="${3:-items}" _msg
case "$_d" in ''|*[!0-9]*) _d=0 ;; esac
case "$_t" in ''|*[!0-9]*) _t=0 ;; esac
if [ "$_t" -gt 0 ]; then
_msg="$_d/$_t $_n ($(( _d * 100 / _t ))%)"
else
_msg="$_d $_n"
fi
if { [ -t 1 ] || [ "${_DOWNPOUR_FORCE_TTY_OUT:-0}" = "1" ]; }; then
printf '\r %s\033[K' "$_msg"
_pt_inline=1
else
log_info "$_msg"
fi
}
_progress_end() {
[ "${_pt_inline:-0}" = "1" ] || return 0
{ [ -t 1 ] || [ "${_DOWNPOUR_FORCE_TTY_OUT:-0}" = "1" ]; } && printf '\n'
_pt_inline=0
_pt_last=0
return 0
}
csv_field() {
local s="$1"
if [ "${#s}" -gt 1048576 ] && [ -z "${_CSV_OVERSIZE_WARNED:-}" ]; then
local _bytes
_bytes=$(printf '%s' "$s" | LC_ALL=C wc -c | LC_ALL=C tr -d ' ')
if [ "${_bytes:-0}" -gt 1048576 ]; then
log_warn "csv_field: emitting an oversized field (${_bytes} bytes) — typical RFC 4180 consumers cannot handle values this large; check the caller for an accidental BLOB pass-through."
_CSV_OVERSIZE_WARNED=1
fi
fi
s="${s//$'\r'/}"
case "$s" in
[=+@-]*|$'\t'*) s="'$s" ;;
esac
case "$s" in
*,*|*'"'*|*$'\n'*)
s="${s//\"/\"\"}"; s="${s//$'\n'/ }"
s="\"$s\"" ;;
esac
if [ -n "${2:-}" ]; then printf -v "$2" '%s' "$s"; return 0; fi
printf '%s' "$s"
}
_sanitize_name() {
local s="$1"
s="${s//\//_}"; s="${s//\\/_}"; s="${s//:/_}"
s="${s//\"/_}"; s="${s//\?/_}"; s="${s//\*/_}"
s="${s//|/_}"; s="${s///_}"
s="${s//$'\t'/ }"; s="${s//$'\n'/ }"; s="${s//$'\r'/ }"
s="${s//[[:cntrl:]]/}"
while [ "${s# }" != "$s" ] || [ "${s#.}" != "$s" ]; do s="${s# }"; s="${s#.}"; done
while [ "${s% }" != "$s" ] || [ "${s%.}" != "$s" ]; do s="${s% }"; s="${s%.}"; done
if [ "${#s}" -gt 50 ]; then
s="$(printf '%s' "$s" | LC_ALL=C awk '{
s = $0
if (length(s) > 200) {
s = substr(s, 1, 200)
last_lead = 0
for (i = length(s); i > 0; i--) {
c = substr(s, i, 1)
if (c ~ /[\200-\277]/) continue
if (c ~ /[\300-\377]/) last_lead = i
break
}
if (last_lead > 0) {
lead = substr(s, last_lead, 1)
expected = 0
if (lead ~ /[\300-\337]/) expected = 1
else if (lead ~ /[\340-\357]/) expected = 2
else if (lead ~ /[\360-\367]/) expected = 3
actual = length(s) - last_lead
if (actual < expected) {
s = substr(s, 1, last_lead - 1)
}
}
}
print s
}')"
fi
while [ "${s% }" != "$s" ] || [ "${s%.}" != "$s" ]; do s="${s% }"; s="${s%.}"; done
[ -z "$s" ] && s="Untitled"
printf '%s' "$s"
}
_snapshot_db_cleanup_partial() {
local _snap="$1" _why="$2"
if ! rm -f -- "$_snap" "$_snap-wal" "$_snap-shm" 2>/dev/null; then
if [ -f "$_snap" ]; then
mv -f -- "$_snap" "$_snap.partial.$$" 2>/dev/null || true
fi
log_warn "snapshot_db: failed to clean up partial snapshot family at $_snap ($_why; manual cleanup may be needed)"
fi
}
snapshot_db() {
local src="$1" src_md5 snap _src_bytes _need_kb _free_kb _src_mb
local _headroom_kb=102400 # 100 MB
local _progress_min_mb=50
[ -f "$src" ] || return 1
{ : < "$src"; } 2>/dev/null || return 2
ensure_scratch
_src_bytes="$(stat -f %z -- "$src" 2>/dev/null || printf 0)"
case "$_src_bytes" in ''|*[!0-9]*) _src_bytes=0 ;; esac
local _sb _sbz
for _sb in "$src-wal" "$src-shm"; do
if [ -f "$_sb" ]; then
_sbz="$(stat -f %z -- "$_sb" 2>/dev/null || printf 0)"
case "$_sbz" in ''|*[!0-9]*) _sbz=0 ;; esac
_src_bytes=$((_src_bytes + _sbz))
fi
done
_need_kb=$((_src_bytes / 1024 + _headroom_kb))
local _df_out
_df_out="$(df -P -k "$SCRATCH_DIR" 2>/dev/null)"
_free_kb="$(printf '%s\n' "$_df_out" | LC_ALL=C awk 'NR==2 && NF >= 4 {print $4+0}')"
if [ -n "$_free_kb" ]; then
if [ "$_free_kb" -eq 0 ]; then
printf ' error: scratch volume reports 0 MB free; cannot snapshot %s\n' "$src" >&2
return 5
elif [ "$_free_kb" -lt "$_need_kb" ]; then
printf ' error: scratch volume has %d MB free; need ~%d MB to snapshot %s\n' \
"$((_free_kb / 1024))" "$((_need_kb / 1024))" "$src" >&2
return 5
fi
fi
local _src_base
_src_base="$(printf '%s' "${src##*/}" | LC_ALL=C tr -c 'A-Za-z0-9._-' '_')"
src_md5="$(printf '%s' "$src" | md5 -q 2>/dev/null)"
case "${src_md5:-}" in
''|*[!0-9a-f]*)
log_err "snapshot_db: md5 output is not a valid hex digest for '$src' (got: ${src_md5:-}; md5 missing, swapped for a different tool, or broken)"
return 6 ;;
esac
[ "${#src_md5}" -eq 32 ] || {
log_err "snapshot_db: md5 output is not 32 chars long for '$src' (got length ${#src_md5}; md5 output format unexpected)"
return 6
}
snap="$SCRATCH_DIR/snap_${_src_base}_$src_md5.db"
local _snap_just_created=0
if [ ! -f "$snap" ]; then
_snap_just_created=1
_src_mb=$((_src_bytes / 1048576))
if [ "$_src_mb" -ge "$_progress_min_mb" ]; then
printf ' Snapshotting %s (%d MB)...\n' "${src##*/}" "$_src_mb" >&2
fi
if ! cp -f -- "$src" "$snap" 2>/dev/null; then
rm -f -- "$snap" 2>/dev/null
return 3
fi
if [ -f "$src-wal" ] && ! cp -f -- "$src-wal" "$snap-wal" 2>/dev/null; then
_snapshot_db_cleanup_partial "$snap" "WAL copy failed"
return 3
fi
if [ -f "$src-shm" ] && ! cp -f -- "$src-shm" "$snap-shm" 2>/dev/null; then
_snapshot_db_cleanup_partial "$snap" "SHM copy failed"
return 3
fi
if [ -f "$snap-wal" ]; then
sqlite3 "$snap" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true
fi
fi
local _qpid _qrc=0 _qwatchdog
trap '' USR1
sqlite3 "$snap" "SELECT 1;" >/dev/null 2>&1 &
_qpid=$!
( exec >/dev/null 2>&1 /dev/null || exit 0
kill -TERM "$_qpid" 2>/dev/null
sleep 1
kill -0 "$_qpid" 2>/dev/null && kill -KILL "$_qpid" 2>/dev/null
) /dev/null 2>&1 &
_qwatchdog=$!
wait "$_qpid" 2>/dev/null; _qrc=$?
pkill -KILL -P "$_qwatchdog" 2>/dev/null || true
kill -KILL "$_qwatchdog" 2>/dev/null
wait "$_qwatchdog" 2>/dev/null || true
trap _skip_handler USR1 2>/dev/null || true
if [ "$_qrc" -ne 0 ]; then
rm -f -- "$snap" "$snap-wal" "$snap-shm" 2>/dev/null
return 4
fi
if [ "$_snap_just_created" -eq 1 ] && [ -n "$_src_mb" ] && [ "$_src_mb" -ge "$_progress_min_mb" ]; then
local _wal_marker=""
[ -f "$snap-wal" ] && _wal_marker=", WAL truncated"
printf ' Snapshot OK: %s (%d MB%s, queryable)\n' \
"${src##*/}" "$_src_mb" "$_wal_marker" >&2
fi
printf '%s' "$snap"
}
snapshot_db_or_die() {
local _src="$1" _feature="$2" _snap _rc
_snap="$(snapshot_db "$_src")"
_rc=$?
case "$_rc" in
0) printf '%s' "$_snap" ;;
1) die "${_feature} source database is missing: $_src" ;;
2) die "${_feature} source database is unreadable: $_src — Full Disk Access required?" ;;
3) die "${_feature} snapshot copy failed: $_src — scratch volume full or permission denied?" ;;
4) die "${_feature} source database is corrupt or unqueryable: $_src — try closing the owning app and retrying" ;;
5) die "${_feature} snapshot needs more scratch space than is available; free up the scratch volume ($TMPDIR) or set TMPDIR to a larger volume" ;;
6) die "${_feature} snapshot failed: $_src — the md5 hashing tool is missing or broken (see the error above)" ;;
*) die "${_feature} snapshot failed (rc=$_rc): $_src" ;;
esac
}
require_db_tables() {
local _snap="$1" _feature="$2"
shift 2
local _t _missing=""
for _t in "$@"; do
case "$_t" in
''|*[!A-Za-z0-9_]*)
die "$_feature: invalid table name in schema check: '$_t'" ;;
esac
sqlite3 -readonly "$_snap" "SELECT 1 FROM \"$_t\" LIMIT 1;" \
>/dev/null 2>&1 || _missing="${_missing:+$_missing }$_t"
done
[ -z "$_missing" ] && return 0
local _present
_present="$(sqlite3 -readonly "$_snap" \
"SELECT GROUP_CONCAT(name, ' ') FROM sqlite_master \
WHERE type='table';" 2>/dev/null)"
die "$_feature: database schema unrecognised — expected table(s) not present: $_missing
Apple may have changed the schema in a macOS update; please report this.
Tables in this database: ${_present:-(none)}"
}
_col_present() {
local snap="$1" table="$2" col="$3"
[ -n "$snap" ] && [ -n "$table" ] && [ -n "$col" ] || return 1
case "$table" in *[!A-Za-z0-9_]*) return 1 ;; esac
case "$col" in *[!A-Za-z0-9_]*) return 1 ;; esac
local _cp_key _cp_var _cp_list
_cp_key="${snap}__${table}"
_cp_key="${_cp_key//[^A-Za-z0-9_]/_}"
_cp_var="_col_present_cache_$_cp_key"
_cp_list="${!_cp_var:-}"
if [ -z "$_cp_list" ]; then
local _cp_raw _cp_rc
_cp_raw="$(sqlite3 -readonly "$snap" \
"SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('$table');" \
2>/dev/null)"
_cp_rc=$?
if [ "$_cp_rc" -ne 0 ]; then
_probe_miss error "$table" "$col"
return 1
fi
_cp_list=",$_cp_raw,"
printf -v "$_cp_var" '%s' "$_cp_list"
fi
case "$_cp_list" in
*,"$col",*) return 0 ;;
esac
_probe_miss absent "$table" "$col"
return 1
}
_PROBE_KNOWN_ABSENT="
ZJOURNALENTRYMO.ZPLACENAME
ZJOURNALENTRYMO.ZSTATEOFMIND
ZJOURNALENTRYMO.ZTIMEZONE
ZJOURNALENTRYMO.ZWEATHERCONDITIONCODE
ZJOURNALENTRYMO.ZWEATHERHIGHTEMPERATURE
ZJOURNALENTRYMO.ZWEATHERHUMIDITY
ZJOURNALENTRYMO.ZWEATHERLOWTEMPERATURE
ZCLOUDRECORDING.ZISFAVORITE
ZCLOUDRECORDING.ZLOCALDELETEDDATE
ZCLOUDRECORDING.ZMODIFIEDDATE
ZCLOUDRECORDING.ZRECORDING_PARENT
ZCLOUDRECORDING.ZTRANSCRIPTION
ZCLOUDRECORDING.ZTRIMENDOFFSET
ZCLOUDRECORDING.ZTRIMSTARTOFFSET
ZFOLDER.ZPARENTFOLDER
ZFAVORITEITEM.ZLASTVISITEDTIME
ZFAVORITEITEM.ZPHONENUMBER
ZFAVORITEITEM.ZURL
ZTEXTREPLACEMENTENTRY.ZSYNCSTATE
ZTEXTREPLACEMENTENTRY.ZWASDELETEDFROMICLOUD
ZSHARE.ZCLOUDOWNERFIRSTNAME
ZSHARE.ZCLOUDOWNERHASHEDPERSONID
ZSHARE.ZCLOUDOWNERLASTNAME
ZREMCDREMINDER.ZRECURRENCERULE
"
_probe_store() {
local _s; _s="$(_report_store)" || return 1
printf '%s' "$_s.probes"
}
_probe_miss() {
[ "${_PROBE_SUPPRESS:-0}" = "1" ] && return 0
local _pk
if [ "${1:-absent}" = "absent" ] && [ -n "${3:-}" ]; then
_pk=$'\n'"${2:-}.${3}"$'\n'
case "${_PROBE_KNOWN_ABSENT:-}" in *"$_pk"*) return 0 ;; esac
fi
local _st; _st="$(_probe_store)" || return 0
printf '%s\t%s\t%s\t%s\n' \
"$(_report_flat "$(_report_subject "${_LEAF_CATEGORY_DIR:-}")")" \
"${1:-absent}" "${2:-}" "${3:-}" >> "$_st" 2>/dev/null || true
return 0
}
_probe_disclose() {
local _dir="${1:-${_LEAF_CATEGORY_DIR:-}}" _st _subj _kind _list _n
_st="$(_probe_store)" || return 0
[ -s "$_st" ] || return 0
_subj="$(_report_flat "$(_report_subject "$_dir")")"
[ -n "$_subj" ] || return 0
for _kind in absent error; do
_list="$(LC_ALL=C awk -F'\t' -v s="$_subj" -v k="$_kind" \
'$1==s && $2==k {print $3 "." $4}' "$_st" 2>/dev/null \
| LC_ALL=C sort -u)"
[ -n "$_list" ] || continue
_n="$(printf '%s\n' "$_list" | LC_ALL=C awk 'END{print NR}')"
if [ "$_kind" = "absent" ]; then
log_info "Schema probes: $_n optional $(_pluralize "$_n" column) absent on this macOS — the fields they carry are blank or omitted in this export$(_disclosure_pointer)."
report_add schema "$_subj" "$_n" "" "$_list"
else
log_info "Schema probes: $_n column $(_pluralize "$_n" check) could not be completed (database busy or errored) — the features they gate were disabled WITHOUT confirming the columns are absent$(_disclosure_pointer)."
report_incomplete "$_subj" "schema probes could not be completed" \
"$_n column $(_pluralize "$_n" check) failed to run (the database was locked or returned an error), so downpour disabled the features they gate without being able to confirm those columns are really missing. Quit the source app and re-run; if the columns do exist, the affected content will be captured." \
"$_list"
fi
done
: > "$_st" 2>/dev/null || true
return 0
}
_col_in_list() {
printf '%s\n' "$1" | LC_ALL=C grep -qFx -- "$2"
}
_pick_first_col() {
local _snap="$1" _table="$2"
shift 2
local _candidate _pfc_prev="${_PROBE_SUPPRESS:-0}"
_PROBE_SUPPRESS=1
for _candidate in "$@"; do
if _col_present "$_snap" "$_table" "$_candidate"; then
_PROBE_SUPPRESS="$_pfc_prev"
printf '%s' "$_candidate"
return 0
fi
done
_PROBE_SUPPRESS="$_pfc_prev"
_probe_miss absent "$_table" "$1"
return 1
}
_resolve_entity_number() {
case "$2" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
local _ent
_ent="$(sqlite3 -readonly "$1" \
"SELECT Z_ENT FROM Z_PRIMARYKEY WHERE Z_NAME='$2' LIMIT 1;" \
2>/dev/null)"
case "$_ent" in ''|*[!0-9]*) return 1 ;; esac
printf '%s' "$_ent"
}
_discover_z_join() {
case "$2" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
local _matches _n _t
_matches="$(sqlite3 -readonly "$1" \
"SELECT name FROM sqlite_master
WHERE type='table' AND name GLOB 'Z_[0-9]*$2'
ORDER BY name;" 2>/dev/null)"
_n="$(printf '%s\n' "$_matches" | grep -c .)"
case "${_n:-0}" in ''|*[!0-9]*) _n=0 ;; esac
[ "$_n" -eq 0 ] && return 1
[ "$_n" -gt 1 ] && return 2
_t="$_matches"
case "$_t" in Z_*) : ;; *) return 1 ;; esac
case "$_t" in *[!A-Za-z0-9_]*) return 1 ;; esac
printf '%s' "$_t"
}
report_no_source() {
local feature="$1" reason="$2" path="${3:-}" icloud_app="${4:-}"
log_warn "$feature $reason on this Mac — nothing to export"
[ -n "$path" ] && log_info "(Looked for: $(tsv_escape "$path"))"
_icloud_sync_hint "$icloud_app"
_flag_no_content
mark_success
print_summary
exit 0
}
report_empty_source() {
local items="$1" icloud_app="${2:-}"
log_warn "No $items to export — nothing to do."
_icloud_sync_hint "$icloud_app"
_flag_no_content
mark_success
print_summary
exit 0
}
_flag_no_content() {
[ -n "${_LEAF_COUNTS_DIR:-}" ] && [ -d "${_LEAF_COUNTS_DIR:-}" ] \
&& [ -w "${_LEAF_COUNTS_DIR:-}" ] || return 0
: > "$_LEAF_COUNTS_DIR/$_LEAF_COUNTS_ID.nocontent" 2>/dev/null || true
}
_icloud_sync_hint() {
[ -n "${1:-}" ] || return 0
log_info "If $1 is in use on another Apple device, this may be an iCloud-sync gap — open $1 once to trigger sync, then re-run."
}
_url_encode_path() {
_uep_in="$1" LC_ALL=C awk 'BEGIN{ for(i=0;i<256;i++) ord[sprintf("%c",i)]=i
s=ENVIRON["_uep_in"]; out=""
for(i=1;i<=length(s);i++){ c=substr(s,i,1)
if (c ~ /[A-Za-z0-9._~\/-]/) out=out c
else out=out sprintf("%%%02X", ord[c]) }
print out }'
}
_bytes_human() {
awk -v b="${1:-0}" 'BEGIN{
split("B KB MB GB TB", u); s=1
while (b>=1024 && s<5){ b=b/1024; s++ }
if (s==1) printf "%d %s", b, u[s]; else printf "%.1f %s", b, u[s]
}'
}
_optimize_storage_prose() {
local _label="$1" _evicted="$2" _total="${3:-}" _recover="${4:-}"
case "${_evicted:-0}" in ''|*[!0-9]*) return 0 ;; esac
_is_int64_safe "$_evicted" || return 0
_evicted=$((10#${_evicted:-0}))
[ "$_evicted" -eq 0 ] && return 0
local _msg="iCloud eviction (Optimize Mac Storage or app-equivalent) appears active for $_label — "
if [ -n "$_total" ] && [ "$_total" != "0" ]; then
case "$_total" in
*[!0-9]*) _total="" ;;
*) if _is_int64_safe "$_total"; then _total=$((10#$_total)); else _total=""; fi ;;
esac
fi
if [ -n "$_total" ] && [ "$_total" -gt 0 ]; then
_msg="$_msg$_evicted of $_total $(_pluralize "$_total" item) cloud-only and not downloaded."
else
_msg="$_msg$_evicted $(_pluralize "$_evicted" item) cloud-only and not downloaded."
fi
if [ -n "$_recover" ]; then
_msg="$_msg To capture them: $_recover, then re-run."
else
_msg="$_msg To capture them: disable iCloud's Optimize Mac Storage for $_label and wait for sync, then re-run."
fi
_OPTIMIZE_STORAGE_PROSE="$_msg"
return 0
}
_disclose_optimize_storage() {
_OPTIMIZE_STORAGE_PROSE=""
_optimize_storage_prose "$@" || return 0
[ -n "$_OPTIMIZE_STORAGE_PROSE" ] || return 0
log_warn "$_OPTIMIZE_STORAGE_PROSE"
}
_disclosure_pointer() {
[ "${_INVOKED_BY_WRAPPER:-0}" = "1" ] && return 0
printf ' — see _downpour_report.txt'
}
_photos_report_header() {
local lib="${1//$'\n'/ }"; lib="${lib//$'\r'/ }"
printf '# downpour-photos-report v1\n'
printf '# library: %s\n' "$lib"
printf '# generated: %s\n' "$(_utc_now_iso8601)"
printf '# host: %s\n' "$(hostname -s 2>/dev/null || printf 'unknown')"
printf '# hash: md5\n'
}
_doc_header() {
local _title="${1//$'\n'/ }"; _title="${_title//$'\r'/ }"; shift
printf '%s\n' "$_title"
if [ $# -ge 2 ]; then
printf '\n'
local _k _v
while [ $# -ge 2 ]; do
_k="${1//$'\n'/ }"; _k="${_k//$'\r'/ }"
_v="${2//$'\n'/ }"; _v="${_v//$'\r'/ }"
printf '%-8s %s\n' "$_k:" "$_v"
shift 2
done
fi
printf '\n'
}
_report_store() {
if [ -n "${_REPORT_STORE:-}" ]; then printf '%s' "$_REPORT_STORE"; return 0; fi
[ -n "${SCRATCH_DIR:-}" ] || return 1
_REPORT_STORE="$SCRATCH_DIR/_report_entries"
printf '%s' "$_REPORT_STORE"
}
_report_store_init() {
_report_store >/dev/null 2>&1 && return 0
SCRATCH_DIR="$(mktemp -d -t downpour 2>/dev/null || printf '')"
if [ -n "$SCRATCH_DIR" ] && [ -d "$SCRATCH_DIR" ]; then
case "$SCRATCH_DIR" in
*[!A-Za-z0-9./_-]*) rm -rf -- "$SCRATCH_DIR" 2>/dev/null || true
SCRATCH_DIR="" ;;
esac
else
SCRATCH_DIR=""
fi
return 0
}
_report_flat() { printf '%s' "${1:-}" | LC_ALL=C tr -d '\011\012\015\035\037\000'; }
_report_subject() {
local _s
if [ -n "${_LEAF_REPORT_CATEGORY:-}" ]; then
_s="$_LEAF_REPORT_CATEGORY"
else
local _d="${1:-}" _rel="${1:-}"
if [ -n "${_EXPORT_ROOT:-}" ]; then
case "$_d/" in "$_EXPORT_ROOT"/*) _rel="${_d#"$_EXPORT_ROOT"/}" ;; esac
fi
_rel="${_rel#/}"; _s="${_rel%%/*}"
fi
case "$_s" in
'Music Playlists') _s='Music playlists' ;;
'Voice Memos') _s='Voice memos' ;;
'Safari') _s='Safari bookmarks' ;;
esac
printf '%s' "$_s"
}
report_add() {
local _sev="${1:-}" _subj="${2:-}" _title="${3:-}" _prose="${4:-}" _items_raw="${5:-}"
local _store; _store="$(_report_store)" || return 0
local _items=""
if [ -n "$_items_raw" ]; then
_items="$(printf '%s' "$_items_raw" | LC_ALL=C tr -d '\015\035\037\000' | LC_ALL=C tr '\012\011' '\035 ')"
_items="${_items%$'\035'}"
fi
printf '%s\037%s\037%s\037%s\037%s\n' \
"$_sev" "$(_report_flat "$_subj")" "$(_report_flat "$_title")" \
"$(_report_flat "$_prose")" "$_items" >> "$_store" 2>/dev/null || true
}
report_incomplete() { report_add incomplete "${1:-}" "${2:-}" "${3:-}" "${4:-}"; }
report_blocked() { report_add blocked "${1:-}" "${2:-}" "${3:-}" "${4:-}"; }
report_skipped() { report_add skipped "${1:-}" "${2:-}" "${3:-}" "${4:-}"; } # retained for API symmetry + pinned by tests/fault-injection.sh (no live caller; skipped is emitted via _write_skipped_report)
report_review() { report_add review "${1:-}" "${2:-}" "${3:-}" "${4:-}"; }
report_warning() { report_add warning "${1:-}" "${2:-}" "" ""; } #
report_status() { report_add "${1:-}" "${2:-}" "" "" ""; } #
_disclose_excluded() {
local _short="${1:-}" _why="${2:-}" _subj
[ -n "$_short" ] || return 0
if [ -n "$_why" ]; then
log_info "Excluded $_short — $_why"
else
log_info "Excluded $_short"
fi
_subj="$(_report_subject "${_LEAF_CATEGORY_DIR:-}")"
[ -n "$_subj" ] && report_add excluded "$_subj" "$_short" "" ""
return 0
}
_report_has_severity() {
local _sev="${1:-}" _dir="${2:-}" _store _subj _us
_store="$(_report_store)" || return 1
[ -f "$_store" ] || return 1
_subj="$(_report_flat "$(_report_subject "$_dir")")"
_us=$'\037'
s="$_sev" j="$_subj" awk -F"$_us" \
'BEGIN { s=ENVIRON["s"]; j=ENVIRON["j"] } $1==s && $2==j { f=1; exit } END { exit !f }' "$_store" 2>/dev/null
}
report_have_incomplete() { _report_has_severity incomplete "${1:-}"; }
report_have_skipped() { _report_has_severity skipped "${1:-}"; }
_write_skipped_report() {
local _dir="${1:-}" _feature="${2:-}" _ev="${3:-}" _tot="${4:-}" _rec="${5:-}"
[ -n "$_dir" ] || { cat >/dev/null; return 0; }
local _items; _items="$(cat)"
local _prose="These were only in iCloud at export time (not on this Mac). Re-download them in Finder or the source app, then re-run."
if [ -n "$_ev" ]; then
_OPTIMIZE_STORAGE_PROSE=""
_optimize_storage_prose "$_feature" "$_ev" "$_tot" "$_rec"
[ -n "$_OPTIMIZE_STORAGE_PROSE" ] && _prose="$_OPTIMIZE_STORAGE_PROSE"
fi
report_add skipped "$(_report_subject "$_dir")" "$_feature — cloud-only items" \
"$_prose" "$_items"
}
note_incomplete() {
local _dir="${1:-}" _what="${2:-}" _why="${3:-}"
[ -n "$_dir" ] || return 0
local _subj; _subj="$(_report_subject "$_dir")"
report_add incomplete "$_subj" "$_what" "$_why" ""
}
_incomplete_push() {
local _ipn="$1"; shift
case "$_ipn" in
''|*[!A-Za-z0-9_]*)
die "_incomplete_push: invalid array name: '$_ipn'" ;;
esac
local _us; _us=$'\x1f'
local _iw="${2:-}"
eval "$_ipn+=( \"\$1\$_us\$_iw\" )"
}
_incomplete_drain() {
local _iddir="${1:-}"; shift 2>/dev/null || true
[ -n "$_iddir" ] || return 0
local _us _ide; _us=$'\x1f'
for _ide in "$@"; do
note_incomplete "$_iddir" "${_ide%%$_us*}" "${_ide#*$_us}"
done
}
_summary_wrap_text() {
IND="$1" _W="${_REPORT_WIDTH:-120}" LC_ALL=C awk '
BEGIN { ind = ENVIRON["IND"]; w = (ENVIRON["_W"] + 0) - length(ind); if (w < 20) w = 20 }
{
nn = split($0, word, " ")
nl = 0; line = ""
for (i = 1; i <= nn; i++) {
if (word[i] == "") continue
if (line == "") line = word[i]
else if (length(line)+1+length(word[i]) > w) { lines[++nl] = line; line = word[i] }
else line = line " " word[i]
}
if (line != "") lines[++nl] = line
if (nl >= 2 && index(lines[nl], " ") == 0) {
p = lines[nl-1]; sp = 0
for (k = length(p); k >= 1; k--) if (substr(p,k,1) == " ") { sp = k; break }
if (sp > 0) {
merged = substr(p, sp+1) " " lines[nl]
if (length(merged) <= w) { lines[nl-1] = substr(p,1,sp-1); lines[nl] = merged }
}
}
for (i = 1; i <= nl; i++) print ind lines[i]
for (k in lines) delete lines[k]
}'
}
_report_band() {
local _label="${1:-}" _n="${2:-}"
if [ -n "$_n" ]; then
printf ' %s \302\267 %s\n' "$_label" "$_n"
else
printf ' %s\n' "$_label"
fi
}
_render_entry() {
local _rec="${1:-}" _title _prose _items _it
_title="${_rec%%$'\037'*}"; _rec="${_rec#*$'\037'}"
_prose="${_rec%%$'\037'*}"; _items="${_rec#*$'\037'}"
printf ' \342\200\242 %s\n' "$_title"
[ -n "$_prose" ] && printf '%s\n' "$_prose" | _summary_wrap_text ' '
if [ -n "$_items" ]; then
local _cap="${DOWNPOUR_REPORT_MAX_ITEMS:-100}" _n=0 _tot
case "$_cap" in ''|*[!0-9]*) _cap=100 ;; esac
_tot="$(printf '%s\n' "$_items" | LC_ALL=C tr '\035' '\012' | LC_ALL=C grep -c . )"
printf '%s\n' "$_items" | LC_ALL=C tr '\035' '\012' | while IFS= read -r _it; do
[ -n "$_it" ] || continue
_n=$((_n + 1))
if [ "$_cap" -gt 0 ] && [ "$_n" -gt "$_cap" ]; then
printf ' \342\200\246 and %d more (set DOWNPOUR_REPORT_MAX_ITEMS=0 for the full list)\n' \
"$(( _tot - _cap ))"
break
fi
printf ' %s\n' "$_it"
done
fi
}
_report_frame() {
local _sub="${1:-}" _rule _rwid _i=0
_rwid="${_REPORT_WIDTH:-120}"
_rule="$(while [ "$_i" -lt "$_rwid" ]; do printf '\342\225\220'; _i=$((_i + 1)); done)"
printf '%s\n' "$_rule"
printf ' downpour report\n'
printf ' %s \302\267 %s\n' "$(date '+%Y-%m-%d %H:%M' 2>/dev/null)" "$_sub"
printf '%s\n' "$_rule"
printf '\n'
}
_lc() { printf '%s' "${1:-}" | LC_ALL=C tr '[:upper:]' '[:lower:]'; }
_report_sort_names() {
_SORTED_NAMES=()
local _l
while IFS= read -r _l; do
[ -n "$_l" ] && _SORTED_NAMES+=( "$_l" )
done
}
_report_collect() {
_store="$(_report_store 2>/dev/null || printf '')"
_att=() _rev=() _warn=() _exp=() _des=() _emp=() _exc=() _sch=()
_seq=0
if [ -n "$_store" ] && [ -f "$_store" ]; then
while IFS=$'\037' read -r _sev _subj _title _prose _items; do
case "$_sev" in
incomplete|blocked|skipped|error)
_seq=$((_seq + 1))
case "$_sev" in blocked) _rk=0 ;; incomplete) _rk=1 ;; skipped) _rk=2 ;; *) _rk=3 ;; esac
_att+=( "$(_lc "$_subj")"$'\037'"$_rk"$'\037'"$(printf '%06d' "$_seq")"$'\037'"$_subj"$'\037'"$_title"$'\037'"$_prose"$'\037'"$_items" ) ;;
review)
_seq=$((_seq + 1))
_rev+=( "$(_lc "$_subj")"$'\037'"0"$'\037'"$(printf '%06d' "$_seq")"$'\037'"$_subj"$'\037'"$_title"$'\037'"$_prose"$'\037'"$_items" ) ;;
warning) case "$_title" in
"$_subj "*) _title="${_title#"$_subj" }" ;;
esac
_warn+=( "$_subj"$'\037'"$_title" ) ;;
exported) _exp+=( "$_subj" ) ;;
deselected) _des+=( "$_subj" ) ;;
excluded) _exc+=( "$_subj"$'\037'"$_title" ) ;;
schema) _sch+=( "$_subj"$'\037'"$_title"$'\037'"$_items" ) ;;
empty) _emp+=( "$_subj" ) ;;
esac
done < "$_store"
fi
_att_n=${#_att[@]} _rev_n=${#_rev[@]} _warn_n=${#_warn[@]}
_exp_n=${#_exp[@]} _des_n=${#_des[@]} _emp_n=${#_emp[@]}
_exc_n=${#_exc[@]}
}
_emit_report_html() {
local -a _att=() _rev=() _warn=() _exp=() _des=() _emp=() _exc=() _sch=()
local _att_n _rev_n _warn_n _exp_n _des_n _emp_n _exc_n _store _seq _rk
local _sev _subj _title _prose _items
_report_collect
local _e _h1 _h2 _h3
if [ "$_att_n" -eq 0 ]; then
printf 'COMPLETE — everything you selected exported in full.
\n'
else
html_escape "$_att_n" _h1
printf 'INCOMPLETE — %s %s your attention.
\n' \
"$_h1" "$(_pluralize "$_att_n" "item needs" "items need")"
fi
_report_html_group() { #
"; in_para = 0 }
next
}
{
if (in_para == 0) { printf ""; in_para = 1 }
else printf "
"
printf "%s", html_esc($0)
}
END { if (in_para) print "
" }
' <<< "$text"
}
_render_body_rich_html() {
local rtf_file="$1"
[ -s "$rtf_file" ] || return 1
local html
html="$(textutil -convert html -stdout -- "$rtf_file" 2>/dev/null)"
[ -z "$html" ] && return 1
case "$html" in
*"
' "$er" "$ej" "$en"
;;
jpg|jpeg|png|gif|webp|tiff|tif|bmp)
printf '
' "$er" "$en"
;;
mov|mp4|m4v|webm)
printf '' "$er" "$en"
;;
m4a|mp3|wav|aac)
printf '' "$er" "$en"
;;
*)
printf '%s' "$er" "$en"
;;
esac
}
JOURNAL_DIR="$HOME/Library/Group Containers/group.com.apple.moments"
MOMENTS_DB="$JOURNAL_DIR/Library/moments.sqlite"
ATTACHMENTS_BASE="$JOURNAL_DIR/Library/Attachments"
_journal_idx_rows() {
local _rel _date _title _jrnl _e1 _e2 _e3
LC_ALL=C sort -t "$(printf '\x1f')" -k2,2 "$_mf" | \
while IFS="$(printf '\x1f')" read -r _rel _date _title; do
printf ''
html_escape "$_date" _e1
printf '| %s | ' "$_e1"
html_escape "${_title:-(untitled)}" _e2
printf '%s | ' "$(_url_encode_path "$_rel")" "$_e2"
if [ "$_multi" = "yes" ]; then
case "$_rel" in */*) _jrnl="${_rel%%/*}" ;; *) _jrnl="" ;; esac
html_escape "${_jrnl:-—}" _e3
printf '%s | ' "$_e3"
fi
printf '
\n'
done
}
_journal_emit_index() {
local _mf="$1" _nc="$2" _root="$3" _multi="$4"
local _idx_title _idx_h1 _idx_extra_css _idx_thead _idx_rows_fn
_idx_title="Journal ($_nc)"
_idx_h1="Journal — $_nc $(_pluralize "$_nc" entry entries)"
_idx_extra_css='td.d{white-space:nowrap;color:#555;}'
_idx_thead="| Date | Entry | $([ "$_multi" = "yes" ] && printf 'Journal | ')
"
_idx_rows_fn=_journal_idx_rows
_emit_html_index "$_root/index.html" "Journal/index.html"
}
journal_main() {
maybe_show_help "$@"
_announce_step "Journal export"
check_dependencies
local out_arg out_dir
out_arg="$(_parse_output_folder_arg "$@")" || exit 1
out_dir="$(resolve_output_dir "$out_arg")" || exit 1
_announce_output_dir "Journal"
require_full_disk_access
warn_if_app_running Journal
if [ ! -f "$MOMENTS_DB" ]; then
report_no_source "Journal" "not configured" "$MOMENTS_DB" "Journal"
fi
local out_root="$out_dir/Journal"
require_no_collision "$out_root"
ensure_scratch
log_info "Snapshotting moments.sqlite..."
log_info "Source: $MOMENTS_DB"
local snap
snap="$(snapshot_db_or_die "$MOMENTS_DB" "Journal")" || exit 1
if ! sqlite3 -readonly "$snap" "SELECT 1 FROM ZJOURNALENTRYMO LIMIT 1;" >/dev/null 2>&1; then
local _avail
_avail="$(sqlite3 -readonly "$snap" \
"SELECT GROUP_CONCAT(name, ', ') FROM sqlite_master \
WHERE type='table' AND name LIKE 'Z%';" 2>/dev/null)"
die "Journal schema not recognised — ZJOURNALENTRYMO not queryable
(Available tables: ${_avail:-none})"
fi
local _att_base_present=0
if [ ! -d "$ATTACHMENTS_BASE" ]; then
log_info "Attachments dir not present — skipping attachment recovery."
log_info "(Looked for: $ATTACHMENTS_BASE)"
else
_att_base_present=1
fi
log_info "Probing schema (columns, asset tables)..."
local _cols _c
local _jrc
_cols="$(sqlite3 -readonly "$snap" \
"SELECT name FROM pragma_table_info('ZJOURNALENTRYMO');" 2>/dev/null)"
_jrc=$?
[ "$_jrc" -ne 0 ] && _probe_miss error ZJOURNALENTRYMO '*'
local jcol_date jcol_title jcol_text jcol_richtext
local jcol_flagged jcol_type
local jcol_deleted jcol_removed jcol_textlen
local jcol_weather jcol_temp jcol_lat jcol_lon jcol_location jcol_timezone jcol_mood
local jcol_moddate jcol_placename
local jcol_weather_high jcol_weather_low jcol_weather_humidity jcol_weather_code
jcol_date="$(printf '%s\n' "$_cols" | grep -m1 '^ZENTRYDATE$')"
jcol_title="$(printf '%s\n' "$_cols" | grep -m1 '^ZTITLE$')"
jcol_text="$(printf '%s\n' "$_cols" | grep -m1 '^ZTEXT$')"
jcol_richtext="$(printf '%s\n' "$_cols" | grep -m1 '^ZRICHTEXTDATA$')"
[ -n "$jcol_richtext" ] && jcol_text="$jcol_richtext"
jcol_flagged="$(printf '%s\n' "$_cols" | grep -m1 '^ZFLAGGED$')"
jcol_type="$(printf '%s\n' "$_cols" | grep -m1 '^ZENTRYTYPE$')"
jcol_deleted="$(printf '%s\n' "$_cols" | grep -m1 '^ZRECENTLYDELETED$')"
jcol_removed="$(printf '%s\n' "$_cols" | grep -m1 '^ZISFULLYREMOVED$')"
jcol_textlen="$(printf '%s\n' "$_cols" | grep -m1 '^ZTEXTLENGTH$')"
jcol_weather="$(printf '%s\n' "$_cols" | grep -m1 '^ZWEATHERCONDITION$')"
jcol_temp="$(printf '%s\n' "$_cols" | grep -m1 '^ZWEATHERTEMPERATURE$')"
jcol_lat="$(printf '%s\n' "$_cols" | grep -m1 '^ZLATITUDE$')"
jcol_lon="$(printf '%s\n' "$_cols" | grep -m1 '^ZLONGITUDE$')"
jcol_location="$(printf '%s\n' "$_cols" | grep -m1 '^ZLOCATION$')"
jcol_timezone="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZTIMEZONE ZTIMEZONENAME)"
jcol_mood="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZSTATEOFMIND ZMOODVALENCE ZMENTALSTATE ZEMOTION)"
jcol_moddate="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZUPDATEDDATE ZENTRYDATAUPDATEDATE \
ZMODIFICATIONDATE ZLASTMODIFIEDDATE)"
jcol_placename="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZPLACENAME ZLOCATIONNAME ZADDRESS ZPLACE)"
jcol_weather_high="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZWEATHERHIGHTEMPERATURE ZWEATHERTEMPERATUREHIGH ZWEATHERHIGHTEMP)"
jcol_weather_low="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZWEATHERLOWTEMPERATURE ZWEATHERTEMPERATURELOW ZWEATHERLOWTEMP)"
jcol_weather_humidity="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZWEATHERHUMIDITY ZWEATHERRELATIVEHUMIDITY)"
jcol_weather_code="$(_pick_first_col "$snap" ZJOURNALENTRYMO \
ZWEATHERCONDITIONCODE ZWEATHERCODE)"
[ -z "$jcol_date" ] && jcol_date="ZENTRYDATE"
[ -z "$jcol_title" ] && jcol_title="ZTITLE"
[ -z "$jcol_text" ] && jcol_text="ZTEXT"
[ -z "$jcol_deleted" ] && jcol_deleted="ZRECENTLYDELETED"
[ -z "$jcol_removed" ] && jcol_removed="ZISFULLYREMOVED"
for _c in "$jcol_date" "$jcol_title" "$jcol_text" "$jcol_flagged" \
"$jcol_type" "$jcol_deleted" \
"$jcol_removed" "$jcol_textlen" "$jcol_weather" \
"$jcol_temp" "$jcol_lat" "$jcol_lon" "$jcol_location" \
"$jcol_timezone" "$jcol_mood" "$jcol_moddate" \
"$jcol_placename" "$jcol_weather_high" "$jcol_weather_low" \
"$jcol_weather_humidity" "$jcol_weather_code"; do
[ -z "$_c" ] && continue
case "$_c" in *[!A-Za-z0-9_]*) die "Journal: unsafe column name '$_c'" ;; esac
done
[ -n "$jcol_timezone" ] && [ "$jcol_timezone" != "ZTIMEZONE" ] && \
log_info "Schema: timezone = $jcol_timezone (legacy)"
[ -n "$jcol_mood" ] && [ "$jcol_mood" != "ZSTATEOFMIND" ] && \
log_info "Schema: mood = $jcol_mood (legacy)"
[ -n "$jcol_moddate" ] && [ "$jcol_moddate" != "ZUPDATEDDATE" ] && \
log_info "Schema: modification-date = $jcol_moddate (legacy)"
[ -n "$jcol_placename" ] && [ "$jcol_placename" != "ZPLACENAME" ] && \
log_info "Schema: place-name = $jcol_placename (legacy)"
local _J_HAS_CONTENT="(COALESCE(ZJOURNALENTRYMO.$jcol_textlen, 0) > 0
OR COALESCE(length(ZJOURNALENTRYMO.$jcol_text), 0) > 0)"
if [ -z "$jcol_textlen" ]; then
_J_HAS_CONTENT="(COALESCE(length(ZJOURNALENTRYMO.$jcol_text), 0) > 0)"
fi
local _J_LIVE_FILTER="(
(COALESCE(ZJOURNALENTRYMO.$jcol_deleted, 0) = 0
AND COALESCE(ZJOURNALENTRYMO.$jcol_removed, 0) = 0)
OR $_J_HAS_CONTENT
)"
local _tables has_asset_table="no" has_file_att_table="no"
local _jt_rc
_tables="$(sqlite3 -readonly "$snap" "SELECT name FROM sqlite_master WHERE type='table';" 2>/dev/null)"
_jt_rc=$?
[ "$_jt_rc" -ne 0 ] && _probe_miss error sqlite_master '*'
if _col_in_list "$_tables" 'ZJOURNALENTRYASSETMO'; then
has_asset_table="yes"
fi
if _col_in_list "$_tables" 'ZJOURNALENTRYASSETFILEATTACHMENTMO'; then
has_file_att_table="yes"
local _fa_cols _fa_need _fa_missing="" _fa_rc
_fa_cols="$(sqlite3 -readonly "$snap" \
"SELECT name FROM pragma_table_info('ZJOURNALENTRYASSETFILEATTACHMENTMO');" \
2>/dev/null)"
_fa_rc=$?
[ "$_fa_rc" -ne 0 ] && _probe_miss error ZJOURNALENTRYASSETFILEATTACHMENTMO '*'
for _fa_need in ZFILEPATH ZNAME ZASSET ZINDEX; do
_col_in_list "$_fa_cols" "$_fa_need" \
|| _fa_missing="$_fa_missing $_fa_need"
done
if [ -n "$_fa_missing" ]; then
log_warn "Journal: ZJOURNALENTRYASSETFILEATTACHMENTMO is missing required column(s):$_fa_missing — falling back to legacy ZJOURNALENTRYASSETMO lookup for attachments"
if [ "${_fa_rc:-0}" -eq 0 ]; then
for _fa_need in $_fa_missing; do
_probe_miss absent ZJOURNALENTRYASSETFILEATTACHMENTMO "$_fa_need"
done
fi
has_file_att_table="no"
fi
fi
local _has_multi_journal=0
local _journal_join_table="" _journal_entries_col="" _journal_journals_col=""
local _journal_count=0
if [ "${JOURNAL_MULTI_FOLDER:-1}" = "1" ] && \
_col_in_list "$_tables" 'ZJOURNALMO'; then
local _journal_join_candidates _jj_rc
_journal_join_table="$(_discover_z_join "$snap" JOURNALS)"; _jj_rc=$?
if [ "$_jj_rc" -eq 2 ]; then
_journal_join_candidates="$(sqlite3 -readonly "$snap" \
"SELECT name FROM sqlite_master
WHERE type='table' AND name GLOB 'Z_[0-9]*JOURNALS'
ORDER BY name;" 2>/dev/null)"
log_info "Schema: multiple Z_JOURNALS join tables present ($_journal_join_candidates) — picking the one with the most rows"
local _jjc _best_count=-1 _best_name=""
while IFS= read -r _jjc; do
[ -z "$_jjc" ] && continue
case "$_jjc" in
*[!A-Za-z0-9_]*) continue ;;
esac
local _jjc_count
_jjc_count="$(sqlite3 -readonly "$snap" "SELECT COUNT(*) FROM $_jjc;" 2>/dev/null)"
case "${_jjc_count:-0}" in ''|*[!0-9]*) _jjc_count=0 ;; esac
if [ "$_jjc_count" -gt "$_best_count" ]; then
_best_count="$_jjc_count"
_best_name="$_jjc"
fi
done <<< "$_journal_join_candidates"
_journal_join_table="$_best_name"
fi
case "$_journal_join_table" in
Z_*JOURNALS) : ;;
*) _journal_join_table="" ;;
esac
case "$_journal_join_table" in
*[!A-Za-z0-9_]*) _journal_join_table="" ;;
esac
if [ -n "$_journal_join_table" ]; then
_journal_entries_col="$(sqlite3 -readonly "$snap" \
"SELECT name FROM pragma_table_info('$_journal_join_table')
WHERE name GLOB 'Z_[0-9]*ENTRIES' LIMIT 1;" 2>/dev/null)"
_journal_journals_col="$(sqlite3 -readonly "$snap" \
"SELECT name FROM pragma_table_info('$_journal_join_table')
WHERE name GLOB 'Z_[0-9]*JOURNALS' LIMIT 1;" 2>/dev/null)"
case "$_journal_entries_col" in *[!A-Za-z0-9_]*) _journal_entries_col="" ;; esac
case "$_journal_journals_col" in *[!A-Za-z0-9_]*) _journal_journals_col="" ;; esac
fi
if [ -n "$_journal_join_table" ] && [ -n "$_journal_entries_col" ] \
&& [ -n "$_journal_journals_col" ]; then
_journal_count="$(sqlite3 -readonly "$snap" \
"SELECT COUNT(*) FROM ZJOURNALMO
WHERE COALESCE(ZUSERDELETED, 0) = 0;" 2>/dev/null)"
case "${_journal_count:-0}" in ''|*[!0-9]*) _journal_count=0 ;; esac
fi
if [ "$_journal_count" -gt 1 ]; then
_has_multi_journal=1
local _jpk _jidx=0
while IFS= read -r _jpk; do
[ -z "$_jpk" ] && continue
case "$_jpk" in *[!0-9]*) continue ;; esac
_jidx=$((_jidx + 1))
printf -v "_jl_$_jpk" '%s' "Journal $_jidx"
done < <(sqlite3 -readonly "$snap" \
"SELECT Z_PK FROM ZJOURNALMO
WHERE COALESCE(ZUSERDELETED, 0) = 0
ORDER BY Z_PK ASC;" 2>/dev/null)
log_info "Multi-journal: $_journal_count journals — entries will land in 'Journal N' subfolders (set JOURNAL_MULTI_FOLDER=0 to force flat output)"
fi
fi
_journal_label_for() {
case "$1" in ''|*[!0-9]*) return 0 ;; esac
local _ref="_jl_$1"
printf '%s' "${!_ref:-}"
}
local _acol_entry _acol_atitle
if [ "$has_asset_table" = "yes" ]; then
_acol_entry="$(_pick_first_col "$snap" ZJOURNALENTRYASSETMO ZENTRY ZENTRY1 ZENTRY2)"
_acol_atitle="$(_pick_first_col "$snap" ZJOURNALENTRYASSETMO ZTITLE ZTITLE1 ZTITLE2)"
[ -z "$_acol_entry" ] && _acol_entry="ZENTRY"
[ -z "$_acol_atitle" ] && _acol_atitle="ZTITLE"
case "$_acol_entry" in *[!A-Za-z0-9_]*) die "Journal: unsafe asset entry column '$_acol_entry'" ;; esac
case "$_acol_atitle" in *[!A-Za-z0-9_]*) die "Journal: unsafe asset title column '$_acol_atitle'" ;; esac
fi
local _J_ATTACH_EXISTS=""
if [ "$has_asset_table" = "yes" ]; then
_J_ATTACH_EXISTS="EXISTS (SELECT 1 FROM ZJOURNALENTRYASSETMO a
WHERE a.$_acol_entry = ZJOURNALENTRYMO.Z_PK)"
_J_HAS_CONTENT="($_J_HAS_CONTENT
OR $_J_ATTACH_EXISTS)"
_J_LIVE_FILTER="(
(COALESCE(ZJOURNALENTRYMO.$jcol_deleted, 0) = 0
AND COALESCE(ZJOURNALENTRYMO.$jcol_removed, 0) = 0)
OR $_J_HAS_CONTENT
)"
fi
local _entry_type_filter=""
if [ -n "$jcol_type" ] && [ "${JOURNAL_INCLUDE_ALL_TYPES:-0}" != "1" ]; then
local _et_dist
_et_dist="$(sqlite3 -readonly -separator $'\t' "$snap" "
SELECT COALESCE($jcol_type, 'NULL') AS t, COUNT(*)
FROM ZJOURNALENTRYMO
WHERE $_J_LIVE_FILTER
GROUP BY t ORDER BY t;" 2>/dev/null)"
local _et_storage
_et_storage="$(sqlite3 -readonly "$snap" "
SELECT typeof($jcol_type) FROM ZJOURNALENTRYMO
WHERE $jcol_type IS NOT NULL
LIMIT 1;" 2>/dev/null)"
local _et_kind="numeric"
case "$_et_storage" in
text) _et_kind="string" ;;
esac
if [ "$_et_kind" = "string" ]; then
local _et_text_guard="COALESCE(ZJOURNALENTRYMO.$jcol_textlen, 0) > 0
OR (ZJOURNALENTRYMO.$jcol_text IS NOT NULL
AND length(ZJOURNALENTRYMO.$jcol_text) > 0)"
if [ -z "$jcol_textlen" ]; then
_et_text_guard="ZJOURNALENTRYMO.$jcol_text IS NOT NULL
AND length(ZJOURNALENTRYMO.$jcol_text) > 0"
fi
[ -n "$_J_ATTACH_EXISTS" ] && _et_text_guard="$_et_text_guard
OR $_J_ATTACH_EXISTS"
_entry_type_filter="AND (ZJOURNALENTRYMO.$jcol_type = 'blankEntry'
OR ZJOURNALENTRYMO.$jcol_type IS NULL
OR ($_et_text_guard))"
else
local _et_text_guard_num="ZJOURNALENTRYMO.$jcol_text IS NOT NULL
AND length(ZJOURNALENTRYMO.$jcol_text) > 0"
[ -n "$jcol_textlen" ] && \
_et_text_guard_num="COALESCE(ZJOURNALENTRYMO.$jcol_textlen, 0) > 0
OR $_et_text_guard_num"
[ -n "$_J_ATTACH_EXISTS" ] && _et_text_guard_num="$_et_text_guard_num
OR $_J_ATTACH_EXISTS"
_entry_type_filter="AND (COALESCE(ZJOURNALENTRYMO.$jcol_type, 0) = 0
OR ($_et_text_guard_num))"
fi
if printf '%s\n' "$_et_dist" | LC_ALL=C awk -F'\t' 'END{exit !(NR>1)}'; then
log_info "ZENTRYTYPE distribution: $(printf '%s' "$_et_dist" | LC_ALL=C tr '\t' '=' | LC_ALL=C tr '\n' ' ')"
if [ "$_et_kind" = "string" ]; then
log_info "Filtering to user-authored entries (ZENTRYTYPE='blankEntry'/NULL or any entry with text). Set JOURNAL_INCLUDE_ALL_TYPES=1 to export everything."
else
log_info "Filtering to ZENTRYTYPE=0 (user-authored). Set JOURNAL_INCLUDE_ALL_TYPES=1 to export everything."
fi
fi
fi
local _rd_skipped_tombstones=0
local _n_tombstone
_n_tombstone="$(sqlite3 -readonly "$snap" "
SELECT COUNT(*) FROM ZJOURNALENTRYMO
WHERE (COALESCE($jcol_deleted, 0) = 1
OR COALESCE($jcol_removed, 0) = 1)
AND NOT $_J_HAS_CONTENT;" 2>/dev/null)"
case "${_n_tombstone:-0}" in
''|*[!0-9]*) : ;;
*) _rd_skipped_tombstones=$_n_tombstone ;;
esac
local n_entries n_entries_discovery_ok=1
n_entries="$(sqlite3 -readonly "$snap" "
SELECT COUNT(*) FROM ZJOURNALENTRYMO
WHERE $_J_LIVE_FILTER
$_entry_type_filter;" 2>/dev/null)"
case "$n_entries" in
''|*[!0-9]*) n_entries_discovery_ok=0 ;;
esac
if [ "$n_entries_discovery_ok" = "0" ]; then
log_warn "Journal entry-count query failed — proceeding without a discovery baseline; the drift check will surface this as a verification gap."
n_entries=0
fi
if [ "$n_entries" = "0" ] && [ "$n_entries_discovery_ok" = "1" ]; then
report_empty_source "journal entries" "Journal"
fi
if [ "$n_entries_discovery_ok" = "1" ]; then
log_info "Found $n_entries $(_pluralize "$n_entries" entry entries)"
else
log_info "Entry count could not be determined — proceeding with per-entry SELECT (may be partial)"
fi
_create_output_dir "$out_root"
local sel_weather="''" sel_temp="''" sel_lat="''" sel_lon="''" sel_loc="''" sel_tz="''" sel_mood="''"
local sel_moddate="''" sel_placename="''"
local sel_flagged="0"
[ -n "$jcol_flagged" ] && sel_flagged="COALESCE(ZJOURNALENTRYMO.$jcol_flagged, 0)"
local sel_weather_high="''" sel_weather_low="''"
local sel_weather_humidity="''" sel_weather_code="''"
[ -n "$jcol_weather" ] && sel_weather="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_weather, '')")"
[ -n "$jcol_temp" ] && sel_temp="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_temp, '')")"
[ -n "$jcol_lat" ] && sel_lat="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_lat, '')")"
[ -n "$jcol_lon" ] && sel_lon="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_lon, '')")"
[ -n "$jcol_location" ] && sel_loc="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_location, '')")"
[ -n "$jcol_timezone" ] && sel_tz="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_timezone, '')")"
[ -n "$jcol_mood" ] && sel_mood="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_mood, '')")"
[ -n "$jcol_moddate" ] && sel_moddate="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_moddate, '')")"
[ -n "$jcol_weather_high" ] && sel_weather_high="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_weather_high, '')")"
[ -n "$jcol_weather_low" ] && sel_weather_low="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_weather_low, '')")"
[ -n "$jcol_weather_humidity" ] && sel_weather_humidity="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_weather_humidity, '')")"
[ -n "$jcol_weather_code" ] && sel_weather_code="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_weather_code, '')")"
if [ -n "$jcol_placename" ]; then
sel_placename="$(_sql_tsv_safe "COALESCE(ZJOURNALENTRYMO.$jcol_placename, '')")"
fi
local exported=0 empty_count=0 total_attachments=0 stub_count=0 _attachments_missing=0 _entry_write_failed=0 _rd_exported=0
local _attachments_copyfail=0
ensure_scratch
local _journal_stubs_tsv="$SCRATCH_DIR/journal_stubs.tsv"
: > "$_journal_stubs_tsv"
local _journal_join_sql="" _journal_pk_sel=", 0 AS journal_pk"
local _journal_group_sql=""
if [ "$_has_multi_journal" = "1" ]; then
_journal_join_sql="
LEFT JOIN $_journal_join_table jt
ON jt.$_journal_entries_col = ZJOURNALENTRYMO.Z_PK
LEFT JOIN ZJOURNALMO j
ON j.Z_PK = jt.$_journal_journals_col
AND COALESCE(j.ZUSERDELETED, 0) = 0"
_journal_pk_sel=", COALESCE(MIN(j.Z_PK), 0) AS journal_pk"
_journal_group_sql="
GROUP BY ZJOURNALENTRYMO.Z_PK"
fi
log_info "Bulk-extracting title and body RTF for $n_entries $(_pluralize "$n_entries" entry entries)..."
if ! sqlite3 -readonly "$snap" "
SELECT writefile('$SCRATCH_DIR/jentry_title_' || Z_PK || '.rtf', $jcol_title)
FROM ZJOURNALENTRYMO
WHERE $_J_LIVE_FILTER
$_entry_type_filter
AND $jcol_title IS NOT NULL
AND length($jcol_title) > 0;
" >/dev/null 2>&1; then
log_warn "Bulk-extract of entry titles failed; per-entry titles may be missing."
note_incomplete "$out_root" "Journal entry titles" \
"the bulk extraction of entry titles from the database failed — per-entry .html files may render as \"(untitled)\" / friendly-name fallback even though the entries themselves were emitted with full body content"
fi
if ! sqlite3 -readonly "$snap" "
SELECT writefile('$SCRATCH_DIR/jentry_body_' || Z_PK || '.rtf', $jcol_text)
FROM ZJOURNALENTRYMO
WHERE $_J_LIVE_FILTER
$_entry_type_filter
AND $jcol_text IS NOT NULL
AND length($jcol_text) > 0;
" >/dev/null 2>&1; then
log_warn "Bulk-extract of entry bodies failed; per-entry bodies may be missing."
note_incomplete "$out_root" "Journal entry body text" \
"the bulk extraction of entry bodies from the database failed — the per-entry .html files may have empty or missing body content even though the entries themselves were emitted"
fi
local _legacy_att_index="" _legacy_att_root="" _legacy_att_index_built=0
_ensure_legacy_att_index() {
[ "$_legacy_att_index_built" = "1" ] && return 0
_legacy_att_index_built=1
if [ -d "$JOURNAL_DIR/Library/Attachments" ]; then
_legacy_att_root="$JOURNAL_DIR/Library/Attachments"
elif [ -d "$JOURNAL_DIR/Library" ]; then
_legacy_att_root="$JOURNAL_DIR/Library"
fi
[ -n "$_legacy_att_root" ] || return 0
_legacy_att_index="$SCRATCH_DIR/journal_legacy_att_index"
find "$_legacy_att_root" -type f ! -type l 2>/dev/null \
| LC_ALL=C awk '
{ n = $0; sub(/.*\//, "", n);
if (!(n in seen)) { seen[n] = 1; print } }' \
> "$_legacy_att_index" 2>/dev/null || _legacy_att_index=""
[ -s "$_legacy_att_index" ] || _legacy_att_index=""
}
log_info "Emitting per-entry HTML..."
local _pt_last=0 _pt_inline=0
local _html_validated=0
local _journal_index="$SCRATCH_DIR/journal_index.tsv"
: > "$_journal_index" 2>/dev/null || true
local pk entry_date_raw flagged j_weather j_temp j_lat j_lon j_loc j_tz j_mood
local j_moddate j_placename j_weather_high j_weather_low j_weather_humidity j_weather_code is_deleted journal_pk
while IFS=$'\x1f' read -r pk entry_date_raw flagged \
j_weather j_temp j_lat j_lon j_loc j_tz j_mood \
j_moddate j_placename j_weather_high j_weather_low \
j_weather_humidity j_weather_code is_deleted journal_pk; do
[ -z "$pk" ] && continue
case "$pk" in *[!0-9]*) continue ;; esac
local _target_dir="$out_root"
if [ "$_has_multi_journal" = "1" ] && [ -n "$journal_pk" ] \
&& [ "$journal_pk" != "0" ]; then
case "$journal_pk" in *[!0-9]*) : ;; *)
local _jlabel
_jlabel="$(_journal_label_for "$journal_pk")"
if [ -n "$_jlabel" ]; then
_target_dir="$out_root/$_jlabel"
if [ ! -d "$_target_dir" ]; then
if mkdir -p "$_target_dir" 2>/dev/null; then
track_output_path "$_target_dir"
else
log_warn "Cannot create journal subfolder $_target_dir; emitting this entry flat under $out_root."
_target_dir="$out_root"
fi
fi
fi
;; esac
fi
if [ "${is_deleted:-0}" = "1" ]; then
_target_dir="$out_root/Recently Deleted"
if [ ! -d "$_target_dir" ]; then
if mkdir -p "$_target_dir" 2>/dev/null; then
track_output_path "$_target_dir"
else
log_warn "Cannot create Recently Deleted subfolder $_target_dir; emitting this entry flat under $out_root."
_target_dir="$out_root"
fi
fi
fi
local _epoch_int="" iso_date="" file_date="undated" entry_date_fmt="(unknown)"
_epoch_int="$(_cd_ts_to_unix "$entry_date_raw")"
local _entry_tz=""
if [ -n "$j_tz" ]; then
case "$j_tz" in
*[!A-Za-z0-9/_+-]*) ;;
*) _entry_tz="$j_tz" ;;
esac
fi
if [ -n "$_epoch_int" ]; then
if [ -n "$_entry_tz" ]; then
iso_date="$(TZ="$_entry_tz" date -r "$_epoch_int" +%Y-%m-%dT%H:%M:%S%z 2>/dev/null)"
file_date="$(TZ="$_entry_tz" date -r "$_epoch_int" +%Y-%m-%d 2>/dev/null)"
entry_date_fmt="$(TZ="$_entry_tz" date -r "$_epoch_int" +'%Y-%m-%d %H:%M:%S %Z' 2>/dev/null)"
else
_epoch_to_utc_iso_into "$_epoch_int" iso_date
file_date="$(date -r "$_epoch_int" +%Y-%m-%d 2>/dev/null)"
entry_date_fmt="$(date -r "$_epoch_int" +'%Y-%m-%d %H:%M:%S' 2>/dev/null)"
fi
fi
[ -z "$file_date" ] && file_date="undated"
[ -z "$entry_date_fmt" ] && entry_date_fmt="(unknown)"
local _mod_iso=""
if [ -n "${j_moddate:-}" ] && [ "$j_moddate" != "0" ]; then
local _mod_epoch=""
_mod_epoch="$(_cd_ts_to_unix "$j_moddate")"
if [ -n "$_mod_epoch" ]; then
if [ -n "$_entry_tz" ]; then
_mod_iso="$(TZ="$_entry_tz" date -r "$_mod_epoch" +%Y-%m-%dT%H:%M:%S%z 2>/dev/null)"
else
_epoch_to_utc_iso_into "$_mod_epoch" _mod_iso
fi
fi
fi
local title_text="" rtf_title="$SCRATCH_DIR/jentry_title_$pk.rtf"
if [ -s "$rtf_title" ]; then
title_text="$(textutil -convert txt -stdout -- "$rtf_title" 2>/dev/null)"
while [ -n "$title_text" ] && \
{ [ "${title_text: -1}" = $'\n' ] || \
[ "${title_text: -1}" = $'\r' ]; }; do
title_text="${title_text%?}"
done
rm -f -- "$rtf_title" 2>/dev/null
fi
[ -z "$title_text" ] && title_text="(untitled)"
local safe_title base_name target
safe_title="$(_sanitize_name "$title_text")"
base_name="${file_date}_${safe_title}_${pk}"
target="$(_unique_path "$_target_dir/$base_name.html")"
local body_text="" body_html_rich="" rtf_body="$SCRATCH_DIR/jentry_body_$pk.rtf"
if [ -s "$rtf_body" ]; then
body_html_rich="$(_render_body_rich_html "$rtf_body")"
body_text="$(textutil -convert txt -stdout -- "$rtf_body" 2>/dev/null)"
rm -f -- "$rtf_body" 2>/dev/null
fi
local _att_stem="${target##*/}"; _att_stem="${_att_stem%.html}"
local att_dir="$_target_dir/$_att_stem.attachments"
local att_count=0 att_html_inline=""
local _src _dst _basename
local _modern_saw_row=0
if [ "$has_file_att_table" = "yes" ] && [ "$has_asset_table" = "yes" ]; then
local _fpath _fname
while IFS=$'\x1f' read -r _fpath _fname; do
[ -z "$_fpath" ] && continue
_modern_saw_row=1
_src=""
if [ -n "$ATTACHMENTS_BASE" ] && [ -f "$ATTACHMENTS_BASE/$_fpath" ]; then
_src="$ATTACHMENTS_BASE/$_fpath"
fi
if [ -z "$_src" ] && [ -n "$ATTACHMENTS_BASE" ]; then
if _icloud_stub_sibling_for "$ATTACHMENTS_BASE/$_fpath" >/dev/null; then
stub_count=$((stub_count + 1))
printf '%s\x1f%s\n' "$base_name" "${_fpath##*/}" \
>> "$_journal_stubs_tsv"
continue
fi
fi
if [ -z "$_src" ]; then
[ "$_att_base_present" = "1" ] && \
_attachments_missing=$((_attachments_missing + 1))
continue
fi
if is_icloud_stub "$_src"; then
stub_count=$((stub_count + 1))
printf '%s\x1f%s\n' "$base_name" \
"$(_strip_icloud_stub_name "${_src##*/}")" \
>> "$_journal_stubs_tsv"
continue
fi
_basename="${_fpath##*/}"
if [ "$att_count" -eq 0 ]; then
if ! mkdir -p "$att_dir" 2>/dev/null; then
log_warn "Failed to create attachment dir for entry $base_name — remaining attachments discarded."
note_incomplete "$out_root" \
"Journal entry $base_name attachments" \
"could not create $att_dir; remaining attachment rows for this entry were dropped"
break
fi
fi
_dst="$(_unique_path "$att_dir/$_basename")"
if _ditto_verified "$_src" "$_dst"; then
att_count=$((att_count + 1))
_make_jpg_sibling_if_heic "$_dst"
att_html_inline="$att_html_inline$(_attachment_tag "$_att_stem.attachments/${_dst##*/}")"$'\n'
else
_attachments_copyfail=$((_attachments_copyfail + 1))
log_warn "Failed to copy attachment '$_basename' for entry $base_name."
fi
done < <(sqlite3 -readonly -separator $'\x1f' "$snap" "
SELECT COALESCE(f.ZFILEPATH, ''),
COALESCE(f.ZNAME, '')
FROM ZJOURNALENTRYASSETFILEATTACHMENTMO f
JOIN ZJOURNALENTRYASSETMO a ON f.ZASSET = a.Z_PK
WHERE a.$_acol_entry = $pk
ORDER BY COALESCE(f.ZINDEX, 0), f.Z_PK ASC;" 2>/dev/null)
fi
local -a _seen_legacy_src=()
if [ "$att_count" -eq 0 ] && [ "$_modern_saw_row" = "0" ] \
&& [ "$has_asset_table" = "yes" ]; then
_ensure_legacy_att_index
local _a_pk _a_title
while IFS=$'\x1f' read -r _a_pk _a_title; do
[ -z "$_a_pk" ] && continue
_src=""
if [ -n "$_a_title" ]; then
local _wanted
_wanted="${_a_title##*/}"
if [ -n "$_legacy_att_index" ]; then
_src="$(_wanted="$_wanted" LC_ALL=C awk '
BEGIN { want = ENVIRON["_wanted"] }
{ n = $0; sub(/.*\//, "", n); if (n == want) { print; exit } }' \
"$_legacy_att_index" 2>/dev/null)"
fi
fi
[ -z "$_src" ] && continue
local _already=0 _ss
for _ss in "${_seen_legacy_src[@]+"${_seen_legacy_src[@]}"}"; do
[ "$_ss" = "$_src" ] && { _already=1; break; }
done
[ "$_already" = "1" ] && continue
_seen_legacy_src+=( "$_src" )
if is_icloud_stub "$_src"; then
stub_count=$((stub_count + 1))
printf '%s\x1f%s\n' "$base_name" \
"$(_strip_icloud_stub_name "${_src##*/}")" \
>> "$_journal_stubs_tsv"
continue
fi
_basename="${_src##*/}"
if [ "$att_count" -eq 0 ]; then
if ! mkdir -p "$att_dir" 2>/dev/null; then
log_warn "Failed to create attachment dir for entry $base_name (legacy path) — remaining attachments discarded."
note_incomplete "$out_root" \
"Journal entry $base_name attachments (legacy path)" \
"could not create $att_dir; remaining attachment rows for this entry were dropped"
break
fi
fi
_dst="$(_unique_path "$att_dir/$_basename")"
if _ditto_verified "$_src" "$_dst"; then
att_count=$((att_count + 1))
_make_jpg_sibling_if_heic "$_dst"
att_html_inline="$att_html_inline$(_attachment_tag "$_att_stem.attachments/${_dst##*/}")"$'\n'
else
_attachments_copyfail=$((_attachments_copyfail + 1))
log_warn "Failed to copy attachment '$_basename' for entry $base_name."
fi
done < <(sqlite3 -readonly -separator $'\x1f' "$snap" "
SELECT a.Z_PK, COALESCE(a.$_acol_atitle, '')
FROM ZJOURNALENTRYASSETMO a
WHERE a.$_acol_entry = $pk
ORDER BY a.Z_PK ASC;" 2>/dev/null)
fi
total_attachments=$((total_attachments + att_count))
if [ "$att_count" -eq 0 ] && [ -d "$att_dir" ]; then
rmdir "$att_dir" 2>/dev/null || true
fi
local body_html=""
if [ -n "$body_html_rich" ]; then
body_html="$body_html_rich"
elif [ -n "$body_text" ]; then
body_html="$(_render_body_html "$body_text")"
fi
if [ -z "$body_html" ] && [ "$att_count" -eq 0 ]; then
empty_count=$((empty_count + 1))
body_html="(empty entry)
"
fi
local meta_block=""
if [ "${flagged:-0}" = "1" ]; then
meta_block="$meta_block Flagged
"
fi
if [ -n "$j_weather" ] && [ "$j_weather" != "0" ]; then
local weather_text="$j_weather"
[ -n "$j_temp" ] && [ "$j_temp" != "0" ] && weather_text="$weather_text, $j_temp"
[ -n "${j_weather_high:-}" ] && [ "$j_weather_high" != "0" ] && \
weather_text="$weather_text, high $j_weather_high"
[ -n "${j_weather_low:-}" ] && [ "$j_weather_low" != "0" ] && \
weather_text="$weather_text, low $j_weather_low"
[ -n "${j_weather_humidity:-}" ] && [ "$j_weather_humidity" != "0" ] && \
weather_text="$weather_text, humidity $j_weather_humidity"
[ -n "${j_weather_code:-}" ] && [ "$j_weather_code" != "0" ] && \
weather_text="$weather_text, code $j_weather_code"
meta_block="$meta_block Weather: $(html_escape "$weather_text")
"
fi
if [ -n "${j_placename:-}" ]; then
meta_block="$meta_block Location: $(html_escape "$j_placename")
"
elif [ -n "$j_loc" ]; then
meta_block="$meta_block Location: $(html_escape "$j_loc")
"
elif [ -n "$j_lat" ] && [ -n "$j_lon" ] && [ "$j_lat" != "0" ] && [ "$j_lon" != "0" ]; then
meta_block="$meta_block Location: $(html_escape "$j_lat, $j_lon")
"
fi
local tmp
if ! tmp="$(mktemp "$target.tmp.XXXXXX" 2>/dev/null)"; then
log_warn "Cannot create temp file for $base_name.html."
_entry_write_failed=$((_entry_write_failed + 1))
note_incomplete "$out_root" "$base_name.html" \
"could not write per-entry HTML — mktemp failed in the export folder (disk full or permission denied)"
[ -d "$att_dir" ] && rm -rf -- "$att_dir"
continue
fi
track_output_path "$tmp"
{
printf '\n\n\n\n'
printf '%s\n' "$(html_escape "$title_text")"
[ -n "$iso_date" ] && printf '\n' "$iso_date"
[ -n "$_mod_iso" ] && printf '\n' "$_mod_iso"
[ "${flagged:-0}" = "1" ] && printf '\n'
[ -n "${j_mood:-}" ] && [ "$j_mood" != "0" ] && \
printf '\n' "$(html_escape "$j_mood")"
printf '\n\n'
printf '%s
\n' "$(html_escape "$title_text")"
printf '%s
\n' "$(html_escape "$entry_date_fmt")"
if [ -n "$meta_block" ]; then
printf '\n'
printf '%s' "$meta_block"
printf '
\n'
fi
printf '%s\n' "$body_html"
if [ -n "$att_html_inline" ]; then
printf '\n'
printf '%s' "$att_html_inline"
printf '
\n'
fi
printf '\n\n'
} > "$tmp"
if ! mv -f -- "$tmp" "$target"; then
log_warn "Failed to write $target"
rm -f -- "$tmp"
_entry_write_failed=$((_entry_write_failed + 1))
note_incomplete "$out_root" "$base_name.html" \
"could not write per-entry HTML — atomic mv into export folder failed (volume ejected, disk full, or permission denied)"
[ -d "$att_dir" ] && rm -rf -- "$att_dir"
continue
fi
_untrack_output_path "$tmp"
if [ "$_html_validated" != "1" ]; then
validate_html_structure "$target"
_html_validated=1
fi
if [ -n "$_epoch_int" ]; then
_apply_touch_t "$(_format_touch_t "$_epoch_int")" "$target"
fi
exported=$((exported + 1))
[ "${is_deleted:-0}" = "1" ] && _rd_exported=$((_rd_exported + 1))
printf '%s\x1f%s\x1f%s\n' \
"${target#"$out_root/"}" "$entry_date_fmt" "$(tsv_escape "$title_text")" \
>> "$_journal_index" 2>/dev/null || true
_progress_tick "$exported" "$n_entries" "entries exported"
done < <(sqlite3 -readonly -separator $'\x1f' "$snap" "
SELECT ZJOURNALENTRYMO.Z_PK,
ZJOURNALENTRYMO.$jcol_date,
$sel_flagged,
$sel_weather,
$sel_temp,
$sel_lat,
$sel_lon,
$sel_loc,
$sel_tz,
$sel_mood,
$sel_moddate,
$sel_placename,
$sel_weather_high,
$sel_weather_low,
$sel_weather_humidity,
$sel_weather_code,
CASE WHEN COALESCE(ZJOURNALENTRYMO.$jcol_deleted, 0) = 1
OR COALESCE(ZJOURNALENTRYMO.$jcol_removed, 0) = 1
THEN 1 ELSE 0 END
$_journal_pk_sel
FROM ZJOURNALENTRYMO $_journal_join_sql
WHERE $_J_LIVE_FILTER
$_entry_type_filter
$_journal_group_sql
ORDER BY ZJOURNALENTRYMO.$jcol_date ASC, ZJOURNALENTRYMO.Z_PK ASC;" 2>/dev/null)
_progress_end
[ "$empty_count" -gt 0 ] && log_info "$empty_count empty / unextractable entries"
[ "$_rd_exported" -gt 0 ] && \
log_info "Exported $_rd_exported recently-deleted journal $(_pluralize "$_rd_exported" entry entries) to Recently Deleted/ — review and discard if not needed."
[ "$_rd_skipped_tombstones" -gt 0 ] && \
_disclose_excluded "$_rd_skipped_tombstones fully-removed $(_pluralize "$_rd_skipped_tombstones" tombstone tombstones)" "deletion records with no remaining content"
log_info "Recovering attachments + writing evicted-attachments list..."
if [ "$stub_count" -gt 0 ]; then
{
printf '# Sections below group by entry filename (attachment filenames within).\n\n'
LC_ALL=C sort -u "$_journal_stubs_tsv" \
| LC_ALL=C awk -F$'\x1f' '
$1 != prev { if (prev != "") printf "\n"; printf "[%s]\n", $1; prev=$1 }
{ print $2 }
'
} | _write_skipped_report "$out_root" "Journal attachments" \
"$stub_count" "" \
"disable Optimize Mac Storage system-wide, open Journal.app once to trigger sync"
fi
if [ "$_attachments_missing" -gt 0 ]; then
log_warn "$_attachments_missing journal $(_pluralize "$_attachments_missing" attachment) had DB pointers but were missing on disk"
note_incomplete "$out_root" "$_attachments_missing journal $(_pluralize "$_attachments_missing" attachment)" \
"the database referenced these attachments but the files were not on disk (not iCloud-evicted stubs) — they are not in the export"
fi
if [ "$_attachments_copyfail" -gt 0 ]; then
log_warn "$_attachments_copyfail journal $(_pluralize "$_attachments_copyfail" attachment) failed to copy"
note_incomplete "$out_root" "$_attachments_copyfail journal $(_pluralize "$_attachments_copyfail" attachment)" \
"the attachment file(s) were on disk but cp failed — the photo / video content was not exported"
fi
log_info "Sanity checks and finalising..."
local _vcp_discovered="$n_entries"
[ "$n_entries_discovery_ok" = "0" ] && _vcp_discovered=""
local _vcp_accounted=$((exported + _entry_write_failed))
validate_count_parity "$_vcp_discovered" "$_vcp_accounted" \
"Journal entry" "$out_root"
if [ "$_entry_write_failed" -gt 0 ]; then
log_warn "$_entry_write_failed $(_pluralize "$_entry_write_failed" entry entries) could not be written$(_disclosure_pointer "Journal/_incomplete.txt")"
fi
local _journal_have_index=0
if [ "$exported" -gt 0 ] && [ -s "$_journal_index" ]; then
local _ji_multi="no"
[ "$_has_multi_journal" = "1" ] && _ji_multi="yes"
_journal_emit_index "$_journal_index" "$exported" "$out_root" "$_ji_multi"
validate_html_structure "$out_root/index.html"
[ -s "$out_root/index.html" ] && _journal_have_index=1
fi
local _stat="$exported $(_pluralize "$exported" entry entries)"
[ "$_journal_have_index" -eq 1 ] && _stat="$_stat + index.html"
[ "$total_attachments" -gt 0 ] && _stat="$_stat, $total_attachments $(_pluralize "$total_attachments" attachment) copied"
[ "$stub_count" -gt 0 ] && _stat="$_stat, $stub_count cloud-only$(_disclosure_pointer "Journal/_skipped.txt")"
_stat="$_stat, $(_dir_size_human "$out_root")"
log_ok "Wrote Journal/ ($_stat)"
mark_success
sweep_junk_files "$out_root"
print_summary
if [ "$ERR_COUNT" -gt 0 ]; then exit 1; fi
exit 0
}
MAIL_DIR="$HOME/Library/Mail"
_parse_emlx_plist() {
local emlx_file="$1" plist_offset="$2"
_emlx_date_recv=""
_emlx_flags=""
tail -c +"$plist_offset" "$emlx_file" > "$_MAIL_PLIST_TMP" 2>/dev/null
[ -s "$_MAIL_PLIST_TMP" ] || return 0
local json
json="$(plutil -convert json -o - "$_MAIL_PLIST_TMP" 2>/dev/null)" || return 0
[ -n "$json" ] || return 0
_emlx_date_recv="$(printf '%s' "$json" \
| sed -n 's/.*"date-received"[[:space:]]*:[[:space:]]*\([0-9.]*\).*/\1/p' | head -1)"
[ -z "$_emlx_date_recv" ] && _emlx_date_recv="$(printf '%s' "$json" \
| sed -n 's/.*"date-received"[[:space:]]*:[[:space:]]*"\([0-9.]*\)".*/\1/p' | head -1)"
_emlx_flags="$(printf '%s' "$json" \
| sed -n 's/.*"flags"[[:space:]]*:[[:space:]]*\([0-9]*\).*/\1/p' | head -1)"
[ -z "$_emlx_flags" ] && _emlx_flags="$(printf '%s' "$json" \
| sed -n 's/.*"flags"[[:space:]]*:[[:space:]]*"\([0-9]*\)".*/\1/p' | head -1)"
}
_emlx_flags_to_mozilla() {
local _f
_f=$((10#${1:-0}))
local _s=0 _s2=0
[ "$((_f & 1))" -ne 0 ] && _s=$((_s | 1)) # read (Apple bit 0)
[ "$((_f & 4))" -ne 0 ] && _s=$((_s | 2)) # replied (Apple bit 2)
[ "$((_f & 16))" -ne 0 ] && _s=$((_s | 4)) # flagged (Apple bit 4)
_moz_status="$(printf '%04x' "$_s")"
_moz_status2="$(printf '%08x' "$_s2")"
}
_emlx_header_parse() {
local _f="$1"
[ -f "$_f" ] || return 1
local _line1
_line1="$(head -1 -- "$_f" 2>/dev/null; printf x)"
_line1="${_line1%x}"
_emlx_byte_count="$(printf '%s' "$_line1" | LC_ALL=C tr -d '[:space:]')"
case "$_emlx_byte_count" in
''|*[!0-9]*) return 1 ;;
esac
[ "$_emlx_byte_count" -eq 0 ] && return 1
_emlx_first_line_bytes="$(printf '%s' "$_line1" | LC_ALL=C wc -c | LC_ALL=C tr -d '[:space:]')"
return 0
}
_emlx_to_mbox() {
local emlx_file="$1" mbox_file="$2"
[ -f "$emlx_file" ] || return 1
local _emlx_is_partial=0
case "${emlx_file##*/}" in
*.partial.emlx) _emlx_is_partial=1 ;;
esac
local _emlx_body_truncated=0
local byte_count first_line_bytes file_size avail
local _emlx_byte_count _emlx_first_line_bytes
_emlx_header_parse "$emlx_file" || return 1
byte_count="$((10#$_emlx_byte_count))"
first_line_bytes="$_emlx_first_line_bytes"
file_size="$(stat -f %z "$emlx_file" 2>/dev/null)"
case "$file_size" in ''|*[!0-9]*) file_size=0 ;; esac
avail=$((file_size - first_line_bytes))
local _emlx_orig_byte_count="$((10#$_emlx_byte_count))"
if [ "$avail" -gt 0 ] && [ "$byte_count" -gt "$avail" ]; then
log_warn "Partial emlx body clamped: $emlx_file (declared $byte_count bytes, only $avail on disk)"
byte_count="$avail"
_emlx_body_truncated=1
elif [ "$avail" -le 0 ]; then
local _avail_disp
if [ "$avail" -lt 0 ]; then _avail_disp=0; else _avail_disp="$avail"; fi
log_warn "Catastrophic emlx truncation: $emlx_file (declared $byte_count bytes; file has ${_avail_disp} body bytes available — first_line_bytes=$first_line_bytes file_size=$file_size)"
return 1
fi
local _actual_size _plist_offset
_actual_size="$file_size"
_plist_offset=$((first_line_bytes + byte_count + 1))
local _emlx_date_recv="" _emlx_flags="" _emlx_no_plist=0
if [ "$_emlx_body_truncated" -eq 1 ]; then
local _emlx_orig_plist_offset
_emlx_orig_plist_offset=$((first_line_bytes + _emlx_orig_byte_count + 1))
if [ "$_actual_size" -gt 0 ] && [ "$_emlx_orig_plist_offset" -le "$_actual_size" ]; then
_parse_emlx_plist "$emlx_file" "$_emlx_orig_plist_offset"
fi
elif [ "$_actual_size" -gt 0 ] && [ "$_plist_offset" -le "$_actual_size" ]; then
_parse_emlx_plist "$emlx_file" "$_plist_offset"
elif [ "$_actual_size" -gt 0 ] && [ "$_plist_offset" -eq "$((_actual_size + 1))" ]; then
_emlx_no_plist=1
else
log_warn "Truncated emlx: $emlx_file (plist offset $_plist_offset > size $_actual_size)"
fi
_emlx_flags=$((10#${_emlx_flags:-0}))
if [ "$((_emlx_flags & 2))" -ne 0 ]; then
return 2
fi
tail -c +"$((first_line_bytes + 1))" "$emlx_file" \
| head -c "$byte_count" > "$_MAIL_MSG_TMP" 2>/dev/null
[ -s "$_MAIL_MSG_TMP" ] || return 1
local from_addr="" from_date="" line="" _in_from=0 _in_date=0
while IFS= read -r line; do
line="${line%$'\r'}"
[ -z "$line" ] && break
case "$line" in
' '*|$'\t'*)
if [ "$_in_from" -eq 1 ] && [ -z "$from_addr" ]; then
local _angle_c
_angle_c="$(printf '%s' "$line" | LC_ALL=C grep -oE '<[^<>]*>' | tail -1)"
if [ -n "$_angle_c" ]; then
_angle_c="${_angle_c#<}"; _angle_c="${_angle_c%>}"
from_addr="$(printf '%s' "$_angle_c" \
| LC_ALL=C grep -oE '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' | head -1)"
fi
if [ -z "$from_addr" ]; then
from_addr="$(printf '%s' "$line" \
| LC_ALL=C grep -oE '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' | head -1)"
fi
elif [ "$_in_date" -eq 1 ] && [ -z "$from_date" ]; then
from_date="${line#"${line%%[![:space:]]*}"}"
fi
continue ;;
[Ff][Rr][Oo][Mm]:*)
_in_from=1; _in_date=0
if [ -z "$from_addr" ]; then
local _angle
_angle="$(printf '%s' "$line" \
| LC_ALL=C grep -oE '<[^<>]*>' | tail -1)"
if [ -n "$_angle" ]; then
_angle="${_angle#<}"; _angle="${_angle%>}"
from_addr="$(printf '%s' "$_angle" \
| LC_ALL=C grep -oE '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' | head -1)"
fi
if [ -z "$from_addr" ]; then
from_addr="$(printf '%s' "$line" \
| LC_ALL=C grep -oE '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' | head -1)"
fi
fi ;;
[Dd][Aa][Tt][Ee]:*)
_in_date=1; _in_from=0
if [ -z "$from_date" ]; then
from_date="${line#*:}"
from_date="${from_date#"${from_date%%[![:space:]]*}"}"
fi ;;
*)
_in_from=0; _in_date=0 ;;
esac
done < "$_MAIL_MSG_TMP"
[ -n "$from_addr" ] || from_addr='unknown@unknown'
local mbox_date=""
if [ -n "$_emlx_date_recv" ]; then
mbox_date="$(date -r "${_emlx_date_recv%%.*}" '+%a %b %e %H:%M:%S %Y' 2>/dev/null)"
fi
if [ -z "$mbox_date" ] && [ -n "$from_date" ]; then
local cleaned
cleaned="$(printf '%s' "$from_date" | LC_ALL=C sed 's/([^)]*)//g; s/[[:space:]]*$//')"
mbox_date="$(date -j -f '%a, %d %b %Y %H:%M:%S %z' "$cleaned" '+%a %b %e %H:%M:%S %Y' 2>/dev/null \
|| date -j -f '%d %b %Y %H:%M:%S %z' "$cleaned" '+%a %b %e %H:%M:%S %Y' 2>/dev/null \
|| date -j -f '%a, %d %b %Y %H:%M:%S' "$cleaned" '+%a %b %e %H:%M:%S %Y' 2>/dev/null)"
fi
if [ -z "$mbox_date" ]; then
if [ -n "$_emlx_date_recv" ] || [ -n "$from_date" ]; then
_emlx_datefail=$(( ${_emlx_datefail:-0} + 1 ))
if [ "$_emlx_datefail" -le "${_MAIL_DATE_WARN_LIMIT:-3}" ]; then
log_warn "Date header unparseable for emlx (plist='$_emlx_date_recv', from='$from_date'); using epoch fallback — message will sort as Jan 2000 in import."
fi
fi
mbox_date='Thu Jan 1 00:00:00 2000'
fi
local _moz_status="0000" _moz_status2="00000000"
_emlx_flags_to_mozilla "$_emlx_flags"
{
printf 'From %s %s\n' "$from_addr" "$mbox_date"
printf 'X-Mozilla-Status: %s\n' "$_moz_status"
printf 'X-Mozilla-Status2: %s\n' "$_moz_status2"
[ "$_emlx_is_partial" -eq 1 ] && printf 'X-Downpour-Partial: true\n'
[ "$_emlx_body_truncated" -eq 1 ] && printf 'X-Downpour-Partial-Body: true\n'
[ "$_emlx_no_plist" -eq 1 ] && printf 'X-Downpour-No-Plist: true\n'
LC_ALL=C sed 's/^\(>*\)From /\1>From /' "$_MAIL_MSG_TMP"
local _last_byte
_last_byte="$(LC_ALL=C tail -c1 "$_MAIL_MSG_TMP" 2>/dev/null | LC_ALL=C od -An -tu1 | LC_ALL=C tr -d '[:space:]')"
[ -n "$_last_byte" ] && [ "$_last_byte" != 10 ] && printf '\n'
printf '\n'
} >> "$mbox_file"
}
_mbox_logical_name() {
local mbox_dir="$1" base_root="$2" rel="" name="" part=""
rel="${mbox_dir#"$base_root"}"
rel="${rel#/}"
[ -n "$rel" ] || rel="${mbox_dir##*/}"
while [ -n "$rel" ]; do
case "$rel" in
*/*) part="${rel%%/*}"; rel="${rel#*/}" ;;
*) part="$rel"; rel="" ;;
esac
case "$part" in
*.mbox) part="${part%.mbox}" ;;
*) continue ;;
esac
[ -n "$part" ] || continue
if [ -n "$name" ]; then
name="${name}-${part}"
else
name="$part"
fi
done
[ -n "$name" ] || name='mailbox'
printf '%s' "$name"
}
_emlx_in_mbox() {
find "$1" \
\( -mindepth 1 -type d -name '*.mbox' -prune \) \
-o \( -type f -name '*.emlx' -print0 \) 2>/dev/null
}
_extract_mime_attachments_from_emlx() {
local emlx="$1"
[ -f "$emlx" ] || return 1
local byte_count first_line_bytes
local _emlx_byte_count _emlx_first_line_bytes
_emlx_header_parse "$emlx" || return 1
byte_count="$((10#$_emlx_byte_count))"
first_line_bytes="$_emlx_first_line_bytes"
local _email="$SCRATCH_DIR/_mime_email_$$.eml"
tail -c +"$((first_line_bytes + 1))" "$emlx" 2>/dev/null \
| head -c "$byte_count" > "$_email" 2>/dev/null
[ -s "$_email" ] || { rm -f -- "$_email" 2>/dev/null; return 1; }
local _stream="$SCRATCH_DIR/_mime_parts_$$.txt"
LC_ALL=C awk '
function get_param(hdr, pname, lc, plc, vv, rs, rl) {
lc = tolower(hdr); plc = tolower(pname)
if (match(lc, "(^|[ \t;])" plc "=\"[^\"]*\"")) {
rs = RSTART; rl = RLENGTH
vv = substr(hdr, rs, rl)
sub(/^[^"]*"/, "", vv); sub(/"$/, "", vv)
return vv
}
if (match(lc, "(^|[ \t;])" plc "=[^ \t;\r\n]+")) {
rs = RSTART; rl = RLENGTH
vv = substr(hdr, rs, rl)
sub(/^[^=]*=/, "", vv)
return vv
}
return ""
}
function rfc2231_pct_decode(raw, decoded, i, c, h1, h2) {
decoded = ""
i = 1
while (i <= length(raw)) {
c = substr(raw, i, 1)
if (c == "%" && i + 2 <= length(raw)) {
h1 = substr(raw, i+1, 1); h2 = substr(raw, i+2, 1)
if ((h1 in hex2) && (h2 in hex2)) {
decoded = decoded sprintf("%c", hex2[h1] * 16 + hex2[h2])
i += 3
continue
}
}
decoded = decoded c
i++
}
return decoded
}
function rfc2231_raw_value(hdr, lc, tok, rs, rl, vv) {
_r2231_found = 0
if (match(lc, "(^|[ \t;])" tok "=\"[^\"]*\"")) {
rs = RSTART; rl = RLENGTH
vv = substr(hdr, rs, rl)
sub(/^[^"]*"/, "", vv); sub(/"$/, "", vv)
_r2231_found = 1
return vv
}
if (match(lc, "(^|[ \t;])" tok "=[^ \t;\r\n]+")) {
rs = RSTART; rl = RLENGTH
vv = substr(hdr, rs, rl)
sub(/^[^=]*=/, "", vv)
_r2231_found = 1
return vv
}
return ""
}
function get_rfc2231_param(hdr, pname, lc, plc, vv, n, parts, charset, raw, i, seg, segraw, charset_done, found_any, esc_seg, esc_seg_star) {
lc = tolower(hdr); plc = tolower(pname)
raw = ""
charset = ""
charset_done = 0
found_any = 0
i = 0
while (1) {
esc_seg_star = plc "\\*" i "\\*"
esc_seg = plc "\\*" i
segraw = rfc2231_raw_value(hdr, lc, esc_seg_star)
if (_r2231_found) {
if (i == 0) {
n = split(segraw, parts, "'\''")
if (n >= 3) {
charset = tolower(parts[1])
seg = parts[3]
for (vv = 4; vv <= n; vv++) seg = seg "'\''" parts[vv]
charset_done = 1
} else {
seg = ""
}
} else {
seg = segraw
}
raw = raw seg
found_any = 1
i++
continue
}
segraw = rfc2231_raw_value(hdr, lc, esc_seg)
if (_r2231_found) {
raw = raw segraw
found_any = 1
i++
continue
}
break
}
if (found_any) {
if (charset_done) {
if (charset != "utf-8" && charset != "us-ascii" && charset != "") return ""
}
return rfc2231_pct_decode(raw)
}
vv = rfc2231_raw_value(hdr, lc, plc "\\*")
if (!_r2231_found) return ""
n = split(vv, parts, "'\''")
if (n < 3) return ""
charset = tolower(parts[1])
if (charset != "utf-8" && charset != "us-ascii" && charset != "") return ""
raw = parts[3]
for (i = 4; i <= n; i++) raw = raw "'\''" parts[i]
return rfc2231_pct_decode(raw)
}
function reset_block() {
cur_ct = "text/plain"; cur_boundary = ""
cur_encoding = "7bit"; cur_filename = ""; cur_disp = ""
cur_cid = ""
}
function image_subtype_ext(ct, st) {
st = ct
gsub(/[ \t]/, "", st)
st = tolower(st)
gsub(/;.*$/, "", st)
gsub(/^image\//, "", st)
gsub(/^x-/, "", st)
if (st == "jpeg" || st == "jpg") return "jpg"
if (st == "png") return "png"
if (st == "gif") return "gif"
if (st == "tiff" || st == "tif") return "tiff"
if (st == "bmp") return "bmp"
if (st == "webp") return "webp"
if (st == "heic") return "heic"
if (st == "heif") return "heif"
if (st == "svg+xml" || st == "svg") return "svg"
return "img"
}
function process_header(line, lc, val, _fn) {
sub(/[\r]+$/, "", line)
lc = tolower(line)
if (lc ~ /^content-type:/) {
val = substr(line, index(line, ":") + 1)
sub(/^[ \t]+/, "", val); sub(/[ \t]+$/, "", val)
cur_ct = val
if (tolower(val) ~ /^multipart\//) cur_boundary = get_param(val, "boundary")
if (cur_filename == "") {
_fn = get_rfc2231_param(val, "name")
if (_fn == "") _fn = get_param(val, "name")
cur_filename = _fn
}
} else if (lc ~ /^content-transfer-encoding:/) {
val = substr(line, index(line, ":") + 1)
sub(/^[ \t]+/, "", val); sub(/[ \t]+$/, "", val)
cur_encoding = tolower(val)
} else if (lc ~ /^content-id:/) {
val = substr(line, index(line, ":") + 1)
sub(/^[ \t]+/, "", val); sub(/[ \t]+$/, "", val)
gsub(/^, "", val); gsub(/>$/, "", val)
cur_cid = val
} else if (lc ~ /^content-disposition:/) {
val = substr(line, index(line, ":") + 1)
sub(/^[ \t]+/, "", val); sub(/[ \t]+$/, "", val)
cur_disp = val
if (val ~ /[Ff][Ii][Ll][Ee][Nn][Aa][Mm][Ee](\*[0-9]+)?\*?=/) {
_fn = get_rfc2231_param(val, "filename")
if (_fn == "") _fn = get_param(val, "filename")
cur_filename = _fn
}
}
}
function end_headers_block( lct, ldisp, is_att, is_inline_image, _ext) {
if (accum != "") { process_header(accum); accum = "" }
lct = tolower(cur_ct)
ldisp = tolower(cur_disp)
if (cur_boundary != "") {
depth++; b_stack[depth] = cur_boundary
state = "scan"
reset_block()
return
}
if (lct ~ /^application\/(x-)?pkcs7-signature/) {
capturing = 0
return
}
is_att = (ldisp ~ /^attachment/ ||
(cur_filename != "" && lct !~ /^(text\/plain|text\/html|multipart\/|message\/)/))
is_inline_image = (cur_filename == "" && lct ~ /^image\// &&
(ldisp ~ /^inline/ || cur_cid != ""))
if (is_att || is_inline_image) {
if (cur_filename == "" && is_inline_image) {
_ext = image_subtype_ext(cur_ct)
if (cur_cid != "") {
cur_filename = cur_cid
if (cur_filename !~ /\.[A-Za-z0-9]+$/)
cur_filename = cur_filename "." _ext
} else {
inline_seq++
cur_filename = sprintf("inline-%03d.%s", inline_seq, _ext)
}
}
gsub(/[\t\r\n]/, "_", cur_filename)
sub(/^[ \t]+/, "", cur_filename); sub(/[ \t]+$/, "", cur_filename)
if (cur_filename == "") cur_filename = "attachment"
print "H\t" cur_filename "\t" cur_encoding
capturing = 1
} else {
capturing = 0
}
}
BEGIN {
state = "headers"; depth = 0; capturing = 0; accum = ""
inline_seq = 0
for (_hi = 0; _hi < 16; _hi++) {
hex2[substr("0123456789ABCDEF", _hi+1, 1)] = _hi
hex2[substr("0123456789abcdef", _hi+1, 1)] = _hi
}
reset_block()
}
state == "headers" {
if ($0 ~ /^\r?$/) {
end_headers_block()
if (state == "headers") state = "body"
next
}
if (depth > 0) {
boundary_cand = $0
sub(/[ \t\r]+$/, "", boundary_cand)
for (b_idx = depth; b_idx >= 1; b_idx--) {
if (boundary_cand == "--" b_stack[b_idx] ||
boundary_cand == "--" b_stack[b_idx] "--") {
end_headers_block()
if (state == "headers") state = "body"
break
}
}
}
if (state == "headers") {
if ($0 ~ /^[ \t]/ && accum != "") {
accum = accum " " $0
next
}
if (accum != "") { process_header(accum) }
accum = $0
next
}
}
state == "scan" || state == "body" {
if (depth > 0) {
boundary_cand = $0
sub(/[ \t\r]+$/, "", boundary_cand)
if (depth < 0) {
depth = 0
}
for (i = depth; i >= 1; i--) {
if (boundary_cand == "--" b_stack[i]) {
if (capturing) { print "E"; capturing = 0 }
depth = i
state = "headers"; accum = ""
reset_block()
next
}
if (boundary_cand == "--" b_stack[i] "--") {
if (capturing) { print "E"; capturing = 0 }
depth = i - 1
state = (depth > 0) ? "body" : "ended"
next
}
}
}
if (capturing) print "B\t" $0
}
END { if (capturing) print "E" }
' "$_email" > "$_stream" 2>/dev/null
rm -f -- "$_email" 2>/dev/null
[ -s "$_stream" ] || { rm -f -- "$_stream" 2>/dev/null; return 0; }
local _cur_enc="" _cur_fn="" _cur_body="$SCRATCH_DIR/_mime_body_$$"
local _line _rest _decoded _final _amd5
local _tab=$'\t'
: > "$_cur_body"
while IFS= read -r _line; do
case "$_line" in
"H$_tab"*)
_rest="${_line#H$_tab}"
_cur_fn="${_rest%%$_tab*}"
_cur_enc="${_rest#*$_tab}"
_cur_fn="${_cur_fn##*/}"
_cur_fn="${_cur_fn%$'\r'}"
_cur_fn="$(_sanitize_name "$_cur_fn")"
[ -z "$_cur_fn" ] && _cur_fn="attachment"
: > "$_cur_body"
;;
"B$_tab"*)
printf '%s\n' "${_line#B$_tab}" >> "$_cur_body"
;;
E)
_decoded="$SCRATCH_DIR/_mime_decoded_$$.bin"
local _decode_rc=0
case "$_cur_enc" in
base64)
LC_ALL=C tr -d ' \t\r' < "$_cur_body" \
| base64 -D > "$_decoded" 2>/dev/null
_decode_rc="${PIPESTATUS[1]}"
;;
quoted-printable)
LC_ALL=C awk '
BEGIN {
for (i = 0; i < 16; i++) {
hex2[substr("0123456789ABCDEF", i+1, 1)] = i
hex2[substr("0123456789abcdef", i+1, 1)] = i
}
cont = ""
}
{
line = $0
sub(/\r$/, "", line)
out = ""
L = length(line)
i = 1
softbreak = 0
while (i <= L) {
c = substr(line, i, 1)
if (c == "=" && i + 2 <= L) {
h1 = substr(line, i+1, 1); h2 = substr(line, i+2, 1)
if ((h1 in hex2) && (h2 in hex2)) {
v = hex2[h1] * 16 + hex2[h2]
out = out sprintf("%c", v)
i += 3
continue
}
}
if (c == "=" && i == L) {
softbreak = 1
break
}
out = out c
i++
}
out = cont out
cont = ""
if (softbreak) { cont = out } else { printf "%s\n", out }
}
END { if (cont != "") printf "%s", cont }
' "$_cur_body" > "$_decoded" 2>/dev/null
_decode_rc="$?"
;;
7bit|8bit|binary|"")
cp -- "$_cur_body" "$_decoded" 2>/dev/null
_decode_rc="$?"
;;
*)
cp -- "$_cur_body" "$_decoded" 2>/dev/null
_decode_rc="$?"
;;
esac
if [ "$_decode_rc" -ne 0 ]; then
rm -f -- "$_decoded" 2>/dev/null
att_failed=$((att_failed + 1))
note_incomplete "$mail_dir" \
"Attachment ${_cur_fn:-attachment} (decode error)" \
"the MIME decoder (base64 or quoted-printable) reported an error — partial body discarded"
elif [ -s "$_decoded" ]; then
_amd5="$(_md5_hex "$_decoded")"
if [ -n "$_amd5" ] && _mail_seen_md5_check "$_amd5"; then
att_dupes=$((att_dupes + 1))
rm -f -- "$_decoded" 2>/dev/null
else
_final="$(_unique_path "$att_dir/$_cur_fn")"
if mv -- "$_decoded" "$_final"; then
att_count=$((att_count + 1))
_mail_seen_md5_mark "$_amd5"
else
att_failed=$((att_failed + 1))
rm -f -- "$_final" 2>/dev/null
rm -f -- "$_decoded" 2>/dev/null
fi
fi
else
rm -f -- "$_decoded" 2>/dev/null
fi
_cur_fn=""; _cur_enc=""
: > "$_cur_body"
;;
esac
done < "$_stream"
rm -f -- "$_stream" "$_cur_body" 2>/dev/null
return 0
}
mail_main() {
maybe_show_help "$@"
_announce_step "Mail export"
check_dependencies
[ "${_INVOKED_BY_WRAPPER:-0}" = "1" ] || timemachine_suspend
local out_arg out_dir
out_arg="$(_parse_output_folder_arg "$@")" || exit 1
out_dir="$(resolve_output_dir "$out_arg")" || exit 1
_announce_output_dir "Mail"
log_info "Checking Full Disk Access and discovering mailboxes..."
require_full_disk_access
warn_if_app_running Mail
if [ ! -d "$MAIL_DIR" ]; then
report_no_source "Mail" "not configured" "$MAIL_DIR"
fi
log_info "Source: $MAIL_DIR"
if [ -d "$MAIL_DIR" ]; then
local _src_kb _avail_kb _need_kb
_src_kb="$(LC_ALL=C du -sk "$MAIL_DIR" 2>/dev/null | awk '{print $1}')"
case "$_src_kb" in ''|*[!0-9]*) _src_kb=0 ;; esac
_avail_kb="$(LC_ALL=C df -P -k "$out_dir" 2>/dev/null | awk 'END{print $4}')"
case "$_avail_kb" in ''|*[!0-9]*) _avail_kb=0 ;; esac
_need_kb=$((_src_kb * 2))
if [ "$_src_kb" -gt 0 ] && [ "$_avail_kb" -gt 0 ] && [ "$_avail_kb" -lt "$_need_kb" ]; then
log_warn "Output volume has $((_avail_kb/1024)) MB free; this Mail export needs roughly $((_need_kb/1024)) MB for the verbatim copy + Thunderbird conversion. The copy may fail partway — free up space or choose another output folder."
fi
fi
local mail_dir="$out_dir/Mail"
mkdir -p "$mail_dir" || die "Cannot create $mail_dir"
track_output_path "$mail_dir"
local out_root="$mail_dir/Apple Mail"
require_no_collision "$out_root"
_create_output_dir "$out_root"
log_info "Copying Apple Mail (verbatim)..."
local total_mboxes=0 nonempty_mboxes=0 empty_mboxes=0
local total_files=0 copy_failures=0
local mbox rel dst f rel_in_mbox dst_in_mbox
local _MAIL_COPY_FAIL_WARN_LIMIT=3
local _mail_copy_first_err=""
while IFS= read -r -d '' mbox; do
total_mboxes=$((total_mboxes + 1))
mark_activity
if find "$mbox" -mindepth 1 -type d -name '*.mbox' -prune -o -name '*.emlx' -print -quit 2>/dev/null | grep -q .; then
nonempty_mboxes=$((nonempty_mboxes + 1))
else
empty_mboxes=$((empty_mboxes + 1))
continue
fi
rel="${mbox#"$MAIL_DIR"/}"
dst="$out_root/$rel"
if ! mkdir -p "$dst"; then
log_warn "Cannot create $dst — skipping mailbox."
note_incomplete "$mail_dir" "Mailbox: $rel" \
"could not create its destination folder under Mail/Apple Mail/ — this mailbox was not mirrored"
continue
fi
while IFS= read -r -d '' f; do
rel_in_mbox="${f#"$mbox"/}"
should_skip_copy "$f" "$rel_in_mbox" && continue
dst_in_mbox="$dst/$rel_in_mbox"
local _tdir="${dst_in_mbox%/*}"
[ -d "$_tdir" ] || mkdir -p "$_tdir" 2>/dev/null
local _copy_err _copy_err_rc
_copy_err="$(ditto -- "$f" "$dst_in_mbox" 2>&1 >/dev/null)"; _copy_err_rc=$?
if [ "$_copy_err_rc" -eq 0 ] && [ -f "$dst_in_mbox" ]; then
total_files=$((total_files + 1))
else
copy_failures=$((copy_failures + 1))
rm -f -- "$dst_in_mbox" 2>/dev/null || true
local _err_msg="${_copy_err##*: }"
[ -z "$_err_msg" ] && _err_msg="(no error message)"
if [ "$copy_failures" -le "$_MAIL_COPY_FAIL_WARN_LIMIT" ]; then
log_warn "Copy failed: $rel/$rel_in_mbox: $_err_msg."
fi
[ -z "$_mail_copy_first_err" ] && _mail_copy_first_err="$_err_msg"
fi
done < <(find "$mbox" -mindepth 1 -type d -name '*.mbox' -prune -o -type f -print0 2>/dev/null)
done < <(find "$MAIL_DIR" -type d -name '*.mbox' -print0 2>/dev/null)
find "$out_root" -mindepth 1 -type d -empty -delete 2>/dev/null || true
if [ "$total_mboxes" -eq 0 ]; then
rmdir "$out_root" 2>/dev/null || true
rmdir "$mail_dir" 2>/dev/null || true
report_no_source "Mail" "has no mailboxes" "$MAIL_DIR"
fi
log_info "Found $total_mboxes $(_pluralize "$total_mboxes" mailbox mailboxes) ($nonempty_mboxes with messages, $empty_mboxes empty)"
if [ "$copy_failures" -gt 0 ]; then
local _suffix=""
[ -n "$_mail_copy_first_err" ] && _suffix=" — first error: $_mail_copy_first_err"
if [ "$copy_failures" -gt "$_MAIL_COPY_FAIL_WARN_LIMIT" ]; then
local _hidden=$((copy_failures - _MAIL_COPY_FAIL_WARN_LIMIT))
log_warn "$copy_failures $(_pluralize "$copy_failures" file) failed to copy ($_hidden similar warns suppressed)$_suffix"
else
log_warn "$copy_failures $(_pluralize "$copy_failures" file) failed to copy$_suffix"
fi
log_info " Common causes: Mail.app actively running and locking message files; iCloud-Mail \"optimized storage\" placeholder files; macOS sandbox / data-vault protection on the mailbox. Quitting Mail.app and re-running usually clears the locked-file class of failures."
note_incomplete "$mail_dir" "$copy_failures message file(s) across the Apple Mail mirror" \
"individual file copies failed (commonly: Mail.app running and locking files, iCloud optimized-storage placeholders, or sandbox protection) — those messages are missing from Mail/Apple Mail/. Quit Mail.app and re-run."
fi
if [ "$nonempty_mboxes" -eq 0 ]; then
log_warn "All mailboxes empty — nothing to export."
rm -rf -- "$out_root" 2>/dev/null
rmdir "$mail_dir" 2>/dev/null || true
_flag_no_content
mark_success
print_summary
exit 0
fi
if [ "$total_files" -eq 0 ]; then
rmdir "$out_root" 2>/dev/null || true
rmdir "$mail_dir" 2>/dev/null || true
log_warn "All file copies failed — Mail/Apple Mail/ left empty."
note_incomplete "$mail_dir" "The entire Apple Mail mirror" \
"every message-file copy failed — Mail/Apple Mail/ is empty. Quit Mail.app (it locks message files while running) and re-run."
else
log_ok "Wrote Mail/Apple Mail/ ($nonempty_mboxes $(_pluralize "$nonempty_mboxes" mailbox mailboxes), $total_files $(_pluralize "$total_files" file), $(_dir_size_human "$out_root"))"
fi
ensure_scratch
local _MAIL_PLIST_TMP="$SCRATCH_DIR/a2t_plist"
local _MAIL_MSG_TMP="$SCRATCH_DIR/a2t_msg"
local _MAIL_DATE_WARN_LIMIT=3
local _emlx_datefail=0
local tb_dir="$mail_dir/Thunderbird"
_create_output_dir "$tb_dir"
log_info "Converting to Thunderbird format..."
local _tb_total
_tb_total="$(find "$out_root" -type d -name '*.mbox' -print0 2>/dev/null | tr -dc '\0' | wc -c | tr -d ' ')"
case "$_tb_total" in ''|*[!0-9]*) _tb_total=0 ;; esac
local tb_mailboxes=0 tb_messages=0 tb_deleted=0 tb_errors=0
local mbox_dir base out_file emlx_file converted deleted errors
local _tb_processed=0
local _pt_last=0 _pt_inline=0
while IFS= read -r -d '' mbox_dir; do
base="$(_mbox_logical_name "$mbox_dir" "$out_root")"
out_file="$(_unique_path "$tb_dir/${base}.mbox")"
: > "$out_file" 2>/dev/null || { log_warn "Cannot create $out_file"; continue; }
converted=0
deleted=0
errors=0
while IFS= read -r -d '' emlx_file; do
_emlx_to_mbox "$emlx_file" "$out_file"
case $? in
0) converted=$((converted + 1)) ;;
2) deleted=$((deleted + 1)) ;;
*) errors=$((errors + 1)) ;;
esac
done < <(_emlx_in_mbox "$mbox_dir" | sort -z)
if [ "$converted" -gt 0 ]; then
tb_mailboxes=$((tb_mailboxes + 1))
tb_messages=$((tb_messages + converted))
else
rm -f -- "$out_file" 2>/dev/null
fi
tb_deleted=$((tb_deleted + deleted))
tb_errors=$((tb_errors + errors))
_tb_processed=$((_tb_processed + 1))
_progress_tick "$_tb_processed" "$_tb_total" "mailboxes converted"
mark_activity
done < <(find "$out_root" -type d -name '*.mbox' -print0 2>/dev/null | sort -z)
_progress_end
if [ "$_emlx_datefail" -gt "$_MAIL_DATE_WARN_LIMIT" ]; then
local _date_hidden=$((_emlx_datefail - _MAIL_DATE_WARN_LIMIT))
log_warn "$_emlx_datefail $(_pluralize "$_emlx_datefail" message) had an unparseable Date: header and were given the epoch fallback ($_date_hidden further warning(s) suppressed)"
fi
if [ "$tb_messages" -gt 0 ]; then
local _tb_stat
_tb_stat="$tb_messages $(_pluralize "$tb_messages" msg) across $tb_mailboxes $(_pluralize "$tb_mailboxes" mailbox mailboxes)"
[ "$tb_deleted" -gt 0 ] && _tb_stat="$_tb_stat, $tb_deleted deleted skipped"
[ "$tb_errors" -gt 0 ] && _tb_stat="$_tb_stat, $tb_errors $(_pluralize "$tb_errors" error)"
log_ok "Wrote Mail/Thunderbird/ ($_tb_stat)"
local _from_count
_from_count=$(LC_ALL=C grep -rh '^From ' "$tb_dir" 2>/dev/null | wc -l | tr -d '[:space:]')
case "$_from_count" in ''|*[!0-9]*) _from_count=0 ;; esac
if [ "$_from_count" -lt "$tb_messages" ]; then
log_warn "Thunderbird parity check: $_from_count From-$(_pluralize "$_from_count" envelope) found, expected at least $tb_messages — possible silent-write failure."
note_incomplete "$mail_dir" "Thunderbird mbox — message-count shortfall" \
"the converted Thunderbird mbox holds $_from_count message envelope(s) but $tb_messages were expected; some messages may not have been written to the Thunderbird/ tree (the verbatim Apple Mail/ mirror is the authoritative copy)"
fi
if [ "$tb_errors" -gt 0 ]; then
log_warn "$tb_errors $(_pluralize "$tb_errors" message) failed to convert to Thunderbird format"
note_incomplete "$mail_dir" "Thunderbird mbox — $tb_errors message-conversion $(_pluralize "$tb_errors" error)" \
"$tb_errors message(s) could not be converted into the Thunderbird/ tree; each is still present in the verbatim Apple Mail/ mirror, which is the authoritative copy of those messages"
fi
log_info "Validating mbox structure (RFC 4155)..."
local _f
while IFS= read -r -d '' _f; do
validate_mbox_rfc4155 "$_f" || true
done < <(find "$tb_dir" -type f -print0 2>/dev/null)
elif [ "$tb_errors" -gt 0 ]; then
log_warn "$tb_errors $(_pluralize "$tb_errors" message) failed to convert to Thunderbird format — the Thunderbird/ tree is empty"
note_incomplete "$mail_dir" "Thunderbird mbox — all $tb_errors message-conversion $(_pluralize "$tb_errors" error)" \
"every message failed to convert into the Thunderbird/ tree; all are still present in the verbatim Apple Mail/ mirror, which is the authoritative copy of those messages"
rmdir "$tb_dir" 2>/dev/null || true
else
rmdir "$tb_dir" 2>/dev/null || true
log_info "No Thunderbird mbox files written (no convertible messages)"
fi
local att_dir="$mail_dir/Attachments"
_create_output_dir "$att_dir"
log_info "Extracting attachments..."
ensure_scratch
_mail_seen_md5_check() {
local _h="$1"
[ -n "$_h" ] || return 1
local _v="_mail_seen_md5_$_h"
[ -n "${!_v+x}" ]
}
_mail_seen_md5_mark() {
local _h="$1"
[ -z "$_h" ] && return 0
local _v="_mail_seen_md5_$_h"
printf -v "$_v" 1
}
local att_count=0 att_failed=0 att_dupes=0 _af _arel _atarget _abase _amd5
while IFS= read -r -d '' _af; do
_abase="${_af##*/}"
_arel="${_af#"$out_root"/}"
should_skip_copy "$_af" "$_arel" && continue
[ "$_abase" = "smime.p7s" ] && continue
_amd5="$(_md5_hex "$_af")"
if [ -n "$_amd5" ] && _mail_seen_md5_check "$_amd5"; then
att_dupes=$((att_dupes + 1))
continue
fi
_atarget="$(_unique_path "$att_dir/$_abase")"
if ditto -- "$_af" "$_atarget" 2>/dev/null; then
att_count=$((att_count + 1))
_mail_seen_md5_mark "$_amd5"
else
att_failed=$((att_failed + 1))
rm -f -- "$_atarget" 2>/dev/null
fi
done < <(find "$out_root" -type f \
\( -path '*.mbox/*/Data/Attachments/*' \
-o -path '*.mbox/*/Data/*/Attachments/*' \) \
-print0 2>/dev/null)
local _att_count_pass1="$att_count"
local _mime_emlx_n=0 _mime_with_att=0 _mime_extract_fail=0
local _emlx
_mime_emlx_n=$(find "$out_root" -type f -name '*.emlx' 2>/dev/null \
| wc -l | LC_ALL=C tr -d ' ')
case "$_mime_emlx_n" in ''|*[!0-9]*) _mime_emlx_n=0 ;; esac
while IFS= read -r -d '' _emlx; do
[ -n "$_emlx" ] || continue
_mime_with_att=$((_mime_with_att + 1))
_extract_mime_attachments_from_emlx "$_emlx" \
|| _mime_extract_fail=$((_mime_extract_fail + 1))
case "$_mime_with_att" in *00) mark_activity ;; esac
done < <(find "$out_root" -type f -name '*.emlx' -print0 2>/dev/null \
| LC_ALL=C xargs -0 grep -liE --null \
'^content-type:.*multipart|^[[:space:]]+multipart/|^content-disposition:.*attachment|filename=|name=' \
/dev/null 2>/dev/null)
local _att_count_pass2=$((att_count - _att_count_pass1))
if [ "$_mime_extract_fail" -gt 0 ]; then
log_warn "$_mime_extract_fail multipart $(_pluralize "$_mime_extract_fail" message) could not be parsed for inline attachments"
note_incomplete "$mail_dir" "Inline attachment extraction — $_mime_extract_fail malformed $(_pluralize "$_mime_extract_fail" message)" \
"$_mime_extract_fail multipart message(s) could not be parsed for inline MIME attachments; each message itself is still present in the verbatim Apple Mail/ mirror, so this is an attachment-extraction gap, not message data loss"
fi
if [ "$att_count" -gt 0 ] || [ "$att_failed" -gt 0 ]; then
log_info "Pass 1 (decoded tree): $_att_count_pass1 $(_pluralize "$_att_count_pass1" file)"
log_info "Pass 2 (inline MIME): $_att_count_pass2 $(_pluralize "$_att_count_pass2" file) from $_mime_with_att scanned candidate $(_pluralize "$_mime_with_att" message) (of $_mime_emlx_n total)"
fi
if [ "$att_count" -gt 0 ]; then
local _att_stat
_att_stat="$att_count $(_pluralize "$att_count" attachment)"
[ "$att_dupes" -gt 0 ] && _att_stat="$_att_stat, $att_dupes $(_pluralize "$att_dupes" content-dupe) skipped"
[ "$att_failed" -gt 0 ] && _att_stat="$_att_stat, $att_failed failed"
_att_stat="$_att_stat, $(_dir_size_human "$att_dir")"
log_ok "Wrote Mail/Attachments/ ($_att_stat)"
if [ "$att_failed" -gt 0 ]; then
log_warn "$att_failed $(_pluralize "$att_failed" attachment) could not be extracted to Mail/Attachments/"
note_incomplete "$mail_dir" "Attachment extraction — $att_failed failed" \
"$att_failed $(_pluralize "$att_failed" attachment) failed to copy/extract into Mail/Attachments/; each affected message is still present in the verbatim Apple Mail/ mirror, so this is an attachment-extraction gap, not message data loss"
fi
elif [ "$att_failed" -gt 0 ]; then
rmdir "$att_dir" 2>/dev/null || true
log_warn "All $att_failed attachment copy(ies) failed — Mail/Attachments/ left empty."
note_incomplete "$mail_dir" "All $att_failed mail attachment(s)" \
"every attachment copy failed — Mail/Attachments/ is empty. Quit Mail.app and re-run."
else
rmdir "$att_dir" 2>/dev/null || true
log_info "No attachments found in any mailbox"
fi
mark_success
sweep_junk_files "$mail_dir"
rmdir "$mail_dir" 2>/dev/null || true
if [ "$total_files" -gt 0 ] || [ "$tb_messages" -gt 0 ] || [ "$att_count" -gt 0 ]; then
local _mail_stat
_mail_stat="$nonempty_mboxes $(_pluralize "$nonempty_mboxes" mailbox mailboxes), $total_files $(_pluralize "$total_files" file)"
[ "$tb_messages" -gt 0 ] && _mail_stat="$_mail_stat, $tb_messages Thunderbird $(_pluralize "$tb_messages" msg)"
[ "$att_count" -gt 0 ] && _mail_stat="$_mail_stat, $att_count $(_pluralize "$att_count" attachment)"
_mail_stat="$_mail_stat, $(_dir_size_human "$mail_dir")"
log_ok "Wrote Mail/ ($_mail_stat)"
fi
print_summary
if [ "$ERR_COUNT" -gt 0 ]; then exit 1; fi
exit 0
}
_maps_has_table() {
case "$1" in *[!A-Za-z0-9_]*) return 1 ;; esac
sqlite3 -readonly "$_snap" \
"SELECT name FROM sqlite_master WHERE type='table' AND name='$1';" 2>/dev/null | grep -q .
}
_maps_has_col() {
_col_present "$_snap" "$1" "$2"
}
export_maps() {
local out_root="$1"
local _maps_root="$out_root/Maps"
local _maps_dir="$HOME/Library/Containers/com.apple.Maps/Data/Maps"
if [ ! -d "$_maps_dir" ]; then
report_no_source "Maps" "not configured" \
"~/Library/Containers/com.apple.Maps/Data/Maps" \
"Maps"
fi
require_no_collision "$_maps_root"
_create_output_dir "$_maps_root" "permission denied / disk full?"
log_info "Source: $_maps_dir"
local _db="" _db_is_cache=0 _db_all _db_count
_db_all=$(find "$_maps_dir" -name 'MapsSync_*' \
-not -name '*_deviceLocalCache*' -not -name '*-wal' -not -name '*-shm' \
-not -name '*-journal' \
-type f 2>/dev/null | LC_ALL=C sort)
_db_count=$(printf '%s\n' "$_db_all" | grep -c .)
case "$_db_count" in ''|*[!0-9]*) _db_count=0 ;; esac
_db=$(printf '%s\n' "$_db_all" | head -1)
if [ "$_db_count" -gt 1 ]; then
ensure_scratch
local _secondaries_with_content=0 _secondaries_unverified=0 _sdb _scnt
local _sdb_list
_sdb_list=$(printf '%s\n' "$_db_all" | tail -n +2)
while IFS= read -r _sdb; do
[ -z "$_sdb" ] && continue
local _sdb_snap _shist _sfav _scounts
_sdb_snap="$(snapshot_db "$_sdb" 2>/dev/null)" || _sdb_snap=""
if [ -z "$_sdb_snap" ]; then
_secondaries_unverified=$((_secondaries_unverified + 1))
continue
fi
_scounts=$(sqlite3 -readonly -separator $'\x1f' "$_sdb_snap" \
"SELECT (SELECT COUNT(*) FROM ZHISTORYITEM),
(SELECT COUNT(*) FROM ZFAVORITEITEM);" 2>/dev/null)
if [ -n "$_scounts" ]; then
_shist="${_scounts%%$'\x1f'*}"
_sfav="${_scounts#*$'\x1f'}"
[ "$_sfav" = "$_scounts" ] && _sfav=""
else
_shist=$(sqlite3 -readonly "$_sdb_snap" \
"SELECT COUNT(*) FROM ZHISTORYITEM;" 2>/dev/null)
_sfav=$(sqlite3 -readonly "$_sdb_snap" \
"SELECT COUNT(*) FROM ZFAVORITEITEM;" 2>/dev/null)
fi
case "${_shist:-0}" in ''|*[!0-9]*) _shist=0 ;; esac
case "${_sfav:-0}" in ''|*[!0-9]*) _sfav=0 ;; esac
_scnt=$((_shist + _sfav))
if [ "$_scnt" -gt 0 ]; then
_secondaries_with_content=$((_secondaries_with_content + 1))
fi
done <<<"$_sdb_list"
if [ "$_secondaries_with_content" -gt 0 ] || [ "$_secondaries_unverified" -gt 0 ]; then
local _smsg="found $_db_count MapsSync_*.sqlite candidates; exported from the first one only"
[ "$_secondaries_with_content" -gt 0 ] && _smsg="$_smsg — $_secondaries_with_content secondary account DB(s) hold user content that was NOT processed"
[ "$_secondaries_unverified" -gt 0 ] && _smsg="$_smsg — $_secondaries_unverified secondary DB(s) could not be snapshotted (locked by a running Maps.app / WAL skew), so their content is UNVERIFIED (not confirmed empty)"
_smsg="$_smsg. Re-run with the desired DB selected if its content matters."
log_warn "$_smsg"
note_incomplete "$_maps_root" "Maps source database selection" "$_smsg"
else
log_info "Found $_db_count MapsSync_*.sqlite candidates under $_maps_dir; secondary DB(s) empty — using the first."
fi
fi
if [ -z "$_db" ]; then
_db=$(find "$_maps_dir" -name 'MapsSync_*_deviceLocalCache.db' \
-type f 2>/dev/null | head -1)
[ -n "$_db" ] && _db_is_cache=1
fi
if [ "$_db_is_cache" = "1" ]; then
log_warn "main MapsSync database not found — exporting from the device-local cache (results may be partial)"
fi
local _fav_count=0 _coll_count=0 _hist_count=0
if [ -n "$_db" ] && [ -f "$_db" ]; then
log_info "Snapshotting MapsSync database and querying saved places / collections / recents..."
ensure_scratch
local _snap=""
_snap=$(snapshot_db "$_db" 2>/dev/null) || _snap=""
if [ -n "$_snap" ]; then
if _maps_has_table ZFAVORITEITEM; then
local _fav_count_q
_fav_count_q=$(sqlite3 -readonly "$_snap" \
"SELECT COUNT(*) FROM ZFAVORITEITEM;" 2>/dev/null)
if [ -z "$_fav_count_q" ]; then
log_warn "Maps: ZFAVORITEITEM count query failed — drift-check skipped."
note_incomplete "$_maps_root" "Saved Places.csv completeness" \
"a completeness check on the Maps database failed, so the export could not verify it captured every saved place — Saved Places.csv may be partial"
_fav_count=0
else
_fav_count="$_fav_count_q"
fi
case "$_fav_count" in ''|*[!0-9]*) _fav_count=0 ;; esac
if [ "$_fav_count" -gt 0 ]; then
local _hidden_clause="" _order="ORDER BY Z_PK"
_maps_has_col ZFAVORITEITEM ZHIDDEN && \
_hidden_clause="WHERE COALESCE(ZHIDDEN, 0) = 0"
_maps_has_col ZFAVORITEITEM ZPOSITIONINDEX && \
_order="ORDER BY ZPOSITIONINDEX, Z_PK"
local _fav_custom=""
_maps_has_col ZFAVORITEITEM ZCUSTOMNAME && _fav_custom="ZCUSTOMNAME"
local _fav_place_name=""
_fav_place_name="$(_pick_first_col "$_snap" ZFAVORITEITEM \
ZMAPITEMNAME ZNAME || true)"
local _fav_name_expr="''"
if [ -n "$_fav_custom" ] && [ -n "$_fav_place_name" ]; then
_fav_name_expr="$(_sql_tsv_safe "COALESCE(NULLIF($_fav_custom, ''), $_fav_place_name, '')")"
elif [ -n "$_fav_custom" ]; then
_fav_name_expr="$(_sql_tsv_safe "COALESCE($_fav_custom, '')")"
elif [ -n "$_fav_place_name" ]; then
_fav_name_expr="$(_sql_tsv_safe "COALESCE($_fav_place_name, '')")"
fi
local _fav_created_expr="NULL"
_maps_has_col ZFAVORITEITEM ZCREATETIME && \
_fav_created_expr="CASE WHEN ZCREATETIME <= 0 THEN NULL ELSE $(_sql_cd_to_unix ZCREATETIME) END"
local _fav_lastvisited_expr="''"
_maps_has_col ZFAVORITEITEM ZLASTVISITEDTIME && \
_fav_lastvisited_expr="COALESCE(datetime(CASE WHEN ZLASTVISITEDTIME <= 0 THEN NULL ELSE $(_sql_cd_to_unix ZLASTVISITEDTIME) END, 'unixepoch', 'localtime'), '')"
local _fav_modified_expr="''"
_maps_has_col ZFAVORITEITEM ZMODIFICATIONTIME && \
_fav_modified_expr="COALESCE(datetime(CASE WHEN ZMODIFICATIONTIME <= 0 THEN NULL ELSE $(_sql_cd_to_unix ZMODIFICATIONTIME) END, 'unixepoch', 'localtime'), '')"
local _fav_phone_expr="''"
if _maps_has_col ZFAVORITEITEM ZPHONENUMBER; then
_fav_phone_expr="$(_sql_tsv_safe "ZPHONENUMBER")"
fi
local _fav_url_expr="''" _fav_url_col
_fav_url_col="$(_pick_first_col "$_snap" ZFAVORITEITEM \
ZURL ZWEBSITEURL || true)"
[ -n "$_fav_url_col" ] && \
_fav_url_expr="$(_sql_tsv_safe "$_fav_url_col")"
local _fav_addr_expr="''" _fav_addr_col
_fav_addr_col="$(_pick_first_col "$_snap" ZFAVORITEITEM \
ZMAPITEMADDRESS ZORIGINATINGADDRESSSTRING ZADDRESS || true)"
[ -n "$_fav_addr_col" ] && \
_fav_addr_expr="$(_sql_tsv_safe "$_fav_addr_col")"
local _fav_lat_expr="''" _fav_lon_expr="''"
_maps_has_col ZFAVORITEITEM ZLATITUDE && \
_fav_lat_expr="COALESCE(ZLATITUDE, '')"
_maps_has_col ZFAVORITEITEM ZLONGITUDE && \
_fav_lon_expr="COALESCE(ZLONGITUDE, '')"
local _fav_csv="$_maps_root/Saved Places.csv"
local _fav_tmp
_fav_tmp="$(mktemp "$_fav_csv.tmp.XXXXXX" 2>/dev/null)" || _fav_tmp="$_fav_csv.tmp.$$"
track_output_path "$_fav_tmp"
{
printf 'ID,Name,Address,Latitude,Longitude,Created,Last Visited,Modified,Phone,URL\r\n'
sqlite3 -readonly "$_snap" "
SELECT 'maps-fav-' || Z_PK,
$_fav_name_expr,
$_fav_addr_expr,
$_fav_lat_expr,
$_fav_lon_expr,
COALESCE(datetime($_fav_created_expr, 'unixepoch', 'localtime'), ''),
$_fav_lastvisited_expr,
$_fav_modified_expr,
$_fav_phone_expr,
$_fav_url_expr
FROM ZFAVORITEITEM
$_hidden_clause
$_order;
" -separator $'\x1f' 2>/dev/null | \
while IFS=$'\x1f' read -r _id _name _addr _lat _lon _ct _lv _mt _ph _url; do
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\r\n' \
"$(csv_field "$_id")" "$(csv_field "$_name")" \
"$(csv_field "$_addr")" \
"$(csv_field "$_lat")" "$(csv_field "$_lon")" \
"$(csv_field "$_ct")" "$(csv_field "$_lv")" \
"$(csv_field "$_mt")" "$(csv_field "$_ph")" \
"$(csv_field "$_url")"
done
} > "$_fav_tmp"
if ! _atomic_commit_tmp $? "$_fav_tmp" "$_fav_csv"; then
note_incomplete "$_maps_root" "Saved Places.csv" \
"the CSV commit failed (disk full / read-only volume) — saved places were not exported"
_fav_count=0
fi
_untrack_output_path "$_fav_tmp"
local _fav_emit
_fav_emit=$(_count_csv_rows "$_fav_csv")
if [ -n "$_hidden_clause" ]; then
local _fav_hidden=$((_fav_count - _fav_emit))
if [ "$_fav_hidden" -gt 0 ]; then
_disclose_excluded "$_fav_hidden hidden saved place(s)" "hidden in Maps"
fi
elif [ "$_fav_emit" -ne "$_fav_count" ]; then
log_warn "Maps drift: discovered $_fav_count saved place(s), emitted $_fav_emit."
note_incomplete "$_maps_root" "Saved Places.csv — discovery and emit counts disagree" \
"discovered $_fav_count saved place(s) but emitted $_fav_emit; Saved Places.csv may be missing rows"
fi
_fav_count="$_fav_emit"
validate_csv_rfc4180 "$_fav_csv" || true
[ "$_fav_count" -eq 0 ] && rm -f -- "$_fav_csv" 2>/dev/null
fi
fi
if _maps_has_table ZCOLLECTION; then
local _coll_count_q
_coll_count_q=$(sqlite3 -readonly "$_snap" \
"SELECT COUNT(*) FROM ZCOLLECTION;" 2>/dev/null)
if [ -z "$_coll_count_q" ]; then
log_warn "Maps: ZCOLLECTION count query failed — drift-check skipped."
note_incomplete "$_maps_root" "Collections.csv completeness" \
"a completeness check on the Maps database failed, so the export could not verify it captured every collection — Collections.csv may be partial"
_coll_count=0
else
_coll_count="$_coll_count_q"
fi
case "$_coll_count" in ''|*[!0-9]*) _coll_count=0 ;; esac
local _coll_items=0
if _maps_has_table ZCOLLECTIONITEM; then
local _coll_items_q
_coll_items_q=$(sqlite3 -readonly "$_snap" \
"SELECT COUNT(*) FROM ZCOLLECTIONITEM;" 2>/dev/null)
if [ -n "$_coll_items_q" ]; then
_coll_items="$_coll_items_q"
else
log_warn "Maps: ZCOLLECTIONITEM count query failed — collection-membership drift-check skipped."
note_incomplete "$_maps_root" "Collections.csv membership completeness" \
"a completeness check on the Maps database failed, so the export could not verify it captured every collection-item association — Collections.csv may be partial"
fi
case "$_coll_items" in ''|*[!0-9]*) _coll_items=0 ;; esac
fi
if [ "$_coll_items" -gt 0 ]; then
local _coll_csv="$_maps_root/Collections.csv"
local _c_title_has=0
_maps_has_col ZCOLLECTION ZTITLE && _c_title_has=1
local _c_title_expr="''" _c_title_order="c.Z_PK"
local _c_titled_pred_c="1=1" _c_titled_pred="1=1"
if [ "$_c_title_has" = "1" ]; then
_c_title_expr="$(_sql_tsv_safe "c.ZTITLE")"
_c_title_order="c.ZTITLE, c.Z_PK"
_c_titled_pred_c="COALESCE(TRIM(c.ZTITLE), '') <> ''"
_c_titled_pred="COALESCE(TRIM(ZTITLE), '') <> ''"
else
log_warn "Maps: ZCOLLECTION has no ZTITLE column on this macOS version — Collections.csv exports without collection names (the Collection column is blank; saved items still export)."
note_incomplete "$_maps_root" "Collections.csv collection names" \
"the Maps collection table is missing its title column on this macOS version, so collection NAMES could not be exported — the saved items still export, but the Collection column is blank for every row"
fi
local _jt _jpc _jcc
_jt="$(_discover_z_join "$_snap" PLACES || true)"
if [ -n "$_jt" ]; then
_jpc=$(sqlite3 -readonly "$_snap" "
SELECT name FROM pragma_table_info('$_jt')
WHERE name GLOB 'Z_[0-9]*PLACES' LIMIT 1;" 2>/dev/null)
_jcc=$(sqlite3 -readonly "$_snap" "
SELECT name FROM pragma_table_info('$_jt')
WHERE name GLOB 'Z_[0-9]*COLLECTIONS' LIMIT 1;" 2>/dev/null)
case "$_jpc" in *[!A-Za-z0-9_]*) _jpc="" ;; esac
case "$_jcc" in *[!A-Za-z0-9_]*) _jcc="" ;; esac
fi
local _ci_custom=""
_maps_has_col ZCOLLECTIONITEM ZCUSTOMNAME && _ci_custom="ci.ZCUSTOMNAME"
local _ci_place_name="" _ci_place_col
_ci_place_col="$(_pick_first_col "$_snap" ZCOLLECTIONITEM \
ZMAPITEMNAME ZNAME || true)"
[ -n "$_ci_place_col" ] && _ci_place_name="ci.$_ci_place_col"
local _ci_name_expr="''"
if [ -n "$_ci_custom" ] && [ -n "$_ci_place_name" ]; then
_ci_name_expr="$(_sql_tsv_safe "COALESCE(NULLIF($_ci_custom, ''), $_ci_place_name, '')")"
elif [ -n "$_ci_custom" ]; then
_ci_name_expr="$(_sql_tsv_safe "COALESCE($_ci_custom, '')")"
elif [ -n "$_ci_place_name" ]; then
_ci_name_expr="$(_sql_tsv_safe "COALESCE($_ci_place_name, '')")"
fi
local _ci_created_expr="NULL"
_maps_has_col ZCOLLECTIONITEM ZCREATETIME && \
_ci_created_expr="CASE WHEN ci.ZCREATETIME <= 0 THEN NULL ELSE $(_sql_cd_to_unix ci.ZCREATETIME) END"
local _ci_lat_expr="''" _ci_lon_expr="''"
_maps_has_col ZCOLLECTIONITEM ZLATITUDE && \
_ci_lat_expr="COALESCE(ci.ZLATITUDE, '')"
_maps_has_col ZCOLLECTIONITEM ZLONGITUDE && \
_ci_lon_expr="COALESCE(ci.ZLONGITUDE, '')"
local _ci_addr_expr="''" _ci_addr_col
_ci_addr_col="$(_pick_first_col "$_snap" ZCOLLECTIONITEM \
ZMAPITEMADDRESS ZORIGINATINGADDRESSSTRING ZADDRESS || true)"
[ -n "$_ci_addr_col" ] && \
_ci_addr_expr="$(_sql_tsv_safe "ci.$_ci_addr_col")"
local _ci_order="ci.Z_PK"
_maps_has_col ZCOLLECTIONITEM ZPOSITIONINDEX && \
_ci_order="ci.ZPOSITIONINDEX, ci.Z_PK"
local _c_desc_expr="''" _c_desc_col
_c_desc_col="$(_pick_first_col "$_snap" ZCOLLECTION \
ZCOLLECTIONDESCRIPTION ZSUBTITLE ZDESCRIPTION || true)"
[ -n "$_c_desc_col" ] && \
_c_desc_expr="$(_sql_tsv_safe "c.$_c_desc_col")"
local _c_created_expr="''"
_maps_has_col ZCOLLECTION ZCREATETIME && \
_c_created_expr="COALESCE(datetime(CASE WHEN c.ZCREATETIME <= 0 THEN NULL ELSE $(_sql_cd_to_unix c.ZCREATETIME) END, 'unixepoch', 'localtime'), '')"
local _c_shared_expr="''"
_maps_has_col ZCOLLECTION ZISSHARED && \
_c_shared_expr="CASE WHEN COALESCE(c.ZISSHARED, 0) = 0 THEN 'no' ELSE 'yes' END"
local _mcoll_tmp
_mcoll_tmp="$(mktemp "$_coll_csv.tmp.XXXXXX" 2>/dev/null)" || _mcoll_tmp="$_coll_csv.tmp.$$"
track_output_path "$_mcoll_tmp"
{
printf 'ID,Collection,Collection Description,Collection Created,Collection Shared,Name,Address,Latitude,Longitude,Created\r\n'
if [ -n "$_jt" ] && [ -n "$_jpc" ] && [ -n "$_jcc" ]; then
sqlite3 -readonly "$_snap" "
SELECT 'maps-coll-item-' || ci.Z_PK || '-' || COALESCE(c.Z_PK, 0),
$_c_title_expr,
$_c_desc_expr,
$_c_created_expr,
$_c_shared_expr,
$_ci_name_expr,
$_ci_addr_expr,
$_ci_lat_expr,
$_ci_lon_expr,
COALESCE(datetime($_ci_created_expr, 'unixepoch', 'localtime'), '')
FROM ZCOLLECTIONITEM ci
LEFT JOIN $_jt jt ON jt.$_jpc = ci.Z_PK
LEFT JOIN ZCOLLECTION c ON c.Z_PK = jt.$_jcc
ORDER BY $_c_title_order, $_ci_order;
" -separator $'\x1f' 2>/dev/null
else
sqlite3 -readonly "$_snap" "
SELECT 'maps-coll-item-' || ci.Z_PK,
'', '', '', '',
$_ci_name_expr,
$_ci_addr_expr,
$_ci_lat_expr,
$_ci_lon_expr,
COALESCE(datetime($_ci_created_expr, 'unixepoch', 'localtime'), '')
FROM ZCOLLECTIONITEM ci
ORDER BY $_ci_order;
" -separator $'\x1f' 2>/dev/null
fi | while IFS=$'\x1f' read -r _id _coll _cdesc _cct _cshr _name _caddr _lat _lon _ct; do
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\r\n' \
"$(csv_field "$_id")" "$(csv_field "$_coll")" \
"$(csv_field "$_cdesc")" "$(csv_field "$_cct")" \
"$(csv_field "$_cshr")" "$(csv_field "$_name")" \
"$(csv_field "$_caddr")" \
"$(csv_field "$_lat")" "$(csv_field "$_lon")" \
"$(csv_field "$_ct")"
done
} > "$_mcoll_tmp"
if ! _atomic_commit_tmp $? "$_mcoll_tmp" "$_coll_csv"; then
note_incomplete "$_maps_root" "Collections.csv" \
"the CSV commit failed (disk full / read-only volume) — collections were not exported"
fi
_untrack_output_path "$_mcoll_tmp"
validate_csv_rfc4180 "$_coll_csv" || true
if [ -n "$_jt" ] && [ -n "$_jpc" ] && [ -n "$_jcc" ]; then
local _empty_colls
_empty_colls=$(sqlite3 -readonly "$_snap" "
SELECT COUNT(*) FROM ZCOLLECTION c
WHERE $_c_titled_pred_c
AND NOT EXISTS (
SELECT 1 FROM $_jt jt
WHERE jt.$_jcc = c.Z_PK);
" 2>/dev/null)
case "$_empty_colls" in ''|*[!0-9]*) _empty_colls=0 ;; esac
if [ "$_empty_colls" -gt 0 ]; then
note_incomplete "$_maps_root" "Collections.csv — empty collections omitted" \
"$_empty_colls titled collection(s) contain no saved items and therefore produce no rows in Collections.csv; the collection name itself is not represented in the export"
fi
else
local _coll_titled
_coll_titled=$(sqlite3 -readonly "$_snap" "
SELECT COUNT(*) FROM ZCOLLECTION
WHERE $_c_titled_pred;
" 2>/dev/null)
case "$_coll_titled" in ''|*[!0-9]*) _coll_titled=0 ;; esac
note_incomplete "$_maps_root" "Collections.csv — collection grouping not exported (fallback path)" \
"the Maps schema on this Mac does not expose the item→collection join table this leaf reads; ${_coll_titled} titled collection(s) exist on the source but the export carries items only (Collection column is blank for every row). Empty collections are silently absent from the export entirely."
fi
_coll_count=$(_count_csv_rows "$_coll_csv")
if [ -z "$_jt" ] || [ -z "$_jpc" ] || [ -z "$_jcc" ]; then
if [ "$_coll_count" -lt "$_coll_items" ]; then
log_warn "Maps drift: $_coll_items collection item(s) discovered, only $_coll_count in Collections.csv."
note_incomplete "$_maps_root" "Collections.csv — discovery and emit counts disagree" \
"discovered $_coll_items collection item(s) but only $_coll_count landed in Collections.csv — rows were dropped between query and emit"
fi
fi
[ "$_coll_count" -eq 0 ] && rm -f -- "$_coll_csv" 2>/dev/null
else
local _coll_titled_empty
if _maps_has_col ZCOLLECTION ZTITLE; then
_coll_titled_empty=$(sqlite3 -readonly "$_snap" "
SELECT COUNT(*) FROM ZCOLLECTION
WHERE COALESCE(TRIM(ZTITLE), '') <> '';
" 2>/dev/null)
else
_coll_titled_empty=$(sqlite3 -readonly "$_snap" \
"SELECT COUNT(*) FROM ZCOLLECTION;" 2>/dev/null)
fi
case "$_coll_titled_empty" in ''|*[!0-9]*) _coll_titled_empty=0 ;; esac
if [ "$_coll_titled_empty" -gt 0 ]; then
note_incomplete "$_maps_root" "Collections.csv — empty collections omitted" \
"$_coll_titled_empty titled collection(s) contain no saved items and therefore produce no rows in Collections.csv; the collection name itself is not represented in the export"
fi
_coll_count=0
fi
fi
if _maps_has_table ZHISTORYITEM; then
local _hist_count_q
_hist_count_q=$(sqlite3 -readonly "$_snap" \
"SELECT COUNT(*) FROM ZHISTORYITEM;" 2>/dev/null)
if [ -z "$_hist_count_q" ]; then
log_warn "Maps: ZHISTORYITEM count query failed — drift-check skipped."
note_incomplete "$_maps_root" "Recents.csv completeness" \
"a completeness check on the Maps database failed, so the export could not verify it captured every recent — Recents.csv may be partial"
_hist_count=0
else
_hist_count="$_hist_count_q"
fi
case "$_hist_count" in ''|*[!0-9]*) _hist_count=0 ;; esac
if [ "$_hist_count" -gt 0 ]; then
local _hist_csv="$_maps_root/Recents.csv"
local _hist_date_expr="NULL" _hist_order="Z_PK"
if _maps_has_col ZHISTORYITEM ZCREATETIME; then
_hist_date_expr="CASE WHEN ZCREATETIME <= 0 THEN NULL ELSE $(_sql_cd_to_unix ZCREATETIME) END"
_hist_order="ZCREATETIME DESC, Z_PK"
fi
local _hist_type_col=""
_hist_type_col="$(_pick_first_col "$_snap" ZHISTORYITEM \
ZTYPE ZHISTORYITEMTYPE ZITEMTYPE || true)"
local _hist_type_expr="''"
if [ -n "$_hist_type_col" ]; then
_hist_type_expr="CASE
WHEN $_hist_type_col IS NULL THEN ''
WHEN $_hist_type_col = 0 THEN 'search'
WHEN $_hist_type_col = 1 THEN 'location'
WHEN $_hist_type_col = 2 THEN 'route'
WHEN $_hist_type_col = 3 THEN 'place'
ELSE '' END"
fi
local _hist_trans_col=""
_hist_trans_col="$(_pick_first_col "$_snap" ZHISTORYITEM \
ZTRANSPORTTYPE ZROUTINGMETHOD || true)"
local _hist_trans_expr="''"
if [ -n "$_hist_trans_col" ]; then
_hist_trans_expr="CASE
WHEN $_hist_trans_col IS NULL THEN ''
WHEN $_hist_trans_col = 0 THEN 'driving'
WHEN $_hist_trans_col = 1 THEN 'walking'
WHEN $_hist_trans_col = 2 THEN 'transit'
WHEN $_hist_trans_col = 3 THEN 'cycling'
ELSE '' END"
fi
local _hist_mod_expr="''"
_maps_has_col ZHISTORYITEM ZMODIFICATIONTIME && \
_hist_mod_expr="COALESCE(datetime(CASE WHEN ZMODIFICATIONTIME <= 0 THEN NULL ELSE $(_sql_cd_to_unix ZMODIFICATIONTIME) END, 'unixepoch', 'localtime'), '')"
local _hist_lat_own="" _hist_lon_own=""
_maps_has_col ZHISTORYITEM ZLATITUDE && _hist_lat_own="ZLATITUDE, "
_maps_has_col ZHISTORYITEM ZLONGITUDE && _hist_lon_own="ZLONGITUDE, "
local _hist_lat_expr="''"
local _hist_lon_expr="''"
[ -n "$_hist_lat_own" ] && _hist_lat_expr="COALESCE(${_hist_lat_own}'')"
[ -n "$_hist_lon_own" ] && _hist_lon_expr="COALESCE(${_hist_lon_own}'')"
if _maps_has_col ZHISTORYITEM ZMAPITEM \
&& _maps_has_table ZMIXINMAPITEM \
&& _maps_has_col ZMIXINMAPITEM ZLATITUDE \
&& _maps_has_col ZMIXINMAPITEM ZLONGITUDE; then
_hist_lat_expr="COALESCE(${_hist_lat_own}(SELECT m.ZLATITUDE FROM ZMIXINMAPITEM m WHERE m.Z_PK = ZHISTORYITEM.ZMAPITEM), '')"
_hist_lon_expr="COALESCE(${_hist_lon_own}(SELECT m.ZLONGITUDE FROM ZMIXINMAPITEM m WHERE m.Z_PK = ZHISTORYITEM.ZMAPITEM), '')"
fi
local _hist_query_expr="''"
_maps_has_col ZHISTORYITEM ZQUERY && \
_hist_query_expr="$(_sql_tsv_safe "ZQUERY")"
local _hist_locdisp_expr="''"
_maps_has_col ZHISTORYITEM ZLOCATIONDISPLAY && \
_hist_locdisp_expr="$(_sql_tsv_safe "ZLOCATIONDISPLAY")"
local _hist_tmp
_hist_tmp="$(mktemp "$_hist_csv.tmp.XXXXXX" 2>/dev/null)" || _hist_tmp="$_hist_csv.tmp.$$"
track_output_path "$_hist_tmp"
{
printf 'ID,Query,Location,Type,Transport,Latitude,Longitude,Date,Modified\r\n'
sqlite3 -readonly "$_snap" "
SELECT 'maps-hist-' || Z_PK,
$_hist_query_expr,
$_hist_locdisp_expr,
$_hist_type_expr,
$_hist_trans_expr,
$_hist_lat_expr,
$_hist_lon_expr,
COALESCE(datetime($_hist_date_expr, 'unixepoch', 'localtime'), ''),
$_hist_mod_expr
FROM ZHISTORYITEM
ORDER BY $_hist_order;
" -separator $'\x1f' 2>/dev/null | \
while IFS=$'\x1f' read -r _id _q _loc _ty _tr _lat _lon _dt _mt; do
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s\r\n' \
"$(csv_field "$_id")" "$(csv_field "$_q")" \
"$(csv_field "$_loc")" "$(csv_field "$_ty")" \
"$(csv_field "$_tr")" \
"$(csv_field "$_lat")" "$(csv_field "$_lon")" \
"$(csv_field "$_dt")" "$(csv_field "$_mt")"
done
} > "$_hist_tmp"
_atomic_commit_tmp $? "$_hist_tmp" "$_hist_csv"
_untrack_output_path "$_hist_tmp"
local _hist_emit
_hist_emit=$(_count_csv_rows "$_hist_csv")
validate_count_parity "$_hist_count" "$_hist_emit" "Maps recents" "$_maps_root"
_hist_count="$_hist_emit"
validate_csv_rfc4180 "$_hist_csv" || true
fi
fi
else
note_incomplete "$_maps_root" "Maps database unreadable" \
"the Maps database at $_db is present but could not be snapshotted (corrupt, locked, or a scratch-copy failure) — no Maps data could be exported"
fi
fi
local _parts=()
[ "$_fav_count" -gt 0 ] && _parts+=("$_fav_count $(_pluralize "$_fav_count" "saved place")")
[ "$_coll_count" -gt 0 ] && _parts+=("$_coll_count $(_pluralize "$_coll_count" "collection entry" "collection entries")")
[ "$_hist_count" -gt 0 ] && _parts+=("$_hist_count $(_pluralize "$_hist_count" recent)")
if [ "${#_parts[@]}" -gt 0 ]; then
local _summary
_summary="$(IFS=, ; printf '%s' "${_parts[*]}")"
_summary="${_summary//,/, }"
log_ok "Wrote Maps/ ($_summary, $(_dir_size_human "$_maps_root"))"
elif report_have_incomplete "$_maps_root"; then
log_warn "Maps: an earlier step could not complete$(_disclosure_pointer "Maps/_incomplete.txt")."
else
rmdir "$_maps_root" 2>/dev/null || true
report_empty_source "saved places, collections, or recents" "Maps"
fi
return 0
}
maps_main() {
maybe_show_help "$@"
_announce_step "Maps export"
check_dependencies
local out_arg out_dir
out_arg="$(_parse_output_folder_arg "$@")" || exit 1
out_dir="$(resolve_output_dir "$out_arg")" || exit 1
_announce_output_dir "Maps"
require_full_disk_access
export_maps "$out_dir"
mark_success
sweep_junk_files "$out_dir"
print_summary
if [ "$ERR_COUNT" -gt 0 ]; then exit 1; fi
exit 0
}
MSG_DIR="$HOME/Library/Messages"
_contact_resolve_cache=""
_contact_resolve_cache_file=""
MESSAGES_USE_IMESSAGE_EXPORTER="${MESSAGES_USE_IMESSAGE_EXPORTER:-1}"
_IME_BIN=""
_IME_VERSION=""
_fetch_imessage_exporter() {
[ "$MESSAGES_USE_IMESSAGE_EXPORTER" = "1" ] || return 1
if ! command -v curl >/dev/null 2>&1; then
log_warn "curl not found — cannot auto-fetch imessage-exporter; HTML conversion skipped"
return 1
fi
ensure_scratch
local arch asset_arch
arch="$(uname -m 2>/dev/null)"
case "$arch" in
arm64) asset_arch="aarch64-apple-darwin" ;;
x86_64) asset_arch="x86_64-apple-darwin" ;;
*)
log_warn "Unrecognised architecture '$arch' — cannot pick an imessage-exporter asset; HTML conversion skipped."
return 1 ;;
esac
local _imessage_exporter_fallback_tag="4.2.0"
local tag
tag="$(curl -fsSL --max-time 10 \
https://api.github.com/repos/ReagentX/imessage-exporter/releases/latest \
2>/dev/null \
| sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \
| head -1)"
if [ -z "$tag" ]; then
log_warn "GitHub API returned no tag (rate-limited or offline) — using fallback $_imessage_exporter_fallback_tag (no cryptographic verification; trust is HTTPS-transport only)."
tag="$_imessage_exporter_fallback_tag"
fi
local asset_name="imessage-exporter-$asset_arch"
local url="https://github.com/ReagentX/imessage-exporter/releases/download/${tag}/${asset_name}"
local bin="$SCRATCH_DIR/$asset_name"
if ! _fetch_download "$url" "$bin" "imessage-exporter $tag" 120; then
log_warn "Download failed: $url; HTML conversion skipped."
return 1
fi
[ -s "$bin" ] || { log_warn "Downloaded binary is empty; HTML conversion skipped"; return 1; }
chmod +x "$bin" 2>/dev/null || {
log_warn "Cannot chmod +x downloaded binary; HTML conversion skipped."
return 1
}
xattr -d com.apple.quarantine "$bin" 2>/dev/null || true
if ! _run_with_timeout 30 "$bin" --help >/dev/null 2>&1; then
log_warn "Downloaded imessage-exporter binary failed or timed out on --help smoke test; HTML conversion skipped."
return 1
fi
_IME_BIN="$bin"
_IME_VERSION="$tag"
return 0
}
_msg_ime_diag_field() { #