Update to v6.0.0
This version includes the following changes - Uses bind mounts instead of symlinks - Flock is used to handle the multiple syncs happening together - Backups are compressed using zstd - The backup process is optimized and uses link-dest flag to decrease the size of backups
This commit is contained in:
Manorit Chawdhry
GitHub
commit
19204c7fd2
17
Makefile
17
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
VERSION = 5.86
|
||||
VERSION = 6.0.0
|
||||
PN = anything-sync-daemon
|
||||
|
||||
PREFIX ?= /usr
|
||||
|
|
@ -25,13 +25,23 @@ INSTALL_DIR = $(INSTALL) -d
|
|||
|
||||
Q = @
|
||||
|
||||
common/$(PN): common/$(PN).in
|
||||
common/$(PN): Makefile common/$(PN).in
|
||||
$(Q)echo -e '\033[1;32mSetting version\033[0m'
|
||||
$(Q)$(SED) 's/@VERSION@/'$(VERSION)'/' common/$(PN).in > common/$(PN)
|
||||
|
||||
help: install
|
||||
|
||||
install-bin: common/$(PN)
|
||||
stop-asd:
|
||||
ifneq ($(PREFIX), /usr)
|
||||
sudo -E asd unsync
|
||||
endif
|
||||
|
||||
disable-systemd:
|
||||
ifeq ($(PREFIX), /usr)
|
||||
systemctl stop asd asd-resync || /bin/true
|
||||
endif
|
||||
|
||||
install-bin: stop-asd disable-systemd common/$(PN)
|
||||
$(Q)echo -e '\033[1;32mInstalling main script...\033[0m'
|
||||
$(INSTALL_DIR) "$(DESTDIR)$(BINDIR)"
|
||||
$(INSTALL_PROGRAM) common/$(PN) "$(DESTDIR)$(BINDIR)/$(PN)"
|
||||
|
|
@ -72,6 +82,7 @@ install-upstart:
|
|||
$(INSTALL_DIR) "$(DESTDIR)$(INITDIR_UPSTART)"
|
||||
$(INSTALL_SCRIPT) init/asd.upstart "$(DESTDIR)$(INITDIR_UPSTART)/asd"
|
||||
|
||||
|
||||
install-systemd-all: install-bin install-man install-systemd
|
||||
|
||||
install-upstart-all: install-bin install-man install-cron install-upstart
|
||||
|
|
|
|||
|
|
@ -13,6 +13,19 @@ debug() {
|
|||
fi
|
||||
}
|
||||
|
||||
set -e
|
||||
Error() {
|
||||
echo "Error occurred at $1"
|
||||
exit 1
|
||||
}
|
||||
trap 'Error $LINENO' ERR
|
||||
|
||||
debug "checking flock"
|
||||
command -v flock >/dev/null 2>&1 || {
|
||||
echo "I require flock but it's not installed. Aborting." >&2
|
||||
exit 1; }
|
||||
debug "flock found"
|
||||
|
||||
BLD="\e[01m"
|
||||
RED="\e[01;31m"
|
||||
GRN="\e[01;32m"
|
||||
|
|
@ -31,10 +44,14 @@ if [[ "$ASDCONF" != "/etc/asd.conf" ]]; then
|
|||
DAEMON_FILE="/run/$ASDNAME"
|
||||
fi
|
||||
fi
|
||||
LOCKFILE="/run/$ASDNAME-lock"
|
||||
# shellcheck disable=SC2015
|
||||
[[ ${FLOCKER} != "$0" ]] && echo -e "${RED}Waiting for lock...${NRM}" && exec env FLOCKER="$0" flock --verbose -e "$LOCKFILE" "$0" "$*" ||
|
||||
|
||||
debug "${RED}ASDNAME:${NRM} $ASDNAME"
|
||||
debug "${BLU}Configuration file:${NRM} $ASDCONF"
|
||||
debug "${GRN}Daemon file:${NRM} $DAEMON_FILE\n"
|
||||
debug "${GRN}Daemon file:${NRM} $DAEMON_FILE"
|
||||
debug "${RED}Lockfile:${NRM} $LOCKFILE\n"
|
||||
|
||||
# Setup check /etc/asd.conf
|
||||
debug "Checking for existence of $ASDCONF"
|
||||
|
|
@ -90,7 +107,9 @@ fi
|
|||
debug "Backup limit: $BACKUP_LIMIT"
|
||||
|
||||
# saving current extended pattern matching setting
|
||||
previous_extglob_setting=$(shopt -p extglob)
|
||||
# command returns non zero exit code if the option is unset hence
|
||||
# pipe with true
|
||||
previous_extglob_setting=$(shopt -p extglob || true)
|
||||
|
||||
# ensuring pattern matching is enabled
|
||||
shopt -s extglob
|
||||
|
|
@ -111,6 +130,9 @@ df -T "$VOLATILE" | grep -m 1 -q '\( tmpfs \|^/dev/zram\)' || {
|
|||
echo "$VOLATILE is not tmpfs/zram so running asd is pointless. Aborting." >&2
|
||||
exit 1; }
|
||||
|
||||
[[ -z "$ENABLE_HARDLINK_SAFETY_CHECK" ]] && ENABLE_HARDLINK_SAFETY_CHECK=1
|
||||
debug "Hardlink safety check: $ENABLE_HARDLINK_SAFETY_CHECK"
|
||||
|
||||
# simple function to determine user intent rather than using a null value
|
||||
case "${USE_OVERLAYFS,,}" in
|
||||
y|yes|true|t|on|1|enabled|enable|use)
|
||||
|
|
@ -185,7 +207,7 @@ header() {
|
|||
}
|
||||
|
||||
dep_check() {
|
||||
# Function is used to insure all dependencies are installed
|
||||
# Function is used to ensure all dependencies are installed
|
||||
debug "\n${BLU}Checking dependencies${NRM}"
|
||||
debug "checking rsync"
|
||||
command -v rsync >/dev/null 2>&1 || {
|
||||
|
|
@ -194,6 +216,15 @@ dep_check() {
|
|||
debug "checking awk"
|
||||
command -v awk >/dev/null 2>&1 || {
|
||||
echo "I require awk but it's not installed. Aborting." >&2; exit 1; }
|
||||
debug "checking pv"
|
||||
command -v pv >/dev/null 2>&1 || {
|
||||
echo "I require pv but it's not installed. Aborting." >&2; exit 1; }
|
||||
debug "checking tar"
|
||||
command -v tar >/dev/null 2>&1 || {
|
||||
echo "I require tar but it's not installed. Aborting." >&2; exit 1; }
|
||||
debug "checking zstd"
|
||||
command -v zstd >/dev/null 2>&1 || {
|
||||
echo "I require zstd but it's not installed. Aborting." >&2; exit 1; }
|
||||
if [[ $OLFS -eq 1 ]]; then
|
||||
[[ $OLFSVER -ge 22 ]] || {
|
||||
echo -e " ${BLD}Your kernel requires either the ${BLU}overlay${NRM}${BLD} or ${BLU}overlayfs${NRM}${BLD} module to use${NRM}"
|
||||
|
|
@ -213,16 +244,27 @@ config_check() {
|
|||
# make sure the user defined real dirs
|
||||
for DIR in "${WHATTOSYNC[@]}"; do
|
||||
debug "DIR: $DIR"
|
||||
[[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" ||
|
||||
BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd"
|
||||
[[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" ||
|
||||
BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old"
|
||||
debug "BACKUP: $BACKUP"
|
||||
debug "BACK_OLD: $BACK_OLD"
|
||||
if [[ ! -d "$DIR" ]]; then
|
||||
[[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" ||
|
||||
BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd"
|
||||
debug "BACKUP: $BACKUP"
|
||||
if [[ ! -d "$BACKUP" ]]; then
|
||||
if [[ ! -d "$BACK_OLD" ]]; then
|
||||
echo -e "${BLD}Bad entry in your WHATTOSYNC array detected:${NRM}"
|
||||
echo -e " ${BLD}${RED}$DIR${NRM}"
|
||||
echo -e "${BLD}Edit ${BLU}$ASDCONF${NRM}${BLD} correcting the mistake and try again.${NRM}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# sanity check for hardlinks
|
||||
if [[ ! -d "$BACK_OLD" && $ENABLE_HARDLINK_SAFETY_CHECK -ne 0 && -n $(find "$DIR" -type f -links +1) ]]; then
|
||||
echo -e "$DIR:\n${RED} Presence of hardlinks found, asd might break them:${NRM}"
|
||||
exit 1
|
||||
else
|
||||
debug "No hardlinks found"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
debug "Configs seem to be fine"
|
||||
|
|
@ -254,70 +296,41 @@ ungraceful_state_check() {
|
|||
# this is the hdd bound backup in case of power failure
|
||||
[[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" ||
|
||||
BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd"
|
||||
[[ ${DIR##*/} == .* ]] && BACK_OVFS="${DIR%/*}/${DIR##*/}-back-ovfs" ||
|
||||
BACK_OVFS="${DIR%/*}/.${DIR##*/}-back-ovfs"
|
||||
[[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" ||
|
||||
BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old"
|
||||
if [[ -d "$BACKUP" ]]; then
|
||||
USER=$(stat -c %U "$BACKUP")
|
||||
else
|
||||
USER=$(stat -c %U "$DIR")
|
||||
fi
|
||||
TMP="$VOLATILE/$ASDNAME-$USER$DIR"
|
||||
debug "DIR: $DIR\nBACKUP: $BACKUP\nBACK_OVFS: $BACK_OVFS\nUSER: $USER\nTMP: $TMP"
|
||||
UPPER="$VOLATILE/$ASDNAME-$USER$DIR-rw"
|
||||
WORK="$VOLATILE/.$ASDNAME-$USER$DIR"
|
||||
debug "DIR: $DIR\nBACKUP: $BACKUP\nBACK_OLD: $BACK_OLD\nUSER: $USER\nTMP: $TMP"
|
||||
|
||||
if [[ -e "$TMP"/.flagged ]]; then
|
||||
if [[ -e "$TMP"/.flagged || ! -d "$BACKUP" ]]; then
|
||||
debug "No ungraceful state detected"
|
||||
# all is well so continue
|
||||
return
|
||||
continue
|
||||
else
|
||||
echo "Ungraceful state detected for $DIR so fixing"
|
||||
NOW=$(date +%Y%m%d_%H%M%S)
|
||||
if [[ -h "$DIR" ]]; then
|
||||
echo "Ungraceful state detected for $DIR so fixing"
|
||||
debug "unlinking $DIR"
|
||||
unlink "$DIR"
|
||||
fi
|
||||
|
||||
if [[ -d "$BACKUP" ]]; then
|
||||
if [[ -d "$BACK_OVFS" ]]; then
|
||||
# always snapshot the most recent of these two dirs...
|
||||
# if using overlayfs $BACK_OVFS and $BACKUP should be compared
|
||||
# against each other to see which is newer and then that should
|
||||
# be what psd snapshots since BACKUP (the lowerdir) is readonly
|
||||
# at the time the user started psd could be many resync cycles
|
||||
# in the past
|
||||
debug "unmounting $DIR"
|
||||
mountpoint -q "$DIR" && umount -R -l "$DIR"
|
||||
debug "unmounting $BACKUP"
|
||||
mountpoint -q "$BACKUP" && umount -R -l "$BACKUP" && rm -rf "$BACKUP" && debug "removed $BACKUP dir"
|
||||
|
||||
BACKUP_TIME=$(stat "$BACKUP" | grep Change | awk '{ print $2,$3 }')
|
||||
BACK_OVFS_TIME=$(stat "$BACK_OVFS" | grep Change | awk '{ print $2,$3 }')
|
||||
mountpoint -q "$TMP" && umount "$TMP" && rm -rf "$TMP" "$UPPER" "$WORK" && debug "unmounted overlay dirs"
|
||||
|
||||
[[ $(date -d "$BACK_OVFS_TIME" "+%s") -ge $(date -d "$BACKUP_TIME" "+%s") ]] &&
|
||||
TARGETTOKEEP="$BACK_OVFS" ||
|
||||
TARGETTOKEEP="$BACKUP"
|
||||
|
||||
if [[ $CRRE -eq 1 ]]; then
|
||||
debug "copying $TARGETTOKEEP to $BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW"
|
||||
cp -a --reflink=auto "$TARGETTOKEEP" "$BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW"
|
||||
fi
|
||||
|
||||
debug "moving $TARGETTOKEEP to $DIR"
|
||||
mv --no-target-directory "$TARGETTOKEEP" "$DIR"
|
||||
|
||||
debug "removing $BACKUP"
|
||||
rm -rf "$BACKUP"
|
||||
else
|
||||
# we only find the BACKUP and no BACKOVFS then either the initial resync
|
||||
# never occurred before the crash using overlayfs or we aren't using overlayfs
|
||||
# at all which can be treated the same way
|
||||
|
||||
if [[ $CRRE -eq 1 ]]; then
|
||||
debug "copying $BACKUP to $BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW"
|
||||
cp -a --reflink=auto "$BACKUP" "$BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW"
|
||||
|
||||
debug "moving $BACKUP to $DIR"
|
||||
mv --no-target-directory "$BACKUP" "$DIR"
|
||||
fi
|
||||
if [[ -d "$BACK_OLD" ]]; then
|
||||
if [[ $CRRE -eq 1 ]]; then
|
||||
debug "copying $BACK_OLD to $BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW.tar.zstd"
|
||||
tar cf - -C "${BACK_OLD%/*}" "${BACK_OLD##*/}" | pv -s "$(du -sb "$BACK_OLD" | awk '{print $1}')" | zstd > "$BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW.tar.zstd"
|
||||
fi
|
||||
rm -rf "$BACK_OLD" && debug "deleting the $BACK_OLD directory"
|
||||
fi
|
||||
fi
|
||||
# if overlayfs was active but is no longer, remove $BACK_OVFS
|
||||
[[ $OLFS -eq 1 ]] || rm -rf "$BACK_OVFS" && debug "removing $BACK_OVFS"
|
||||
done
|
||||
}
|
||||
|
||||
|
|
@ -335,7 +348,7 @@ cleanup() {
|
|||
WORK="$VOLATILE/.$ASDNAME-$USER$DIR"
|
||||
debug "DIR: $DIR\nUSER: $USER\nGROUP: $GROUP\nTMP: $TMP\nUPPER: $UPPER\nWORK: $WORK"
|
||||
|
||||
mapfile -t CRASHArr < <(find "${BACKUP%/*}" -type d -maxdepth 1 -name "$BACKUP-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r)
|
||||
mapfile -t CRASHArr < <(find "${BACKUP%/*}" -maxdepth 1 -name "${BACKUP##*/}-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r)
|
||||
|
||||
if [[ ${#CRASHArr[@]} -gt 0 ]]; then
|
||||
echo -e "${BLD}Deleting ${#CRASHArr[@]} crashrecovery dir(s) for sync target ${BLU}$DIR${NRM}"
|
||||
|
|
@ -360,7 +373,7 @@ enforce() {
|
|||
[[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" ||
|
||||
BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd"
|
||||
debug "DIR: $DIR\nBACKUP: $BACKUP"
|
||||
mapfile -t CRASHArr < <(find "${BACKUP%/*}" -type d -maxdepth 1 -name "$BACKUP-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r)
|
||||
mapfile -t CRASHArr < <(find "${BACKUP%/*}" -maxdepth 1 -name "${BACKUP##*/}-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r)
|
||||
|
||||
if [[ ${#CRASHArr[@]} -gt $BACKUP_LIMIT ]]; then
|
||||
debug "The backups are greater than $BACKUP_LIMIT"
|
||||
|
|
@ -377,8 +390,6 @@ enforce() {
|
|||
|
||||
do_sync() {
|
||||
debug "\n${GRN}Syncing files${NRM}"
|
||||
debug "creating $DAEMON_FILE"
|
||||
touch "$DAEMON_FILE"
|
||||
|
||||
# make a snapshot of /etc/asd.conf and redefine its location to tmpfs while
|
||||
# asd is running to keep any edits made to the live /etc/asd.conf from
|
||||
|
|
@ -391,8 +402,8 @@ do_sync() {
|
|||
# this is the hdd bound backup in case of power failure
|
||||
[[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" ||
|
||||
BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd"
|
||||
[[ ${DIR##*/} == .* ]] && BACK_OVFS="${DIR%/*}/${DIR##*/}-back-ovfs" ||
|
||||
BACK_OVFS="${DIR%/*}/.${DIR##*/}-back-ovfs"
|
||||
[[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" ||
|
||||
BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old"
|
||||
USER=$(stat -c %U "$DIR")
|
||||
GROUP=$(id -g "$USER")
|
||||
TMP="$VOLATILE/$ASDNAME-$USER$DIR"
|
||||
|
|
@ -405,6 +416,7 @@ do_sync() {
|
|||
# retain permissions on sync target
|
||||
PREFIXP=$(stat -c %a "$DIR")
|
||||
[[ -r "$TMP" ]] || install -dm"$PREFIXP" --owner="$USER" --group="$GROUP" "$TMP"
|
||||
[[ -r "$BACKUP" ]] || install -dm"$PREFIXP" --owner="$USER" --group="$GROUP" "$BACKUP"
|
||||
|
||||
if [[ $OLFS -eq 1 ]]; then
|
||||
debug "ensuring overlay directories"
|
||||
|
|
@ -416,26 +428,33 @@ do_sync() {
|
|||
fi
|
||||
fi
|
||||
|
||||
# backup target and link to tmpfs container
|
||||
if [[ $(readlink "$DIR") != "$TMP" ]]; then
|
||||
debug "moving $DIR to $BACKUP"
|
||||
mv --no-target-directory "$DIR" "$BACKUP"
|
||||
debug "symlinking $TMP -> $DIR"
|
||||
ln -s "$TMP" "$DIR"
|
||||
debug "Changing ownership of $DIR to $USER and $GROUP"
|
||||
chown -h "$USER":"$GROUP" "$DIR"
|
||||
fi
|
||||
|
||||
# sync the tmpfs targets to the disc
|
||||
if [[ -e "$TMP"/.flagged ]]; then
|
||||
if [[ $OLFS -eq 1 ]]; then
|
||||
debug "Syncing $(readlink "$DIR") and $BACK_OVFS"
|
||||
rsync -aogX --delete-after --inplace --no-whole-file --exclude .flagged "$DIR/" "$BACK_OVFS/"
|
||||
else
|
||||
debug "Syncing $(readlink "$DIR") and $BACKUP"
|
||||
rsync -aogX --delete-after --inplace --no-whole-file --exclude .flagged "$DIR/" "$BACKUP/"
|
||||
fi
|
||||
debug "Syncing $TMP and $BACKUP"
|
||||
# don't do inplace sync
|
||||
set -x
|
||||
rsync -aX --delete-after --exclude .flagged "$TMP/" "$BACKUP/" --info=progress2
|
||||
set +x
|
||||
else
|
||||
# backup target and link to tmpfs container
|
||||
debug "Bind mounting $DIR -> $BACKUP"
|
||||
mount --rbind --make-private -o noatime "$DIR" "$BACKUP"
|
||||
if [[ $CRRE -eq 1 ]];then
|
||||
debug "Creating new linked backup directory $BACK_OLD"
|
||||
tempfile=$(mktemp)
|
||||
|
||||
set -x
|
||||
# this copies all the files
|
||||
find "$BACKUP" -type l -printf '%P\n' > "$tempfile"
|
||||
rm -rf "$BACK_OLD" && rsync -aX --no-links --link-dest="$DIR" "$DIR/" "$BACK_OLD/" --info=progress2
|
||||
|
||||
# this is used to handle the symlinks
|
||||
rsync -aXl --files-from="$tempfile" "$DIR/" "$BACK_OLD/" --info=progress2
|
||||
|
||||
set +x
|
||||
rm "$tempfile"
|
||||
fi
|
||||
|
||||
# initial sync
|
||||
if [[ $OLFS -eq 1 ]]; then
|
||||
debug "Mounting overlay directory"
|
||||
|
|
@ -446,27 +465,33 @@ do_sync() {
|
|||
fi
|
||||
else
|
||||
debug "Doing initial sync with $BACKUP and $TMP"
|
||||
rsync -aogX --inplace --no-whole-file "$BACKUP/" "$TMP/"
|
||||
set -x
|
||||
rsync -aXl --append "$BACKUP/" "$TMP/" --info=progress2
|
||||
set +x
|
||||
fi
|
||||
touch "$DIR"/.flagged
|
||||
debug "bind mounting $TMP -> $DIR"
|
||||
mount --rbind --make-private -o noatime "$TMP" "$DIR"
|
||||
|
||||
touch "$TMP"/.flagged
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo -e "${BLD}Sync successful${NRM}"
|
||||
|
||||
debug "creating $DAEMON_FILE"
|
||||
touch "$DAEMON_FILE"
|
||||
}
|
||||
|
||||
do_unsync() {
|
||||
debug "\n${RED}Unsyncing files${NRM}"
|
||||
debug "Removing $DAEMON_FILE and ${DAEMON_FILE}.conf"
|
||||
rm -f "$DAEMON_FILE" "${DAEMON_FILE}.conf"
|
||||
|
||||
local DIR USER BACKUP TMP
|
||||
for DIR in "${WHATTOSYNC[@]}"; do
|
||||
# this is the hdd bound backup in case of power failure
|
||||
[[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" ||
|
||||
BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd"
|
||||
[[ ${DIR##*/} == .* ]] && BACK_OVFS="${DIR%/*}/${DIR##*/}-back-ovfs" ||
|
||||
BACK_OVFS="${DIR%/*}/.${DIR##*/}-back-ovfs"
|
||||
[[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" ||
|
||||
BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old"
|
||||
USER=$(stat -c %U "$DIR")
|
||||
GROUP=$(id -g "$USER")
|
||||
TMP="$VOLATILE/$ASDNAME-$USER$DIR"
|
||||
|
|
@ -475,31 +500,40 @@ do_unsync() {
|
|||
debug "DIR: $DIR\nUSER: $USER\nGROUP: $GROUP\nTMP: $TMP\nUPPER: $UPPER\nWORK: $WORK\n"
|
||||
|
||||
# remove link and move data from tmpfs to disk
|
||||
if [[ -h "$DIR" ]]; then
|
||||
debug "unlinking $DIR"
|
||||
unlink "$DIR"
|
||||
if mountpoint -q "$DIR"; then
|
||||
# this assumes that the backup is always
|
||||
# updated so be sure to invoke a sync before an unsync
|
||||
debug "unmounting $DIR"
|
||||
umount -R -f -l "$DIR"
|
||||
debug "unmounting $BACKUP"
|
||||
umount -R -f -l "$BACKUP" && rm -rf "$BACKUP" && debug "removing $BACKUP"
|
||||
|
||||
# restore original dirtree
|
||||
[[ -d "$BACKUP" ]] && mv --no-target-directory "$BACKUP" "$DIR" && debug "move $BACKUP to $DIR"
|
||||
if [[ $OLFS -eq 1 ]] && mountpoint -q "$TMP"; then
|
||||
rsync -aogX --delete-after --inplace --no-whole-file --exclude .flagged "$BACK_OVFS/" "$DIR/" && debug "sync $BACK_OVFS with $DIR"
|
||||
umount -l "$TMP" && debug "unmount $TMP"
|
||||
rm -rf "$VOLATILE/$ASDNAME-$USER" "$VOLATILE/$ASDNAME-$USER-rw" "$VOLATILE/.$ASDNAME-$USER" && debug "removing overlayfs folders"
|
||||
rm -rf "$TMP" "$UPPER" "$WORK" && debug "removing overlayfs folders"
|
||||
else
|
||||
[[ -d "$TMP" ]] && rm -rf "$VOLATILE/$ASDNAME-$USER" && debug "removing $VOLATILE/$ASDNAME-$USER"
|
||||
[[ -d "$TMP" ]] && rm -rf "$TMP" && debug "removing $TMP"
|
||||
fi
|
||||
[[ $CRRE -eq 1 ]] && rm -rf "$BACK_OLD" && debug "removing $BACK_OLD"
|
||||
fi
|
||||
done
|
||||
|
||||
# delete daemon file in the end, so that unsync can be run again
|
||||
# incase of some failure midway
|
||||
# since unsync also requires sync before, if any of the DIRS are unsynced during last run,
|
||||
# they will be synced again before unsyncing not leading to any breakage
|
||||
|
||||
debug "Removing $DAEMON_FILE and ${DAEMON_FILE}.conf"
|
||||
rm -f "$DAEMON_FILE" "${DAEMON_FILE}.conf"
|
||||
|
||||
echo -e "${BLD}Unsync successful${NRM}"
|
||||
}
|
||||
|
||||
parse() {
|
||||
if [[ -f /usr/lib/systemd/system/asd.service ]]; then
|
||||
# running systemd
|
||||
asd_state=$(systemctl is-active asd)
|
||||
resync_state=$(systemctl is-active asd-resync.timer)
|
||||
asd_state=$(systemctl show -p ActiveState --value asd)
|
||||
resync_state=$(systemctl show -p ActiveState --value asd-resync.timer)
|
||||
[[ "$asd_state" = "active" ]] && asd_color="${GRN}" || asd_color="${RED}"
|
||||
[[ "$resync_state" = "active" ]] && resync_color="${GRN}" || resync_color="${RED}"
|
||||
echo -e " ${BLD}Systemd service is currently ${asd_color}$asd_state${NRM}${BLD}.${NRM}"
|
||||
|
|
@ -548,7 +582,7 @@ parse() {
|
|||
echo -e "$(tput cr)$(tput cuf 20) $rwsize${NRM}"
|
||||
fi
|
||||
echo -en " ${BLD}recovery dirs:"
|
||||
mapfile -t CRASHArr < <(find "${BACKUP%/*}" -type d -maxdepth 1 -name "*asd-$CRASH_RECOVERY_SUFFIX*" 2>/dev/null|sort -r)
|
||||
mapfile -t CRASHArr < <(find "${BACKUP%/*}" -maxdepth 1 -name "${BACKUP##*/}-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r)
|
||||
if [[ "${#CRASHArr[@]}" -eq 0 ]]; then
|
||||
echo -e "$(tput cr)$(tput cuf 20) none${NRM}"
|
||||
else
|
||||
|
|
@ -593,6 +627,7 @@ case "$1" in
|
|||
resync)
|
||||
if [[ -f "$DAEMON_FILE" ]]; then
|
||||
root_check
|
||||
ungraceful_state_check
|
||||
do_sync
|
||||
else
|
||||
echo -e "${RED}ASD is not running${NRM}"
|
||||
|
|
@ -602,6 +637,7 @@ case "$1" in
|
|||
# make sure the daemon ran to setup the links
|
||||
if [[ -f "$DAEMON_FILE" ]]; then
|
||||
root_check
|
||||
ungraceful_state_check
|
||||
do_sync
|
||||
do_unsync
|
||||
else
|
||||
|
|
|
|||
|
|
@ -30,6 +30,13 @@ WHATTOSYNC=()
|
|||
#
|
||||
#VOLATILE="/tmp"
|
||||
|
||||
# ASD can break hardlinks present in the system, it has a safety check to ensure
|
||||
# that the synced directories don't have the presence of any hardlinks in them
|
||||
# by default, incase you want the directory to work standalone
|
||||
# and not affect any other hardlinks of the files present in the synced directory,
|
||||
# you can disable this safety check
|
||||
# ENABLE_HARDLINK_SAFETY_CHECK=1
|
||||
|
||||
# Uncomment and set to yes to use an overlayfs instead of a full copy to reduce
|
||||
# the memory costs and to improve sync/unsync operations.
|
||||
#
|
||||
|
|
|
|||
25
doc/asd.1
25
doc/asd.1
|
|
@ -4,7 +4,7 @@
|
|||
\fBanything-sync-daemon \fP- Symlinks and syncs user specified dirs to RAM thus reducing HDD/SDD calls and speeding-up the system.
|
||||
\fB
|
||||
.SH DESCRIPTION
|
||||
Anything-sync-daemon (asd) is a tiny pseudo-daemon designed to manage user specified directories referred to as sync targets from here on out, in tmpfs and to periodically sync them back to the physical disc (HDD/SSD). This is accomplished via a symlinking step and an innovative use of rsync to maintain synchronization between a tmpfs copy and media-bound backups. Additionally, asd features several crash-recovery features.
|
||||
Anything-sync-daemon (asd) is a tiny pseudo-daemon designed to manage user specified directories referred to as sync targets from here on out, in tmpfs and to periodically sync them back to the physical disc (HDD/SSD). This is accomplished via a bind mounting step and an innovative use of rsync to maintain synchronization between a tmpfs copy and media-bound backups. Additionally, asd features several crash-recovery features.
|
||||
.PP
|
||||
Design goals of asd:
|
||||
.RS
|
||||
|
|
@ -130,7 +130,6 @@ Note that for these init systems, the supplied cron script (installed to /etc/cr
|
|||
At this time, the following distros are officially supported but there is no reason to think that asd will not run on another distro:
|
||||
.IP \(bu 3
|
||||
Arch Linux
|
||||
.IP \(bu 3
|
||||
.SH FAQ
|
||||
Q1: What is overlayfs mode?
|
||||
.PP
|
||||
|
|
@ -140,17 +139,17 @@ There are several versions of overlayfs available to the Linux kernel in product
|
|||
.PP
|
||||
See the example in the PREVIEW MODE section above which shows a system using overlayfs to illustrate the memory savings that can be achieved. Note the "overlayfs size" report compared to the total "dir size" report for each sync target. Be aware that these numbers will change depending on just how much data is written to the sync target, but in common use cases, the overlayfs size will always be less than the dir size.
|
||||
.PP
|
||||
Q2: Why do I see another directory ".foo-back-ovfs" when I enable overlayfs?
|
||||
Q2: Why do I see directory ".foo-backup_asd" ".foo-backup_asd-old"?
|
||||
.PP
|
||||
A2: The way overlayfs works is to mount a read-only base copy (so-called lower dir) of the target, and manage the new data on top of that. In order to avoid resyncing to the read-only file system, a copy is used instead. So using overlayfs is a trade off: faster initial sync times and less memory usage vs. disk space.
|
||||
A2: The way the backup process of asd works is that it creates a hard linked clone of the original directory; this is known as .foo-backup_asd-old. The other .foo-backup_asd is just a bind mount to the original directory link which is used to access the contents of the original directory for overlay purposes.
|
||||
.PP
|
||||
Q3: My system crashed and asd didn't sync back. What do I do?
|
||||
.PP
|
||||
A3: The "last good" backup of your sync targets is just fine still sitting happily on your filesystem. Upon restarting asd (on a reboot for example), a check is preformed to see if the symlink to the tmpfs copy of your sync target is valid. If it is invalid, asd will snapshot the "last good" backup before it rotates it back into place. This is more for a sanity check that asd did no harm and that any data loss was a function of something else.
|
||||
A3: The "last good" backup of your sync targets is just fine still sitting happily on your filesystem. Upon restarting asd (on a reboot for example), a check is preformed to see if asd was exited in some corrupted state. If it is detected, asd will snapshot the "last good" backup before it rotates it back into place. Note that, since asd tries to decrease the disk usage, it never really "copies" the full contents of the directory and just uses the hardlinks to the previous files. And during the rsync step, it creates new files so that the previous hardlinks are untouched. So trying to modify the directory during the time asd is trying to backup can leave the directory in some corrupted state.
|
||||
.PP
|
||||
Q4: Where can I find this snapshot?
|
||||
.PP
|
||||
A4: You will find the snapshot in the same directory as the sync target and it will contain a date-time-stamp that corresponds to the time at which the recovery took place. For example, a /foo/bar snapshot will be /foo/.bar-backup_asd-crashrecovery-20141221_070112 -- of course, the date_time suffix will be different for you.
|
||||
A4: You will find the snapshot in the same directory as the sync target and it will contain a date-time-stamp that corresponds to the time at which the recovery took place. For example, a /foo/bar snapshot will be /foo/.bar-backup_asd-crashrecovery-20141221_070112.tar.zstd -- of course, the date_time suffix will be different for you.
|
||||
.PP
|
||||
Q5: How can I restore the snapshot?
|
||||
.PP
|
||||
|
|
@ -159,19 +158,21 @@ A5: Follow these steps:
|
|||
.IP 1. 4
|
||||
Stop asd.
|
||||
.IP 2. 4
|
||||
Confirm that there is no symlink to the sync target. If there is, asd did not stop correctly for other reasons.
|
||||
Confirm that the directories created by asd is not present. If they are, asd did not stop correctly for other reasons.
|
||||
.IP 3. 4
|
||||
Move the "bad" copy of the sync taget to a backup (don't blindly delete anything).
|
||||
.IP 4. 4
|
||||
Copy the snapshot directory to the expected sync target.
|
||||
Untar the snapshot directory to the expected sync target.
|
||||
.RE
|
||||
.PP
|
||||
Example using /foo/bar:
|
||||
.RS
|
||||
.IP 1. 4
|
||||
mv /foo/bar /for/bar-bad
|
||||
cd /foo
|
||||
.IP 1. 4
|
||||
mv bar bar-bad
|
||||
.IP 2. 4
|
||||
cp \fB-a\fP /foo/.bar-backup_asd-crashrecovery-20141221_070112 /foo/bar
|
||||
tar -xvf .bar-backup_asd-crashrecovery-20141221_070112.tar.zstd
|
||||
.RE
|
||||
.PP
|
||||
At this point, check that everything is fine with the data on /foo/bar and, if all is well, it is safe to delete the snapshot.
|
||||
|
|
@ -186,8 +187,6 @@ Discover a bug? Please open an issue on the project page linked below.
|
|||
.RS
|
||||
.IP \(bu 3
|
||||
Currently, asd cannot handle open files on a sync target so if a hung process has something open there, it can be messy.
|
||||
.IP \(bu 3
|
||||
If syncing a path where pacman (Arch Linux package manager) is expected to install files, pacman will stop the update since version 4.2 of pacman will refuse to install to a symlink. If you are syncing a path like this, you will need to stop asd prior to the package update.
|
||||
.SH ONLINE
|
||||
.IP \(bu 3
|
||||
Project page: https://github.com/graysky2/anything-sync-daemon
|
||||
|
|
@ -195,3 +194,5 @@ Project page: https://github.com/graysky2/anything-sync-daemon
|
|||
Wiki page: https://wiki.archlinux.org/index.php/Anything-sync-daemon
|
||||
.SH AUTHOR
|
||||
graysky (graysky AT archlinux DOT us)
|
||||
.SH MAINTAINER
|
||||
Manorit Chawdhry (manorit2001@gmail.com)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Wants=asd-resync.timer
|
|||
PartOf=asd.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
Environment="DEBUG=1"
|
||||
ExecStart=/usr/bin/anything-sync-daemon resync
|
||||
|
||||
[Install]
|
||||
|
|
|
|||
Loading…
Reference in New Issue