diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README.md | 24 | ||||
| -rwxr-xr-x | rsync_tmbackup.sh | 95 | ||||
| -rw-r--r-- | tests/populate_dest.php | 30 |
4 files changed, 120 insertions, 31 deletions
@@ -1,3 +1,5 @@ .idea test.sh *~ +tests/TestDest/ +tests/TestSource/ @@ -22,6 +22,10 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it --log-dir Set the log file directory. If this flag is set, generated files will not be managed by the script - in particular they will not be automatically deleted. + --strategy Set the expiration strategy. Default: "1:1 30:7 365:30" means after one + day, keep one backup per day. After 30 days, keep one backup every 7 days. + After 365 days keep one backup every 30 days. + --no-auto-expire Set option to disable automatically purging old backups when out of space. ## Features @@ -66,13 +70,15 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it ## Backup expiration logic -The script automatically deletes old backups using the following logic: -- Within the last 24 hours, all the backups are kept. -- Within the last 31 days, the most recent backup of each day is kept. -- After 31 days, only the most recent backup of each month is kept. -- Additionally, if the backup destination directory is full, the oldest backups are deleted until enough space is available. +Backup sets are automatically deleted following a simple expiration strategy defined with the `--strategy` flag. This strategy is a series of time intervals with each item being defined as `x:y`, which means "after x days, keep one backup every y days". The default strategy is `1:1 30:7 365:30`, which means: -## Exclude file +- After **1** day, keep one backup every **1** day (**1:1**). +- After **30** days, keep one backup every **7** days (**30:7**). +- After **365** days, keep one backup every **30** days (**365:30**). + +Before the first interval (i.e. by default within the first 24h) it is implied that all backup sets are kept. Additionally, if the backup destination directory is full, the oldest backups are deleted until enough space is available. + +## Exclusion file An optional exclude file can be provided as a third parameter. It should be compatible with the `--exclude-from` parameter of rsync. See [this tutorial](https://sites.google.com/site/rsync2u/home/rsync-tutorial/the-exclude-from-option) for more information. @@ -86,6 +92,10 @@ To display the rsync options that are used for backup, run `./rsync_tmbackup.sh rsync_tmbackup --rsync-set-flags "--numeric-ids --links --hard-links \ --one-file-system --archive --no-perms --no-groups --itemize-changes" /src /dest + +## No automatic backup expiration + +An option to diable the default behaviour to purge old backups when out of space. This option is set with the `--no-auto-expire` flag. ## How to restore @@ -107,7 +117,7 @@ The script creates a backup in a regular directory so you can simply copy the fi The MIT License (MIT) -Copyright (c) 2013-2017 Laurent Cozic +Copyright (c) 2013-2018 Laurent Cozic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index 07289d4..63a863f 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -43,6 +43,11 @@ fn_display_usage() { echo " not be managed by the script - in particular they will not be" echo " automatically deleted." echo " Default: $LOG_DIR" + echo " --strategy Set the expiration strategy. Default: \"1:1 30:7 365:30\" means after one" + echo " day, keep one backup per day. After 30 days, keep one backup every 7 days." + echo " After 365 days keep one backup every 30 days." + echo " --no-auto-expire Disable automatically deleting backups when out of space. Instead an error" + echo " is logged, and the backup is aborted." echo "" echo "For more detailed help, please see the README file:" echo "" @@ -83,6 +88,56 @@ fn_expire_backup() { fn_rm_dir "$1" } +fn_expire_backups() { + local current_timestamp=$EPOCH + local last_kept_timestamp=9999999999 + + # Process each backup dir from most recent to oldest + for backup_dir in $(fn_find_backups | sort -r); do + local backup_date=$(basename "$backup_dir") + local backup_timestamp=$(fn_parse_date "$backup_date") + + # Skip if failed to parse date... + if [ -z "$backup_timestamp" ]; then + fn_log_warn "Could not parse date: $backup_dir" + continue + fi + + # Find which strategy token applies to this particular backup + for strategy_token in $(echo $EXPIRATION_STRATEGY | tr " " "\n" | sort -r -n); do + IFS=':' read -r -a t <<< "$strategy_token" + + # After which date (relative to today) this token applies (X) + local cut_off_timestamp=$((current_timestamp - ${t[0]} * 86400)) + + # Every how many days should a backup be kept past the cut off date (Y) + local cut_off_interval=$((${t[1]} * 86400)) + + # If we've found the strategy token that applies to this backup + if [ "$backup_timestamp" -le "$cut_off_timestamp" ]; then + + # Special case: if Y is "0" we delete every time + if [ $cut_off_interval -eq "0" ]; then + fn_expire_backup "$backup_dir" + break + fi + + # Check if the current backup is in the interval between + # the last backup that was kept and Y + local interval_since_last_kept=$((last_kept_timestamp - backup_timestamp)) + if [ "$interval_since_last_kept" -lt "$cut_off_interval" ]; then + # Yes: Delete that one + fn_expire_backup "$backup_dir" + else + # No: Keep it + last_kept_timestamp=$backup_timestamp + fi + break + fi + done + done +} + fn_parse_ssh() { # To keep compatibility with bash version < 3, we use grep if echo "$DEST_FOLDER"|grep -Eq '^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$' @@ -157,6 +212,8 @@ DEST_FOLDER="" EXCLUSION_FILE="" LOG_DIR="$HOME/.$APPNAME" AUTO_DELETE_LOG="1" +EXPIRATION_STRATEGY="1:1 30:7 365:30" +AUTO_EXPIRE="1" RSYNC_FLAGS="-D --compress --numeric-ids --links --hard-links --one-file-system --itemize-changes --times --recursive --perms --owner --group --stats --human-readable" @@ -179,11 +236,18 @@ while :; do shift RSYNC_FLAGS="$1" ;; + --strategy) + shift + EXPIRATION_STRATEGY="$1" + ;; --log-dir) shift LOG_DIR="$1" AUTO_DELETE_LOG="0" ;; + --no-auto-expire) + AUTO_EXPIRE="0" + ;; --) shift SRC_FOLDER="$1" @@ -359,30 +423,7 @@ while : ; do # Purge certain old backups before beginning new backup. # ----------------------------------------------------------------------------- - # Default value for $PREV ensures that the most recent backup is never deleted. - PREV="0000-00-00-000000" - for FILENAME in $(fn_find_backups | sort -r); do - BACKUP_DATE=$(basename "$FILENAME") - TIMESTAMP=$(fn_parse_date $BACKUP_DATE) - - # Skip if failed to parse date... - if [ -z "$TIMESTAMP" ]; then - fn_log_warn "Could not parse date: $FILENAME" - continue - fi - - if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then - true - elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then - # Delete all but the most recent of each day. - [ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME" - else - # Delete all but the most recent of each month. - [ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME" - fi - - PREV=$BACKUP_DATE - done + fn_expire_backups # ----------------------------------------------------------------------------- # Start backup @@ -420,6 +461,12 @@ while : ; do NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")" if [ -n "$NO_SPACE_LEFT" ]; then + + if [[ $AUTO_EXPIRE == "0" ]]; then + fn_log_error "No space left on device, and automatic purging of old backups is disabled." + exit 1 + fi + fn_log_warn "No space left on device - removing oldest backup and resuming." if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then diff --git a/tests/populate_dest.php b/tests/populate_dest.php new file mode 100644 index 0000000..0603c27 --- /dev/null +++ b/tests/populate_dest.php @@ -0,0 +1,30 @@ +<?php + +// This PHP script can be used to test the expiration strategy. +// It is going to populate a directory with fake backup sets (directories named Y-m-d-His) over several months. +// Then the backup script can be run on it to check what directories are going to be deleted. + +// rm -rf ./tests/TestDest/201* && php ./tests/populate_dest.php && ./rsync_tmbackup.sh ./tests/TestSource/ ./tests/TestDest/ + +$baseDir = dirname(__FILE__); +$destDir = $baseDir . '/TestDest'; + +$backupsPerDay = 2; +$totalDays = 500; + +$intervalBetweenBackups = null; +if ($backupsPerDay === 1) { + $intervalBetweenBackups = 'PT1D'; +} else if ($backupsPerDay === 2) { + $intervalBetweenBackups = 'PT12H'; +} else { + throw new Exception('Not implemented'); +} + +$d = new DateTime(); +$d->sub(new DateInterval('P' . $totalDays . 'D')); + +for ($i = 0; $i < $backupsPerDay * $totalDays; $i++) { + $d->add(new DateInterval($intervalBetweenBackups)); + mkdir($destDir . '/' . $d->format('Y-m-d-His'), 0777, true); +}
\ No newline at end of file |