Merge pull request #120 from laurent22/expiration-strategy

Expiration strategy

4 files changed, 120 insertions, 31 deletions

diff --git a/.gitignore b/.gitignore
index aa54d41..44ac0df 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 .idea
 test.sh
 *~
+tests/TestDest/
+tests/TestSource/
diff --git a/README.md b/README.md
index 1b93a2e..df76a6a 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,10 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it
 	 --log-dir            Set the log file directory. If this flag is set, generated files will
 	                      not be managed by the script - in particular they will not be
 	                      automatically deleted.
+	 --strategy           Set the expiration strategy. Default: "1:1 30:7 365:30" means after one
+	                      day, keep one backup per day. After 30 days, keep one backup every 7 days.
+	                      After 365 days keep one backup every 30 days.
+	 --no-auto-expire     Set option to disable automatically purging old backups when out of space.
 
 ## Features
 
@@ -66,13 +70,15 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it
 
 ## Backup expiration logic
 
-The script automatically deletes old backups using the following logic:
-- Within the last 24 hours, all the backups are kept.
-- Within the last 31 days, the most recent backup of each day is kept.
-- After 31 days, only the most recent backup of each month is kept.
-- Additionally, if the backup destination directory is full, the oldest backups are deleted until enough space is available.
+Backup sets are automatically deleted following a simple expiration strategy defined with the `--strategy` flag. This strategy is a series of time intervals with each item being defined as `x:y`, which means "after x days, keep one backup every y days". The default strategy is `1:1 30:7 365:30`, which means:
 
-## Exclude file
+- After **1** day, keep one backup every **1** day (**1:1**).
+- After **30** days, keep one backup every **7** days (**30:7**).
+- After **365** days, keep one backup every **30** days (**365:30**).
+
+Before the first interval (i.e. by default within the first 24h) it is implied that all backup sets are kept. Additionally, if the backup destination directory is full, the oldest backups are deleted until enough space is available.
+
+## Exclusion file
 
 An optional exclude file can be provided as a third parameter. It should be compatible with the `--exclude-from` parameter of rsync. See [this tutorial](https://sites.google.com/site/rsync2u/home/rsync-tutorial/the-exclude-from-option) for more information.
 
@@ -86,6 +92,10 @@ To display the rsync options that are used for backup, run `./rsync_tmbackup.sh
 
 	rsync_tmbackup --rsync-set-flags "--numeric-ids --links --hard-links \
 	--one-file-system --archive --no-perms --no-groups --itemize-changes" /src /dest
+
+## No automatic backup expiration
+
+An option to diable the default behaviour to purge old backups when out of space. This option is set with the `--no-auto-expire` flag.
 	
 	
 ## How to restore
@@ -107,7 +117,7 @@ The script creates a backup in a regular directory so you can simply copy the fi
 
 The MIT License (MIT)
 
-Copyright (c) 2013-2017 Laurent Cozic
+Copyright (c) 2013-2018 Laurent Cozic
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh
index 07289d4..63a863f 100755
--- a/rsync_tmbackup.sh
+++ b/rsync_tmbackup.sh
@@ -43,6 +43,11 @@ fn_display_usage() {
 	echo "                      not be managed by the script - in particular they will not be"
 	echo "                      automatically deleted."
 	echo "                      Default: $LOG_DIR"
+	echo " --strategy           Set the expiration strategy. Default: \"1:1 30:7 365:30\" means after one"
+	echo "                      day, keep one backup per day. After 30 days, keep one backup every 7 days."
+	echo "                      After 365 days keep one backup every 30 days."
+	echo " --no-auto-expire     Disable automatically deleting backups when out of space. Instead an error"
+	echo "                      is logged, and the backup is aborted."
 	echo ""
 	echo "For more detailed help, please see the README file:"
 	echo ""
@@ -83,6 +88,56 @@ fn_expire_backup() {
 	fn_rm_dir "$1"
 }
 
+fn_expire_backups() {
+	local current_timestamp=$EPOCH
+	local last_kept_timestamp=9999999999
+
+	# Process each backup dir from most recent to oldest
+	for backup_dir in $(fn_find_backups | sort -r); do
+		local backup_date=$(basename "$backup_dir")
+		local backup_timestamp=$(fn_parse_date "$backup_date")
+
+		# Skip if failed to parse date...
+		if [ -z "$backup_timestamp" ]; then
+			fn_log_warn "Could not parse date: $backup_dir"
+			continue
+		fi
+
+		# Find which strategy token applies to this particular backup
+		for strategy_token in $(echo $EXPIRATION_STRATEGY | tr " " "\n" | sort -r -n); do
+			IFS=':' read -r -a t <<< "$strategy_token"
+
+			# After which date (relative to today) this token applies (X)
+			local cut_off_timestamp=$((current_timestamp - ${t[0]} * 86400))
+
+			# Every how many days should a backup be kept past the cut off date (Y)
+			local cut_off_interval=$((${t[1]} * 86400))
+
+			# If we've found the strategy token that applies to this backup
+			if [ "$backup_timestamp" -le "$cut_off_timestamp" ]; then
+
+				# Special case: if Y is "0" we delete every time
+				if [ $cut_off_interval -eq "0" ]; then
+					fn_expire_backup "$backup_dir"
+					break
+				fi
+
+				# Check if the current backup is in the interval between
+				# the last backup that was kept and Y
+				local interval_since_last_kept=$((last_kept_timestamp - backup_timestamp))
+				if [ "$interval_since_last_kept" -lt "$cut_off_interval" ]; then
+					# Yes: Delete that one
+					fn_expire_backup "$backup_dir"
+				else
+					# No: Keep it
+					last_kept_timestamp=$backup_timestamp
+				fi
+				break
+			fi
+		done
+	done
+}
+
 fn_parse_ssh() {
 	# To keep compatibility with bash version < 3, we use grep
 	if echo "$DEST_FOLDER"|grep -Eq '^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$'
@@ -157,6 +212,8 @@ DEST_FOLDER=""
 EXCLUSION_FILE=""
 LOG_DIR="$HOME/.$APPNAME"
 AUTO_DELETE_LOG="1"
+EXPIRATION_STRATEGY="1:1 30:7 365:30"
+AUTO_EXPIRE="1"
 
 RSYNC_FLAGS="-D --compress --numeric-ids --links --hard-links --one-file-system --itemize-changes --times --recursive --perms --owner --group --stats --human-readable"
 
@@ -179,11 +236,18 @@ while :; do
 			shift
 			RSYNC_FLAGS="$1"
 			;;
+		--strategy)
+			shift
+			EXPIRATION_STRATEGY="$1"
+			;;
 		--log-dir)
 			shift
 			LOG_DIR="$1"
 			AUTO_DELETE_LOG="0"
 			;;
+		--no-auto-expire)
+			AUTO_EXPIRE="0"
+			;;
 		--)
 			shift
 			SRC_FOLDER="$1"
@@ -359,30 +423,7 @@ while : ; do
 	# Purge certain old backups before beginning new backup.
 	# -----------------------------------------------------------------------------
 
-	# Default value for $PREV ensures that the most recent backup is never deleted.
-	PREV="0000-00-00-000000"
-	for FILENAME in $(fn_find_backups | sort -r); do
-		BACKUP_DATE=$(basename "$FILENAME")
-		TIMESTAMP=$(fn_parse_date $BACKUP_DATE)
-
-		# Skip if failed to parse date...
-		if [ -z "$TIMESTAMP" ]; then
-			fn_log_warn "Could not parse date: $FILENAME"
-			continue
-		fi
-
-		if   [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then
-			true
-		elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then
-			# Delete all but the most recent of each day.
-			[ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME"
-		else
-			# Delete all but the most recent of each month.
-			[ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME"
-		fi
-
-		PREV=$BACKUP_DATE
-	done
+	fn_expire_backups
 
 	# -----------------------------------------------------------------------------
 	# Start backup
@@ -420,6 +461,12 @@ while : ; do
 	NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")"
 
 	if [ -n "$NO_SPACE_LEFT" ]; then
+
+		if [[ $AUTO_EXPIRE == "0" ]]; then
+			fn_log_error "No space left on device, and automatic purging of old backups is disabled."
+			exit 1
+		fi
+
 		fn_log_warn "No space left on device - removing oldest backup and resuming."
 
 		if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then
diff --git a/tests/populate_dest.php b/tests/populate_dest.php
new file mode 100644
index 0000000..0603c27
--- /dev/null
+++ b/tests/populate_dest.php
@@ -0,0 +1,30 @@
+<?php
+
+// This PHP script can be used to test the expiration strategy.
+// It is going to populate a directory with fake backup sets (directories named Y-m-d-His) over several months.
+// Then the backup script can be run on it to check what directories are going to be deleted.
+
+// rm -rf ./tests/TestDest/201* && php ./tests/populate_dest.php && ./rsync_tmbackup.sh ./tests/TestSource/ ./tests/TestDest/
+
+$baseDir = dirname(__FILE__);
+$destDir = $baseDir . '/TestDest';
+
+$backupsPerDay = 2;
+$totalDays = 500;
+
+$intervalBetweenBackups = null;
+if ($backupsPerDay === 1) {
+	$intervalBetweenBackups = 'PT1D';
+} else if ($backupsPerDay === 2) {
+	$intervalBetweenBackups = 'PT12H';
+} else {
+	throw new Exception('Not implemented');
+}
+
+$d = new DateTime();
+$d->sub(new DateInterval('P' . $totalDays . 'D'));
+
+for ($i = 0; $i < $backupsPerDay * $totalDays; $i++) {
+	$d->add(new DateInterval($intervalBetweenBackups));
+	mkdir($destDir . '/' . $d->format('Y-m-d-His'), 0777, true);
+}
\ No newline at end of file