diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2012-05-25 15:48:38 +0300 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2012-05-25 16:21:32 +0300 |
commit | 74940a305bb009a690e56d6d6562b3fd24269ae0 (patch) | |
tree | 91e110503760b00817da16e9fcfe8536435ba352 | |
parent | bb2c9e35cd16f12ec085a4ab46b7744a556a7f1a (diff) | |
download | beam-74940a305bb009a690e56d6d6562b3fd24269ae0.tar.gz beam-74940a305bb009a690e56d6d6562b3fd24269ae0.tar.bz2 |
Add mysql.sh (untested). More configuration examples.
* Makefile.am: Install backup-cleanup.
* backup-cleanup: New file.
* backup.in (tar_version_options): Remove.
(taroptions): Add -c.
* examples/Makefile.am (EXTRA_DIST): Add s3mount.in
(noinst_SCRIPTS): Add s3mount.sh
* examples/backup.conf.s3 (prologue): Cleanup old files.
* examples/s3mount.in: New file.
* examples/.gitignore: New file.
* lib/backup/Makefile.am (libbackup_SCRIPTS)
(EXTRA_DIST): Add mysql.sh
* lib/backup/fs.sh (fs_backup): Remove -c option: it is
present in $taroptions by default.
* lib/backup/mysql.sh: New file.
* lib/backup/postgres.sh (postgres_backup): Fix variable name,
* restore.in: More error checking.
-rw-r--r-- | Makefile.am | 3 | ||||
-rwxr-xr-x | backup-cleanup | 67 | ||||
-rw-r--r-- | backup.in | 15 | ||||
-rw-r--r-- | examples/.gitignore | 1 | ||||
-rw-r--r-- | examples/Makefile.am | 5 | ||||
-rw-r--r-- | examples/backup.conf.s3 | 7 | ||||
-rw-r--r-- | examples/s3mount.in | 53 | ||||
-rw-r--r-- | lib/backup/Makefile.am | 2 | ||||
-rwxr-xr-x | lib/backup/fs.sh | 2 | ||||
-rw-r--r-- | lib/backup/mysql.sh | 76 | ||||
-rwxr-xr-x | lib/backup/postgres.sh | 2 | ||||
-rw-r--r-- | restore.in | 2 |
12 files changed, 217 insertions, 18 deletions
diff --git a/Makefile.am b/Makefile.am index 9a6e098..e86de86 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,6 @@ SUBDIRS=. lib examples doc noinst_SCRIPTS=build.sed backup.sh restore.sh +bin_SCRIPTS=backup-cleanup CLEANFILES=backup.sh restore.sh install-exec-hook: for file in backup restore; \ @@ -21,7 +22,7 @@ uninstall-hook: rm -f "$(DESTDIR)$(bindir)/$$file"; \ done backup.sh restore.sh: build.sed -EXTRA_DIST=backup.in restore.in backup.conf +EXTRA_DIST=backup.in restore.in backup.conf backup-cleanup distuninstallcheck_listfiles=find $(DESTDIR)$(prefix) -type f -not -wholename '$(DESTDIR)$(sysconfdir)/backup.conf' $(top_builddir)/build.sed: Makefile diff --git a/backup-cleanup b/backup-cleanup new file mode 100755 index 0000000..31483c9 --- /dev/null +++ b/backup-cleanup @@ -0,0 +1,67 @@ +#! /bin/sh + +dir= +suffix= +retainweeks=3 +dry_run= +verbose=: + +help() { + cat <<EOF +usage: $0 [OPTIONS] DIR +cleans up old backup files in DIR + +OPTIONS are: + -s, --suffix SUF consider only file names ending in SUF + -r, --retain N retain N last weeks of backups (default $retainweeks) + -v, --verbose verbosely list what is being done + -n, --dry-run do nothing, print what would have been done + -h, --help print this help list + +EOF + exit 0 +} + +while [ $# -ne 0 ] +do + case $1 in + -s|--suffix) shift; suffix=$1;; + -r|--retain) shift; retainweeks=$1;; + -v|--verbose) verbose=echo;; + -n|--dry-run) dry_run=echo; verbose=echo;; + -h|--help) help;; + --) shift; break;; + -*) echo >&2 "$0: unrecognized option $1" + exit 1;; + *) break + esac + shift +done + +case $# in +0) echo >&2 "$0: not enough arguments" + exit 1;; +1) dir=$1;; +*) echo >&2 "$0: too many arguments" + exit 1;; +esac + +thisweek=$(date +%U) +lastweek=$((thisweek - retainweeks)) + +$verbose \# removing files older than week $lastweek + +if [ -z "$suffix" ]; then + find $dir -type f -printf '%f\n' +else + find $dir -type f -name "*$suffix" -printf '%f\n' +fi | + while read name + do + $verbose \# considering $name + week=$(expr "$name" : '[^-][^-]*-\([0-9][0-9]*\)-.*') + if [ $week -le $lastweek ]; then + $verbose \# removing $name + $dry_run rm $dir/$name + fi + done @@ -10,19 +10,8 @@ set +e ts=`date +%Y%m%d`; week=`date +%U` -# Add version-specific tar options -tar_version_options() { - set -- $(tar --version | awk 'NR==1 {gsub(/\./, " ",$NF); print $NF}') - if [ $# -ge 2 ]; then - v=$((1 * 1000 + $2)) - if [ $v -gt 1025 ]; then - taroptions="$taroptions --level=$level" - fi - fi -} - load_config -taroptions="-Hpax $backup_tar_options $backup_rsh_command" +taroptions="-c -Hpax $backup_tar_options $backup_rsh_command" ########################################################### # Global variables @@ -86,8 +75,6 @@ do shift done -tar_version_options - if [ -n "$backup_logfile" ]; then exec >>$backup_logfile exec 2>&1 diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000..6bef033 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1 @@ +s3mount.sh diff --git a/examples/Makefile.am b/examples/Makefile.am index c5a0a76..2236b11 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -1 +1,4 @@ -EXTRA_DIST=backup.conf.s3 +EXTRA_DIST=backup.conf.s3 s3mount.in +noinst_SCRIPTS=s3mount.sh +CLEANFILES=s3mount.sh +include $(top_srcdir)/Make.rules diff --git a/examples/backup.conf.s3 b/examples/backup.conf.s3 index 07b3d68..668a383 100644 --- a/examples/backup.conf.s3 +++ b/examples/backup.conf.s3 @@ -27,6 +27,8 @@ umount_list="" # Prologue function is responsible for mounting the s3-backed fs. prologue() { + local cleanup=1 + # Sample mount output, split into several lines: # http://finox-backup-fs.s3.amazonaws.com/ on /mnt/s3backer type fuse.s3backer # (rw,nosuid,nodev,allow_other,default_permissions) @@ -45,6 +47,7 @@ prologue() { case $(basename $0) in restore) mountopt=",ro";; backup) mountopt=",rw,data=writeback";; + s3mount) cleanup=0;; *) error "called as $0: assuming default mount options" esac # NOTE: For ext4 add the journal_async_commit option. @@ -54,6 +57,10 @@ prologue() { else mp_s3=$1 fi + if test $cleanup -eq 1; then + backup-cleanup --verbose --suffix .tar.bz2 $mp_s3 + backup-cleanup --verbose --suffix .db $backup_snapshot_dir + fi } # Epilogue function unmounts all fs mounted so far. diff --git a/examples/s3mount.in b/examples/s3mount.in new file mode 100644 index 0000000..261cd72 --- /dev/null +++ b/examples/s3mount.in @@ -0,0 +1,53 @@ +#! /bin/bash +set -e +test -z "$BACKUP_CONFIG" && BACKUP_CONFIG=@SYSCONFDIR@/backup.conf + +if [ ! -r $BACKUP_CONFIG ]; then + echo >&2 "$0: configuration file $BACKUP_CONFIG is not found or is unreadable" + exit 1 +fi + +. @LIBDIR@/backup/common.sh +. $BACKUP_CONFIG + +if [ -z "$prologue_hook" ]; then + echo >&2 "$0: s3 mount does not seem to be configured (examine $BACKUP_CONFIG)" + exit 1 +fi + +getmpoint() +{ + case $1 in + backer) + mount -tfuse.s3backer | + awk '/https?:\/\/'$backup_bucket_name'/ { print $3 }';; + s3) + mount | grep "^${mp_s3backer}/file" | awk '{ print $3 }';; + *) + abent 1 "invalid usage of getmpoint" + esac +} + +case $1 in +start) + $prologue_hook + echo "$backup_bucket_name mounted under $mp_s3";; +stop) + for id in s3 backer + do + mpoint=$(getmpoint $id) + test -n "$mpoint" && umount $mpoint + done;; +status) + for id in s3 backer + do + mpoint=$(getmpoint $id) + if [ -n "$mpoint" ]; then + echo "$id is mounted on $mpoint" + fi + done;; + +*) + echo "$0 mounts or unmounts the backup s3 bucket, as configured in $BACKUP_CONFIG" + echo "usage: $0 start|stop" +esac diff --git a/lib/backup/Makefile.am b/lib/backup/Makefile.am index ec4cb91..38a8085 100644 --- a/lib/backup/Makefile.am +++ b/lib/backup/Makefile.am @@ -2,10 +2,12 @@ libbackupdir=$(libdir)/backup libbackup_SCRIPTS=\ common.sh\ fs.sh\ + mysql.sh\ postgres.sh EXTRA_DIST=\ common.in\ fs.sh\ + mysql.sh\ postgres.sh DISTCLEANFILES=common.sh include $(top_srcdir)/Make.rules diff --git a/lib/backup/fs.sh b/lib/backup/fs.sh index 3950e37..2ac9ad1 100755 --- a/lib/backup/fs.sh +++ b/lib/backup/fs.sh @@ -55,7 +55,7 @@ fs_backup() { initdb $1 logit "backing up $text ($basename.$tar_suffix)" $dry_run tar $verbose $taroptions \ - -c -f $backup_archive_dir/$basename.$tar_suffix \ + -f $backup_archive_dir/$basename.$tar_suffix \ --listed=$backup_snapshot_dir/$basename.db \ -C $root $files tarcode $? diff --git a/lib/backup/mysql.sh b/lib/backup/mysql.sh new file mode 100644 index 0000000..5538ea8 --- /dev/null +++ b/lib/backup/mysql.sh @@ -0,0 +1,76 @@ +#! /bin/bash + +# mysql_check item +mysql_check() { + eval database=\$${1}_database + test -z "$database" && error "${1}_database not set" && return 1 + return 0 +} + +# mysql_backup item +mysql_backup() { + local database + + logit "backing up MySQL database $1" + eval database=\$${1}_database + test -z "$database" && abend 1 "${1}_database not set" + cmd="mysqldump" + eval defaults_file=\$${1}_defaults_file + if [ -n "$defaults_file" ]; then + cmd="$cmd --defaults-file=$defaults_file" + fi + cmd="$cmd --add-drop-database --databases" + if [ -z "$dry_run" ]; then + $cmd $database > $backup_snapshot_dir/$1-$week-$round-$level + else + echo "$cmd $database > $backup_snapshot_dir/$1-$week-$round-$level" + fi + + if [ $? -ne 0 ]; then + tarerror=$((tarerror + 1)) + echo >&2 "`date`: failed" + else + echo "`date`: creating $1-$week-$round-$level.$tar_suffix" + $dry_run tar $verbose $taroptions \ + -f $backup_archive_dir/$1-$week-$round-$level.$tar_suffix \ + -C $backup_snapshot_dir $1-$week-$round-$level + tarcode $? + $dry_run rm $backup_snapshot_dir/dbdump-$week-$round-$level + fi +} + +mysql_restore() { + local u database + + eval database=\$${1}_database + logit "restoring MySQL database $database" + u=$(umask) + trap "umask $u" 1 2 3 13 15 + umask 077 + $dry_run tar $verbose $taroptions \ + -f $backup_archive_dir/$1-$week-$round-$level.$tar_suffix + e=$? + tarcode $e + if [ $e -eq 0 ]; then + logit "restoring database from the dump" + cmd="mysql -A --batch" + eval defaults_file=\$${1}_defaults_file + if [ -n "$defaults_file" ]; then + cmd="$cmd --defaults-file=$defaults_file" + fi + if [ -n "$dry_run" ]; then + echo "$cmd < $1-$week-$round-$level" + elif [ -r $1-$week-$round-$level ]; then + $cmd < $1-$week-$round-$level > db-$1.log + if grep ERROR db-$1.log >/dev/null; then + error "errors occurred during restore; see db-$1.log for details" + error "dump preserved in file $1-$week-$round-$level" + tarerror=$((tarerror + 1)) + else + rm $1-$week-$round-$level + fi + fi + fi + umask $u + trap - 1 2 3 13 15 +} diff --git a/lib/backup/postgres.sh b/lib/backup/postgres.sh index 953c467..f233615 100755 --- a/lib/backup/postgres.sh +++ b/lib/backup/postgres.sh @@ -29,7 +29,7 @@ postgres_backup() { -f $backup_archive_dir/$1-$week-$round-$level.$tar_suffix \ -C $backup_snapshot_dir $1-$week-$round-$level tarcode $? - $dry_run rm $snapshotdir/dbdump-$week-$round-$level + $dry_run rm $backup_snapshot_dir/dbdump-$week-$round-$level fi } @@ -104,6 +104,7 @@ if [ -z "$round" ]; then sed 's/.*-'$week'-\([0-9][0-9]*\)-[0-9][0-9]*\..*/\1/' | sort +0 -1 | tail -1) + test -z "$round" && abend 1 "cannot determine last round number" fi if [ -z "$level" ]; then @@ -113,6 +114,7 @@ if [ -z "$level" ]; then sed 's/.*-'$week-$round'-\([0-9][0-9]*\)\..*/\1/' | sort +0 -1 | tail -1) + test -z "$level" && abend 1 "cannot determine last level number" fi echo "$0: target root directory $root" |