diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2012-05-28 18:37:26 +0300 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2012-05-28 18:46:14 +0300 |
commit | 8ceb3484f218d25e025deb618ae3d202e2599dab (patch) | |
tree | 21b551664e24a420d1c967ace2144ee12f3adecd | |
parent | 74940a305bb009a690e56d6d6562b3fd24269ae0 (diff) | |
download | beam-8ceb3484f218d25e025deb618ae3d202e2599dab.tar.gz beam-8ceb3484f218d25e025deb618ae3d202e2599dab.tar.bz2 |
Incorporate s3 support. Change hook handling.
(Additionally: mysql.sh has been tested.)
* .gitignore: Add s3mount.sh
* Makefile.am: Build and install s3mount (conditionally).
* backup-cleanup: Rename to backup-cleaner.
* backup.conf (backup_retain_interval)
(backup_bucket_name,backup_mp_s3backer)
(backup_mp_s3): New variables.
* backup.in: Change hook handling: each hook is a list
of commands which are executed in succession.
New option --version (-V).
Run backup-cleaner if backup_retain_interval is set.
* restore.in: Change hook handling.
New option --version (-V).
* configure.ac: Change version to 1.1.
New option --with-s3.
* doc/backup.conf.5in: Document new variables and changes in the
hook handling.
* examples/.gitignore: Remove.
* examples/Makefile.am: Remove s3mount.
* examples/s3mount.in: Remove.
* lib/backup/Makefile.am (libbackup_SCRIPTS, EXTRA_DIST): Add s3.sh
* lib/backup/common.in: Source backup/s3 if backup_bucket_name is
set.
(runhook,print_version): New function.
* lib/backup/s3.sh: New file.
* s3mount.in: New file.
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | Makefile.am | 24 | ||||
-rwxr-xr-x | backup-cleaner (renamed from backup-cleanup) | 0 | ||||
-rw-r--r-- | backup.conf | 22 | ||||
-rw-r--r-- | backup.in | 16 | ||||
-rw-r--r-- | configure.ac | 8 | ||||
-rw-r--r-- | doc/backup.conf.5in | 50 | ||||
-rw-r--r-- | examples/.gitignore | 1 | ||||
-rw-r--r-- | examples/Makefile.am | 5 | ||||
-rw-r--r-- | examples/s3mount.in | 53 | ||||
-rw-r--r-- | lib/backup/Makefile.am | 2 | ||||
-rw-r--r-- | lib/backup/common.in | 26 | ||||
-rw-r--r-- | lib/backup/s3.sh | 69 | ||||
-rw-r--r-- | restore.in | 9 | ||||
-rw-r--r-- | s3mount.in | 37 |
15 files changed, 244 insertions, 79 deletions
@@ -20,6 +20,7 @@ configure core backup.sh restore.sh +s3mount.sh build.sed *.tar.* tmp diff --git a/Makefile.am b/Makefile.am index e86de86..f3569b8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,9 +1,13 @@ SUBDIRS=. lib examples doc -noinst_SCRIPTS=build.sed backup.sh restore.sh -bin_SCRIPTS=backup-cleanup -CLEANFILES=backup.sh restore.sh +if COND_S3MOUNT + S3MOUNT=s3mount + S3MOUNT_SH=s3mount.sh +endif +noinst_SCRIPTS=build.sed backup.sh restore.sh $(S3MOUNT_SH) +bin_SCRIPTS=backup-cleaner +CLEANFILES=backup.sh restore.sh $(S3MOUNT_SH) install-exec-hook: - for file in backup restore; \ + for file in backup restore $(S3MOUNT); \ do \ test -z "$(DESTDIR)$(bindir)" || \ $(mkdir_p) "$(DESTDIR)$(bindir)"; \ @@ -17,12 +21,12 @@ install-data-hook: ${INSTALL} -m 644 $(top_srcdir)/backup.conf $(DESTDIR)$(sysconfdir);\ fi uninstall-hook: - for file in backup restore; \ + for file in backup restore $(S3MOUNT); \ do \ rm -f "$(DESTDIR)$(bindir)/$$file"; \ done -backup.sh restore.sh: build.sed -EXTRA_DIST=backup.in restore.in backup.conf backup-cleanup +backup.sh restore.sh $(S3MOUNT_SH): build.sed +EXTRA_DIST=backup.in restore.in backup.conf backup-cleaner s3mount.in distuninstallcheck_listfiles=find $(DESTDIR)$(prefix) -type f -not -wholename '$(DESTDIR)$(sysconfdir)/backup.conf' $(top_builddir)/build.sed: Makefile @@ -30,6 +34,12 @@ $(top_builddir)/build.sed: Makefile echo 's|@''SYSCONFDIR''@|${sysconfdir}|g';\ echo 's|@''LIBDIR''@|${libdir}|g'; \ echo 's|@''BINDIR''@|${bindir}|g'; \ + echo 's|@''PACKAGE_BUGREPORT''@|$(PACKAGE_BUGREPORT)|g'; \ + echo 's|@''PACKAGE_NAME''@|$(PACKAGE_NAME)|g';\ + echo 's|@''PACKAGE_STRING''@|$(PACKAGE_STRING)|g';\ + echo 's|@''PACKAGE_TARNAME''@|$(PACKAGE_STRING)|g';\ + echo 's|@''PACKAGE_URL''@|$(PACKAGE_URL)|g';\ + echo 's|@''PACKAGE_VERSION''@|$(PACKAGE_VERSION)|g';\ } > $(top_builddir)/build.sed DISTCLEANFILES=build.sed backup.sh restore.sh include $(top_srcdir)/Make.rules diff --git a/backup-cleanup b/backup-cleaner index 31483c9..31483c9 100755 --- a/backup-cleanup +++ b/backup-cleaner diff --git a/backup.conf b/backup.conf index 297024f..4d12eca 100644 --- a/backup.conf +++ b/backup.conf @@ -23,6 +23,28 @@ prologue_hook= epilogue_hook= ########################################################################## +# Archivation setup +########################################################################## + +# Interval in weeks during which old backups and snapshots are retained. +# If zero or empty, backups are retained forever. +backup_retain_interval= + + +########################################################################## +# S3 configuration. +########################################################################## + +# To use S3 as the backup storage, define this variable to the name of the +# bucket to keep backups in. +backup_bucket_name= + +# Mountpoint for the raw bucket. +#backup_mp_s3backer=/mnt/s3backer +# Mountpoint for the actual file system. +#backup_mp_s3=/mnt/s3 + +########################################################################## # Tar setup. # # Tar setup variables configure invocation of tar. @@ -51,6 +51,8 @@ OPTIONS: -R, --round N mark archives with incremental round N --week N force using week number N instead of the current one -h, --help produce this help list + +Report bugs to <@PACKAGE_BUGREPORT@> EOF exit 0 } @@ -67,6 +69,7 @@ do -n|--dry-run) dry_run=echo;; -N|--no-logfile) backup_logfile=;; -h|--help) help;; + -V|--version) print_version;; -L|--level) shift; level=$1;; -R|--round) shift; round=$1;; --week) shift; week=$1;; @@ -81,8 +84,15 @@ if [ -n "$backup_logfile" ]; then fi logit "started" -$prologue_hook -test -z "$epilogue_hook" && trap "$epilogue_hook" EXIT INT QUIT TERM +if test -n "$backup_retain_interval" && test $backup_retain_interval -gt 0; then + backup-cleaner --retain $backup_retain_interval --verbose \ + --suffix .tar.bz2 $backup_archive_dir + backup-cleaner --retain $backup_retain_interval --verbose \ + --suffix .db $backup_snapshot_dir +fi + +runhook prologue_hook +trap "runhook epilogue_hook" EXIT INT QUIT TERM for item in $backup_items do @@ -90,8 +100,8 @@ do ${type}_backup $item done -$epilogue_hook trap '' EXIT INT QUIT TERM +runhook epilogue_hook logit "finished" diff --git a/configure.ac b/configure.ac index b58cc91..b8c341a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ(2.63) -AC_INIT(backup, 1.0, gray@gnu.org.ua) +AC_INIT(backup, 1.1, gray@gnu.org.ua) AC_CONFIG_SRCDIR(backup.in) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([1.11 tar-ustar silent-rules]) @@ -8,6 +8,12 @@ AM_INIT_AUTOMAKE([1.11 tar-ustar silent-rules]) # Enable silent rules by default: AM_SILENT_RULES([yes]) +AC_ARG_WITH([s3], + AC_HELP_STRING([--with-s3], + [use s3 utilities]), + [s3_status=$withval]) +AM_CONDITIONAL([COND_S3MOUNT],[test "$s3_status"=yes]) + AC_OUTPUT(Makefile lib/Makefile lib/backup/Makefile diff --git a/doc/backup.conf.5in b/doc/backup.conf.5in index aa0bfe8..5f3c707 100644 --- a/doc/backup.conf.5in +++ b/doc/backup.conf.5in @@ -46,6 +46,39 @@ utilities will first look for an executable file named If not found, they will fall back to using \fB/usr/bin/ssh\fR. It is the responsibility of the administrator to provide for authentication on the remote system, e.g. by using shared-key authentication. +.TP +.B backup_retain_interval +Sets the interval, in weeks, during which old backups and snapshots +are retained. Empty value or zero means never delete the backups. + +This variable is used by +.BR backup (1), +which calls +.BR backup-cleaner (1) +if it is set. +.SH S3 SETUP +To keep archives in a +.B s3 +bucket you will need the +.BR s3backer (1) +utility. +.TP +.B backup_bucket_name +Sets the name of the +.B s3 +bucket to keep the archives in. +.TP +.B backup_mp_s3backer +Defines mount point for the "raw" bucket. + +Default value is +.BR /mnt/s3backer . +.TP +.B backup_mp_s3 +Defines mount point for the actual s3-backed filesystem. + +Default value is +.BR /mnt/s3 . .SH TAR SETUP Tar setup variables control additional options and arguments passed to .BR tar (1). @@ -180,14 +213,16 @@ Two special variables, called hooks, allow you to supply arbitrary procedures to be run before and after backup. .TP .B prologue_hook -This variable contains the name of a shell procedure or external -program to be called before starting backup (or restore). The hook is -called without arguments. By default, this variable is empty. +This variable contains a whitespace-separated list of the commands to +be invoked before starting backup (or restore). The commands are +started in order of their appearance in the list and are called +without arguments. By default, this variable is empty. .TP .B epilogue_hook -This variable contains the name of a shell procedure or external -program to be called after backup (or restore) is finished. The hook -is called without arguments. By default, this variable is empty. +This variable contains a whitespace-separated list of the commands to +be invoked after the backup (or restore) has finished. The commands are +started in order of their appearance in the list and are called +without arguments. By default, this variable is empty. .SH EXAMPLES A simple configuration file for backing up the contents of .BR /etc, @@ -222,7 +257,8 @@ system_files="etc home" .sp .SH "SEE ALSO" .BR restore (1), -.BR backup (1). +.BR backup (1), +.BR backup-cleaner (1). .SH AUTHORS Sergey Poznyakoff .SH "BUG REPORTS" diff --git a/examples/.gitignore b/examples/.gitignore deleted file mode 100644 index 6bef033..0000000 --- a/examples/.gitignore +++ /dev/null @@ -1 +0,0 @@ -s3mount.sh diff --git a/examples/Makefile.am b/examples/Makefile.am index 2236b11..c5a0a76 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -1,4 +1 @@ -EXTRA_DIST=backup.conf.s3 s3mount.in -noinst_SCRIPTS=s3mount.sh -CLEANFILES=s3mount.sh -include $(top_srcdir)/Make.rules +EXTRA_DIST=backup.conf.s3 diff --git a/examples/s3mount.in b/examples/s3mount.in deleted file mode 100644 index 261cd72..0000000 --- a/examples/s3mount.in +++ /dev/null @@ -1,53 +0,0 @@ -#! /bin/bash -set -e -test -z "$BACKUP_CONFIG" && BACKUP_CONFIG=@SYSCONFDIR@/backup.conf - -if [ ! -r $BACKUP_CONFIG ]; then - echo >&2 "$0: configuration file $BACKUP_CONFIG is not found or is unreadable" - exit 1 -fi - -. @LIBDIR@/backup/common.sh -. $BACKUP_CONFIG - -if [ -z "$prologue_hook" ]; then - echo >&2 "$0: s3 mount does not seem to be configured (examine $BACKUP_CONFIG)" - exit 1 -fi - -getmpoint() -{ - case $1 in - backer) - mount -tfuse.s3backer | - awk '/https?:\/\/'$backup_bucket_name'/ { print $3 }';; - s3) - mount | grep "^${mp_s3backer}/file" | awk '{ print $3 }';; - *) - abent 1 "invalid usage of getmpoint" - esac -} - -case $1 in -start) - $prologue_hook - echo "$backup_bucket_name mounted under $mp_s3";; -stop) - for id in s3 backer - do - mpoint=$(getmpoint $id) - test -n "$mpoint" && umount $mpoint - done;; -status) - for id in s3 backer - do - mpoint=$(getmpoint $id) - if [ -n "$mpoint" ]; then - echo "$id is mounted on $mpoint" - fi - done;; - -*) - echo "$0 mounts or unmounts the backup s3 bucket, as configured in $BACKUP_CONFIG" - echo "usage: $0 start|stop" -esac diff --git a/lib/backup/Makefile.am b/lib/backup/Makefile.am index 38a8085..d1f687d 100644 --- a/lib/backup/Makefile.am +++ b/lib/backup/Makefile.am @@ -1,11 +1,13 @@ libbackupdir=$(libdir)/backup libbackup_SCRIPTS=\ common.sh\ + s3.sh\ fs.sh\ mysql.sh\ postgres.sh EXTRA_DIST=\ common.in\ + s3.sh\ fs.sh\ mysql.sh\ postgres.sh diff --git a/lib/backup/common.in b/lib/backup/common.in index ab80204..11cb2d0 100644 --- a/lib/backup/common.in +++ b/lib/backup/common.in @@ -83,4 +83,30 @@ $type" test -n "$delayed_exit" && abend 1 "aborting" tar_suffix=${backup_suffix:-.tar} + + if [ -n "$backup_bucket_name" ]; then + . @LIBDIR@/backup/s3.sh + prologue_hook="$prologue_hook s3_mount" + fi +} + +runhook() { + local hook_list + + eval hook_list=\$$1 + for hook in $hook_list + do + $hook + done } + +print_version() { + cat <<EOF +$0 (@PACKAGE_NAME@) @PACKAGE_VERSION@ +Copyright (C) 2012 Sergey Poznyakoff +License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +EOF + exit 0 +}
\ No newline at end of file diff --git a/lib/backup/s3.sh b/lib/backup/s3.sh new file mode 100644 index 0000000..e463e76 --- /dev/null +++ b/lib/backup/s3.sh @@ -0,0 +1,69 @@ +# The configuration variable "backup_bucket_name" must contain the name +# of the s3 bucket to use. + +# Raw bucket is mounted to /mnt/s3backer +test -z "$backup_mp_s3backer" && backup_mp_s3backer=/mnt/s3backer +# The actual file system is mounted to /mnt/s3 +test -z "$backup_mp" && backup_mp_s3=/mnt/s3 + +# This variable is populated by s3_mount and is used by s3_unmount to unmount +# s3-backed file system. +umount_list="" + +s3_mount() { + # Make sure both mountpoints exist + test -d $backup_mp_s3backer || mkdir $backup_mp_s3backer + test -d $backup_mp_s3 || mkdir $backup_mp_s3backer + +# Sample mount output, split into several lines: +# http://finox-backup-fs.s3.amazonaws.com/ on /mnt/s3backer type fuse.s3backer +# (rw,nosuid,nodev,allow_other,default_permissions) + set -- $(mount -tfuse.s3backer | + awk '/https?:\/\/'$backup_bucket_name'/ { print $3 }') + if test -z "$1"; then + $dry_run s3backer --accessFile=/root/.s3backer_passwd --vhost \ + $backup_bucket_name $backup_mp_s3backer || + abend 1 "unable to mount $backup_bucket_name" + umount_list="$backup_mp_s3backer" + else + backup_mp_s3backer=$1 + fi + set -- $(mount | grep "^${backup_mp_s3backer}/file" | awk '{ print $3 }') + if test -z "$1"; then + case $(basename $0) in + restore) mountopt=",ro";; + backup) mountopt=",rw,data=writeback";; + s3mount) ;; + *) error "called as $0: assuming default mount options" + esac + # NOTE: For ext4 add the journal_async_commit option. + $dry_run mount -oloop$mountopt $backup_mp_s3backer/file $backup_mp_s3 || + abend 1 "unable to mount $backup_mp_s3backer/file" + umount_list="$backup_mp_s3 $umount_list" + else + backup_mp_s3=$1 + fi + epilogue_hook="s3_unmount $epilogue_hook" +} + +s3_getmpoint() +{ + case $1 in + backer) + mount -tfuse.s3backer | + awk '/https?:\/\/'$backup_bucket_name'/ { print $3 }';; + s3) + mount | grep "^${backup_mp_s3backer}/file" | awk '{ print $3 }';; + *) + abent 1 "invalid usage of getmpoint" + esac +} + +s3_unmount() +{ + for id in s3 backer + do + mpoint=$(s3_getmpoint $id) + test -n "$mpoint" && umount $mpoint + done +} @@ -43,6 +43,8 @@ OPTIONS: -h, --help produce this help list Valid ITEMs are: $backup_items + +Report bugs to <@PACKAGE_BUGREPORT@> EOT exit 0 } @@ -57,6 +59,7 @@ do -l|--logfile) shift; logfile=$1; confirm=0;; -n|--dry-run) dry_run=echo;; -h|--help) help;; + -V|--version) print_version;; -L|--level) shift; level=$1;; -R|--round) shift; round=$1;; -W|--week) shift; week=$1;; @@ -84,8 +87,8 @@ if [ -n "$logfile" ]; then exec 2>&1 fi -$prologue_hook -test -z "$epilogue_hook" && trap "$epilogue_hook" EXIT INT QUIT TERM +runhook prologue_hook +trap "runhook epilogue_hook" EXIT INT QUIT TERM # Guess missing values if [ -z "$week" ]; then @@ -134,8 +137,8 @@ do ${type}_restore $item done -$epilogue_hook trap '' EXIT INT QUIT TERM +runhook epilogue_hook logit "finished" diff --git a/s3mount.in b/s3mount.in new file mode 100644 index 0000000..c6e5083 --- /dev/null +++ b/s3mount.in @@ -0,0 +1,37 @@ +#! /bin/bash +set -e +test -z "$BACKUP_CONFIG" && BACKUP_CONFIG=@SYSCONFDIR@/backup.conf + +if [ ! -r $BACKUP_CONFIG ]; then + echo >&2 "$0: configuration file $BACKUP_CONFIG is not found or is unreadable" + exit 1 +fi + +. @LIBDIR@/backup/common.sh +. $BACKUP_CONFIG +. @LIBDIR@/backup/s3.sh + +if [ -z "$backup_bucket_name" ]; then + echo >&2 "$0: backup_bucket_name is not defined (examine $BACKUP_CONFIG)" + exit 1 +fi + +case $1 in +start) + s3_mount + echo "$backup_bucket_name mounted under $backup_mp_s3";; +stop) + s3_unmount;; +status) + for id in s3 backer + do + mpoint=$(s3_getmpoint $id) + if [ -n "$mpoint" ]; then + echo "$id is mounted on $mpoint" + fi + done;; + +*) + echo "$0 mounts or unmounts a backup s3 bucket, as configured in $BACKUP_CONFIG" + echo "usage: $0 start|stop" +esac |