diff options
Diffstat (limited to 'lib/beam')
| -rw-r--r-- | lib/beam/.gitignore | 1 | ||||
| -rw-r--r-- | lib/beam/Makefile.am | 31 | ||||
| -rw-r--r-- | lib/beam/common.in | 135 | ||||
| -rwxr-xr-x | lib/beam/fs.sh | 107 | ||||
| -rw-r--r-- | lib/beam/mysql.sh | 91 | ||||
| -rwxr-xr-x | lib/beam/postgres.sh | 87 | ||||
| -rw-r--r-- | lib/beam/s3.sh | 85 |
7 files changed, 537 insertions, 0 deletions
diff --git a/lib/beam/.gitignore b/lib/beam/.gitignore new file mode 100644 index 0000000..5621a6e --- /dev/null +++ b/lib/beam/.gitignore @@ -0,0 +1 @@ +common.sh diff --git a/lib/beam/Makefile.am b/lib/beam/Makefile.am new file mode 100644 index 0000000..d490acb --- /dev/null +++ b/lib/beam/Makefile.am @@ -0,0 +1,31 @@ +# This file is part of BEAM +# Copyright (C) 2012 Sergey Poznyakoff +# +# BEAM is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# BEAM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BEAM. If not, see <http://www.gnu.org/licenses/>. + +libbackupdir=$(libdir)/beam +libbackup_SCRIPTS=\ + common.sh\ + s3.sh\ + fs.sh\ + mysql.sh\ + postgres.sh +EXTRA_DIST=\ + common.in\ + s3.sh\ + fs.sh\ + mysql.sh\ + postgres.sh +DISTCLEANFILES=common.sh +include $(top_srcdir)/Make.rules diff --git a/lib/beam/common.in b/lib/beam/common.in new file mode 100644 index 0000000..49b41eb --- /dev/null +++ b/lib/beam/common.in @@ -0,0 +1,135 @@ +#! /bin/bash +# This file is part of BEAM +# Copyright (C) 2012 Sergey Poznyakoff +# +# BEAM is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# BEAM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BEAM. If not, see <http://www.gnu.org/licenses/>. + +# Force C locale +LC_ALL=C +export LC_ALL + +prologue_hook= +epilogue_hook= + +# User configuration variables +backup_tar_options= +backup_suffix= +backup_archive_dir= +backup_snapshot_dir= +backup_verbose= +backup_logfile="/var/log/backup" + +error() { + echo >&2 $0: $* +} + +logit() { + echo `date`: $* +} + +abend() { + ec=$1 + shift + error $@ + exit $ec +} + +tarcode() { + case $1 in + 0) echo "`date`: success";; + 1) echo "`date`: some files changed while being archived";; + 2) echo "`date`: fatal error occurred, but trying to continue anyway" + tarerror=$((tarerror + 1));; + *) echo "`date`: unexpected error code $1" + tarerror=$((tarerror + 1)); + esac +} + +load_config() { + local delayed_exit + + test -z "$BEAM_CONFIG" && BEAM_CONFIG=@SYSCONFDIR@/beam.conf + if [ -r $BEAM_CONFIG ]; then + . $BEAM_CONFIG + else + abend 1 "configuration file $BEAM_CONFIG does not exist or is unreadable" + fi + + if [ -z "$backup_items" ]; then + abend 1 "backup_items not specified" + fi + + delayed_exit= + loaded_types= + for item in $backup_items + do + eval type=\$${item}_type + if [ -z "$type" ]; then + error "${item}_type not set" + delayed_exit=1 + continue + fi + + if echo "$loaded_types" | grep -wq $type; then + : + elif [ -x $libdir/${type}.sh ]; then + . $libdir/${type}.sh || delayed_exit=1 + loaded_types="$loaded_files +$type" + else + error "$libdir/${type}.sh not found" + delayed_exit=1 + fi + + ${type}_check $item || delayed_exit=1 + done + + test -n "$delayed_exit" && abend 1 "aborting" + + tar_suffix=${backup_suffix:-.tar} + + if [ -n "$backup_bucket_name" ]; then + . @LIBDIR@/beam/s3.sh + prologue_hook="$prologue_hook s3_mount" + fi +} + +runhook() { + local hook_list + + eval hook_list=\$$1 + for hook in $hook_list + do + $hook + done +} + +print_version() { + name=$(basename $0) + cat <<EOF +$name (@PACKAGE_NAME@) @PACKAGE_VERSION@ +Copyright (C) 2012 Sergey Poznyakoff +License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +EOF + exit 0 +} + +wtf() { + l=`echo "${1#beam-}"|sed "s|.|.|g"` + s=$(echo " " | sed "s|$l|${1#beam-}|") + shift + echo " $s $@" +} diff --git a/lib/beam/fs.sh b/lib/beam/fs.sh new file mode 100755 index 0000000..6d7d4c7 --- /dev/null +++ b/lib/beam/fs.sh @@ -0,0 +1,107 @@ +#! /bin/bash +# This file is part of BEAM +# Copyright (C) 2012 Sergey Poznyakoff +# +# BEAM is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# BEAM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BEAM. If not, see <http://www.gnu.org/licenses/>. + +# initdb item +# Initializes snapshot for the given basename. +initdb() { + local filename + + if [ -n "$dry_run" ]; then + logit "initializing snapshot for $1" + return + fi + + if [ $level -eq 0 ]; then + filename=$backup_snapshot_dir/$1-$week-$round-$level.db + test -r $filename && rm $filename + else + if [ $level -eq 1 ]; then + filename=$backup_snapshot_dir/$1-$week-0-0.db + else + filename=$backup_snapshot_dir/$1-$week-$round-$((level - 1)).db + fi + if [ -r $filename ]; then + cp $filename $backup_snapshot_dir/$1-$week-$round-$level.db + else + abend 1 "previous snapshot file $filename not found; cannot backup at level $level" + exit 1 + fi + fi +} + +# fs_check item +fs_check() { + local rc=0 + + eval root=\$${1}_dir + eval files=\$${1}_files + + test -z "$root" && rc=1 && error "${1}_dir not set" + test -z "$files" && rc=1 && error "${1}_files not set" + return $rc +} + +# fs_backup item +fs_backup() { + local basename text root files + + basename=$1-$week-$round-$level + eval text=\$${1}_text + eval root=\$${1}_dir + eval files=\$${1}_files + + test -z "$root" && abend 1 "${1}_dir not set" + test -z "$files" && abend 1 "${1}_files not set" + test -z "$text" && text="$1" + initdb $1 + logit "backing up $text ($basename.$tar_suffix)" + $dry_run tar $verbose $taroptions \ + -f $backup_archive_dir/$basename.$tar_suffix \ + --listed=$backup_snapshot_dir/$basename.db \ + -C $root $files + tarcode $? +} + +# fs_restore item +fs_restore() { + local i text root files tarcommand + + eval text=\$${1}_text + eval root=\$${1}_dir + eval files=\$${1}_files + + test -z "$root" && abend 1 "${1}_dir not set" + test -z "$files" && abend 1 "${1}_files not set" + test -z "$text" && text="$1" + + tarcommand="tar $verbose $taroptions -C $root --listed-incremental=/dev/null -f" + + logit "restoring $text" + logit "restoring from level 0 backup" + + $dry_run $tarcommand $backup_archive_dir/$1-$week-0-0.$tar_suffix + tarcode $? + + for i in $(seq 1 $level) + do + logit "restoring from the $round/$i backup" + $dry_run $tarcommand $backup_archive_dir/$1-$week-$round-$i.tar.bz2 + tarcode $? + done + logit "finished restoring $text" +} + diff --git a/lib/beam/mysql.sh b/lib/beam/mysql.sh new file mode 100644 index 0000000..c33adb0 --- /dev/null +++ b/lib/beam/mysql.sh @@ -0,0 +1,91 @@ +#! /bin/bash +# This file is part of BEAM +# Copyright (C) 2012 Sergey Poznyakoff +# +# BEAM is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# BEAM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BEAM. If not, see <http://www.gnu.org/licenses/>. + +# mysql_check item +mysql_check() { + eval database=\$${1}_database + test -z "$database" && error "${1}_database not set" && return 1 + return 0 +} + +# mysql_backup item +mysql_backup() { + local database + + logit "backing up MySQL database $1" + eval database=\$${1}_database + test -z "$database" && abend 1 "${1}_database not set" + cmd="mysqldump" + eval defaults_file=\$${1}_defaults_file + if [ -n "$defaults_file" ]; then + cmd="$cmd --defaults-file=$defaults_file" + fi + cmd="$cmd --add-drop-database --databases" + if [ -z "$dry_run" ]; then + $cmd $database > $backup_snapshot_dir/$1-$week-$round-$level + else + echo "$cmd $database > $backup_snapshot_dir/$1-$week-$round-$level" + fi + + if [ $? -ne 0 ]; then + tarerror=$((tarerror + 1)) + echo >&2 "`date`: failed" + else + echo "`date`: creating $1-$week-$round-$level.$tar_suffix" + $dry_run tar $verbose $taroptions \ + -f $backup_archive_dir/$1-$week-$round-$level.$tar_suffix \ + -C $backup_snapshot_dir $1-$week-$round-$level + tarcode $? + $dry_run rm $backup_snapshot_dir/dbdump-$week-$round-$level + fi +} + +mysql_restore() { + local u database + + eval database=\$${1}_database + logit "restoring MySQL database $database" + u=$(umask) + trap "umask $u" 1 2 3 13 15 + umask 077 + $dry_run tar $verbose $taroptions \ + -f $backup_archive_dir/$1-$week-$round-$level.$tar_suffix + e=$? + tarcode $e + if [ $e -eq 0 ]; then + logit "restoring database from the dump" + cmd="mysql -A --batch" + eval defaults_file=\$${1}_defaults_file + if [ -n "$defaults_file" ]; then + cmd="$cmd --defaults-file=$defaults_file" + fi + if [ -n "$dry_run" ]; then + echo "$cmd < $1-$week-$round-$level" + elif [ -r $1-$week-$round-$level ]; then + $cmd < $1-$week-$round-$level > db-$1.log + if grep ERROR db-$1.log >/dev/null; then + error "errors occurred during restore; see db-$1.log for details" + error "dump preserved in file $1-$week-$round-$level" + tarerror=$((tarerror + 1)) + else + rm $1-$week-$round-$level + fi + fi + fi + umask $u + trap - 1 2 3 13 15 +} diff --git a/lib/beam/postgres.sh b/lib/beam/postgres.sh new file mode 100755 index 0000000..0072da6 --- /dev/null +++ b/lib/beam/postgres.sh @@ -0,0 +1,87 @@ +#! /bin/bash +# This file is part of BEAM +# Copyright (C) 2012 Sergey Poznyakoff +# +# BEAM is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# BEAM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BEAM. If not, see <http://www.gnu.org/licenses/>. + +# postgres_check item +postgres_check() { + eval database=\$${1}_database + test -z "$database" && error "${1}_database not set" && return 1 + return 0 +} + +# postgres_backup item +postgres_backup() { + local database + + logit "backing up PostgreSQL $1" + eval database=\$${1}_database + test -z "$database" && abend 1 "${1}_database not set" + if [ -z "$dry_run" ]; then + su postgres -c "pg_dump $verbose $database" > $backup_snapshot_dir/$1-$week-$round-$level + else + echo "su postgres -c \"pg_dump $verbose $database\" > $backup_snapshot_dir/$1-$week-$round-$level" + fi + + if [ $? -ne 0 ]; then + tarerror=$((tarerror + 1)) + echo >&2 "`date`: failed" + else + echo "`date`: creating $1-$week-$round-$level.$tar_suffix" + $dry_run tar $verbose $taroptions \ + -f $backup_archive_dir/$1-$week-$round-$level.$tar_suffix \ + -C $backup_snapshot_dir $1-$week-$round-$level + tarcode $? + $dry_run rm $backup_snapshot_dir/dbdump-$week-$round-$level + fi +} + +postgres_restore() { + local u database + + eval database=\$${1}_database + logit "restoring PostgreSQL database $database" + u=$(umask) + trap "umask $u" 1 2 3 13 15 + umask 077 + $dry_run tar $verbose $taroptions \ + -f $backup_archive_dir/$1-$week-$round-$level.$tar_suffix + e=$? + tarcode $e + if [ $e -eq 0 ]; then + logit "restoring database from the dump" + if [ -n "$dry_run" ]; then + cat <<-EOT + su postgres -c "dropdb $database" + su postgres -c "createdb $database" + su postgres -c "psql -d $database -f $1-$week-$round-$level" + rm $1-$week-$round-$level +EOT + elif [ -r $1-$week-$round-$level ]; then + su postgres -c "dropdb $database" + su postgres -c "createdb $database" + su postgres -c "psql -d $database -f $1-$week-$round-$level" > db-$1.log + if grep ERROR db-$1.log >/dev/null; then + error "errors occurred during restore; see db-$1.log for details" + error "dump preserved in file $1-$week-$round-$level" + tarerror=$((tarerror + 1)) + else + rm $1-$week-$round-$level + fi + fi + fi + umask $u + trap - 1 2 3 13 15 +} diff --git a/lib/beam/s3.sh b/lib/beam/s3.sh new file mode 100644 index 0000000..5f7235c --- /dev/null +++ b/lib/beam/s3.sh @@ -0,0 +1,85 @@ +# This file is part of BEAM -*- shell-script -*- +# Copyright (C) 2012 Sergey Poznyakoff +# +# BEAM is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# BEAM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BEAM. If not, see <http://www.gnu.org/licenses/>. + +# The configuration variable "backup_bucket_name" must contain the name +# of the s3 bucket to use. + +# Raw bucket is mounted to /mnt/s3backer +test -z "$backup_mp_s3backer" && backup_mp_s3backer=/mnt/s3backer +# The actual file system is mounted to /mnt/s3 +test -z "$backup_mp" && backup_mp_s3=/mnt/s3 + +# This variable is populated by s3_mount and is used by s3_unmount to unmount +# s3-backed file system. +umount_list="" + +s3_mount() { + # Make sure both mountpoints exist + test -d $backup_mp_s3backer || mkdir $backup_mp_s3backer + test -d $backup_mp_s3 || mkdir $backup_mp_s3backer + +# Sample mount output, split into several lines: +# http://finox-backup-fs.s3.amazonaws.com/ on /mnt/s3backer type fuse.s3backer +# (rw,nosuid,nodev,allow_other,default_permissions) + set -- $(mount -tfuse.s3backer | + awk '/https?:\/\/'$backup_bucket_name'/ { print $3 }') + if test -z "$1"; then + $dry_run s3backer $backup_s3backer_options \ + $backup_bucket_name $backup_mp_s3backer || + abend 1 "unable to mount $backup_bucket_name" + umount_list="$backup_mp_s3backer" + else + backup_mp_s3backer=$1 + fi + set -- $(mount | grep "^${backup_mp_s3backer}/file" | awk '{ print $3 }') + if test -z "$1"; then + case $(basename $0) in + beam-restore|restore) mountopt=",ro";; + beam-backup|backup) mountopt=",rw,data=writeback";; + beam-s3) ;; + *) error "called as $0: assuming default mount options" + esac + # NOTE: For ext4 add the journal_async_commit option. + $dry_run mount -oloop$mountopt $backup_mp_s3backer/file $backup_mp_s3 || + abend 1 "unable to mount $backup_mp_s3backer/file" + umount_list="$backup_mp_s3 $umount_list" + else + backup_mp_s3=$1 + fi + epilogue_hook="s3_unmount $epilogue_hook" +} + +s3_getmpoint() +{ + case $1 in + backer) + mount -tfuse.s3backer | + awk '/https?:\/\/'$backup_bucket_name'/ { print $3 }';; + s3) + mount | grep "^${backup_mp_s3backer}/file" | awk '{ print $3 }';; + *) + abent 1 "invalid usage of getmpoint" + esac +} + +s3_unmount() +{ + for id in s3 backer + do + mpoint=$(s3_getmpoint $id) + test -n "$mpoint" && umount $mpoint + done +} |