From 507ae64c3e886b82435cbc963594ed5e2af3458f Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Fri, 7 Feb 2020 11:53:08 +0100 Subject: Implement the "safe upgrade" mode In safe upgrade mode, all the necessary package tarballs are first downloaded to the spool directory. Once downloaded, the program starts upgrading using these tarballs. Each tarball is removed after being processed. The old ("incremental") mode is used if there is not enough disk space for full download. --- slackupgrade | 288 +++++++++++++++++++++++++++++++++++++++++++++++---------- slackupgrade.8 | 158 ++++++++++++++++++++++--------- 2 files changed, 353 insertions(+), 93 deletions(-) diff --git a/slackupgrade b/slackupgrade index 662b325..ff4d46f 100644 --- a/slackupgrade +++ b/slackupgrade @@ -1,9 +1,9 @@ #!/bin/bash # slackupgrade - full upgrade of a Slackware installation -# Copyright (C) 2019 Sergey Poznyakoff. +# Copyright (C) 2019-2020 Sergey Poznyakoff. # # Slackware-upgrade-system is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as published +# modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 3, or (at your option) # any later version. # @@ -20,6 +20,7 @@ set -e # Configuration directory : ${SLACKUPGRADE_CONFDIR:=/etc/slackupgrade} +: ${SLACKUPGRADE_PKGDIR:=/var/slackupgrade} # Slackware root directory rooturl= # Log file name @@ -41,10 +42,25 @@ install_packages= keep_file= # Max. count of backup files to keep max_backup=4 +# Display progress bar when downloading +progressbar=1 +# Operating mode. Possible values: +# INCR Incremental mode. Archive for each package is downloaded prior +# to its nstallation and removed immediately afterwards. This +# ensures minimal disk space requirements. +# SAFE Safe mode. All archives are first downloaded to +# SLACKUPGRADE_PKGDIR and then installed. Each archive is removed +# immediately after installing from it. This requires some 2.5G +# of disk space at the beginning, which will be freed by the end +# of upgrade. +# AUTO Select incremental mode if there is enough disk space. If not, +# ask the user if it is OK to proceed in incremental mode. +opmode=AUTO # Internal variables remote= tempdir=${TMP:-/tmp}/slackupg.$$ +strip_series=0 installed_list=$tempdir/installed.list avail_index=$tempdir/avail.index avail_list=$tempdir/avail.list @@ -132,6 +148,8 @@ function error() { fi } +# Abnormal termination: print error message, remove temporary directory +# and terminate with status 1. function abend() { error "$@" tempdir_remove @@ -143,14 +161,12 @@ function package_file_name() { if [ -z "$pkg" ]; then error "package $name not found in index (should not happen!)" else - echo $pkg - fi -} - -function package_name_md5sum() { - pkg=$(awk -vname=$1 '$1==name { print $3 }' $avail_index) + echo $pkg + fi } +# check_package_md5sum PKG ARCHIVE +# Verifies the MD5 sum of the ARCHIVE file for Slackware package PKG function check_package_md5sum() { if [ -n "$checksums" ]; then awk -vcname="$1" -vdname="$2" \ @@ -159,10 +175,14 @@ function check_package_md5sum() { fi } +# all_package_names +# Lists all packages from the Slackware distribution. function all_package_names() { grep -v '.*/kde[^/]*/' $avail_index | cut -d ' ' -f 1 } +# series_package_names S +# Lists the names of packages in Slackware series S. function series_package_names() { grep '.*/'"$1/" $avail_index | cut -d ' ' -f 1 } @@ -171,10 +191,14 @@ function catfile() { echo $rooturl/${1#./} } +# download_curl FILE URL +# Downloads file from URL to FILE using curl. function download_curl { curl $CURL_OPTIONS -L -sS -o$1 $2 } +# download_wget LOCAL URL +# Downloads file from URL to FILE using wget. function download_wget { if ! wget $WGET_OPTIONS -nv -o wget.log -O$1 $2; then grep -i "failed\|error" wget.log @@ -182,6 +206,8 @@ function download_wget { fi } +# dnfunc_init +# Initializes the downloader function to wget or curl. function dnfunc_init() { if [ -z "$dnfunc" ]; then if wget --version >/dev/null 2>&1; then @@ -194,34 +220,45 @@ function dnfunc_init() { fi } +# download FILE +# Downloads package FILE from the remote server. On success, returns the +# full pathname of the downloaded copy. function download() { - local name=$(basename $1) + local name=$SLACKUPGRADE_PKGDIR/$(basename $1) local url=$(catfile $1) if $dnfunc $name $url; then - echo $name + echo $name fi } +# getfile PKG [GPG] +# Retrieves the package archive file for the package PKG. If second argument +# is non-empty, verifies the GPG signature of the file. +# If checksums file is available, verifies also the MD5 checksum of the file. +# +# Returns full pathname of the retrieved file. function getfile() { local name=$(if [ -n "$remote" ]; then - download $1 - else - catfile $1 - fi) - + download $1 + elif [ $strip_series -eq 1 ]; then + catfile $(basename $1) + else + catfile $1 + fi) + if [ -n "$2" ]; then ascname=$(if [ -n "$remote" ]; then - download $1.asc - else - catfile $1.asc - fi) + download $1.asc + else + catfile $1.asc + fi) if [ -n "$ascname" ] \ && ${GPG:-gpg} --verify $ascname $name 2>/dev/null; then - : - else + : + else error "gpg verification failed for $name" return - fi + fi fi if [ -n "$checksums" ] && ! check_package_md5sum $1 $name; then @@ -231,12 +268,17 @@ function getfile() { echo $name } +# dropfile FILE +# Removes FILE, if it has been downloaded from the remote repository, function dropfile() { - if [ -n "$remote" ]; then + if [ -n "$remote" ] || [ $strip_series -eq 1 ]; then rm $1 fi -} +} +# upgrade_package [OPTIONS] FILE +# Runs upgradepkg with the supplied arguments and captures its output to +# the log file. In verbose mode, filters parts of it to the stdout. function upgrade_local() { if [ -n "$dry_run" ]; then echo "upgradepkg $@" @@ -249,6 +291,8 @@ function upgrade_local() { fi } +# upgrade_package NAME [OPTIONS] +# Upgrades the package NAME. OPTIONS will be passed to upgradepkg verbatim. function upgrade_package() { local name=$1 shift @@ -263,6 +307,8 @@ function upgrade_package() { fi } +# version_gt A B +# Returns true if version A is greater than B. function version_gt() { test "$(echo "$@" | tr " " "\n" | sort -V | head -n 1)" != "$1" } @@ -290,25 +336,79 @@ function backup() { sort +0 -1 -n -r |\ while read n stem do - if [ -n "$max_backup" ] && [ $n -ge $max_backup ]; then - $p rm $dir/${stem}~$n - else - $p mv $dir/${stem}~$n $dir/${stem}~$((n + 1)) - fi - done + if [ -n "$max_backup" ] && [ $n -ge $max_backup ]; then + $p rm $dir/${stem}~$n + else + $p mv $dir/${stem}~$n $dir/${stem}~$((n + 1)) + fi + done $p mv "$1~" "$1~1" - fi + fi $p mv "$1" "$1~" fi } +# disk_avail_size DIR +# Returns the available disk size (in kilobytes) on the device where +# DIR is located +function disk_avail_size() { + df -k --output=target,avail | \ + sed 1d | \ + awk -vdir=$1 \ + 'BEGIN { dirlen=length(dir) } + { len = length($1) + if (len <= dirlen && substr(dir, 1, len) == $1) { + if (mp_len < len) { + mp_len = len + mp_avail = $2 + } + } + } + END { print mp_avail }' +} + +# Downloads all selected packages to the spool directory. +function download_all() { + info "downloading packages" + if [ ${COLUMNS:-0} -lt 10 ]; then + progressbar=0 + fi + if [ $progressbar -eq 1 ]; then + total=$(wc -l candidates | cut -d ' ' -f 1) + width=$(( $COLUMNS - 7 )) + fi + i=0 + cat candidates | + while read pkg + do + if [ $progressbar -eq 1 ]; then + n=$(( $i * $width / $total )) + s=$(printf "%${n}s" ' '|sed 's/./=/g') + printf "\r% 3d%% %s>" $(( $i * 100 / $total)) $s + fi + if [ -z "$dry_run" ]; then + file=$(getfile $(package_file_name $pkg)) + if [ -z "$file" ]; then + abend "failed to download $pkg" + fi + fi + i=$(( $i + 1 )) + done || exit $? + if [ $progressbar -eq 1 ]; then + s=$(printf "%${width}s" ' '|sed 's/./=/g') + printf "\r% 3d%% %s|\n" 100 $s + fi +} + # ########## # Main # ########## -while getopts "ahknp:qs:vy" OPTION +while getopts "ahIknp:qSs:vy" OPTION do case $OPTION in + I) opmode=INCR;; + S) opmode=SAFE;; a) install_all=y;; h) usage exit 0;; @@ -354,6 +454,10 @@ case "$(uname -m)" in *) abend "architecture $(uname -m) is not yet supported" esac +if [ ! -d $SLACKUPGRADE_PKGDIR ]; then + mkdir -p $SLACKUPGRADE_PKGDIR || abend "can't create $SLACKUPGRADE_PKGDIR" +fi + tempdir_create cd $tempdir @@ -376,7 +480,7 @@ if [ -z "$rooturl" ]; then else abend "can't find distribution newer than $VERSION; please supply URL if you have any" fi -fi +fi # Check if rooturl is local or remote case $rooturl in @@ -394,6 +498,8 @@ if [ -z "$remote" ]; then if [ ! -d $rooturl ]; then abend "$rooturl does not exist" fi + # Safe mode is meaningless with local repository + opmode=INCR fi # Check if rooturl contains all we need @@ -409,14 +515,14 @@ fi announce=$(tail +13 $checksums | \ sed -n -r\ - -e 's/^[0-9a-fA-F]+[[:space:]]+(\.\/ANNOUNCE\.[[:digit:]_]+)$/\1/p') + -e 's/^[0-9a-fA-F]+[[:space:]]+(\.\/ANNOUNCE\.[[:digit:]_]+)$/\1/p') if [ -z "$announce" ]; then abend "ANNOUNCE not found in $rooturl" -fi +fi file=$(getfile $announce) if [ -z "$file" ]; then abend "file $announce not found in $rooturl" -fi +fi newversion=$(echo "$announce" | sed -e 's/\.\/ANNOUNCE\.//' -e 's/_/./g') if [ -z "$newversion" ]; then @@ -451,7 +557,7 @@ do abend "exiting" fi dropfile $file -done +done # Build a list of installed packages ls /var/log/packages |\ @@ -461,9 +567,9 @@ ls /var/log/packages |\ # Build a list of packages to install (if [ -n "$install_all" ]; then all_package_names -else +else comm -1 -2 $installed_list $avail_list -fi +fi for s in $install_series do series_package_names $s @@ -477,7 +583,7 @@ fi) | sort -u > candidates.$$ comm -2 -3 $installed_list candidates.$$ > remove.list.$$ if [ -s "$keep_file" ]; then grep -v '^#' $keep_file | \ - tr -s '\n' | sort | comm -2 -3 remove.list.$$ - > remove.list + tr -s '\n' | sort -u | comm -2 -3 remove.list.$$ - > remove.list rm remove.list.$$ else mv remove.list.$$ remove.list @@ -488,11 +594,11 @@ if [ -f "$SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl" ]; then info "reading $SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl" # Update candidates and save the replacement map in a temporary. awk '{ sub(/#.*$/,""); sub(/[[:space:]]+$/,"") } - /\\$/ { sub(/\\$/,""); p = p $0; next } - NF == 0 { if (p) print p; p = ""; next } - { print p $0; p="" }' \ + /\\$/ { sub(/\\$/,""); p = p $0; next } + NF == 0 { if (p) print p; p = ""; next } + { print p $0; p="" }' \ $SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl | \ - sort +0 -1 | \ + sort +0 -1 | \ tee rename.$$ | \ join - remove.list | \ cut -d ' ' -f 2- | \ @@ -501,9 +607,9 @@ if [ -f "$SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl" ]; then sort -u > candidates # Use the temporary to remove the original package names from the report. awk '{print $1}' rename.$$ | \ - comm -13 - remove.list > $remove_report + comm -13 - remove.list > $remove_report # Clean up - rm rename.$$ + rm rename.$$ else cp remove.list $remove_report mv candidates.$$ candidates @@ -519,7 +625,7 @@ restore them after the upgrade. The package names are preserved for you in the file $remove_report: EOF -cat $remove_report) | ${PAGER:-more} + cat $remove_report) | ${PAGER:-more} fi if ! version_gt $newversion $VERSION; then @@ -536,6 +642,89 @@ if ! getyn n "Ready for upgrade from $VERSION to $newversion. Continue"; then exit 0 fi +if [ $opmode != INCR ]; then + # Automatic mode selection. Compute total compressed size of packages + # to be installed and compare it with available disk space on the device + # hosting SLACKUPGRADE_PKGDIR. Select safe mode if there is enough space, + # otherwise ask if the user wishes to continue in incremental mode. + packages=$(download PACKAGES.TXT) + if [ -z "$packages" ]; then + abend "PACKAGES.TXT not found in $rooturl" + fi + download_size=$(sed -n -r \ + -e '/PACKAGE NAME:[[:space:]]*/{' \ + -e 's///' \ + -e 's/^(.*)-[^-]+-(i386|x86(_64)?|arm|noarch|fw)-[[:digit:]]+(_.*)?\.t.z$/\1 &/' \ + -e h \ + -e '}' \ + -e '/PACKAGE SIZE \(compressed\):[[:space:]]*/{' \ + -e 's///' \ + -e H \ + -e x \ + -e 's/\n/ /' \ + -e p \ + -e '}' \ + $packages | \ + sort +0 -1 | \ + join candidates - | \ + awk '{ if ($4 == "M") s += $3*1024 + else if ($4 == "G") s += $3*1024*1024 + else s += $3 } + END { print s }') + avail_size=$(disk_avail_size $SLACKUPGRADE_PKGDIR) + if [ $avail_size -lt $download_size ]; then + error "not enough free space in $SLACKUPGRADE_PKGDIR" + error "needed ${download_size}K, but only ${avail_size}K available" + if [ $opmode = INCR ]; then + abend "terminating" + fi + + cat >&2 <&2 <. -.TH SLACKUPGRADE 8 "September 23, 2019" "SLACKUPGRADE" "System Manager's Manual" +.TH SLACKUPGRADE 8 "February 7, 2020" "SLACKUPGRADE" "System Manager's Manual" .SH NAME -slackwupgrade \- do a full upgrade of a Slackware installation +slackupgrade \- do a full upgrade of a Slackware installation .SH SYNOPSIS .nh .na \fBslackupgrade\fR\ - [\fB\-anqvy\fR]\ + [\fB\-aInSqvy\fR]\ [\fB\-k \fIFILE\fR]\ [\fB\-p \fIPACKAGE\fR\]\ [\fB\-s \fISERIES\fR]\ @@ -48,7 +48,7 @@ directories. The file \fBCHECKSUMS.md5\fR and its GPG signature are downloaded first. Then, the program verifies that the signature is correct. For this to succeed, you must have the Slackware Linux Project -public key in your keyring. If you don't, run +public key in your GPG keyring. If you don't, run .PP .EX curl -o - https://www.slackware.com/gpg-key | gpg --import @@ -56,22 +56,15 @@ curl -o - https://www.slackware.com/gpg-key | gpg --import .PP When this initial check is passed, the program constructs two lists of packages: a list of currently installed packages and a list of -packages available in the distribution. When constructing the list of -available packages, known differences between Slackware releases are -taken into account. For example, consider upgrade from version 14.1 to -14.2. It is known that the \fIportmap\fR package from 14.1 is -replaced with the \fIrpcbind\fR in version 14.2. Consequently, if the -program sees that \fIportmap\fR is installed on the system, it will -include \fIrpcbind\fR to the list of installation candidates. -Information about package differences in various versions is kept in -\fIreplacement map files\fR. See the section \fBREPLACEMENT MAP\fR, -for a discussion of these files. +packages available in the distribution. The intersection of these two +is the list of \fIupgrade candidates\fR, i.e. packages that will be +upgraded. .PP -The difference between these two lists, is the set of installed +The difference between these lists is the set of installed packages that have no equivalent in the available package list. Those are \fIorphaned packages\fR, which were either removed from the Slackware distribution, or were installed from third-party sources. It is -unpredictable whether or not these will work on the newly upgraded +unpredictable whether these will work on the newly upgraded system, therefore they will be removed after a successful upgrade. Before proceeding, the program will display this list on the screen and save it in file @@ -82,32 +75,21 @@ for your consideration (here, \fIOLD\fR and \fINEW\fR stand for the current and new Slackware version numbers, correspondingly). After the upgrade, you can re-install them, if necessary. .PP -After this step, the program will print the current Slackware version -and the version you are going to upgrade to, and will ask you to -confirm that you really want to upgrade. This is the right moment to -quit if you decide to modify program invocation in order to handle -orphaned packages. +Known differences between Slackware releases are taken into account +when building lists of upgrade and delete candidates. For example, +consider an upgrade from version 14.1 to 14.2. It is known that the +\fIportmap\fR package from 14.1 is replaced with the \fIrpcbind\fR in +version 14.2. Consequently, if the program sees that \fIportmap\fR is +installed on the system, it will include \fIrpcbind\fR to the list of +installation candidates. Information about package differences in +various versions is kept in \fIreplacement map files\fR. See the +section \fBREPLACEMENT MAP\fR, for a discussion of these files. .PP -If you do, type \fBno\fR. You have two options. First, if there are -any orphaned packages that you want to keep in place, create a -\fIkeep-list\fR file. This file should contain names of those -packages, each name on a separate line. When you restart the program -use the \fB\-k \fIFILE\fR\fR option to instruct it to use this file. -.PP -Secondly, if you are upgrading to the version for which there is no -replacement map, there can be replacement packages for some of the -orphaned ones. You can create a replacement map and save it to -the \fB/etc/slackupgrade\fR directory (see the section -\fBREPLACEMENT MAP\fR for details). If you do, please drop me a note -so that your changes become available for other users (see the -\fBBUGS\fR section, for contact information). You can also use the -\fB\-p\fR command line option to provide the names of replacement -packages from the command line. -.PP -By default, the program will upgrade only the packages actually -installed on your system. You can request to install additional -series from the distribution with the \fB\-s\fR option. E.g., it is -often a good idea to install all packages from series \fBl\fR, like that: +In addition to upgrading already installed packages you can request +the program to install additional series or individual packages +from the distribution with the \fB\-s\fR and \fB\-p\fR options. E.g., +it is often a good idea to install all packages from series \fBl\fR (ell), +like that: .PP .EX slackupgrade \-s l @@ -117,6 +99,51 @@ Otherwise, you can request to install all missing packages from all series, excepting \fBkde*\fR, by running the command with the \fB\-a\fR option. .PP +Once the package lists are ready, the program prints the current +Slackware version and the version it is going to upgrade to, and asks +you to confirm that you really want to upgrade. This is the right +moment to quit if you decide to modify program invocation in order to +handle orphaned packages. See the subsection +.B Handling of orphaned packages +for a discussion. +.PP +If you decide to continue and \fBslackupgrade\fR uses the remote +repository, it will select the upgrade mode to use. There are two +possibilities: +.TP +.I Incremental mode +In this mode, the archive for each package will be downloaded +immediately prior to its installation and removed afterwards. The +only exception are several basic packages that are needed for the +upgrade process and which will be downloaded and installed in the +first place. + +This mode requires little additional disk space, but it is somewhat +risky. If something happens to your internet connection during the +upgrade, you'll end up with partly upgraded (and consequently, +unstable) system. +.TP +.I Safe mode +In safe mode, \fBslackupgrade\fR first downloads all needed packages +from the remote repository and then proceeds to upgrade using local +disk copies. Downloaded files are removed when no longer needed, i.e. +immediately after the corresponding package has been installed. + +This mode requires around 2.5G of free disk space on the device which +hosts the \fBslackupgrade\fR spool directory (by default -- +.BR /var/slackupgrade ). +You can select another location by setting the +\fBSLACKUPGRADE_PKGDIR\fR environment variable to the full pathname +of the spool directory you wish to use. + +The advantage of the safe mode is that even if something goes wrong +during the upgrade, you have all the necessary stuff at hand to resume +upgrade from where you left off. +.PP +Safe mode is assumed if there is enough disk space. Otherwise, the +program will ask you whether you wish to continue in incremental mode +and will act accordingly. +.PP At the end of the run, the program prints additional instructions and leaves the detailed log in file .nh @@ -131,6 +158,24 @@ conflict with the existing files, they will be stored alongside the original files, with the \fB.new\fR extension. The list of these files will be stored in file \fB/var/log/slackupgrade\-\fIOLD\fB\-\fINEW\fB.new\fR. +.SS Handling of orphaned packages +There are two possible ways to handle installed packages that have no +installation candidate in the new Slackware release. + +First, if you want to keep in place any of them, create a +\fIkeep-list\fR file. This file is a list of package names, each on a +separate line. When you start \fBslackupgrade\fR, use the +\fB\-k \fIFILE\fR\fR option to instruct it to use this file. +.PP +Secondly, if you are upgrading to the version for which there is no +replacement map, there can be replacement packages for some of the +orphaned ones. You can create a replacement map and save it to +the \fB/etc/slackupgrade\fR directory (see the section +\fBREPLACEMENT MAP\fR for details). If you do, please drop me a note +so that your changes become available for other users (see the +\fBBUGS\fR section, for contact information). You can also use the +\fB\-p\fR command line option to provide the names of replacement +packages from the command line. .SH OPTIONS .TP .B \-a @@ -139,6 +184,9 @@ Install all series except \fBkde*\fR. .B \-h Display a short help summary and exit. .TP +.B \-I +Force using incremental mode (see also \fB\-S\fR). +.TP \fB\-k \fIFILE\fR After successful upgrade, \fBslackupgrade\fR will remove previously installed packages that are not available in the new @@ -169,6 +217,9 @@ Install additional package. .B \-q Quiet mode: suppress all messages, except error diagnostics. .TP +.B \-S +Force using safe mode. Abort if not enough disk space is available. +.TP \fB\-s \fISERIES\fR Additionally install all packages from \fISERIES\fR. .sp @@ -219,6 +270,12 @@ nearest newline character. Empty lines and comments are ignored. \fB/etc/slackupgrade/\fIOLD\fB\-\fINEW\fB.repl\fR Replacement map for upgrades from version \fIOLD\fR to \fINEW\fR. .TP +.B /var/slackupgrade +Default spool directory. This is where all downloaded files are +stored. You can override the default location by setting the +.B SLACKUPGRADE_PKGDIR +environment variable. +.TP \fB/var/log/slackupgrade\-\fIOLD\fB\-\fINEW\fB.log\fR Detailed log of operations performed during the upgrade from version \fIOLD\fR to \fINEW\fR. @@ -250,6 +307,21 @@ name. E.g. \fBslackupgrade-14.1-14.2.log\fR is renamed to using the following pattern: \fBX~\fR becomes \fBX~1\fR, \fBX~1\fR becomes \fBX~2\fR and so on. At most five backup copies are kept (from \fBX~\fR up to \fBX~4\fR). +.SH ENVIRONMENT +.TP +.B SLACKUPGRADE_CONFDIR +Location of the configuration directory. This is the place where +replacement maps are stored. The default is +.BR /etc/slackupgrade . +.TP +.B SLACKUPGRADE_PKGDIR +Location of the \fBslackupgrade\fR package spool directory. This is +the temporary storage for downloaded package tarballs. The default is +.BR /var/slackupgrade . +.TP +.B TMP +Location of the temporary directory. The default is +.BR /tmp . .SH "SEE ALSO" The \fBUPGRADE.TXT\fR document, outlining the procedure as a whole: .nh @@ -263,7 +335,7 @@ subdirectory is not used. .PP Report bugs to . .SH COPYRIGHT -Copyright \(co 2019 Sergey Poznyakoff +Copyright \(co 2019 \(em 2020 Sergey Poznyakoff .br .na License GPLv3+: GNU GPL version 3 or later -- cgit v1.2.1