Implement the "safe upgrade" mode

In safe upgrade mode, all the necessary package tarballs are first
downloaded to the spool directory.  Once downloaded, the program
starts upgrading using these tarballs.  Each tarball is removed after
being processed.

The old ("incremental") mode is used if there is not enough disk space
for full download.

1 files changed, 238 insertions, 50 deletions

diff --git a/slackupgrade b/slackupgrade
index 662b325..ff4d46f 100644
--- a/slackupgrade
+++ b/slackupgrade
@@ -1,9 +1,9 @@
 #!/bin/bash
 # slackupgrade - full upgrade of a Slackware installation
-# Copyright (C) 2019 Sergey Poznyakoff.
+# Copyright (C) 2019-2020 Sergey Poznyakoff.
 #
 # Slackware-upgrade-system is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published 
+# modify it under the terms of the GNU General Public License as published
 # by the Free Software Foundation; either version 3, or (at your option)
 # any later version.
 #
@@ -20,6 +20,7 @@ set -e
 
 # Configuration directory
 : ${SLACKUPGRADE_CONFDIR:=/etc/slackupgrade}
+: ${SLACKUPGRADE_PKGDIR:=/var/slackupgrade}
 # Slackware root directory
 rooturl=
 # Log file name
@@ -41,10 +42,25 @@ install_packages=
 keep_file=
 # Max. count of backup files to keep
 max_backup=4
+# Display progress bar when downloading
+progressbar=1
+# Operating mode. Possible values:
+#  INCR       Incremental mode.  Archive for each package is downloaded prior
+#             to its nstallation and removed immediately afterwards.  This
+#             ensures minimal disk space requirements.
+#  SAFE       Safe mode.  All archives are first downloaded to
+#             SLACKUPGRADE_PKGDIR and then installed.  Each archive is removed
+#             immediately after installing from it.  This requires some 2.5G
+#             of disk space at the beginning, which will be freed by the end
+#             of upgrade.
+#  AUTO       Select incremental mode if there is enough disk space.  If not,
+#             ask the user if it is OK to proceed in incremental mode.
+opmode=AUTO
 
 # Internal variables
 remote=
 tempdir=${TMP:-/tmp}/slackupg.$$
+strip_series=0
 installed_list=$tempdir/installed.list
 avail_index=$tempdir/avail.index
 avail_list=$tempdir/avail.list
@@ -132,6 +148,8 @@ function error() {
     fi
 }
 
+# Abnormal termination: print error message, remove temporary directory
+# and terminate with status 1.
 function abend() {
     error "$@"
     tempdir_remove
@@ -143,14 +161,12 @@ function package_file_name() {
     if [ -z "$pkg" ]; then
 	error "package $name not found in index (should not happen!)"
     else
-        echo $pkg
-    fi	    
-}
-
-function package_name_md5sum() {
-    pkg=$(awk -vname=$1 '$1==name { print $3 }' $avail_index)
+	echo $pkg
+    fi
 }
 
+# check_package_md5sum PKG ARCHIVE
+# Verifies the MD5 sum of the ARCHIVE file for Slackware package PKG
 function check_package_md5sum() {
     if [ -n "$checksums" ]; then
 	awk -vcname="$1" -vdname="$2" \
@@ -159,10 +175,14 @@ function check_package_md5sum() {
     fi
 }
 
+# all_package_names
+# Lists all packages from the Slackware distribution.
 function all_package_names() {
     grep -v '.*/kde[^/]*/' $avail_index | cut -d ' ' -f 1
 }
 
+# series_package_names S
+# Lists the names of packages in Slackware series S.
 function series_package_names() {
     grep '.*/'"$1/" $avail_index | cut -d ' ' -f 1
 }
@@ -171,10 +191,14 @@ function catfile() {
     echo $rooturl/${1#./}
 }
 
+# download_curl FILE URL
+# Downloads file from URL to FILE using curl.
 function download_curl {
     curl $CURL_OPTIONS -L -sS -o$1 $2
 }
 
+# download_wget LOCAL URL
+# Downloads file from URL to FILE using wget.
 function download_wget {
     if ! wget $WGET_OPTIONS -nv -o wget.log -O$1 $2; then
 	grep -i "failed\|error" wget.log
@@ -182,6 +206,8 @@ function download_wget {
     fi
 }
 
+# dnfunc_init
+# Initializes the downloader function to wget or curl.
 function dnfunc_init() {
     if [ -z "$dnfunc" ]; then
 	if wget --version >/dev/null 2>&1; then
@@ -194,34 +220,45 @@ function dnfunc_init() {
     fi
 }
 
+# download FILE
+# Downloads package FILE from the remote server.  On success, returns the
+# full pathname of the downloaded copy.
 function download() {
-    local name=$(basename $1)
+    local name=$SLACKUPGRADE_PKGDIR/$(basename $1)
     local url=$(catfile $1)
     if $dnfunc $name $url; then
-        echo $name
+	echo $name
     fi
 }
 
+# getfile PKG [GPG]
+# Retrieves the package archive file for the package PKG.  If second argument
+# is non-empty, verifies the GPG signature of the file.
+# If checksums file is available, verifies also the MD5 checksum of the file.
+#
+# Returns full pathname of the retrieved file.
 function getfile() {
     local name=$(if [ -n "$remote" ]; then
-	            download $1
-                 else
-	            catfile $1
-                 fi)
-    
+		    download $1
+		 elif [ $strip_series -eq 1 ]; then
+		    catfile $(basename $1)
+		 else
+		    catfile $1
+		 fi)
+
     if [ -n "$2" ]; then
 	ascname=$(if [ -n "$remote" ]; then
-	            download $1.asc
-                  else
-	            catfile $1.asc
-                  fi)
+		    download $1.asc
+		  else
+		    catfile $1.asc
+		  fi)
 	if [ -n "$ascname" ] \
 	    && ${GPG:-gpg} --verify $ascname $name 2>/dev/null; then
-  	    :
-        else
+	    :
+	else
 	    error "gpg verification failed for $name"
 	    return
-    	fi
+	fi
     fi
 
     if [ -n "$checksums" ] && ! check_package_md5sum $1 $name; then
@@ -231,12 +268,17 @@ function getfile() {
     echo $name
 }
 
+# dropfile FILE
+# Removes FILE, if it has been downloaded from the remote repository,
 function dropfile() {
-    if [ -n "$remote" ]; then
+    if [ -n "$remote" ] || [ $strip_series -eq 1 ]; then
 	rm $1
     fi
-}    
+}
 
+# upgrade_package [OPTIONS] FILE
+# Runs upgradepkg with the supplied arguments and captures its output to
+# the log file.  In verbose mode, filters parts of it to the stdout.
 function upgrade_local() {
     if [ -n "$dry_run" ]; then
 	echo "upgradepkg $@"
@@ -249,6 +291,8 @@ function upgrade_local() {
     fi
 }
 
+# upgrade_package NAME [OPTIONS]
+# Upgrades the package NAME.  OPTIONS will be passed to upgradepkg verbatim.
 function upgrade_package() {
     local name=$1
     shift
@@ -263,6 +307,8 @@ function upgrade_package() {
     fi
 }
 
+# version_gt A B
+# Returns true if version A is greater than B.
 function version_gt() {
     test "$(echo "$@" | tr " " "\n" | sort -V | head -n 1)" != "$1"
 }
@@ -290,25 +336,79 @@ function backup() {
 		sort +0 -1 -n -r |\
 		while read n stem
 		do
-	            if [ -n "$max_backup" ] && [ $n -ge $max_backup ]; then
-		        $p rm $dir/${stem}~$n
-	            else
-	                $p mv $dir/${stem}~$n $dir/${stem}~$((n + 1))
-	            fi
-	        done
+		    if [ -n "$max_backup" ] && [ $n -ge $max_backup ]; then
+			$p rm $dir/${stem}~$n
+		    else
+			$p mv $dir/${stem}~$n $dir/${stem}~$((n + 1))
+		    fi
+		done
 	    $p mv "$1~" "$1~1"
-        fi
+	fi
 	$p mv "$1" "$1~"
     fi
 }
 
+# disk_avail_size DIR
+# Returns the available disk size (in kilobytes) on the device where
+# DIR is located
+function disk_avail_size() {
+    df -k --output=target,avail | \
+      sed 1d | \
+      awk -vdir=$1 \
+	  'BEGIN { dirlen=length(dir) }
+	   { len = length($1)
+	     if (len <= dirlen && substr(dir, 1, len) == $1) {
+		 if (mp_len < len) {
+		     mp_len = len
+		     mp_avail = $2
+		 }
+	     }
+	   }
+	   END { print mp_avail }'
+}
+
+# Downloads all selected packages to the spool directory.
+function download_all() {
+    info "downloading packages"
+    if [ ${COLUMNS:-0} -lt 10 ]; then
+	progressbar=0
+    fi
+    if [ $progressbar -eq 1 ]; then
+	total=$(wc -l candidates | cut -d ' ' -f 1)
+	width=$(( $COLUMNS - 7 ))
+    fi
+    i=0
+    cat candidates |
+	 while read pkg
+	 do
+	     if [ $progressbar -eq 1 ]; then
+		 n=$(( $i * $width / $total ))
+		 s=$(printf "%${n}s" ' '|sed 's/./=/g')
+		 printf "\r% 3d%% %s>" $(( $i * 100 / $total)) $s
+	     fi
+	     if [ -z "$dry_run" ]; then
+		 file=$(getfile $(package_file_name $pkg))
+		 if [ -z "$file" ]; then
+		     abend "failed to download $pkg"
+		 fi
+	     fi
+	     i=$(( $i + 1 ))
+	 done || exit $?
+    if [ $progressbar -eq 1 ]; then
+	s=$(printf "%${width}s" ' '|sed 's/./=/g')
+	printf "\r% 3d%% %s|\n" 100 $s
+    fi
+}
+
 # ##########
 # Main
 # ##########
 
-while getopts "ahknp:qs:vy" OPTION
+while getopts "ahIknp:qSs:vy" OPTION
 do
     case $OPTION in
+	I) opmode=INCR;;
+	S) opmode=SAFE;;
 	a) install_all=y;;
 	h) usage
 	   exit 0;;
@@ -354,6 +454,10 @@ case "$(uname -m)" in
     *)      abend "architecture $(uname -m) is not yet supported"
 esac
 
+if [ ! -d $SLACKUPGRADE_PKGDIR ]; then
+    mkdir -p $SLACKUPGRADE_PKGDIR || abend "can't create $SLACKUPGRADE_PKGDIR"
+fi
+
 tempdir_create
 cd $tempdir
 
@@ -376,7 +480,7 @@ if [ -z "$rooturl" ]; then
     else
 	abend "can't find distribution newer than $VERSION; please supply URL if you have any"
     fi
-fi    
+fi
 
 # Check if rooturl is local or remote
 case $rooturl in
@@ -394,6 +498,8 @@ if [ -z "$remote" ]; then
     if [ ! -d $rooturl ]; then
 	abend "$rooturl does not exist"
     fi
+    # Safe mode is meaningless with local repository
+    opmode=INCR
 fi
 
 # Check if rooturl contains all we need
@@ -409,14 +515,14 @@ fi
 
 announce=$(tail +13 $checksums | \
        sed -n -r\
-           -e 's/^[0-9a-fA-F]+[[:space:]]+(\.\/ANNOUNCE\.[[:digit:]_]+)$/\1/p')
+	   -e 's/^[0-9a-fA-F]+[[:space:]]+(\.\/ANNOUNCE\.[[:digit:]_]+)$/\1/p')
 if [ -z "$announce" ]; then
     abend "ANNOUNCE not found in $rooturl"
-fi    
+fi
 file=$(getfile $announce)
 if [ -z "$file" ]; then
     abend "file $announce not found in $rooturl"
-fi    
+fi
 
 newversion=$(echo "$announce" | sed -e 's/\.\/ANNOUNCE\.//' -e 's/_/./g')
 if [ -z "$newversion" ]; then
@@ -451,7 +557,7 @@ do
 	abend "exiting"
     fi
     dropfile $file
-done    
+done
 
 # Build a list of installed packages
 ls /var/log/packages |\
@@ -461,9 +567,9 @@ ls /var/log/packages |\
 # Build a list of packages to install
 (if [ -n "$install_all" ]; then
     all_package_names
-else    
+else
     comm -1 -2 $installed_list $avail_list
-fi    
+fi
 for s in $install_series
 do
     series_package_names $s
@@ -477,7 +583,7 @@ fi) | sort -u  > candidates.$$
 comm -2 -3 $installed_list candidates.$$ > remove.list.$$
 if [ -s "$keep_file" ]; then
     grep -v '^#' $keep_file | \
-	tr -s '\n' | sort | comm -2 -3 remove.list.$$ - > remove.list
+	tr -s '\n' | sort -u | comm -2 -3 remove.list.$$ - > remove.list
     rm remove.list.$$
 else
     mv remove.list.$$ remove.list
@@ -488,11 +594,11 @@ if [ -f "$SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl" ]; then
     info "reading $SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl"
     # Update candidates and save the replacement map in a temporary.
     awk '{ sub(/#.*$/,""); sub(/[[:space:]]+$/,"") }
-         /\\$/ { sub(/\\$/,""); p = p $0; next }
-         NF == 0 { if (p) print p; p = ""; next }
-         { print p $0; p="" }' \
+	 /\\$/ { sub(/\\$/,""); p = p $0; next }
+	 NF == 0 { if (p) print p; p = ""; next }
+	 { print p $0; p="" }' \
 	     $SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl | \
-        sort +0 -1 | \
+	sort +0 -1 | \
 	tee rename.$$ | \
 	join - remove.list | \
 	cut -d ' ' -f 2- | \
@@ -501,9 +607,9 @@ if [ -f "$SLACKUPGRADE_CONFDIR/$VERSION-$newversion.repl" ]; then
 	sort -u > candidates
     # Use the temporary to remove the original package names from the report.
     awk '{print $1}' rename.$$ | \
-        comm -13 - remove.list > $remove_report
+	comm -13 - remove.list > $remove_report
     # Clean up
-    rm rename.$$    
+    rm rename.$$
 else
     cp remove.list $remove_report
     mv candidates.$$ candidates
@@ -519,7 +625,7 @@ restore them after the upgrade.  The package names are preserved for you
 in the file $remove_report:
 
 EOF
-cat $remove_report) | ${PAGER:-more}
+     cat $remove_report) | ${PAGER:-more}
 fi
 
 if ! version_gt $newversion $VERSION; then
@@ -536,6 +642,89 @@ if ! getyn n "Ready for upgrade from $VERSION to $newversion. Continue"; then
     exit 0
 fi
 
+if [ $opmode != INCR ]; then
+    # Automatic mode selection.  Compute total compressed size of packages
+    # to be installed and compare it with available disk space on the device
+    # hosting SLACKUPGRADE_PKGDIR.  Select safe mode if there is enough space,
+    # otherwise ask if the user wishes to continue in incremental mode.
+    packages=$(download PACKAGES.TXT)
+    if [ -z "$packages" ]; then
+	abend "PACKAGES.TXT not found in $rooturl"
+    fi
+    download_size=$(sed -n -r \
+	-e '/PACKAGE NAME:[[:space:]]*/{' \
+	-e   's///' \
+	-e   's/^(.*)-[^-]+-(i386|x86(_64)?|arm|noarch|fw)-[[:digit:]]+(_.*)?\.t.z$/\1 &/' \
+	-e   h \
+	-e '}' \
+	-e '/PACKAGE SIZE \(compressed\):[[:space:]]*/{' \
+	-e   's///' \
+	-e   H \
+	-e   x \
+	-e   's/\n/ /' \
+	-e   p \
+	-e '}' \
+	$packages | \
+      sort +0 -1 | \
+      join candidates - | \
+      awk '{ if ($4 == "M") s += $3*1024
+	     else if ($4 == "G") s += $3*1024*1024
+	     else s += $3 }
+	   END { print s }')
+    avail_size=$(disk_avail_size $SLACKUPGRADE_PKGDIR)
+    if [ $avail_size -lt $download_size ]; then
+	error "not enough free space in $SLACKUPGRADE_PKGDIR"
+	error "needed ${download_size}K, but only ${avail_size}K available"
+	if [ $opmode = INCR ]; then
+	    abend "terminating"
+	fi
+
+	cat >&2 <<EOT
+
+Safe mode is the default for slackupgrade.  In this mode, the program would
+first download all packages from the remote site, verify them and then
+proceed to actual upgrade.  This ensures that even if something goes wrong
+during the upgrade, you have all tarballs at hand and can resume from
+where you left after fixing the problem.
+
+This mode cannot be used due to the disk space shortage.  Slackupgrade can
+nevertheless continue in incremental download mode.  In this mode, it will
+download and install packages one by one.  This takes more time and is less
+error prone.  If you choose this mode, type 'Y' at the prompt below.  Other-
+wise, type 'N' to terminate the program, and follow the instructions you
+will get.
+
+EOT
+
+	if getyn n "Continue in incremental download mode"; then
+	    opmode=INCR
+	else
+	    cat >&2 <<EOT
+
+Please make sure the slackupgrade spool directory can accomodate at least
+${download_size}K of data.  To do so, either free sufficient disk space on
+the device where $SLACKUPGRADE_PKGDIR resides, or set the SLACKUPGRADE_PKGDIR
+environment variable to the full pathname of the new spool directory,
+
+EOT
+	    error "Terminating."
+	    exit 0
+	fi
+    else
+	# Download all packages
+	download_all
+	# Switch to the local installation mode
+	rooturl=$SLACKUPGRADE_PKGDIR
+	opmode=INCR
+	# Instruct getfile to strip off the series directory from the
+	# package names.
+	strip_series=1
+	# Disable remote indicator and checksum verification (all archives
+	# have already been verified during the download).
+	unset remote checksums
+    fi
+fi
+
 info "starting upgrade of Slackware $VERSION to $newversion${dry_run:+ (DRY RUN MODE)}"
 
 pkg_names="glibc-solibs pkgtools tar xz findutils"
@@ -544,7 +733,7 @@ if [ -n "$remote" ]; then
     # depends on.  Wget is preferred over curl because of its limited
     # number of dependencies.
     pkg_names="$pkg_names aaa_elflibs wget"
-fi    
+fi
 pkg_re='^('$(echo "$pkg_names" | sed -r -e 's/ +/|/g')')$'
 
 pkg_files=
@@ -565,7 +754,7 @@ if [ -n "$remote" ]; then
     # Until now we have been using whatever downloader that is available
     # (curl or wget).  Switch to wget: it has all its dependecies installed.
     dnfunc=download_wget
-fi    
+fi
 
 # Upgrade the rest of packages
 egrep -v "$pkg_re" candidates |\
@@ -630,4 +819,3 @@ bootloader has been updated for the new kernel!
 Good luck!
 EOF
 fi
-