Further fixes in get and sync

* lib/App/Glacier/Command.pm (glacier_eval): Check for return context
and return appropriate type.
* lib/App/Glacier/Command/Get.pm (run): Minor simplification.
(run): Delete failed jobs.
(download): Revamp multi-part download algorithm.
* lib/App/Glacier/Command/ListVault.pm (getopt): Use
Command::set_time_style_option to set the --time-style option.
* lib/App/Glacier/Command/Sync.pm (sync): Restart expired jobs.
(_sync): Actually store the changed DB records.
* lib/App/Glacier/DateTime.pm (format_can): New format: standard
* lib/App/Glacier/Job.pm (new): Set _job => undef, to avoid
spurious warnings.
(_get_job): Handle invalidate for cached values as well
(is_finished, status): New methods.
(forget): Rename to delete.

1 files changed, 55 insertions, 40 deletions

diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm
index dd3ca37..b00b658 100644
--- a/lib/App/Glacier/Command/Get.pm
+++ b/lib/App/Glacier/Command/Get.pm
@@ -57,6 +57,9 @@ sub run {
 	or die "unexpected failure";
 
     my ($filename, $ver) = ($+{file}, $+{ver});
+    # Reset $ver and $filespec for error reporting
+    $ver = 1 unless defined $ver;
+    $filespec = "$filespec;$ver";
     
     $localname = $filename unless defined($localname);
 
@@ -78,7 +81,12 @@ sub run {
 	$self->error("downloading file $filename initialized on",
 		     $job->get('CreationDate')->canned_format('full-iso'));
 	$self->error("job id:", $job->id);
-	$self->error("current status:", $job->get('StatusCode'));
+	my ($status, $message) = $job->status;
+	$self->error("current status:", $status);
+	if ($message) {
+	    $self->error("status message: $message\n");
+	}
+	
 	if ($job->is_completed) {
 	    $self->error("completed on",
 			 $job->get('CompletionDate')->canned_format('full-iso'));
@@ -89,10 +97,18 @@ sub run {
     if ($job->is_completed) {
 	$self->download($job, $localname);
     } else {
-	$self->abend(EX_TEMPFAIL,
-		     "archive retrieval job for $vaultname:$filespec initiated at " .
-		     $job->get('CreationDate')->canned_format
-		     . "; please retry later to download the file");
+	my ($status, $message) = $job->status;
+	if ($status eq 'InProgress') {
+	    $self->abend(EX_TEMPFAIL,
+			 "archive retrieval job for $vaultname:$filespec initiated at " .
+			 $job->get('CreationDate')->canned_format
+			 . "; please retry later to download the file");
+	} else {  
+	    $self->error("archive retrieval job for $vaultname:$filespec: $status: $message");
+	    $self->error("deleting job", $job->id);
+	    $job->delete;
+	    exit (EX_FAILURE);
+	}
     }
 }
 
@@ -109,8 +125,16 @@ sub download {
     truncate($fd, 0);
     
     my $archive_size = $job->get('ArchiveSizeInBytes');
-    my $njobs = $self->{_options}{jobs} || 1;
-    my $part_size = $self->{_options}{part_size} || 10*1024*1024*1024; # FIXME
+
+    my $njobs;
+    my $part_size;
+    if ($self->{_options}{jobs}) {
+	$njobs = $self->{_options}{jobs};
+	$part_size = int(($archive_size + $njobs - 1) / $njobs);
+    } else {
+	$part_size = $self->{_options}{part_size} || 10*1024*1024*1024; # FIXME
+	$njobs = int(($archive_size + $part_size - 1) / $part_size);
+    }
 
     my $glacier = $self->{_glacier};
 
@@ -119,51 +143,42 @@ sub download {
     if ($njobs <= 1 || $archive_size < $part_size) {
 	# simple download 
 	my $res;
+	$self->debug(1, "downloading", $job->file_name(1), "in single part");
 	($res, $tree_hash) = $glacier->get_job_output($job->vault, $job->id);
 	syswrite($fd, $res);
     } else {
 	use Fcntl qw(SEEK_SET);
-	
+
 	$self->debug(1,
-		     "downloading ".$job->file_name(1)." to $localname in chunks of $part_size bytes, in $njobs jobs");
+		     "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs");
 
 	my @part_hashes = ();
 	my $read_bytes;
 	my $rest_size = $archive_size;
 	my $off = 0;
-	my $part_idx = 0;
-	while ($rest_size) {
-	    for (my $i = 0; $i < $njobs && $rest_size; $i++) {
-		if ($rest_size < $part_size) {
-		    $part_size = $rest_size;
-		}
-		my ($thr) = threads->create(
-		    sub {
-			my ($part_idx, $off) = @_;
-			my $range = $off . '-' . ($off + $part_size);
-			my ($res, $hash) =
-			    $glacier->get_job_output($job->vault,
-						     $job->id, $range);
-			lock $fd;
-			seek($fd, $off, SEEK_SET);
-			syswrite($fd, $res);
-			return ($part_idx, $hash);
-		    },
-		    $part_idx, $off);
-		$part_idx++;
-		$off += $part_size;
-		$rest_size -= $part_size;
+	for (my $i = 0; $i < $njobs; $i++, $off += $part_size, $rest_size -= $part_size) {
+	    if ($rest_size < $part_size) {
+		$part_size = $rest_size;
 	    }
+	    my ($thr) = threads->create(
+		sub {
+		    my ($part_idx, $off) = @_;
+		    my $range = $off . '-' . ($off + $part_size);
+		    my ($res, $hash) =
+			$glacier->get_job_output($job->vault, $job->id, $range);
+		    lock $fd;
+		    seek($fd, $off, SEEK_SET);
+		    syswrite($fd, $res);
+		    return ($part_idx, $hash);
+		},
+		$i, $off);
+	}
 
-	    $self->debug(2, "waiting for the bunch to finish");
-	    while (threads->list()) {
-		foreach my $thr (threads->list(threads::joinable)) {
-		    # FIXME: error handling
-		    my ($idx, $hash) = $thr->join()
-			or croak "thread $thr failed";
-		    $part_hashes[$idx] = $hash;
-		}
-	    }
+	$self->debug(2, "waiting for download to finish");
+	foreach my $thr (threads->list()) {
+	    # FIXME: error handling
+	    my ($idx, $hash) = $thr->join() or croak "thread $thr failed";
+	    $part_hashes[$idx] = $hash;
 	}
 	$tree_hash = $glacier->_tree_hash_from_array_ref(\@part_hashes);
     }