diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2017-05-23 11:48:09 +0300 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2017-05-23 11:49:43 +0300 |
commit | a9c85d6351239bee6e6692986b22efe99ecd9fba (patch) | |
tree | 6bdb0318d67bcf57dc564b5757851b59d8f4d5f3 /lib | |
parent | e184e9cc590cd702d79e983c033df0d60a49f3df (diff) | |
download | glacier-a9c85d6351239bee6e6692986b22efe99ecd9fba.tar.gz glacier-a9c85d6351239bee6e6692986b22efe99ecd9fba.tar.bz2 |
get: improve part size computation
* lib/App/Glacier/Command/Get.pm: Redo the part size calculation in
the similar vein as in Put.pm
Retry if downloading fails
Diffstat (limited to 'lib')
-rw-r--r-- | lib/App/Glacier/Command/Get.pm | 104 |
1 files changed, 64 insertions, 40 deletions
diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm index 5666d36..ce4c533 100644 --- a/lib/App/Glacier/Command/Get.pm +++ b/lib/App/Glacier/Command/Get.pm @@ -170,55 +170,79 @@ sub _download_multipart { my $archive_size = $job->get('ArchiveSizeInBytes'); my $part_size; # Compute approximate part size - $part_size = int(($archive_size + $njobs - 1) / $njobs); + $part_size = ($archive_size - 1) / 10000; + if ($part_size < TWOMB) { + $part_size = TWOMB; + } else { + # Make sure the chunk is Tree-Hash aligned + # http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations-range.html?shortFooter=true#checksum-calculations-upload-archive-with-ranges + $part_size = TWOMB * 2 ** int(log($part_size / TWOMB) / log(2) + 1); + } + # Number of parts to download: + my $total_parts = int(($archive_size + $part_size - 1) / $part_size); + # Compute the number of parts per job + my $job_parts = int(($total_parts + $njobs - 1) / $njobs); + + $self->debug(1, + "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); + return if $self->dry_run; - # Make sure the chunk is Tree-Hash aligned - # http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations-range.html?shortFooter=true#checksum-calculations-upload-archive-with-ranges - $part_size = TWOMB * 2 ** int(log($part_size / TWOMB) / log(2) + 1); - # Adjust the number of jobs - $njobs = int(($archive_size + $part_size - 1) / $part_size); + use Fcntl qw(SEEK_SET); - if ($njobs <= 1) { - return $self->_download_simple($job, $localname); - } else { - use Fcntl qw(SEEK_SET); - - $self->debug(1, - "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs"); - return if $self->dry_run; - - my $fd = $self->_open_output($localname); - my @part_hashes :shared = (); - my $read_bytes; - my $rest_size = $archive_size; - my $off = 0; - for (my $i = 0; $i < $njobs; $i++, $off += $part_size, $rest_size -= $part_size) { - if ($rest_size < $part_size) { - $part_size = $rest_size; - } - my ($thr) = threads->create( - sub { - my ($part_idx, $off) = @_; + my $fd = $self->_open_output($localname); + my @part_hashes :shared = (); + for (my $i = 0; $i < $njobs; $i++) { + my ($thr) = threads->create( + sub { + my ($job_idx) = @_; + # Number of part to start from + my $part_idx = $job_idx * $job_parts; + # Offset in file + my $off = $part_idx * $part_size; + + for (my $j = 0; $j < $job_parts; + $j++, $part_idx++, $off += $part_size) { + last if $off >= $archive_size; + if ($part_size > $archive_size - $off) { + $part_size = $archive_size - $off; + } my $range = 'bytes=' . $off . '-' . ($off + $part_size - 1); - my ($res, $hash) = - $glacier->get_job_output($job->vault, $job->id, $range); + my ($res, $hash); + for (my $try = 0;;) { + ($res, $hash) = $self->glacier_eval('get_job_output', + $job->vault, + $job->id, $range); + if ($self->lasterr) { + if (++$try < 10) { + $self->debug("part $part_idx: ", + $self->last_error_message); + $self->debug("retrying"); + } else { + $self->error("failed to download part $part_idx: ", + $self->last_error_message); + return 0; + } + } else { + last; + } + } + lock @part_hashes; seek($fd, $off, SEEK_SET); syswrite($fd, $res); $part_hashes[$part_idx] = $hash; - return 1; - }, - $i, $off); - } + } + return 1; + }, $i); + } - $self->debug(2, "waiting for download to finish"); - foreach my $thr (threads->list()) { - # FIXME: error handling - $thr->join() or croak "thread $thr failed"; - } - close($fd); - return $glacier->_tree_hash_from_array_ref(\@part_hashes); + $self->debug(2, "waiting for download to finish"); + foreach my $thr (threads->list()) { + # FIXME: error handling + $thr->join() or croak "thread $thr failed"; } + close($fd); + return $glacier->_tree_hash_from_array_ref(\@part_hashes); } 1; |