aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org.ua>2017-05-23 11:48:09 +0300
committerSergey Poznyakoff <gray@gnu.org.ua>2017-05-23 11:49:43 +0300
commita9c85d6351239bee6e6692986b22efe99ecd9fba (patch)
tree6bdb0318d67bcf57dc564b5757851b59d8f4d5f3
parente184e9cc590cd702d79e983c033df0d60a49f3df (diff)
downloadglacier-a9c85d6351239bee6e6692986b22efe99ecd9fba.tar.gz
glacier-a9c85d6351239bee6e6692986b22efe99ecd9fba.tar.bz2
get: improve part size computation
* lib/App/Glacier/Command/Get.pm: Redo the part size calculation in the similar vein as in Put.pm Retry if downloading fails
-rw-r--r--lib/App/Glacier/Command/Get.pm104
1 files changed, 64 insertions, 40 deletions
diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm
index 5666d36..ce4c533 100644
--- a/lib/App/Glacier/Command/Get.pm
+++ b/lib/App/Glacier/Command/Get.pm
@@ -170,55 +170,79 @@ sub _download_multipart {
my $archive_size = $job->get('ArchiveSizeInBytes');
my $part_size;
# Compute approximate part size
- $part_size = int(($archive_size + $njobs - 1) / $njobs);
+ $part_size = ($archive_size - 1) / 10000;
+ if ($part_size < TWOMB) {
+ $part_size = TWOMB;
+ } else {
+ # Make sure the chunk is Tree-Hash aligned
+ # http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations-range.html?shortFooter=true#checksum-calculations-upload-archive-with-ranges
+ $part_size = TWOMB * 2 ** int(log($part_size / TWOMB) / log(2) + 1);
+ }
+ # Number of parts to download:
+ my $total_parts = int(($archive_size + $part_size - 1) / $part_size);
+ # Compute the number of parts per job
+ my $job_parts = int(($total_parts + $njobs - 1) / $njobs);
+
+ $self->debug(1,
+ "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job");
+ return if $self->dry_run;
- # Make sure the chunk is Tree-Hash aligned
- # http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations-range.html?shortFooter=true#checksum-calculations-upload-archive-with-ranges
- $part_size = TWOMB * 2 ** int(log($part_size / TWOMB) / log(2) + 1);
- # Adjust the number of jobs
- $njobs = int(($archive_size + $part_size - 1) / $part_size);
+ use Fcntl qw(SEEK_SET);
- if ($njobs <= 1) {
- return $self->_download_simple($job, $localname);
- } else {
- use Fcntl qw(SEEK_SET);
-
- $self->debug(1,
- "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs");
- return if $self->dry_run;
-
- my $fd = $self->_open_output($localname);
- my @part_hashes :shared = ();
- my $read_bytes;
- my $rest_size = $archive_size;
- my $off = 0;
- for (my $i = 0; $i < $njobs; $i++, $off += $part_size, $rest_size -= $part_size) {
- if ($rest_size < $part_size) {
- $part_size = $rest_size;
- }
- my ($thr) = threads->create(
- sub {
- my ($part_idx, $off) = @_;
+ my $fd = $self->_open_output($localname);
+ my @part_hashes :shared = ();
+ for (my $i = 0; $i < $njobs; $i++) {
+ my ($thr) = threads->create(
+ sub {
+ my ($job_idx) = @_;
+ # Number of part to start from
+ my $part_idx = $job_idx * $job_parts;
+ # Offset in file
+ my $off = $part_idx * $part_size;
+
+ for (my $j = 0; $j < $job_parts;
+ $j++, $part_idx++, $off += $part_size) {
+ last if $off >= $archive_size;
+ if ($part_size > $archive_size - $off) {
+ $part_size = $archive_size - $off;
+ }
my $range = 'bytes=' . $off . '-' . ($off + $part_size - 1);
- my ($res, $hash) =
- $glacier->get_job_output($job->vault, $job->id, $range);
+ my ($res, $hash);
+ for (my $try = 0;;) {
+ ($res, $hash) = $self->glacier_eval('get_job_output',
+ $job->vault,
+ $job->id, $range);
+ if ($self->lasterr) {
+ if (++$try < 10) {
+ $self->debug("part $part_idx: ",
+ $self->last_error_message);
+ $self->debug("retrying");
+ } else {
+ $self->error("failed to download part $part_idx: ",
+ $self->last_error_message);
+ return 0;
+ }
+ } else {
+ last;
+ }
+ }
+
lock @part_hashes;
seek($fd, $off, SEEK_SET);
syswrite($fd, $res);
$part_hashes[$part_idx] = $hash;
- return 1;
- },
- $i, $off);
- }
+ }
+ return 1;
+ }, $i);
+ }
- $self->debug(2, "waiting for download to finish");
- foreach my $thr (threads->list()) {
- # FIXME: error handling
- $thr->join() or croak "thread $thr failed";
- }
- close($fd);
- return $glacier->_tree_hash_from_array_ref(\@part_hashes);
+ $self->debug(2, "waiting for download to finish");
+ foreach my $thr (threads->list()) {
+ # FIXME: error handling
+ $thr->join() or croak "thread $thr failed";
}
+ close($fd);
+ return $glacier->_tree_hash_from_array_ref(\@part_hashes);
}
1;

Return to:

Send suggestions and report system problems to the System administrator.