diff options
author | Sergey Poznyakoff <gray@gnu.org> | 2018-12-19 14:35:49 +0200 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org> | 2018-12-19 14:55:09 +0200 |
commit | ea2c058ce96484937fcfe52dc7382a461a9d72e3 (patch) | |
tree | ae86e1115c3ef91bc03d2cfcb7df4f7ff8d19ce9 | |
parent | 249512af92d3f592ae683279c2cd1f54a7c1a726 (diff) | |
download | glacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.gz glacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.bz2 |
Bugfixes.
* lib/App/Glacier/Command.pm (option,check_job): New methods.
* lib/App/Glacier/Command/Get.pm: Rewrite debug messages (fixes
temporary solution in 6c6dab5d).
* lib/App/Glacier/Command/Jobs.pm: Rewrite the db->foreach sub.
* lib/App/Glacier/Command/Periodic.pm: Likewise.
* lib/App/Glacier/DB/GDBM.pm (CLONE_SKIP): New method.
* lib/App/Glacier/Job.pm: Overload the "" operator.
* lib/App/Glacier/Roster.pm (foreach): Overload method.
-rw-r--r-- | lib/App/Glacier/Command.pm | 43 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Get.pm | 11 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Jobs.pm | 31 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Periodic.pm | 38 | ||||
-rw-r--r-- | lib/App/Glacier/DB/GDBM.pm | 11 | ||||
-rw-r--r-- | lib/App/Glacier/Job.pm | 5 | ||||
-rw-r--r-- | lib/App/Glacier/Roster.pm | 9 |
7 files changed, 77 insertions, 71 deletions
diff --git a/lib/App/Glacier/Command.pm b/lib/App/Glacier/Command.pm index 7178a75..1f9a130 100644 --- a/lib/App/Glacier/Command.pm +++ b/lib/App/Glacier/Command.pm @@ -163,4 +163,12 @@ sub clone { } +sub option { + my ($self, $opt, $val) = @_; + if (defined($val)) { + $self->{_options}{$opt} = $val; + } + return $self->{_options}{$opt}; +} + sub touchdir { my ($self, $dir) = @_; @@ -277,3 +285,38 @@ sub archive_cache_filename { } +sub check_job { + my ($self, $key, $descr, $vault) = @_; + + $self->debug(2, "$descr->{JobId} $descr->{Action} $vault"); + if ($descr->{StatusCode} eq 'Failed') { + $self->debug(1, + "deleting failed $key $vault " + . ($descr->{JobDescription} || $descr->{Action}) + . ' ' + . $descr->{JobId}); + $self->jobdb()->delete($key) unless $self->dry_run; + return; + } + + my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); + if ($self->glacier->lasterr) { + if ($self->glacier->lasterr('code') == 404) { + $self->debug(1, + "deleting expired $key $vault " + . ($descr->{JobDescription} || $descr->{Action}) + . ' ' + . $descr->{JobId}); + App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() + unless $self->dry_run; + } else { + $self->error("can't describe job $descr->{JobId}: ", + $self->glacier->last_error_message); + } + return; + } elsif (ref($res) ne 'HASH') { + croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; + } + return $res; +} + 1; diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm index 0d17659..82b32c8 100644 --- a/lib/App/Glacier/Command/Get.pm +++ b/lib/App/Glacier/Command/Get.pm @@ -161,4 +161,6 @@ sub run { my $cache_file = $job->cache_file; if (-f $cache_file) { + $self->debug(1, "$job: copying from $cache_file"); + return if $self->dry_run; unless (copy($cache_file, $localname)) { $self->abend(EX_FAILURE, @@ -194,4 +196,5 @@ use constant TWOMB => 2*MB; sub download { my ($self, $job, $localname) = @_; + my $archive_size = $job->get('ArchiveSizeInBytes'); if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) { @@ -215,7 +218,5 @@ sub _download_simple { my ($self, $job, $localname) = @_; - eval { # FIXME: file_name might be absent - $self->debug(1, "downloading", $job->file_name(1), "in single part"); - }; + $self->debug(1, "$job: downloading in single part"); return if $self->dry_run; my $fd = $self->_open_output($localname); @@ -257,6 +258,6 @@ sub _download_multipart { my $job_parts = int(($total_parts + $njobs - 1) / $njobs); - $self->debug(1, - "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); + $self->debug(1, "$job: downloading in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); + return if $self->dry_run; diff --git a/lib/App/Glacier/Command/Jobs.pm b/lib/App/Glacier/Command/Jobs.pm index 5a5f488..7dc5d06 100644 --- a/lib/App/Glacier/Command/Jobs.pm +++ b/lib/App/Glacier/Command/Jobs.pm @@ -117,35 +117,11 @@ sub list { my $db = $self->jobdb(); $db->foreach(sub { - my ($key, $descr) = @_; - my $vault = $descr->{VaultARN}; - $vault =~ s{.*:vaults/}{}; + my ($key, $descr, $vault) = @_; return if (@vault_names && ! grep { $_ eq $vault } @vault_names); unless ($self->{_options}{cached}) { - if ($descr->{StatusCode} eq 'Failed') { - $self->debug(1, "deleting failed $key $vault " . - ($descr->{JobDescription} || $descr->{Action}) . - $descr->{JobId}); - $db->delete($key) unless $self->dry_run; - return; - } - - my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); - if ($self->glacier->lasterr) { - if ($self->glacier->lasterr('code') == 404) { - $self->debug(1, "deleting expired $key $vault " . - ($descr->{JobDescription} || $descr->{Action}) . - $descr->{JobId}); - App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() - unless $self->dry_run; - return; - } else { - $self->error("can't describe job $descr->{JobId}: ", - $self->glacier->last_error_message); - } - } elsif (ref($res) ne 'HASH') { - croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; - } else { + my $res = $self->check_job($key, $descr, $vault) + or return; $res = timestamp_deserialize($res); $self->debug(2, $res->{StatusCode}); @@ -153,5 +129,4 @@ sub list { $descr = $res; } - } my $started = $self->format_date_time($descr, 'CreationDate'); diff --git a/lib/App/Glacier/Command/Periodic.pm b/lib/App/Glacier/Command/Periodic.pm index 5a4dc2a..39a2d7a 100644 --- a/lib/App/Glacier/Command/Periodic.pm +++ b/lib/App/Glacier/Command/Periodic.pm @@ -8,4 +8,5 @@ use Data::Dumper; use File::Basename; use App::Glacier::Job; +use App::Glacier::Command::Get; =head1 NAME @@ -39,37 +40,8 @@ sub run { my $db = $self->jobdb(); $db->foreach(sub { - my ($key, $descr) = @_; - my $vault = $descr->{VaultARN}; - $vault =~ s{.*:vaults/}{}; - - my $completed = $descr->{Completed}; - - $self->debug(2, "$descr->{JobId} $descr->{Action} $vault"); - if ($descr->{StatusCode} eq 'Failed') { - $self->debug(1, - "deleting failed $key $vault " - . ($descr->{JobDescription} || $descr->{Action}) - . ' ' - . $descr->{JobId}); - $db->delete($key) unless $self->dry_run; - } + my ($key, $descr, $vault) = @_; - my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); - if ($self->glacier->lasterr) { - if ($self->glacier->lasterr('code') == 404) { - $self->debug(1, - "deleting expired $key $vault " - . ($descr->{JobDescription} || $descr->{Action}) - . ' ' - . $descr->{JobId}); - App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() - unless $self->dry_run; - } else { - $self->error("can't describe job $descr->{JobId}: ", - $self->glacier->last_error_message); - } - } elsif (ref($res) ne 'HASH') { - croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; - } elsif ($res->{Completed} ne $completed) { + my $res = $self->check_job($key, $descr, $vault); + if ($res && $res->{Completed} ne $descr->{Completed}) { $self->debug(2, $res->{StatusCode}); if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') { @@ -87,6 +59,6 @@ sub run { $self->touchdir(dirname($localname)); - require App::Glacier::Command::Get; my $get = clone App::Glacier::Command::Get($self); + $get->option(quiet => 1); $get->download($job, $localname); } diff --git a/lib/App/Glacier/DB/GDBM.pm b/lib/App/Glacier/DB/GDBM.pm index 0e14a65..f7cc29a 100644 --- a/lib/App/Glacier/DB/GDBM.pm +++ b/lib/App/Glacier/DB/GDBM.pm @@ -7,4 +7,8 @@ use File::Basename; use File::Path qw(make_path); +# Avoid coredumps in threaded code. +# See https://rt.perl.org/Public/Bug/Display.html?id=61912. +sub CLONE_SKIP { 1 } + sub new { my $class = shift; @@ -53,9 +57,6 @@ sub configtest { } -# We can't tie the DB to $self->{_map} at once, in the new method, because -# this will cause coredumps in threaded code (see -# https://rt.perl.org/Public/Bug/Display.html?id=61912). So, the following -# auxiliary method is used, which calls &$code with $self->{_map} tied -# to the DB. +# Tie in the database, run $code, and untie it again. Correctly handle +# nested invocations to avoid deadlocking. sub _tied { my ($self, $code) = @_; diff --git a/lib/App/Glacier/Job.pm b/lib/App/Glacier/Job.pm index ae12b22..9478b20 100644 --- a/lib/App/Glacier/Job.pm +++ b/lib/App/Glacier/Job.pm @@ -109,4 +109,9 @@ sub get { } +sub as_string { shift->get('JobDescription') } + +use overload + '""' => \&as_string; + sub is_finished { my $self = shift; diff --git a/lib/App/Glacier/Roster.pm b/lib/App/Glacier/Roster.pm index ee56ff5..32c2b08 100644 --- a/lib/App/Glacier/Roster.pm +++ b/lib/App/Glacier/Roster.pm @@ -2,3 +2,12 @@ package App::Glacier::Roster; use parent 'App::Glacier::DB'; +sub foreach { + my ($self, $fun) = @_; + $self->SUPER::foreach(sub { + my ($key, $descr) = @_; + (my $vault = $descr->{VaultARN}) =~ s{.*:vaults/}{}; + &{$fun}($key, $descr, $vault); + }); +} + 1; |