diff options
author | Sergey Poznyakoff <gray@gnu.org> | 2018-12-11 21:48:37 +0200 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org> | 2018-12-11 21:48:37 +0200 |
commit | 6c6dab5d1784a57a22600af4fd7202293df18fa7 (patch) | |
tree | 928978e042f2fa7af004e4b238141ae8159d3bf1 /lib | |
parent | a8494c5196f7b0135a50577c8d90584a3c9d62f6 (diff) | |
download | glacier-6c6dab5d1784a57a22600af4fd7202293df18fa7.tar.gz glacier-6c6dab5d1784a57a22600af4fd7202293df18fa7.tar.bz2 |
Implement the 'periodic' command
* lib/App/Glacier.pm: Implement new command.
* lib/App/Glacier/Command.pm: New configuration statement
transfer.download.cachedir
(archive_cache_filename): New method.
* lib/App/Glacier/Command/Get.pm (run): Use cache file, if available.
(_open_output): As a temporary solution, protect the call to
$job->file_name by eval.
* lib/App/Glacier/Command/Periodic.pm: New file.
* lib/App/Glacier/Job.pm (fromdb): New class method.
(cache_file): New method.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/App/Glacier.pm | 6 | ||||
-rw-r--r-- | lib/App/Glacier/Command.pm | 12 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Get.pm | 23 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Periodic.pm | 103 | ||||
-rw-r--r-- | lib/App/Glacier/Job.pm | 21 | ||||
-rw-r--r-- | lib/App/Glacier/Job/FileRetrieval.pm | 3 |
6 files changed, 159 insertions, 9 deletions
diff --git a/lib/App/Glacier.pm b/lib/App/Glacier.pm index 8d62f56..e8c6cf9 100644 --- a/lib/App/Glacier.pm +++ b/lib/App/Glacier.pm @@ -47,7 +47,11 @@ my %comtab = ( jobs => sub { require App::Glacier::Command::Jobs; return new App::Glacier::Command::Jobs(@_); - } + }, + periodic => sub { + require App::Glacier::Command::Periodic; + return new App::Glacier::Command::Periodic(@_); + } ); sub getcom { diff --git a/lib/App/Glacier/Command.pm b/lib/App/Glacier/Command.pm index a641f34..5dd9eb8 100644 --- a/lib/App/Glacier/Command.pm +++ b/lib/App/Glacier/Command.pm @@ -6,6 +6,7 @@ use Carp; use App::Glacier::Core; use parent 'App::Glacier::Core'; use File::Basename; +use File::Spec; use App::Glacier::EclatCreds; use App::Glacier::Config; use Net::Amazon::Glacier; @@ -69,8 +70,9 @@ my %parameters = ( 'single-part-size' => { check => \&ck_size }, 'jobs' => { check => \&ck_number }, 'retries' => { check => \&ck_number }, + 'cachedir' => { default => '/var/lib/glacier/cache' } } - } + } } }, database => { @@ -161,6 +163,7 @@ sub clone { my $self = $class->SUPER::clone($orig); $self->{_config} = $orig->config; $self->{_glacier} = $orig->{_glacier}; + $self->{_jobdb} = $orig->{_jobdb}; $self } @@ -316,4 +319,11 @@ sub format_date_time { return $obj->{$field}->canned_format($self->{_options}{time_style}); } +sub archive_cache_filename { + my ($self, $vault_name, $archive_id) = @_; + return File::Spec->catfile($self->cfget(qw(transfer download cachedir)), + $vault_name, + $archive_id); +} + 1; diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm index c7199e3..0fb1afb 100644 --- a/lib/App/Glacier/Command/Get.pm +++ b/lib/App/Glacier/Command/Get.pm @@ -10,6 +10,7 @@ use App::Glacier::Progress; use parent qw(App::Glacier::Command); use Carp; use Scalar::Util; +use File::Copy; =head1 NAME @@ -157,11 +158,19 @@ sub run { } if ($job->is_completed) { - my $tree_hash = $self->download($job, $localname); - if (!$self->dry_run - && $tree_hash ne $job->get('ArchiveSHA256TreeHash')) { - unlink $localname; - $self->abend(EX_SOFTWARE, "downloaded file is corrupt"); + my $cache_file = $job->cache_file; + if (-f $cache_file) { + unless (copy($cache_file, $localname)) { + $self->abend(EX_FAILURE, + "can't copy $cache_file to $localname: $!"); + } + } else { + my $tree_hash = $self->download($job, $localname); + if (!$self->dry_run + && $tree_hash ne $job->get('ArchiveSHA256TreeHash')) { + unlink $localname; + $self->abend(EX_SOFTWARE, "downloaded file is corrupt"); + } } } else { my ($status, $message) = $job->status; @@ -205,7 +214,9 @@ sub _open_output { sub _download_simple { my ($self, $job, $localname) = @_; - $self->debug(1, "downloading", $job->file_name(1), "in single part"); + eval { # FIXME: file_name might be absent + $self->debug(1, "downloading", $job->file_name(1), "in single part"); + }; return if $self->dry_run; my $fd = $self->_open_output($localname); my ($res, $tree_hash) = $self->glacier_eval('get_job_output', diff --git a/lib/App/Glacier/Command/Periodic.pm b/lib/App/Glacier/Command/Periodic.pm new file mode 100644 index 0000000..85f3cc0 --- /dev/null +++ b/lib/App/Glacier/Command/Periodic.pm @@ -0,0 +1,103 @@ +package App::Glacier::Command::Periodic; +use strict; +use warnings; +use App::Glacier::Core; +use parent qw(App::Glacier::Command); +use Carp; +use Data::Dumper; +use File::Basename; +use File::Path qw(make_path); +use App::Glacier::Timestamp; + +=head1 NAME + +glacier periodic - periodic cronjob for Glacier + +=head1 SYNOPSIS + +B<glacier periodic> + +=head1 DESCRIPTION + +Scans pending glacier jobs. For each job, checks its current status and if +it the job is completed, retrieves the result. Typical usage is in the +crontab. + +=cut + +sub run { + my $self = shift; + + my $db = $self->jobdb(); + $db->foreach(sub { + my ($key, $descr) = @_; + my $vault = $descr->{VaultARN}; + $vault =~ s{.*:vaults/}{}; + + return if $descr->{Completed}; + + $self->debug(2, "$descr->{JobId} $descr->{Action} $vault"); + if ($descr->{StatusCode} eq 'Failed') { + $self->debug(1, "deleting failed $key $vault " . + ($descr->{JobDescription} || $descr->{Action}) . + $descr->{JobId}); + $db->delete($key) unless $self->dry_run; + } + + my $res = $self->glacier_eval('describe_job', + $vault, + $descr->{JobId}); + if ($self->lasterr) { + if ($self->lasterr('code') == 404) { + $self->debug(1, "deleting expired $key $vault " . + ($descr->{JobDescription} || $descr->{Action}) . + $descr->{JobId}); + $db->delete($key) unless $self->dry_run; + } else { + $self->error("can't describe job $descr->{JobId}: ", + $self->last_error_message); + } + } elsif (ref($res) ne 'HASH') { + croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; + } else { + $self->debug(2, $res->{StatusCode}); + if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') { + $self->debug(1, "$descr->{JobId}: processing $descr->{Action} for $vault"); + return if $self->dry_run; + if ($res->{Action} eq 'InventoryRetrieval') { + require App::Glacier::Command::Sync; + my $sync = clone App::Glacier::Command::Sync($self); + $sync->sync($vault); + } elsif ($res->{Action} eq 'ArchiveRetrieval') { + my $job = App::Glacier::Job->fromdb($self, $vault, + $key, $res); + my $localname = $self->archive_cache_filename($vault, + $res->{ArchiveId}); + my $dir = dirname($localname); + unless (-d $dir) { + make_path($dir, { error => \my $err }); + if (@$err) { + for my $diag (@$err) { + my ($dir, $message) = %$diag; + if ($dir eq '') { + $self->error($message); + } else { + $self->error("can't create directory $dir: $message"); + } + } + } + } + + require App::Glacier::Command::Get; + my $get = clone App::Glacier::Command::Get($self); + $get->download($job, $localname); + } + } + $db->store($key, $res); + } + }); +} + +1; + + diff --git a/lib/App/Glacier/Job.pm b/lib/App/Glacier/Job.pm index 1906de1..38de5f0 100644 --- a/lib/App/Glacier/Job.pm +++ b/lib/App/Glacier/Job.pm @@ -28,6 +28,14 @@ sub new { _invalidate => $invalidate }, $class; } +sub fromdb { + my ($class, $cmd, $vault, $key, $job) = @_; + return bless { _cmd => $cmd, + _vault => $vault, + _key => $key, + _job => $job }, $class; +} + sub _get_db { my ($self) = @_; return $self->{_cmd}->jobdb(); @@ -142,10 +150,23 @@ sub vault { sub delete { my $self = shift; + if (my $cache = $self->cache_file) { + if (-f $cache) { + unlink($cache); + } + } my $db = $self->_get_db; $db->delete($self->{_key}); } +sub cache_file { + my $self = shift; + my $aid = $self->get('ArchiveId') or return; + my $vault = $self->get('VaultARN') or return; + $vault =~ s{.*:vaults/}{}; + return $self->{_cmd}->archive_cache_filename($vault, $aid); +} + 1; diff --git a/lib/App/Glacier/Job/FileRetrieval.pm b/lib/App/Glacier/Job/FileRetrieval.pm index 25454a2..b9437e6 100644 --- a/lib/App/Glacier/Job/FileRetrieval.pm +++ b/lib/App/Glacier/Job/FileRetrieval.pm @@ -46,4 +46,5 @@ sub file_version { my ($self) = @_; return $self->{_fileversion}; } - + +1; |