aboutsummaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org>2018-12-11 21:48:37 +0200
committerSergey Poznyakoff <gray@gnu.org>2018-12-11 21:48:37 +0200
commit6c6dab5d1784a57a22600af4fd7202293df18fa7 (patch)
tree928978e042f2fa7af004e4b238141ae8159d3bf1 /lib
parenta8494c5196f7b0135a50577c8d90584a3c9d62f6 (diff)
downloadglacier-6c6dab5d1784a57a22600af4fd7202293df18fa7.tar.gz
glacier-6c6dab5d1784a57a22600af4fd7202293df18fa7.tar.bz2
Implement the 'periodic' command
* lib/App/Glacier.pm: Implement new command. * lib/App/Glacier/Command.pm: New configuration statement transfer.download.cachedir (archive_cache_filename): New method. * lib/App/Glacier/Command/Get.pm (run): Use cache file, if available. (_open_output): As a temporary solution, protect the call to $job->file_name by eval. * lib/App/Glacier/Command/Periodic.pm: New file. * lib/App/Glacier/Job.pm (fromdb): New class method. (cache_file): New method.
Diffstat (limited to 'lib')
-rw-r--r--lib/App/Glacier.pm6
-rw-r--r--lib/App/Glacier/Command.pm12
-rw-r--r--lib/App/Glacier/Command/Get.pm23
-rw-r--r--lib/App/Glacier/Command/Periodic.pm103
-rw-r--r--lib/App/Glacier/Job.pm21
-rw-r--r--lib/App/Glacier/Job/FileRetrieval.pm3
6 files changed, 159 insertions, 9 deletions
diff --git a/lib/App/Glacier.pm b/lib/App/Glacier.pm
index 8d62f56..e8c6cf9 100644
--- a/lib/App/Glacier.pm
+++ b/lib/App/Glacier.pm
@@ -47,7 +47,11 @@ my %comtab = (
jobs => sub {
require App::Glacier::Command::Jobs;
return new App::Glacier::Command::Jobs(@_);
- }
+ },
+ periodic => sub {
+ require App::Glacier::Command::Periodic;
+ return new App::Glacier::Command::Periodic(@_);
+ }
);
sub getcom {
diff --git a/lib/App/Glacier/Command.pm b/lib/App/Glacier/Command.pm
index a641f34..5dd9eb8 100644
--- a/lib/App/Glacier/Command.pm
+++ b/lib/App/Glacier/Command.pm
@@ -6,6 +6,7 @@ use Carp;
use App::Glacier::Core;
use parent 'App::Glacier::Core';
use File::Basename;
+use File::Spec;
use App::Glacier::EclatCreds;
use App::Glacier::Config;
use Net::Amazon::Glacier;
@@ -69,8 +70,9 @@ my %parameters = (
'single-part-size' => { check => \&ck_size },
'jobs' => { check => \&ck_number },
'retries' => { check => \&ck_number },
+ 'cachedir' => { default => '/var/lib/glacier/cache' }
}
- }
+ }
}
},
database => {
@@ -161,6 +163,7 @@ sub clone {
my $self = $class->SUPER::clone($orig);
$self->{_config} = $orig->config;
$self->{_glacier} = $orig->{_glacier};
+ $self->{_jobdb} = $orig->{_jobdb};
$self
}
@@ -316,4 +319,11 @@ sub format_date_time {
return $obj->{$field}->canned_format($self->{_options}{time_style});
}
+sub archive_cache_filename {
+ my ($self, $vault_name, $archive_id) = @_;
+ return File::Spec->catfile($self->cfget(qw(transfer download cachedir)),
+ $vault_name,
+ $archive_id);
+}
+
1;
diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm
index c7199e3..0fb1afb 100644
--- a/lib/App/Glacier/Command/Get.pm
+++ b/lib/App/Glacier/Command/Get.pm
@@ -10,6 +10,7 @@ use App::Glacier::Progress;
use parent qw(App::Glacier::Command);
use Carp;
use Scalar::Util;
+use File::Copy;
=head1 NAME
@@ -157,11 +158,19 @@ sub run {
}
if ($job->is_completed) {
- my $tree_hash = $self->download($job, $localname);
- if (!$self->dry_run
- && $tree_hash ne $job->get('ArchiveSHA256TreeHash')) {
- unlink $localname;
- $self->abend(EX_SOFTWARE, "downloaded file is corrupt");
+ my $cache_file = $job->cache_file;
+ if (-f $cache_file) {
+ unless (copy($cache_file, $localname)) {
+ $self->abend(EX_FAILURE,
+ "can't copy $cache_file to $localname: $!");
+ }
+ } else {
+ my $tree_hash = $self->download($job, $localname);
+ if (!$self->dry_run
+ && $tree_hash ne $job->get('ArchiveSHA256TreeHash')) {
+ unlink $localname;
+ $self->abend(EX_SOFTWARE, "downloaded file is corrupt");
+ }
}
} else {
my ($status, $message) = $job->status;
@@ -205,7 +214,9 @@ sub _open_output {
sub _download_simple {
my ($self, $job, $localname) = @_;
- $self->debug(1, "downloading", $job->file_name(1), "in single part");
+ eval { # FIXME: file_name might be absent
+ $self->debug(1, "downloading", $job->file_name(1), "in single part");
+ };
return if $self->dry_run;
my $fd = $self->_open_output($localname);
my ($res, $tree_hash) = $self->glacier_eval('get_job_output',
diff --git a/lib/App/Glacier/Command/Periodic.pm b/lib/App/Glacier/Command/Periodic.pm
new file mode 100644
index 0000000..85f3cc0
--- /dev/null
+++ b/lib/App/Glacier/Command/Periodic.pm
@@ -0,0 +1,103 @@
+package App::Glacier::Command::Periodic;
+use strict;
+use warnings;
+use App::Glacier::Core;
+use parent qw(App::Glacier::Command);
+use Carp;
+use Data::Dumper;
+use File::Basename;
+use File::Path qw(make_path);
+use App::Glacier::Timestamp;
+
+=head1 NAME
+
+glacier periodic - periodic cronjob for Glacier
+
+=head1 SYNOPSIS
+
+B<glacier periodic>
+
+=head1 DESCRIPTION
+
+Scans pending glacier jobs. For each job, checks its current status and if
+it the job is completed, retrieves the result. Typical usage is in the
+crontab.
+
+=cut
+
+sub run {
+ my $self = shift;
+
+ my $db = $self->jobdb();
+ $db->foreach(sub {
+ my ($key, $descr) = @_;
+ my $vault = $descr->{VaultARN};
+ $vault =~ s{.*:vaults/}{};
+
+ return if $descr->{Completed};
+
+ $self->debug(2, "$descr->{JobId} $descr->{Action} $vault");
+ if ($descr->{StatusCode} eq 'Failed') {
+ $self->debug(1, "deleting failed $key $vault " .
+ ($descr->{JobDescription} || $descr->{Action}) .
+ $descr->{JobId});
+ $db->delete($key) unless $self->dry_run;
+ }
+
+ my $res = $self->glacier_eval('describe_job',
+ $vault,
+ $descr->{JobId});
+ if ($self->lasterr) {
+ if ($self->lasterr('code') == 404) {
+ $self->debug(1, "deleting expired $key $vault " .
+ ($descr->{JobDescription} || $descr->{Action}) .
+ $descr->{JobId});
+ $db->delete($key) unless $self->dry_run;
+ } else {
+ $self->error("can't describe job $descr->{JobId}: ",
+ $self->last_error_message);
+ }
+ } elsif (ref($res) ne 'HASH') {
+ croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\"";
+ } else {
+ $self->debug(2, $res->{StatusCode});
+ if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') {
+ $self->debug(1, "$descr->{JobId}: processing $descr->{Action} for $vault");
+ return if $self->dry_run;
+ if ($res->{Action} eq 'InventoryRetrieval') {
+ require App::Glacier::Command::Sync;
+ my $sync = clone App::Glacier::Command::Sync($self);
+ $sync->sync($vault);
+ } elsif ($res->{Action} eq 'ArchiveRetrieval') {
+ my $job = App::Glacier::Job->fromdb($self, $vault,
+ $key, $res);
+ my $localname = $self->archive_cache_filename($vault,
+ $res->{ArchiveId});
+ my $dir = dirname($localname);
+ unless (-d $dir) {
+ make_path($dir, { error => \my $err });
+ if (@$err) {
+ for my $diag (@$err) {
+ my ($dir, $message) = %$diag;
+ if ($dir eq '') {
+ $self->error($message);
+ } else {
+ $self->error("can't create directory $dir: $message");
+ }
+ }
+ }
+ }
+
+ require App::Glacier::Command::Get;
+ my $get = clone App::Glacier::Command::Get($self);
+ $get->download($job, $localname);
+ }
+ }
+ $db->store($key, $res);
+ }
+ });
+}
+
+1;
+
+
diff --git a/lib/App/Glacier/Job.pm b/lib/App/Glacier/Job.pm
index 1906de1..38de5f0 100644
--- a/lib/App/Glacier/Job.pm
+++ b/lib/App/Glacier/Job.pm
@@ -28,6 +28,14 @@ sub new {
_invalidate => $invalidate }, $class;
}
+sub fromdb {
+ my ($class, $cmd, $vault, $key, $job) = @_;
+ return bless { _cmd => $cmd,
+ _vault => $vault,
+ _key => $key,
+ _job => $job }, $class;
+}
+
sub _get_db {
my ($self) = @_;
return $self->{_cmd}->jobdb();
@@ -142,10 +150,23 @@ sub vault {
sub delete {
my $self = shift;
+ if (my $cache = $self->cache_file) {
+ if (-f $cache) {
+ unlink($cache);
+ }
+ }
my $db = $self->_get_db;
$db->delete($self->{_key});
}
+sub cache_file {
+ my $self = shift;
+ my $aid = $self->get('ArchiveId') or return;
+ my $vault = $self->get('VaultARN') or return;
+ $vault =~ s{.*:vaults/}{};
+ return $self->{_cmd}->archive_cache_filename($vault, $aid);
+}
+
1;
diff --git a/lib/App/Glacier/Job/FileRetrieval.pm b/lib/App/Glacier/Job/FileRetrieval.pm
index 25454a2..b9437e6 100644
--- a/lib/App/Glacier/Job/FileRetrieval.pm
+++ b/lib/App/Glacier/Job/FileRetrieval.pm
@@ -46,4 +46,5 @@ sub file_version {
my ($self) = @_;
return $self->{_fileversion};
}
-
+
+1;

Return to:

Send suggestions and report system problems to the System administrator.