aboutsummaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/App/Glacier/Command.pm43
-rw-r--r--lib/App/Glacier/Command/Get.pm11
-rw-r--r--lib/App/Glacier/Command/Jobs.pm39
-rw-r--r--lib/App/Glacier/Command/Periodic.pm38
-rw-r--r--lib/App/Glacier/DB/GDBM.pm11
-rw-r--r--lib/App/Glacier/Job.pm5
-rw-r--r--lib/App/Glacier/Roster.pm9
7 files changed, 81 insertions, 75 deletions
diff --git a/lib/App/Glacier/Command.pm b/lib/App/Glacier/Command.pm
index 7178a75..1f9a130 100644
--- a/lib/App/Glacier/Command.pm
+++ b/lib/App/Glacier/Command.pm
@@ -159,12 +159,20 @@ sub clone {
$self->{_config} = $orig->config;
$self->{_glacier} = $orig->{_glacier};
$self->{_jobdb} = $orig->{_jobdb};
$self
}
+sub option {
+ my ($self, $opt, $val) = @_;
+ if (defined($val)) {
+ $self->{_options}{$opt} = $val;
+ }
+ return $self->{_options}{$opt};
+}
+
sub touchdir {
my ($self, $dir) = @_;
unless (-d $dir) {
make_path($dir, {error=>\my $err});
if (@$err) {
for my $diag (@$err) {
@@ -273,7 +281,42 @@ sub archive_cache_filename {
my ($self, $vault_name, $archive_id) = @_;
return File::Spec->catfile($self->cfget(qw(transfer download cachedir)),
$vault_name,
$archive_id);
}
+sub check_job {
+ my ($self, $key, $descr, $vault) = @_;
+
+ $self->debug(2, "$descr->{JobId} $descr->{Action} $vault");
+ if ($descr->{StatusCode} eq 'Failed') {
+ $self->debug(1,
+ "deleting failed $key $vault "
+ . ($descr->{JobDescription} || $descr->{Action})
+ . ' '
+ . $descr->{JobId});
+ $self->jobdb()->delete($key) unless $self->dry_run;
+ return;
+ }
+
+ my $res = $self->glacier->Describe_job($vault, $descr->{JobId});
+ if ($self->glacier->lasterr) {
+ if ($self->glacier->lasterr('code') == 404) {
+ $self->debug(1,
+ "deleting expired $key $vault "
+ . ($descr->{JobDescription} || $descr->{Action})
+ . ' '
+ . $descr->{JobId});
+ App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete()
+ unless $self->dry_run;
+ } else {
+ $self->error("can't describe job $descr->{JobId}: ",
+ $self->glacier->last_error_message);
+ }
+ return;
+ } elsif (ref($res) ne 'HASH') {
+ croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\"";
+ }
+ return $res;
+}
+
1;
diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm
index 0d17659..82b32c8 100644
--- a/lib/App/Glacier/Command/Get.pm
+++ b/lib/App/Glacier/Command/Get.pm
@@ -157,12 +157,14 @@ sub run {
exit(0);
}
if ($job->is_completed) {
my $cache_file = $job->cache_file;
if (-f $cache_file) {
+ $self->debug(1, "$job: copying from $cache_file");
+ return if $self->dry_run;
unless (copy($cache_file, $localname)) {
$self->abend(EX_FAILURE,
"can't copy $cache_file to $localname: $!");
}
} else {
my $tree_hash = $self->download($job, $localname);
@@ -190,12 +192,13 @@ sub run {
use constant MB => 1024*1024;
use constant TWOMB => 2*MB;
sub download {
my ($self, $job, $localname) = @_;
+
my $archive_size = $job->get('ArchiveSizeInBytes');
if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) {
# simple download
$self->_download_simple($job, $localname);
} else {
$self->_download_multipart($job, $localname);
@@ -211,15 +214,13 @@ sub _open_output {
return $fd;
}
sub _download_simple {
my ($self, $job, $localname) = @_;
- eval { # FIXME: file_name might be absent
- $self->debug(1, "downloading", $job->file_name(1), "in single part");
- };
+ $self->debug(1, "$job: downloading in single part");
return if $self->dry_run;
my $fd = $self->_open_output($localname);
my ($res, $tree_hash) = $self->glacier->Get_job_output($job->vault,
$job->id);
if ($self->glacier->lasterr) {
$self->abend(EX_FAILURE, "downoad failed: ",
@@ -253,14 +254,14 @@ sub _download_multipart {
}
# Number of parts to download:
my $total_parts = int(($archive_size + $part_size - 1) / $part_size);
# Compute the number of parts per job
my $job_parts = int(($total_parts + $njobs - 1) / $njobs);
- $self->debug(1,
- "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job");
+ $self->debug(1, "$job: downloading in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job");
+
return if $self->dry_run;
use Fcntl qw(SEEK_SET);
my $fd = $self->_open_output($localname);
my @part_hashes :shared = ();
diff --git a/lib/App/Glacier/Command/Jobs.pm b/lib/App/Glacier/Command/Jobs.pm
index 5a5f488..7dc5d06 100644
--- a/lib/App/Glacier/Command/Jobs.pm
+++ b/lib/App/Glacier/Command/Jobs.pm
@@ -113,48 +113,23 @@ sub run {
sub list {
my ($self, @vault_names) = @_;
my $db = $self->jobdb();
$db->foreach(sub {
- my ($key, $descr) = @_;
- my $vault = $descr->{VaultARN};
- $vault =~ s{.*:vaults/}{};
+ my ($key, $descr, $vault) = @_;
return if (@vault_names && ! grep { $_ eq $vault } @vault_names);
unless ($self->{_options}{cached}) {
- if ($descr->{StatusCode} eq 'Failed') {
- $self->debug(1, "deleting failed $key $vault " .
- ($descr->{JobDescription} || $descr->{Action}) .
- $descr->{JobId});
- $db->delete($key) unless $self->dry_run;
- return;
- }
-
- my $res = $self->glacier->Describe_job($vault, $descr->{JobId});
- if ($self->glacier->lasterr) {
- if ($self->glacier->lasterr('code') == 404) {
- $self->debug(1, "deleting expired $key $vault " .
- ($descr->{JobDescription} || $descr->{Action}) .
- $descr->{JobId});
- App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete()
- unless $self->dry_run;
- return;
- } else {
- $self->error("can't describe job $descr->{JobId}: ",
- $self->glacier->last_error_message);
- }
- } elsif (ref($res) ne 'HASH') {
- croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\"";
- } else {
- $res = timestamp_deserialize($res);
- $self->debug(2, $res->{StatusCode});
- $db->store($key, $res) unless $self->dry_run;
- $descr = $res;
- }
+ my $res = $self->check_job($key, $descr, $vault)
+ or return;
+ $res = timestamp_deserialize($res);
+ $self->debug(2, $res->{StatusCode});
+ $db->store($key, $res) unless $self->dry_run;
+ $descr = $res;
}
my $started = $self->format_date_time($descr, 'CreationDate');
print "$started - ";
if ($descr->{Completed} && $descr->{StatusCode} eq 'Succeeded') {
diff --git a/lib/App/Glacier/Command/Periodic.pm b/lib/App/Glacier/Command/Periodic.pm
index 5a4dc2a..39a2d7a 100644
--- a/lib/App/Glacier/Command/Periodic.pm
+++ b/lib/App/Glacier/Command/Periodic.pm
@@ -4,12 +4,13 @@ use warnings;
use App::Glacier::Core;
use parent qw(App::Glacier::Command);
use Carp;
use Data::Dumper;
use File::Basename;
use App::Glacier::Job;
+use App::Glacier::Command::Get;
=head1 NAME
glacier periodic - periodic cronjob for Glacier
=head1 SYNOPSIS
@@ -35,45 +36,16 @@ your crontab, e.g.:
sub run {
my $self = shift;
my $db = $self->jobdb();
$db->foreach(sub {
- my ($key, $descr) = @_;
- my $vault = $descr->{VaultARN};
- $vault =~ s{.*:vaults/}{};
+ my ($key, $descr, $vault) = @_;
- my $completed = $descr->{Completed};
-
- $self->debug(2, "$descr->{JobId} $descr->{Action} $vault");
- if ($descr->{StatusCode} eq 'Failed') {
- $self->debug(1,
- "deleting failed $key $vault "
- . ($descr->{JobDescription} || $descr->{Action})
- . ' '
- . $descr->{JobId});
- $db->delete($key) unless $self->dry_run;
- }
-
- my $res = $self->glacier->Describe_job($vault, $descr->{JobId});
- if ($self->glacier->lasterr) {
- if ($self->glacier->lasterr('code') == 404) {
- $self->debug(1,
- "deleting expired $key $vault "
- . ($descr->{JobDescription} || $descr->{Action})
- . ' '
- . $descr->{JobId});
- App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete()
- unless $self->dry_run;
- } else {
- $self->error("can't describe job $descr->{JobId}: ",
- $self->glacier->last_error_message);
- }
- } elsif (ref($res) ne 'HASH') {
- croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\"";
- } elsif ($res->{Completed} ne $completed) {
+ my $res = $self->check_job($key, $descr, $vault);
+ if ($res && $res->{Completed} ne $descr->{Completed}) {
$self->debug(2, $res->{StatusCode});
if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') {
$self->debug(1, "$descr->{JobId}: processing $descr->{Action} for $vault");
return if $self->dry_run;
if ($res->{Action} eq 'InventoryRetrieval') {
require App::Glacier::Command::Sync;
@@ -83,14 +55,14 @@ sub run {
my $job = App::Glacier::Job->fromdb($self, $vault,
$key, $res);
my $localname = $self->archive_cache_filename($vault,
$res->{ArchiveId});
$self->touchdir(dirname($localname));
- require App::Glacier::Command::Get;
my $get = clone App::Glacier::Command::Get($self);
+ $get->option(quiet => 1);
$get->download($job, $localname);
}
}
$db->store($key, $res);
}
});
diff --git a/lib/App/Glacier/DB/GDBM.pm b/lib/App/Glacier/DB/GDBM.pm
index 0e14a65..f7cc29a 100644
--- a/lib/App/Glacier/DB/GDBM.pm
+++ b/lib/App/Glacier/DB/GDBM.pm
@@ -3,12 +3,16 @@ use strict;
use warnings;
use GDBM_File;
use Carp;
use File::Basename;
use File::Path qw(make_path);
+# Avoid coredumps in threaded code.
+# See https://rt.perl.org/Public/Bug/Display.html?id=61912.
+sub CLONE_SKIP { 1 }
+
sub new {
my $class = shift;
local %_ = @_;
my $file = delete $_{file} // croak "filename is required";
unless (-f $file) {
if (defined(my $create = delete $_{create})) {
@@ -49,17 +53,14 @@ my %lexicon = (
sub configtest {
my ($class, $cfg, @path) = @_;
$cfg->lint(\%lexicon, @path);
}
-# We can't tie the DB to $self->{_map} at once, in the new method, because
-# this will cause coredumps in threaded code (see
-# https://rt.perl.org/Public/Bug/Display.html?id=61912). So, the following
-# auxiliary method is used, which calls &$code with $self->{_map} tied
-# to the DB.
+# Tie in the database, run $code, and untie it again. Correctly handle
+# nested invocations to avoid deadlocking.
sub _tied {
my ($self, $code) = @_;
croak "argument must be a CODE ref" unless ref($code) eq 'CODE';
if ($self->{_nref}++ == 0) {
my $n = 0;
while (! tie %{$self->{_map}}, 'GDBM_File', $self->{_filename},
diff --git a/lib/App/Glacier/Job.pm b/lib/App/Glacier/Job.pm
index ae12b22..9478b20 100644
--- a/lib/App/Glacier/Job.pm
+++ b/lib/App/Glacier/Job.pm
@@ -105,12 +105,17 @@ sub get {
my ($self, $key) = @_;
my $job = $self->_get_job;
return undef unless exists $job->{$key};
return $job->{$key};
}
+sub as_string { shift->get('JobDescription') }
+
+use overload
+ '""' => \&as_string;
+
sub is_finished {
my $self = shift;
return defined($self->get('StatusCode'));
}
sub is_completed {
diff --git a/lib/App/Glacier/Roster.pm b/lib/App/Glacier/Roster.pm
index ee56ff5..32c2b08 100644
--- a/lib/App/Glacier/Roster.pm
+++ b/lib/App/Glacier/Roster.pm
@@ -1,4 +1,13 @@
package App::Glacier::Roster;
use parent 'App::Glacier::DB';
+sub foreach {
+ my ($self, $fun) = @_;
+ $self->SUPER::foreach(sub {
+ my ($key, $descr) = @_;
+ (my $vault = $descr->{VaultARN}) =~ s{.*:vaults/}{};
+ &{$fun}($key, $descr, $vault);
+ });
+}
+
1;

Return to:

Send suggestions and report system problems to the System administrator.

Generated by cgit.