summaryrefslogtreecommitdiffabout
authorSergey Poznyakoff <gray@gnu.org>2018-12-19 12:35:49 (GMT)
committer Sergey Poznyakoff <gray@gnu.org>2018-12-19 12:55:09 (GMT)
commitea2c058ce96484937fcfe52dc7382a461a9d72e3 (patch) (unidiff)
treeae86e1115c3ef91bc03d2cfcb7df4f7ff8d19ce9
parent249512af92d3f592ae683279c2cd1f54a7c1a726 (diff)
downloadglacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.gz
glacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.bz2
Bugfixes.
* lib/App/Glacier/Command.pm (option,check_job): New methods. * lib/App/Glacier/Command/Get.pm: Rewrite debug messages (fixes temporary solution in 6c6dab5d). * lib/App/Glacier/Command/Jobs.pm: Rewrite the db->foreach sub. * lib/App/Glacier/Command/Periodic.pm: Likewise. * lib/App/Glacier/DB/GDBM.pm (CLONE_SKIP): New method. * lib/App/Glacier/Job.pm: Overload the "" operator. * lib/App/Glacier/Roster.pm (foreach): Overload method.
Diffstat (more/less context) (ignore whitespace changes)
-rw-r--r--lib/App/Glacier/Command.pm43
-rw-r--r--lib/App/Glacier/Command/Get.pm11
-rw-r--r--lib/App/Glacier/Command/Jobs.pm39
-rw-r--r--lib/App/Glacier/Command/Periodic.pm38
-rw-r--r--lib/App/Glacier/DB/GDBM.pm11
-rw-r--r--lib/App/Glacier/Job.pm5
-rw-r--r--lib/App/Glacier/Roster.pm9
7 files changed, 81 insertions, 75 deletions
diff --git a/lib/App/Glacier/Command.pm b/lib/App/Glacier/Command.pm
index 7178a75..1f9a130 100644
--- a/lib/App/Glacier/Command.pm
+++ b/lib/App/Glacier/Command.pm
@@ -162,6 +162,14 @@ sub clone {
162 $self 162 $self
163} 163}
164 164
165sub option {
166 my ($self, $opt, $val) = @_;
167 if (defined($val)) {
168 $self->{_options}{$opt} = $val;
169 }
170 return $self->{_options}{$opt};
171}
172
165sub touchdir { 173sub touchdir {
166 my ($self, $dir) = @_; 174 my ($self, $dir) = @_;
167 unless (-d $dir) { 175 unless (-d $dir) {
@@ -276,4 +284,39 @@ sub archive_cache_filename {
276 $archive_id); 284 $archive_id);
277} 285}
278 286
287sub check_job {
288 my ($self, $key, $descr, $vault) = @_;
289
290 $self->debug(2, "$descr->{JobId} $descr->{Action} $vault");
291 if ($descr->{StatusCode} eq 'Failed') {
292 $self->debug(1,
293 "deleting failed $key $vault "
294 . ($descr->{JobDescription} || $descr->{Action})
295 . ' '
296 . $descr->{JobId});
297 $self->jobdb()->delete($key) unless $self->dry_run;
298 return;
299 }
300
301 my $res = $self->glacier->Describe_job($vault, $descr->{JobId});
302 if ($self->glacier->lasterr) {
303 if ($self->glacier->lasterr('code') == 404) {
304 $self->debug(1,
305 "deleting expired $key $vault "
306 . ($descr->{JobDescription} || $descr->{Action})
307 . ' '
308 . $descr->{JobId});
309 App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete()
310 unless $self->dry_run;
311 } else {
312 $self->error("can't describe job $descr->{JobId}: ",
313 $self->glacier->last_error_message);
314 }
315 return;
316 } elsif (ref($res) ne 'HASH') {
317 croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\"";
318 }
319 return $res;
320}
321
2791; 3221;
diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm
index 0d17659..82b32c8 100644
--- a/lib/App/Glacier/Command/Get.pm
+++ b/lib/App/Glacier/Command/Get.pm
@@ -160,6 +160,8 @@ sub run {
160 if ($job->is_completed) { 160 if ($job->is_completed) {
161 my $cache_file = $job->cache_file; 161 my $cache_file = $job->cache_file;
162 if (-f $cache_file) { 162 if (-f $cache_file) {
163 $self->debug(1, "$job: copying from $cache_file");
164 return if $self->dry_run;
163 unless (copy($cache_file, $localname)) { 165 unless (copy($cache_file, $localname)) {
164 $self->abend(EX_FAILURE, 166 $self->abend(EX_FAILURE,
165 "can't copy $cache_file to $localname: $!"); 167 "can't copy $cache_file to $localname: $!");
@@ -193,6 +195,7 @@ use constant TWOMB => 2*MB;
193 195
194sub download { 196sub download {
195 my ($self, $job, $localname) = @_; 197 my ($self, $job, $localname) = @_;
198
196 my $archive_size = $job->get('ArchiveSizeInBytes'); 199 my $archive_size = $job->get('ArchiveSizeInBytes');
197 if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) { 200 if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) {
198 # simple download 201 # simple download
@@ -214,9 +217,7 @@ sub _open_output {
214sub _download_simple { 217sub _download_simple {
215 my ($self, $job, $localname) = @_; 218 my ($self, $job, $localname) = @_;
216 219
217 eval { # FIXME: file_name might be absent 220 $self->debug(1, "$job: downloading in single part");
218 $self->debug(1, "downloading", $job->file_name(1), "in single part");
219 };
220 return if $self->dry_run; 221 return if $self->dry_run;
221 my $fd = $self->_open_output($localname); 222 my $fd = $self->_open_output($localname);
222 my ($res, $tree_hash) = $self->glacier->Get_job_output($job->vault, 223 my ($res, $tree_hash) = $self->glacier->Get_job_output($job->vault,
@@ -256,8 +257,8 @@ sub _download_multipart {
256 # Compute the number of parts per job 257 # Compute the number of parts per job
257 my $job_parts = int(($total_parts + $njobs - 1) / $njobs); 258 my $job_parts = int(($total_parts + $njobs - 1) / $njobs);
258 259
259 $self->debug(1, 260 $self->debug(1, "$job: downloading in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job");
260 "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); 261
261 return if $self->dry_run; 262 return if $self->dry_run;
262 263
263 use Fcntl qw(SEEK_SET); 264 use Fcntl qw(SEEK_SET);
diff --git a/lib/App/Glacier/Command/Jobs.pm b/lib/App/Glacier/Command/Jobs.pm
index 5a5f488..7dc5d06 100644
--- a/lib/App/Glacier/Command/Jobs.pm
+++ b/lib/App/Glacier/Command/Jobs.pm
@@ -116,42 +116,17 @@ sub list {
116 116
117 my $db = $self->jobdb(); 117 my $db = $self->jobdb();
118 $db->foreach(sub { 118 $db->foreach(sub {
119 my ($key, $descr) = @_; 119 my ($key, $descr, $vault) = @_;
120 my $vault = $descr->{VaultARN};
121 $vault =~ s{.*:vaults/}{};
122 120
123 return if (@vault_names && ! grep { $_ eq $vault } @vault_names); 121 return if (@vault_names && ! grep { $_ eq $vault } @vault_names);
124 122
125 unless ($self->{_options}{cached}) { 123 unless ($self->{_options}{cached}) {
126 if ($descr->{StatusCode} eq 'Failed') { 124 my $res = $self->check_job($key, $descr, $vault)
127 $self->debug(1, "deleting failed $key $vault " . 125 or return;
128 ($descr->{JobDescription} || $descr->{Action}) . 126 $res = timestamp_deserialize($res);
129 $descr->{JobId}); 127 $self->debug(2, $res->{StatusCode});
130 $db->delete($key) unless $self->dry_run; 128 $db->store($key, $res) unless $self->dry_run;
131 return; 129 $descr = $res;
132 }
133
134 my $res = $self->glacier->Describe_job($vault, $descr->{JobId});
135 if ($self->glacier->lasterr) {
136 if ($self->glacier->lasterr('code') == 404) {
137 $self->debug(1, "deleting expired $key $vault " .
138 ($descr->{JobDescription} || $descr->{Action}) .
139 $descr->{JobId});
140 App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete()
141 unless $self->dry_run;
142 return;
143 } else {
144 $self->error("can't describe job $descr->{JobId}: ",
145 $self->glacier->last_error_message);
146 }
147 } elsif (ref($res) ne 'HASH') {
148 croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\"";
149 } else {
150 $res = timestamp_deserialize($res);
151 $self->debug(2, $res->{StatusCode});
152 $db->store($key, $res) unless $self->dry_run;
153 $descr = $res;
154 }
155 } 130 }
156 131
157 my $started = $self->format_date_time($descr, 'CreationDate'); 132 my $started = $self->format_date_time($descr, 'CreationDate');
diff --git a/lib/App/Glacier/Command/Periodic.pm b/lib/App/Glacier/Command/Periodic.pm
index 5a4dc2a..39a2d7a 100644
--- a/lib/App/Glacier/Command/Periodic.pm
+++ b/lib/App/Glacier/Command/Periodic.pm
@@ -7,6 +7,7 @@ use Carp;
7use Data::Dumper; 7use Data::Dumper;
8use File::Basename; 8use File::Basename;
9use App::Glacier::Job; 9use App::Glacier::Job;
10use App::Glacier::Command::Get;
10 11
11=head1 NAME 12=head1 NAME
12 13
@@ -38,39 +39,10 @@ sub run {
38 39
39 my $db = $self->jobdb(); 40 my $db = $self->jobdb();
40 $db->foreach(sub { 41 $db->foreach(sub {
41 my ($key, $descr) = @_; 42 my ($key, $descr, $vault) = @_;
42 my $vault = $descr->{VaultARN};
43 $vault =~ s{.*:vaults/}{};
44 43
45 my $completed = $descr->{Completed}; 44 my $res = $self->check_job($key, $descr, $vault);
46 45 if ($res && $res->{Completed} ne $descr->{Completed}) {
47 $self->debug(2, "$descr->{JobId} $descr->{Action} $vault");
48 if ($descr->{StatusCode} eq 'Failed') {
49 $self->debug(1,
50 "deleting failed $key $vault "
51 . ($descr->{JobDescription} || $descr->{Action})
52 . ' '
53 . $descr->{JobId});
54 $db->delete($key) unless $self->dry_run;
55 }
56
57 my $res = $self->glacier->Describe_job($vault, $descr->{JobId});
58 if ($self->glacier->lasterr) {
59 if ($self->glacier->lasterr('code') == 404) {
60 $self->debug(1,
61 "deleting expired $key $vault "
62 . ($descr->{JobDescription} || $descr->{Action})
63 . ' '
64 . $descr->{JobId});
65 App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete()
66 unless $self->dry_run;
67 } else {
68 $self->error("can't describe job $descr->{JobId}: ",
69 $self->glacier->last_error_message);
70 }
71 } elsif (ref($res) ne 'HASH') {
72 croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\"";
73 } elsif ($res->{Completed} ne $completed) {
74 $self->debug(2, $res->{StatusCode}); 46 $self->debug(2, $res->{StatusCode});
75 if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') { 47 if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') {
76 $self->debug(1, "$descr->{JobId}: processing $descr->{Action} for $vault"); 48 $self->debug(1, "$descr->{JobId}: processing $descr->{Action} for $vault");
@@ -86,8 +58,8 @@ sub run {
86 $res->{ArchiveId}); 58 $res->{ArchiveId});
87 $self->touchdir(dirname($localname)); 59 $self->touchdir(dirname($localname));
88 60
89 require App::Glacier::Command::Get;
90 my $get = clone App::Glacier::Command::Get($self); 61 my $get = clone App::Glacier::Command::Get($self);
62 $get->option(quiet => 1);
91 $get->download($job, $localname); 63 $get->download($job, $localname);
92 } 64 }
93 } 65 }
diff --git a/lib/App/Glacier/DB/GDBM.pm b/lib/App/Glacier/DB/GDBM.pm
index 0e14a65..f7cc29a 100644
--- a/lib/App/Glacier/DB/GDBM.pm
+++ b/lib/App/Glacier/DB/GDBM.pm
@@ -6,6 +6,10 @@ use Carp;
6use File::Basename; 6use File::Basename;
7use File::Path qw(make_path); 7use File::Path qw(make_path);
8 8
9# Avoid coredumps in threaded code.
10# See https://rt.perl.org/Public/Bug/Display.html?id=61912.
11sub CLONE_SKIP { 1 }
12
9sub new { 13sub new {
10 my $class = shift; 14 my $class = shift;
11 local %_ = @_; 15 local %_ = @_;
@@ -52,11 +56,8 @@ sub configtest {
52 $cfg->lint(\%lexicon, @path); 56 $cfg->lint(\%lexicon, @path);
53} 57}
54 58
55# We can't tie the DB to $self->{_map} at once, in the new method, because 59# Tie in the database, run $code, and untie it again. Correctly handle
56# this will cause coredumps in threaded code (see 60# nested invocations to avoid deadlocking.
57# https://rt.perl.org/Public/Bug/Display.html?id=61912). So, the following
58# auxiliary method is used, which calls &$code with $self->{_map} tied
59# to the DB.
60sub _tied { 61sub _tied {
61 my ($self, $code) = @_; 62 my ($self, $code) = @_;
62 croak "argument must be a CODE ref" unless ref($code) eq 'CODE'; 63 croak "argument must be a CODE ref" unless ref($code) eq 'CODE';
diff --git a/lib/App/Glacier/Job.pm b/lib/App/Glacier/Job.pm
index ae12b22..9478b20 100644
--- a/lib/App/Glacier/Job.pm
+++ b/lib/App/Glacier/Job.pm
@@ -108,6 +108,11 @@ sub get {
108 return $job->{$key}; 108 return $job->{$key};
109} 109}
110 110
111sub as_string { shift->get('JobDescription') }
112
113use overload
114 '""' => \&as_string;
115
111sub is_finished { 116sub is_finished {
112 my $self = shift; 117 my $self = shift;
113 return defined($self->get('StatusCode')); 118 return defined($self->get('StatusCode'));
diff --git a/lib/App/Glacier/Roster.pm b/lib/App/Glacier/Roster.pm
index ee56ff5..32c2b08 100644
--- a/lib/App/Glacier/Roster.pm
+++ b/lib/App/Glacier/Roster.pm
@@ -1,4 +1,13 @@
1package App::Glacier::Roster; 1package App::Glacier::Roster;
2use parent 'App::Glacier::DB'; 2use parent 'App::Glacier::DB';
3 3
4sub foreach {
5 my ($self, $fun) = @_;
6 $self->SUPER::foreach(sub {
7 my ($key, $descr) = @_;
8 (my $vault = $descr->{VaultARN}) =~ s{.*:vaults/}{};
9 &{$fun}($key, $descr, $vault);
10 });
11 }
12
41; 131;

Return to:

Send suggestions and report system problems to the System administrator.