diff options
author | Sergey Poznyakoff <gray@gnu.org> | 2018-12-19 14:35:49 +0200 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org> | 2018-12-19 14:55:09 +0200 |
commit | ea2c058ce96484937fcfe52dc7382a461a9d72e3 (patch) | |
tree | ae86e1115c3ef91bc03d2cfcb7df4f7ff8d19ce9 | |
parent | 249512af92d3f592ae683279c2cd1f54a7c1a726 (diff) | |
download | glacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.gz glacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.bz2 |
Bugfixes.
* lib/App/Glacier/Command.pm (option,check_job): New methods.
* lib/App/Glacier/Command/Get.pm: Rewrite debug messages (fixes
temporary solution in 6c6dab5d).
* lib/App/Glacier/Command/Jobs.pm: Rewrite the db->foreach sub.
* lib/App/Glacier/Command/Periodic.pm: Likewise.
* lib/App/Glacier/DB/GDBM.pm (CLONE_SKIP): New method.
* lib/App/Glacier/Job.pm: Overload the "" operator.
* lib/App/Glacier/Roster.pm (foreach): Overload method.
-rw-r--r-- | lib/App/Glacier/Command.pm | 43 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Get.pm | 11 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Jobs.pm | 39 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Periodic.pm | 38 | ||||
-rw-r--r-- | lib/App/Glacier/DB/GDBM.pm | 11 | ||||
-rw-r--r-- | lib/App/Glacier/Job.pm | 5 | ||||
-rw-r--r-- | lib/App/Glacier/Roster.pm | 9 |
7 files changed, 81 insertions, 75 deletions
diff --git a/lib/App/Glacier/Command.pm b/lib/App/Glacier/Command.pm index 7178a75..1f9a130 100644 --- a/lib/App/Glacier/Command.pm +++ b/lib/App/Glacier/Command.pm | |||
@@ -163,4 +163,12 @@ sub clone { | |||
163 | } | 163 | } |
164 | 164 | ||
165 | sub option { | ||
166 | my ($self, $opt, $val) = @_; | ||
167 | if (defined($val)) { | ||
168 | $self->{_options}{$opt} = $val; | ||
169 | } | ||
170 | return $self->{_options}{$opt}; | ||
171 | } | ||
172 | |||
165 | sub touchdir { | 173 | sub touchdir { |
166 | my ($self, $dir) = @_; | 174 | my ($self, $dir) = @_; |
@@ -277,3 +285,38 @@ sub archive_cache_filename { | |||
277 | } | 285 | } |
278 | 286 | ||
287 | sub check_job { | ||
288 | my ($self, $key, $descr, $vault) = @_; | ||
289 | |||
290 | $self->debug(2, "$descr->{JobId} $descr->{Action} $vault"); | ||
291 | if ($descr->{StatusCode} eq 'Failed') { | ||
292 | $self->debug(1, | ||
293 | "deleting failed $key $vault " | ||
294 | . ($descr->{JobDescription} || $descr->{Action}) | ||
295 | . ' ' | ||
296 | . $descr->{JobId}); | ||
297 | $self->jobdb()->delete($key) unless $self->dry_run; | ||
298 | return; | ||
299 | } | ||
300 | |||
301 | my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); | ||
302 | if ($self->glacier->lasterr) { | ||
303 | if ($self->glacier->lasterr('code') == 404) { | ||
304 | $self->debug(1, | ||
305 | "deleting expired $key $vault " | ||
306 | . ($descr->{JobDescription} || $descr->{Action}) | ||
307 | . ' ' | ||
308 | . $descr->{JobId}); | ||
309 | App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() | ||
310 | unless $self->dry_run; | ||
311 | } else { | ||
312 | $self->error("can't describe job $descr->{JobId}: ", | ||
313 | $self->glacier->last_error_message); | ||
314 | } | ||
315 | return; | ||
316 | } elsif (ref($res) ne 'HASH') { | ||
317 | croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; | ||
318 | } | ||
319 | return $res; | ||
320 | } | ||
321 | |||
279 | 1; | 322 | 1; |
diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm index 0d17659..82b32c8 100644 --- a/lib/App/Glacier/Command/Get.pm +++ b/lib/App/Glacier/Command/Get.pm | |||
@@ -161,4 +161,6 @@ sub run { | |||
161 | my $cache_file = $job->cache_file; | 161 | my $cache_file = $job->cache_file; |
162 | if (-f $cache_file) { | 162 | if (-f $cache_file) { |
163 | $self->debug(1, "$job: copying from $cache_file"); | ||
164 | return if $self->dry_run; | ||
163 | unless (copy($cache_file, $localname)) { | 165 | unless (copy($cache_file, $localname)) { |
164 | $self->abend(EX_FAILURE, | 166 | $self->abend(EX_FAILURE, |
@@ -194,4 +196,5 @@ use constant TWOMB => 2*MB; | |||
194 | sub download { | 196 | sub download { |
195 | my ($self, $job, $localname) = @_; | 197 | my ($self, $job, $localname) = @_; |
198 | |||
196 | my $archive_size = $job->get('ArchiveSizeInBytes'); | 199 | my $archive_size = $job->get('ArchiveSizeInBytes'); |
197 | if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) { | 200 | if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) { |
@@ -215,7 +218,5 @@ sub _download_simple { | |||
215 | my ($self, $job, $localname) = @_; | 218 | my ($self, $job, $localname) = @_; |
216 | 219 | ||
217 | eval { # FIXME: file_name might be absent | 220 | $self->debug(1, "$job: downloading in single part"); |
218 | $self->debug(1, "downloading", $job->file_name(1), "in single part"); | ||
219 | }; | ||
220 | return if $self->dry_run; | 221 | return if $self->dry_run; |
221 | my $fd = $self->_open_output($localname); | 222 | my $fd = $self->_open_output($localname); |
@@ -257,6 +258,6 @@ sub _download_multipart { | |||
257 | my $job_parts = int(($total_parts + $njobs - 1) / $njobs); | 258 | my $job_parts = int(($total_parts + $njobs - 1) / $njobs); |
258 | 259 | ||
259 | $self->debug(1, | 260 | $self->debug(1, "$job: downloading in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); |
260 | "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); | 261 | |
261 | return if $self->dry_run; | 262 | return if $self->dry_run; |
262 | 263 | ||
diff --git a/lib/App/Glacier/Command/Jobs.pm b/lib/App/Glacier/Command/Jobs.pm index 5a5f488..7dc5d06 100644 --- a/lib/App/Glacier/Command/Jobs.pm +++ b/lib/App/Glacier/Command/Jobs.pm | |||
@@ -117,40 +117,15 @@ sub list { | |||
117 | my $db = $self->jobdb(); | 117 | my $db = $self->jobdb(); |
118 | $db->foreach(sub { | 118 | $db->foreach(sub { |
119 | my ($key, $descr) = @_; | 119 | my ($key, $descr, $vault) = @_; |
120 | my $vault = $descr->{VaultARN}; | ||
121 | $vault =~ s{.*:vaults/}{}; | ||
122 | 120 | ||
123 | return if (@vault_names && ! grep { $_ eq $vault } @vault_names); | 121 | return if (@vault_names && ! grep { $_ eq $vault } @vault_names); |
124 | 122 | ||
125 | unless ($self->{_options}{cached}) { | 123 | unless ($self->{_options}{cached}) { |
126 | if ($descr->{StatusCode} eq 'Failed') { | 124 | my $res = $self->check_job($key, $descr, $vault) |
127 | $self->debug(1, "deleting failed $key $vault " . | 125 | or return; |
128 | ($descr->{JobDescription} || $descr->{Action}) . | 126 | $res = timestamp_deserialize($res); |
129 | $descr->{JobId}); | 127 | $self->debug(2, $res->{StatusCode}); |
130 | $db->delete($key) unless $self->dry_run; | 128 | $db->store($key, $res) unless $self->dry_run; |
131 | return; | 129 | $descr = $res; |
132 | } | ||
133 | |||
134 | my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); | ||
135 | if ($self->glacier->lasterr) { | ||
136 | if ($self->glacier->lasterr('code') == 404) { | ||
137 | $self->debug(1, "deleting expired $key $vault " . | ||
138 | ($descr->{JobDescription} || $descr->{Action}) . | ||
139 | $descr->{JobId}); | ||
140 | App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() | ||
141 | unless $self->dry_run; | ||
142 | return; | ||
143 | } else { | ||
144 | $self->error("can't describe job $descr->{JobId}: ", | ||
145 | $self->glacier->last_error_message); | ||
146 | } | ||
147 | } elsif (ref($res) ne 'HASH') { | ||
148 | croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; | ||
149 | } else { | ||
150 | $res = timestamp_deserialize($res); | ||
151 | $self->debug(2, $res->{StatusCode}); | ||
152 | $db->store($key, $res) unless $self->dry_run; | ||
153 | $descr = $res; | ||
154 | } | ||
155 | } | 130 | } |
156 | 131 | ||
diff --git a/lib/App/Glacier/Command/Periodic.pm b/lib/App/Glacier/Command/Periodic.pm index 5a4dc2a..39a2d7a 100644 --- a/lib/App/Glacier/Command/Periodic.pm +++ b/lib/App/Glacier/Command/Periodic.pm | |||
@@ -8,4 +8,5 @@ use Data::Dumper; | |||
8 | use File::Basename; | 8 | use File::Basename; |
9 | use App::Glacier::Job; | 9 | use App::Glacier::Job; |
10 | use App::Glacier::Command::Get; | ||
10 | 11 | ||
11 | =head1 NAME | 12 | =head1 NAME |
@@ -39,37 +40,8 @@ sub run { | |||
39 | my $db = $self->jobdb(); | 40 | my $db = $self->jobdb(); |
40 | $db->foreach(sub { | 41 | $db->foreach(sub { |
41 | my ($key, $descr) = @_; | 42 | my ($key, $descr, $vault) = @_; |
42 | my $vault = $descr->{VaultARN}; | ||
43 | $vault =~ s{.*:vaults/}{}; | ||
44 | 43 | ||
45 | my $completed = $descr->{Completed}; | 44 | my $res = $self->check_job($key, $descr, $vault); |
46 | 45 | if ($res && $res->{Completed} ne $descr->{Completed}) { | |
47 | $self->debug(2, "$descr->{JobId} $descr->{Action} $vault"); | ||
48 | if ($descr->{StatusCode} eq 'Failed') { | ||
49 | $self->debug(1, | ||
50 | "deleting failed $key $vault " | ||
51 | . ($descr->{JobDescription} || $descr->{Action}) | ||
52 | . ' ' | ||
53 | . $descr->{JobId}); | ||
54 | $db->delete($key) unless $self->dry_run; | ||
55 | } | ||
56 | |||
57 | my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); | ||
58 | if ($self->glacier->lasterr) { | ||
59 | if ($self->glacier->lasterr('code') == 404) { | ||
60 | $self->debug(1, | ||
61 | "deleting expired $key $vault " | ||
62 | . ($descr->{JobDescription} || $descr->{Action}) | ||
63 | . ' ' | ||
64 | . $descr->{JobId}); | ||
65 | App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() | ||
66 | unless $self->dry_run; | ||
67 | } else { | ||
68 | $self->error("can't describe job $descr->{JobId}: ", | ||
69 | $self->glacier->last_error_message); | ||
70 | } | ||
71 | } elsif (ref($res) ne 'HASH') { | ||
72 | croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; | ||
73 | } elsif ($res->{Completed} ne $completed) { | ||
74 | $self->debug(2, $res->{StatusCode}); | 46 | $self->debug(2, $res->{StatusCode}); |
75 | if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') { | 47 | if ($res->{Completed} && $res->{StatusCode} eq 'Succeeded') { |
@@ -87,6 +59,6 @@ sub run { | |||
87 | $self->touchdir(dirname($localname)); | 59 | $self->touchdir(dirname($localname)); |
88 | 60 | ||
89 | require App::Glacier::Command::Get; | ||
90 | my $get = clone App::Glacier::Command::Get($self); | 61 | my $get = clone App::Glacier::Command::Get($self); |
62 | $get->option(quiet => 1); | ||
91 | $get->download($job, $localname); | 63 | $get->downl |