diff options
author | Sergey Poznyakoff <gray@gnu.org> | 2018-12-19 14:35:49 +0200 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org> | 2018-12-19 14:55:09 +0200 |
commit | ea2c058ce96484937fcfe52dc7382a461a9d72e3 (patch) | |
tree | ae86e1115c3ef91bc03d2cfcb7df4f7ff8d19ce9 | |
parent | 249512af92d3f592ae683279c2cd1f54a7c1a726 (diff) | |
download | glacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.gz glacier-ea2c058ce96484937fcfe52dc7382a461a9d72e3.tar.bz2 |
Bugfixes.
* lib/App/Glacier/Command.pm (option,check_job): New methods.
* lib/App/Glacier/Command/Get.pm: Rewrite debug messages (fixes
temporary solution in 6c6dab5d).
* lib/App/Glacier/Command/Jobs.pm: Rewrite the db->foreach sub.
* lib/App/Glacier/Command/Periodic.pm: Likewise.
* lib/App/Glacier/DB/GDBM.pm (CLONE_SKIP): New method.
* lib/App/Glacier/Job.pm: Overload the "" operator.
* lib/App/Glacier/Roster.pm (foreach): Overload method.
-rw-r--r-- | lib/App/Glacier/Command.pm | 43 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Get.pm | 11 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Jobs.pm | 31 | ||||
-rw-r--r-- | lib/App/Glacier/Command/Periodic.pm | 38 | ||||
-rw-r--r-- | lib/App/Glacier/DB/GDBM.pm | 11 | ||||
-rw-r--r-- | lib/App/Glacier/Job.pm | 5 | ||||
-rw-r--r-- | lib/App/Glacier/Roster.pm | 9 |
7 files changed, 77 insertions, 71 deletions
diff --git a/lib/App/Glacier/Command.pm b/lib/App/Glacier/Command.pm index 7178a75..1f9a130 100644 --- a/lib/App/Glacier/Command.pm +++ b/lib/App/Glacier/Command.pm | |||
@@ -162,6 +162,14 @@ sub clone { | |||
162 | $self | 162 | $self |
163 | } | 163 | } |
164 | 164 | ||
165 | sub option { | ||
166 | my ($self, $opt, $val) = @_; | ||
167 | if (defined($val)) { | ||
168 | $self->{_options}{$opt} = $val; | ||
169 | } | ||
170 | return $self->{_options}{$opt}; | ||
171 | } | ||
172 | |||
165 | sub touchdir { | 173 | sub touchdir { |
166 | my ($self, $dir) = @_; | 174 | my ($self, $dir) = @_; |
167 | unless (-d $dir) { | 175 | unless (-d $dir) { |
@@ -276,4 +284,39 @@ sub archive_cache_filename { | |||
276 | $archive_id); | 284 | $archive_id); |
277 | } | 285 | } |
278 | 286 | ||
287 | sub check_job { | ||
288 | my ($self, $key, $descr, $vault) = @_; | ||
289 | |||
290 | $self->debug(2, "$descr->{JobId} $descr->{Action} $vault"); | ||
291 | if ($descr->{StatusCode} eq 'Failed') { | ||
292 | $self->debug(1, | ||
293 | "deleting failed $key $vault " | ||
294 | . ($descr->{JobDescription} || $descr->{Action}) | ||
295 | . ' ' | ||
296 | . $descr->{JobId}); | ||
297 | $self->jobdb()->delete($key) unless $self->dry_run; | ||
298 | return; | ||
299 | } | ||
300 | |||
301 | my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); | ||
302 | if ($self->glacier->lasterr) { | ||
303 | if ($self->glacier->lasterr('code') == 404) { | ||
304 | $self->debug(1, | ||
305 | "deleting expired $key $vault " | ||
306 | . ($descr->{JobDescription} || $descr->{Action}) | ||
307 | . ' ' | ||
308 | . $descr->{JobId}); | ||
309 | App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() | ||
310 | unless $self->dry_run; | ||
311 | } else { | ||
312 | $self->error("can't describe job $descr->{JobId}: ", | ||
313 | $self->glacier->last_error_message); | ||
314 | } | ||
315 | return; | ||
316 | } elsif (ref($res) ne 'HASH') { | ||
317 | croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; | ||
318 | } | ||
319 | return $res; | ||
320 | } | ||
321 | |||
279 | 1; | 322 | 1; |
diff --git a/lib/App/Glacier/Command/Get.pm b/lib/App/Glacier/Command/Get.pm index 0d17659..82b32c8 100644 --- a/lib/App/Glacier/Command/Get.pm +++ b/lib/App/Glacier/Command/Get.pm | |||
@@ -160,6 +160,8 @@ sub run { | |||
160 | if ($job->is_completed) { | 160 | if ($job->is_completed) { |
161 | my $cache_file = $job->cache_file; | 161 | my $cache_file = $job->cache_file; |
162 | if (-f $cache_file) { | 162 | if (-f $cache_file) { |
163 | $self->debug(1, "$job: copying from $cache_file"); | ||
164 | return if $self->dry_run; | ||
163 | unless (copy($cache_file, $localname)) { | 165 | unless (copy($cache_file, $localname)) { |
164 | $self->abend(EX_FAILURE, | 166 | $self->abend(EX_FAILURE, |
165 | "can't copy $cache_file to $localname: $!"); | 167 | "can't copy $cache_file to $localname: $!"); |
@@ -193,6 +195,7 @@ use constant TWOMB => 2*MB; | |||
193 | 195 | ||
194 | sub download { | 196 | sub download { |
195 | my ($self, $job, $localname) = @_; | 197 | my ($self, $job, $localname) = @_; |
198 | |||
196 | my $archive_size = $job->get('ArchiveSizeInBytes'); | 199 | my $archive_size = $job->get('ArchiveSizeInBytes'); |
197 | if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) { | 200 | if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) { |
198 | # simple download | 201 | # simple download |
@@ -214,9 +217,7 @@ sub _open_output { | |||
214 | sub _download_simple { | 217 | sub _download_simple { |
215 | my ($self, $job, $localname) = @_; | 218 | my ($self, $job, $localname) = @_; |
216 | 219 | ||
217 | eval { # FIXME: file_name might be absent | 220 | $self->debug(1, "$job: downloading in single part"); |
218 | $self->debug(1, "downloading", $job->file_name(1), "in single part"); | ||
219 | }; | ||
220 | return if $self->dry_run; | 221 | return if $self->dry_run; |
221 | my $fd = $self->_open_output($localname); | 222 | my $fd = $self->_open_output($localname); |
222 | my ($res, $tree_hash) = $self->glacier->Get_job_output($job->vault, | 223 | my ($res, $tree_hash) = $self->glacier->Get_job_output($job->vault, |
@@ -256,8 +257,8 @@ sub _download_multipart { | |||
256 | # Compute the number of parts per job | 257 | # Compute the number of parts per job |
257 | my $job_parts = int(($total_parts + $njobs - 1) / $njobs); | 258 | my $job_parts = int(($total_parts + $njobs - 1) / $njobs); |
258 | 259 | ||
259 | $self->debug(1, | 260 | $self->debug(1, "$job: downloading in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); |
260 | "downloading", $job->file_name(1), "to $localname in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job"); | 261 | |
261 | return if $self->dry_run; | 262 | return if $self->dry_run; |
262 | 263 | ||
263 | use Fcntl qw(SEEK_SET); | 264 | use Fcntl qw(SEEK_SET); |
diff --git a/lib/App/Glacier/Command/Jobs.pm b/lib/App/Glacier/Command/Jobs.pm index 5a5f488..7dc5d06 100644 --- a/lib/App/Glacier/Command/Jobs.pm +++ b/lib/App/Glacier/Command/Jobs.pm | |||
@@ -116,43 +116,18 @@ sub list { | |||
116 | 116 | ||
117 | my $db = $self->jobdb(); | 117 | my $db = $self->jobdb(); |
118 | $db->foreach(sub { | 118 | $db->foreach(sub { |
119 | my ($key, $descr) = @_; | 119 | my ($key, $descr, $vault) = @_; |
120 | my $vault = $descr->{VaultARN}; | ||
121 | $vault =~ s{.*:vaults/}{}; | ||
122 | 120 | ||
123 | return if (@vault_names && ! grep { $_ eq $vault } @vault_names); | 121 | return if (@vault_names && ! grep { $_ eq $vault } @vault_names); |
124 | 122 | ||
125 | unless ($self->{_options}{cached}) { | 123 | unless ($self->{_options}{cached}) { |
126 | if ($descr->{StatusCode} eq 'Failed') { | 124 | my $res = $self->check_job($key, $descr, $vault) |
127 | $self->debug(1, "deleting failed $key $vault " . | 125 | or return; |
128 | ($descr->{JobDescription} || $descr->{Action}) . | ||
129 | $descr->{JobId}); | ||
130 | $db->delete($key) unless $self->dry_run; | ||
131 | return; | ||
132 | } | ||
133 | |||
134 | my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); | ||
135 | if ($self->glacier->lasterr) { | ||
136 | if ($self->glacier->lasterr('code') == 404) { | ||
137 | $self->debug(1, "deleting expired $key $vault " . | ||
138 | ($descr->{JobDescription} || $descr->{Action}) . | ||
139 | $descr->{JobId}); | ||
140 | App::Glacier::Job->fromdb($self, $vault, $key, $res)->delete() | ||
141 | unless $self->dry_run; | ||
142 | return; | ||
143 | } else { | ||
144 | $self->error("can't describe job $descr->{JobId}: ", | ||
145 | $self->glacier->last_error_message); | ||
146 | } | ||
147 | } elsif (ref($res) ne 'HASH') { | ||
148 | croak "describe_job returned wrong datatype (".ref($res).") for \"$descr->{JobId}\""; | ||
149 | } else { | ||
150 | $res = timestamp_deserialize($res); | 126 | $res = timestamp_deserialize($res); |
151 | $self->debug(2, $res->{StatusCode}); | 127 | $self->debug(2, $res->{StatusCode}); |
152 | $db->store($key, $res) unless $self->dry_run; | 128 | $db->store($key, $res) unless $self->dry_run; |
153 | $descr = $res; | 129 | $descr = $res; |
154 | } | 130 | } |
155 | } | ||
156 | 131 | ||
157 | my $started = $self->format_date_time($descr, 'CreationDate'); | 132 | my $started = $self->format_date_time($descr, 'CreationDate'); |
158 | 133 | ||
diff --git a/lib/App/Glacier/Command/Periodic.pm b/lib/App/Glacier/Command/Periodic.pm index 5a4dc2a..39a2d7a 100644 --- a/lib/App/Glacier/Command/Periodic.pm +++ b/lib/App/Glacier/Command/Periodic.pm | |||
@@ -7,6 +7,7 @@ use Carp; | |||
7 | use Data::Dumper; | 7 | use Data::Dumper; |
8 | use File::Basename; | 8 | use File::Basename; |
9 | use App::Glacier::Job; | 9 | use App::Glacier::Job; |
10 | use App::Glacier::Command::Get; | ||
10 | 11 | ||
11 | =head1 NAME | 12 | =head1 NAME |
12 | 13 | ||
@@ -38,39 +39,10 @@ sub run { | |||
38 | 39 | ||
39 | my $db = $self->jobdb(); | 40 | my $db = $self->jobdb(); |
40 | $db->foreach(sub { | 41 | $db->foreach(sub { |
41 | my ($key, $descr) = @_; | 42 | my ($key, $descr, $vault) = @_; |
42 | my $vault = $descr->{VaultARN}; | ||
43 | $vault =~ s{.*:vaults/}{}; | ||
44 | |||
45 | my $completed = $descr->{Completed}; | ||
46 | |||
47 | $self->debug(2, "$descr->{JobId} $descr->{Action} $vault"); | ||
48 | if ($descr->{StatusCode} eq 'Failed') { | ||
49 | $self->debug(1, | ||
50 | "deleting failed $key $vault " | ||
51 | . ($descr->{JobDescription} || $descr->{Action}) | ||
52 | . ' ' | ||
53 | . $descr->{JobId}); | ||
54 | $db->delete($key) unless $self->dry_run; | ||
55 | } | ||
56 | 43 | ||
57 | my $res = $self->glacier->Describe_job($vault, $descr->{JobId}); | 44 | my $res = $self->check_job($key, $descr, $vault); |
58 | if ($self->glacier->lasterr) { | 45 | if ($res && $res->{Completed} ne $descr->{Completed}) { |
59 | if ($self->glacier->lasterr('code') == 404) { | ||
60 | $self->debug(1, | ||
61 | "deleting expired $key $vault " | ||