aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org>2018-03-14 17:22:18 +0200
committerSergey Poznyakoff <gray@gnu.org>2018-03-14 23:21:26 +0200
commit649a04b9deec5d111e735bc47c37adca7e76f39e (patch)
treec651192f145575fc7a49d591738f9cfc6e80e6d8
parentae194ce922bff039e9003f1e84d44132aab63c3e (diff)
downloadeclat-649a04b9deec5d111e735bc47c37adca7e76f39e.tar.gz
eclat-649a04b9deec5d111e735bc47c37adca7e76f39e.tar.bz2
Implement exponential backoff with jitter to handle RequestLimitExceeded
* src/eclat.h (eclat_command_env) <curl>: Remove. <xmltree>: New field. * src/util.c (translate_ids): Close the map when no longer needed (eclat_send_request): Implement exponential backoff. * src/ec2map.c: Reflect the above changes. * src/eclat.c: Likewise. * lib/libeclat.h (eclat_request_dup): New proto. * lib/reqcreat.c (eclat_request_dup): New function. * src/config.c: New configuration statements: max-retry-interval and retry-timeout.
-rw-r--r--lib/istore.c2
-rw-r--r--lib/libeclat.h3
-rw-r--r--lib/reqcreat.c51
-rw-r--r--lib/reqsign.c4
-rw-r--r--lib/trace.c3
-rw-r--r--src/config.c8
-rw-r--r--src/ec2map.c22
-rw-r--r--src/eclat.c21
-rw-r--r--src/eclat.h8
-rw-r--r--src/igw.c3
-rw-r--r--src/ispeek.c5
-rw-r--r--src/mkvpc-cl.opt3
-rw-r--r--src/util.c164
13 files changed, 189 insertions, 108 deletions
diff --git a/lib/istore.c b/lib/istore.c
index dea3a53..8b46b64 100644
--- a/lib/istore.c
+++ b/lib/istore.c
@@ -29,7 +29,6 @@ acc_cb(void *ptr, size_t size, size_t nmemb, void *data)
CURL *
instance_store_curl_new(struct grecs_txtacc *acc)
{
- CURLcode res;
CURL *curl;
curl = curl_easy_init();
@@ -46,7 +45,6 @@ instance_store_read(const char *url, CURL *curl)
{
CURLcode res;
long http_resp;
- char *text;
curl_easy_setopt(curl, CURLOPT_URL, url);
diff --git a/lib/libeclat.h b/lib/libeclat.h
index b865e00..6f1acf6 100644
--- a/lib/libeclat.h
+++ b/lib/libeclat.h
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -104,6 +104,7 @@ struct ec2_request {
struct ec2_request *eclat_request_create(int flags, const char *endpoint,
const char *uri, char const *region,
char const *access_key, char const *token);
+struct ec2_request *eclat_request_dup(struct ec2_request const *src);
void eclat_request_free(struct ec2_request *);
void eclat_request_add_param0(struct ec2_request *req, const char *name,
const char *value, int encoded);
diff --git a/lib/reqcreat.c b/lib/reqcreat.c
index 58d7829..753a415 100644
--- a/lib/reqcreat.c
+++ b/lib/reqcreat.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -27,6 +27,15 @@ ec2_param_free(void *ptr)
free(p);
}
+static int
+ec2_param_dup(void *sym, void *data)
+{
+ struct ec2_param *p = sym;
+ struct ec2_request *req = data;
+ eclat_request_add_param(req, p->name, p->value);
+ return 0;
+}
+
struct ec2_request *
eclat_request_create(int flags, const char *endpoint, const char *uri,
char const *region, char const *access_key,
@@ -49,3 +58,43 @@ eclat_request_create(int flags, const char *endpoint, const char *uri,
req->token = token ? grecs_strdup(token) : NULL;
return req;
}
+
+static inline char *
+safe_strdup(char const *a)
+{
+ return a ? grecs_strdup(a) : NULL;
+}
+
+struct ec2_request *
+eclat_request_dup(struct ec2_request const *src)
+{
+ struct ec2_request *dst = grecs_zalloc(sizeof(*dst));
+ dst->flags = src->flags;
+ dst->endpoint = safe_strdup(src->endpoint);
+ dst->uri = safe_strdup(src->uri);
+ dst->params = grecs_symtab_create(sizeof(struct ec2_param),
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ ec2_param_free);
+
+ grecs_symtab_enumerate(src->params, ec2_param_dup, dst);
+ if (src->headers == NULL) {
+ dst->headers = NULL;
+ } else {
+ struct grecs_list_entry *ep;
+ for (ep = src->headers->head; ep; ep = ep->next) {
+ struct ec2_param *ent = ep->data;
+ eclat_request_add_header(dst, ent->name, ent->value);
+ }
+ }
+ dst->postdata = safe_strdup(src->postdata);
+ dst->region = safe_strdup(src->region);
+ dst->access_key = safe_strdup(src->access_key);
+ dst->token = safe_strdup(src->token);
+ dst->ttl = src->ttl;
+
+ return dst;
+}
+
diff --git a/lib/reqsign.c b/lib/reqsign.c
index 9572961..05988ed 100644
--- a/lib/reqsign.c
+++ b/lib/reqsign.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -216,7 +216,7 @@ requestsign4(struct ec2_request *req, char *secret)
grecs_txtacc_grow_char(acc, '\n');
grecs_txtacc_grow_string(acc, req->uri);
grecs_txtacc_grow_char(acc, '\n');
- /* Append a canonicalized request string */
+ /* Append canonicalized request string */
for (i = 0; i < n; i++) {
struct ec2_param *p, key;
diff --git a/lib/trace.c b/lib/trace.c
index 4438822..6b79e58 100644
--- a/lib/trace.c
+++ b/lib/trace.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
along with Eclat. If not, see <http://www.gnu.org/licenses/>. */
#include "libeclat.h"
+#include <ctype.h>
#include <curl/curl.h>
static void
diff --git a/src/config.c b/src/config.c
index 49b47f9..6fac842 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -396,6 +396,12 @@ static struct grecs_keyword eclat_kw[] = {
"Specifies a file containing `accessID:accessKey' pairs.",
grecs_type_string, GRECS_DFLT, &access_file_name, 0,
cb_access_file },
+ { "max-retry-interval", "seconds",
+ "Maximum interval between retries in exponential backoff algorithm.",
+ grecs_type_ulong, GRECS_DFLT, &max_retry_sleep },
+ { "retry-timeout", "seconds",
+ "Give up retrying after this many seconds",
+ grecs_type_ulong, GRECS_DFLT, &max_retry_time },
{ "default-region", "name",
"Define default AWS region",
grecs_type_string, GRECS_DFLT, &region_name },
diff --git a/src/ec2map.c b/src/ec2map.c
index 6e2be10..3dbf4f1 100644
--- a/src/ec2map.c
+++ b/src/ec2map.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -32,7 +32,6 @@ struct ec2_map {
char *action;
struct grecs_list *args;
char *ret;
- struct eclat_io *io;
};
static struct grecs_keyword ec2_map_kw[] = {
@@ -56,7 +55,6 @@ static void
ec2_map_free(int dbg, void *data)
{
struct ec2_map *map = data;
- eclat_io_free(map->io);
free(map->action);
free(map->ret);
grecs_list_free(map->args);
@@ -114,23 +112,12 @@ ec2_map_config(int dbg, struct grecs_node *node, void *data)
static int
ec2_map_open(int dbg, void *data)
{
- struct ec2_map *map = data;
-
- map->io = eclat_io_init(0);
- if (!map->io) {
- err("cannot open EC2 database");
- return eclat_map_failure;
- }
return eclat_map_ok;
}
static int
ec2_map_close(int dbg, void *data)
{
- struct ec2_map *map = data;
-
- eclat_io_free(map->io);
- map->io = NULL;
return 0;
}
@@ -184,14 +171,9 @@ ec2_map_get(int dbg, int dir, void *data, const char *key, char **return_value)
return eclat_map_failure;
}
- rc = eclat_send_request(map->io->curl, q);
-
- if (rc)
+ if (eclat_send_request(q, &tree))
return eclat_map_failure;
- tree = eclat_io_finish(map->io);
-
-
node = grecs_find_node(tree, map->ret);
if (debug_category[dbg].level > 1) {
diff --git a/src/eclat.c b/src/eclat.c
index 8d4838b..5326c9e 100644
--- a/src/eclat.c
+++ b/src/eclat.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -712,7 +712,7 @@ eclat_do_command(eclat_command_env_t *env, struct eclat_command *command,
if (!eclat_confirm(confirm_mode,
"Proceed with %s", command->ident))
die(EX_CANCELLED, "command not confirmed");
- rc = eclat_send_request(env->curl, env->request);
+ rc = eclat_send_request(env->request, &env->xmltree);
}
return rc;
}
@@ -722,11 +722,10 @@ main(int argc, char **argv)
{
int rc;
struct grecs_node *tree;
- struct eclat_io *io;
- struct grecs_node *xmltree;
forlan_eval_env_t env = NULL;
struct eclat_command *command = NULL;
eclat_command_env_t cmdenv;
+ struct timeval tv;
set_program_name(argv[0]);
proginfo.print_help_hook = listcmdhook;
@@ -746,6 +745,8 @@ main(int argc, char **argv)
eclat_map_drv_register(&eclat_map_drv_ec2);
sortcmds();
config_init();
+ gettimeofday(&tv, NULL);
+ srandom((unsigned int)tv.tv_usec);
parse_options(&argc, &argv);
if (match_command_mode) {
@@ -848,34 +849,30 @@ main(int argc, char **argv)
}
debug(ECLAT_DEBCAT_MAIN, 1, ("using access key %s", access_key));
- io = eclat_io_init(1);
-
if (confirm_mode == eclat_confirm_unspecified)
confirm_mode = command->confirm;
/* Prepare environment */
memset(&cmdenv, 0, sizeof(cmdenv));
cmdenv.cmd = command;
- cmdenv.curl = io->curl;
rc = eclat_do_command(&cmdenv, command, argc, argv);
if (rc)
exit(rc);
- xmltree = eclat_io_finish(io);
-
if (xml_dump_file)
fclose(xml_dump_file);
if (sort_option)
- grecs_tree_sort(xmltree, node_ident_cmp);
+ grecs_tree_sort(cmdenv.xmltree, node_ident_cmp);
if (dry_run_mode)
/* nothing */;
else if (env) {
- rc = forlan_run(env, xmltree);
+ rc = forlan_run(env, cmdenv.xmltree);
} else {
- grecs_print_node(xmltree, GRECS_NODE_FLAG_DEFAULT, stdout);
+ grecs_print_node(cmdenv.xmltree, GRECS_NODE_FLAG_DEFAULT,
+ stdout);
fputc('\n', stdout);
}
diff --git a/src/eclat.h b/src/eclat.h
index 5a067f4..221347f 100644
--- a/src/eclat.h
+++ b/src/eclat.h
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -70,6 +70,8 @@ extern char *instance_store_base_url;
extern unsigned short instance_store_port;
extern char *instance_store_document_path;
extern char *instance_store_credentials_path;
+extern unsigned long max_retry_sleep;
+extern unsigned long max_retry_time;
typedef int (*config_finish_hook_t) (void*);
@@ -82,8 +84,8 @@ int run_config_finish_hooks(void);
struct eclat_command_env {
struct eclat_command const *cmd;
- CURL *curl;
struct ec2_request *request;
+ struct grecs_node *xmltree;
};
typedef struct eclat_command_env eclat_command_env_t;
@@ -221,7 +223,7 @@ void describe_request_create(eclat_command_env_t *env, int argc, char **argv,
void describe_request_update(eclat_command_env_t *env, int argc, char **argv,
const char *uparm, int n_in, int *n_out);
-int eclat_send_request(CURL *curl, struct ec2_request *q);
+int eclat_send_request(struct ec2_request *q, struct grecs_node **ret);
char *eclat_get_instance_zone(void);
void eclat_get_instance_creds(char *id,
char **access_key_ptr, char **secret_key_ptr,
diff --git a/src/igw.c b/src/igw.c
index a0f6f48..6e13c13 100644
--- a/src/igw.c
+++ b/src/igw.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -20,7 +20,6 @@ int
eclat_create_internet_gateway(eclat_command_env_t *env, int argc, char **argv)
{
int i;
- struct ec2_request *q = env->request;
generic_proginfo->args_doc = NULL;
generic_parse_options(env->cmd,
"create internet gateway",
diff --git a/src/ispeek.c b/src/ispeek.c
index efd18cd..e602125 100644
--- a/src/ispeek.c
+++ b/src/ispeek.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -78,8 +78,6 @@ static void print_dir(const char *path, struct closure *cl);
static void
list(const char *path, struct closure *cl)
{
- char *text;
-
cl->text = read_from(path, cl->curl, cl->acc);
if (!cl->text)
return;
@@ -198,7 +196,6 @@ print_file(const char *path, struct closure *cl)
static void
ispeek_do(char **argv)
{
- CURLcode res;
struct grecs_txtacc *acc;
CURL *curl;
const char *path = *argv++;
diff --git a/src/mkvpc-cl.opt b/src/mkvpc-cl.opt
index 03611f0..740374e 100644
--- a/src/mkvpc-cl.opt
+++ b/src/mkvpc-cl.opt
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2013-2015 Sergey Poznyakoff.
+ Copyright (C) 2013-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -14,7 +14,6 @@
You should have received a copy of the GNU General Public License
along with Eclat. If not, see <http://www.gnu.org/licenses/>. */
-static int vpc_dry_run;
static char *vpc_tenancy;
ECLAT_CL_BEGIN([<create VPC>],
diff --git a/src/util.c b/src/util.c
index 7bf13ac..9f8533f 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1,5 +1,5 @@
/* This file is part of Eclat.
- Copyright (C) 2012-2015 Sergey Poznyakoff.
+ Copyright (C) 2012-2018 Sergey Poznyakoff.
Eclat is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -213,79 +213,130 @@ describe_request_update(eclat_command_env_t *env, int argc, char **argv,
void
describe_request_create(eclat_command_env_t *env, int argc, char **argv,
- const char *uparm)
+ const char *uparm)
{
describe_request_update(env, argc, argv, uparm, 1, NULL);
}
-
+
+unsigned long max_retry_sleep = 600;
+unsigned long max_retry_time = 1800;
+
int
-eclat_send_request(CURL *curl, struct ec2_request *req)
+eclat_send_request(struct ec2_request *orig, struct grecs_node **ret_tree)
{
char *url;
CURLcode res;
- int rc = 0;
+ int ret = 0;
struct curl_slist *headers = NULL;
+ struct eclat_io *io;
+ time_t endtime;
+ unsigned long tts, t;
- /* Prepare the request */
- if (req->flags & EC2_RF_POST) {
- eclat_request_finalize(req);
- curl_easy_setopt(curl, CURLOPT_POST, 1);
- curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req->postdata);
- curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
- strlen(req->postdata));
+ io = eclat_io_init(0);
+ if (!io) {
+ err("cannot initialize IO structure");
+ return -1;
}
- eclat_request_sign(req, secret_key, signature_version);
- url = eclat_request_to_url(req);
- curl_easy_setopt(curl, CURLOPT_URL, url);
- debug(ECLAT_DEBCAT_MAIN, 1, ("using URL: %s", url));
- free(url);
- if (req->headers) {
- struct grecs_list_entry *ep;
- struct grecs_txtacc *acc;
- int rc;
-
- acc = grecs_txtacc_create();
-
- for (ep = req->headers->head; ep; ep = ep->next) {
- struct ec2_param *p = ep->data;
- char *str;
-
- grecs_txtacc_grow_string(acc, p->name);
- grecs_txtacc_grow_char(acc, ':');
- grecs_txtacc_grow_string(acc, p->value);
- grecs_txtacc_grow_char(acc, 0);
- str = grecs_txtacc_finish(acc, 0);
- debug(ECLAT_DEBCAT_MAIN, 1, ("HDR: %s", str));
- headers = curl_slist_append(headers, str);
- grecs_txtacc_free_string(acc, str);
+ endtime = time(NULL) + max_retry_time;
+ tts = 1;
+ do {
+ struct grecs_node *node;
+ /* Prepare the request */
+ struct ec2_request *req = eclat_request_dup(orig);
+ if (req->flags & EC2_RF_POST) {
+ eclat_request_finalize(orig);
+ curl_easy_setopt(io->curl, CURLOPT_POST, 1);
+ curl_easy_setopt(io->curl, CURLOPT_POSTFIELDS,
+ req->postdata);
+ curl_easy_setopt(io->curl, CURLOPT_POSTFIELDSIZE,
+ strlen(req->postdata));
}
+ eclat_request_sign(req, secret_key, signature_version);
+ url = eclat_request_to_url(req);
+ curl_easy_setopt(io->curl, CURLOPT_URL, url);
+ debug(ECLAT_DEBCAT_MAIN, 1, ("using URL: %s", url));
+ free(url);
+ if (req->headers) {
+ struct grecs_list_entry *ep;
+ struct grecs_txtacc *acc;
+ int rc;
+
+ acc = grecs_txtacc_create();
+
+ for (ep = req->headers->head; ep; ep = ep->next) {
+ struct ec2_param *p = ep->data;
+ char *str;
+
+ grecs_txtacc_grow_string(acc, p->name);
+ grecs_txtacc_grow_char(acc, ':');
+ grecs_txtacc_grow_string(acc, p->value);
+ grecs_txtacc_grow_char(acc, 0);
+ str = grecs_txtacc_finish(acc, 0);
+ debug(ECLAT_DEBCAT_MAIN, 1, ("HDR: %s", str));
+
+ headers = curl_slist_append(headers, str);
+ grecs_txtacc_free_string(acc, str);
+ }
- rc = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
- grecs_txtacc_free(acc);
+ rc = curl_easy_setopt(io->curl, CURLOPT_HTTPHEADER,
+ headers);
+ grecs_txtacc_free(acc);
- if (rc)
- die(EX_SOFTWARE,
- "failed to add headers: %s",
- curl_easy_strerror(rc));
- }
+ if (rc)
+ die(EX_SOFTWARE,
+ "failed to add headers: %s",
+ curl_easy_strerror(rc));
+ }
- if (req->flags & EC2_RF_POST)
- debug(ECLAT_DEBCAT_MAIN, 1, ("DATA: %s", req->postdata));
+ if (req->flags & EC2_RF_POST)
+ debug(ECLAT_DEBCAT_MAIN, 1,
+ ("DATA: %s", req->postdata));
- if (dry_run_mode)
- debug(ECLAT_DEBCAT_MAIN, 1, ("not sending request"));
- else {
- res = curl_easy_perform(curl);
+ if (dry_run_mode)
+ debug(ECLAT_DEBCAT_MAIN, 1, ("not sending request"));
+ else {
+ res = curl_easy_perform(io->curl);
+ if (res == CURLE_OK) {
+ *ret_tree = eclat_io_finish(io);
+ } else {
+ err("CURL: %s", curl_easy_strerror(res));
+ ret = 1;
+ }
+ }
+
+ curl_slist_free_all(headers);
+ eclat_request_free(req);
+
+ if (ret)
+ break;
- if (res != CURLE_OK) {
- err("CURL: %s", curl_easy_strerror(res));
- rc = 1;
+ node = grecs_find_node(*ret_tree,
+ ".Response.Errors.Error.Code");
+ if (!node)
+ break;
+ if (node->type != grecs_node_stmt
+ || node->v.value->type == GRECS_TYPE_STRING) {
+ err("unexpectedly formatted error code");
+ break;
}
- }
- eclat_request_free(req);
- curl_slist_free_all(headers);
- return rc;
+
+ if (strcmp(node->v.value->v.string,
+ "Client.RequestLimitExceeded"))
+ break;
+
+ t = random() % tts;
+ sleep(t);
+ if (tts < max_retry_sleep) {
+ tts <<= 1;
+ if (tts == 0 || tts > max_retry_sleep)
+ tts = max_retry_sleep;
+ }
+ } while (time(NULL) < endtime);
+
+ eclat_io_free(io);
+
+ return ret;
}
int
@@ -429,7 +480,6 @@ char *instance_store_credentials_path = "meta-data/iam/security-credentials";
static CURL *
get_curl(struct grecs_txtacc *acc)
{
- CURLcode res;
CURL *curl = instance_store_curl_new(acc);
eclat_set_curl_trace(curl, debug_level(ECLAT_DEBCAT_CURL));

Return to:

Send suggestions and report system problems to the System administrator.