From 649a04b9deec5d111e735bc47c37adca7e76f39e Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Wed, 14 Mar 2018 17:22:18 +0200 Subject: Implement exponential backoff with jitter to handle RequestLimitExceeded * src/eclat.h (eclat_command_env) : Remove. : New field. * src/util.c (translate_ids): Close the map when no longer needed (eclat_send_request): Implement exponential backoff. * src/ec2map.c: Reflect the above changes. * src/eclat.c: Likewise. * lib/libeclat.h (eclat_request_dup): New proto. * lib/reqcreat.c (eclat_request_dup): New function. * src/config.c: New configuration statements: max-retry-interval and retry-timeout. --- lib/istore.c | 2 - lib/libeclat.h | 3 +- lib/reqcreat.c | 51 ++++++++++++++++- lib/reqsign.c | 4 +- lib/trace.c | 3 +- src/config.c | 8 ++- src/ec2map.c | 22 +------- src/eclat.c | 21 +++---- src/eclat.h | 8 ++- src/igw.c | 3 +- src/ispeek.c | 5 +- src/mkvpc-cl.opt | 3 +- src/util.c | 164 ++++++++++++++++++++++++++++++++++++------------------- 13 files changed, 189 insertions(+), 108 deletions(-) diff --git a/lib/istore.c b/lib/istore.c index dea3a53..8b46b64 100644 --- a/lib/istore.c +++ b/lib/istore.c @@ -29,7 +29,6 @@ acc_cb(void *ptr, size_t size, size_t nmemb, void *data) CURL * instance_store_curl_new(struct grecs_txtacc *acc) { - CURLcode res; CURL *curl; curl = curl_easy_init(); @@ -46,7 +45,6 @@ instance_store_read(const char *url, CURL *curl) { CURLcode res; long http_resp; - char *text; curl_easy_setopt(curl, CURLOPT_URL, url); diff --git a/lib/libeclat.h b/lib/libeclat.h index b865e00..6f1acf6 100644 --- a/lib/libeclat.h +++ b/lib/libeclat.h @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -104,6 +104,7 @@ struct ec2_request { struct ec2_request *eclat_request_create(int flags, const char *endpoint, const char *uri, char const *region, char const *access_key, char const *token); +struct ec2_request *eclat_request_dup(struct ec2_request const *src); void eclat_request_free(struct ec2_request *); void eclat_request_add_param0(struct ec2_request *req, const char *name, const char *value, int encoded); diff --git a/lib/reqcreat.c b/lib/reqcreat.c index 58d7829..753a415 100644 --- a/lib/reqcreat.c +++ b/lib/reqcreat.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -27,6 +27,15 @@ ec2_param_free(void *ptr) free(p); } +static int +ec2_param_dup(void *sym, void *data) +{ + struct ec2_param *p = sym; + struct ec2_request *req = data; + eclat_request_add_param(req, p->name, p->value); + return 0; +} + struct ec2_request * eclat_request_create(int flags, const char *endpoint, const char *uri, char const *region, char const *access_key, @@ -49,3 +58,43 @@ eclat_request_create(int flags, const char *endpoint, const char *uri, req->token = token ? grecs_strdup(token) : NULL; return req; } + +static inline char * +safe_strdup(char const *a) +{ + return a ? grecs_strdup(a) : NULL; +} + +struct ec2_request * +eclat_request_dup(struct ec2_request const *src) +{ + struct ec2_request *dst = grecs_zalloc(sizeof(*dst)); + dst->flags = src->flags; + dst->endpoint = safe_strdup(src->endpoint); + dst->uri = safe_strdup(src->uri); + dst->params = grecs_symtab_create(sizeof(struct ec2_param), + NULL, + NULL, + NULL, + NULL, + ec2_param_free); + + grecs_symtab_enumerate(src->params, ec2_param_dup, dst); + if (src->headers == NULL) { + dst->headers = NULL; + } else { + struct grecs_list_entry *ep; + for (ep = src->headers->head; ep; ep = ep->next) { + struct ec2_param *ent = ep->data; + eclat_request_add_header(dst, ent->name, ent->value); + } + } + dst->postdata = safe_strdup(src->postdata); + dst->region = safe_strdup(src->region); + dst->access_key = safe_strdup(src->access_key); + dst->token = safe_strdup(src->token); + dst->ttl = src->ttl; + + return dst; +} + diff --git a/lib/reqsign.c b/lib/reqsign.c index 9572961..05988ed 100644 --- a/lib/reqsign.c +++ b/lib/reqsign.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -216,7 +216,7 @@ requestsign4(struct ec2_request *req, char *secret) grecs_txtacc_grow_char(acc, '\n'); grecs_txtacc_grow_string(acc, req->uri); grecs_txtacc_grow_char(acc, '\n'); - /* Append a canonicalized request string */ + /* Append canonicalized request string */ for (i = 0; i < n; i++) { struct ec2_param *p, key; diff --git a/lib/trace.c b/lib/trace.c index 4438822..6b79e58 100644 --- a/lib/trace.c +++ b/lib/trace.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,6 +15,7 @@ along with Eclat. If not, see . */ #include "libeclat.h" +#include #include static void diff --git a/src/config.c b/src/config.c index 49b47f9..6fac842 100644 --- a/src/config.c +++ b/src/config.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -396,6 +396,12 @@ static struct grecs_keyword eclat_kw[] = { "Specifies a file containing `accessID:accessKey' pairs.", grecs_type_string, GRECS_DFLT, &access_file_name, 0, cb_access_file }, + { "max-retry-interval", "seconds", + "Maximum interval between retries in exponential backoff algorithm.", + grecs_type_ulong, GRECS_DFLT, &max_retry_sleep }, + { "retry-timeout", "seconds", + "Give up retrying after this many seconds", + grecs_type_ulong, GRECS_DFLT, &max_retry_time }, { "default-region", "name", "Define default AWS region", grecs_type_string, GRECS_DFLT, ®ion_name }, diff --git a/src/ec2map.c b/src/ec2map.c index 6e2be10..3dbf4f1 100644 --- a/src/ec2map.c +++ b/src/ec2map.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -32,7 +32,6 @@ struct ec2_map { char *action; struct grecs_list *args; char *ret; - struct eclat_io *io; }; static struct grecs_keyword ec2_map_kw[] = { @@ -56,7 +55,6 @@ static void ec2_map_free(int dbg, void *data) { struct ec2_map *map = data; - eclat_io_free(map->io); free(map->action); free(map->ret); grecs_list_free(map->args); @@ -114,23 +112,12 @@ ec2_map_config(int dbg, struct grecs_node *node, void *data) static int ec2_map_open(int dbg, void *data) { - struct ec2_map *map = data; - - map->io = eclat_io_init(0); - if (!map->io) { - err("cannot open EC2 database"); - return eclat_map_failure; - } return eclat_map_ok; } static int ec2_map_close(int dbg, void *data) { - struct ec2_map *map = data; - - eclat_io_free(map->io); - map->io = NULL; return 0; } @@ -184,14 +171,9 @@ ec2_map_get(int dbg, int dir, void *data, const char *key, char **return_value) return eclat_map_failure; } - rc = eclat_send_request(map->io->curl, q); - - if (rc) + if (eclat_send_request(q, &tree)) return eclat_map_failure; - tree = eclat_io_finish(map->io); - - node = grecs_find_node(tree, map->ret); if (debug_category[dbg].level > 1) { diff --git a/src/eclat.c b/src/eclat.c index 8d4838b..5326c9e 100644 --- a/src/eclat.c +++ b/src/eclat.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -712,7 +712,7 @@ eclat_do_command(eclat_command_env_t *env, struct eclat_command *command, if (!eclat_confirm(confirm_mode, "Proceed with %s", command->ident)) die(EX_CANCELLED, "command not confirmed"); - rc = eclat_send_request(env->curl, env->request); + rc = eclat_send_request(env->request, &env->xmltree); } return rc; } @@ -722,11 +722,10 @@ main(int argc, char **argv) { int rc; struct grecs_node *tree; - struct eclat_io *io; - struct grecs_node *xmltree; forlan_eval_env_t env = NULL; struct eclat_command *command = NULL; eclat_command_env_t cmdenv; + struct timeval tv; set_program_name(argv[0]); proginfo.print_help_hook = listcmdhook; @@ -746,6 +745,8 @@ main(int argc, char **argv) eclat_map_drv_register(&eclat_map_drv_ec2); sortcmds(); config_init(); + gettimeofday(&tv, NULL); + srandom((unsigned int)tv.tv_usec); parse_options(&argc, &argv); if (match_command_mode) { @@ -848,34 +849,30 @@ main(int argc, char **argv) } debug(ECLAT_DEBCAT_MAIN, 1, ("using access key %s", access_key)); - io = eclat_io_init(1); - if (confirm_mode == eclat_confirm_unspecified) confirm_mode = command->confirm; /* Prepare environment */ memset(&cmdenv, 0, sizeof(cmdenv)); cmdenv.cmd = command; - cmdenv.curl = io->curl; rc = eclat_do_command(&cmdenv, command, argc, argv); if (rc) exit(rc); - xmltree = eclat_io_finish(io); - if (xml_dump_file) fclose(xml_dump_file); if (sort_option) - grecs_tree_sort(xmltree, node_ident_cmp); + grecs_tree_sort(cmdenv.xmltree, node_ident_cmp); if (dry_run_mode) /* nothing */; else if (env) { - rc = forlan_run(env, xmltree); + rc = forlan_run(env, cmdenv.xmltree); } else { - grecs_print_node(xmltree, GRECS_NODE_FLAG_DEFAULT, stdout); + grecs_print_node(cmdenv.xmltree, GRECS_NODE_FLAG_DEFAULT, + stdout); fputc('\n', stdout); } diff --git a/src/eclat.h b/src/eclat.h index 5a067f4..221347f 100644 --- a/src/eclat.h +++ b/src/eclat.h @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -70,6 +70,8 @@ extern char *instance_store_base_url; extern unsigned short instance_store_port; extern char *instance_store_document_path; extern char *instance_store_credentials_path; +extern unsigned long max_retry_sleep; +extern unsigned long max_retry_time; typedef int (*config_finish_hook_t) (void*); @@ -82,8 +84,8 @@ int run_config_finish_hooks(void); struct eclat_command_env { struct eclat_command const *cmd; - CURL *curl; struct ec2_request *request; + struct grecs_node *xmltree; }; typedef struct eclat_command_env eclat_command_env_t; @@ -221,7 +223,7 @@ void describe_request_create(eclat_command_env_t *env, int argc, char **argv, void describe_request_update(eclat_command_env_t *env, int argc, char **argv, const char *uparm, int n_in, int *n_out); -int eclat_send_request(CURL *curl, struct ec2_request *q); +int eclat_send_request(struct ec2_request *q, struct grecs_node **ret); char *eclat_get_instance_zone(void); void eclat_get_instance_creds(char *id, char **access_key_ptr, char **secret_key_ptr, diff --git a/src/igw.c b/src/igw.c index a0f6f48..6e13c13 100644 --- a/src/igw.c +++ b/src/igw.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -20,7 +20,6 @@ int eclat_create_internet_gateway(eclat_command_env_t *env, int argc, char **argv) { int i; - struct ec2_request *q = env->request; generic_proginfo->args_doc = NULL; generic_parse_options(env->cmd, "create internet gateway", diff --git a/src/ispeek.c b/src/ispeek.c index efd18cd..e602125 100644 --- a/src/ispeek.c +++ b/src/ispeek.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -78,8 +78,6 @@ static void print_dir(const char *path, struct closure *cl); static void list(const char *path, struct closure *cl) { - char *text; - cl->text = read_from(path, cl->curl, cl->acc); if (!cl->text) return; @@ -198,7 +196,6 @@ print_file(const char *path, struct closure *cl) static void ispeek_do(char **argv) { - CURLcode res; struct grecs_txtacc *acc; CURL *curl; const char *path = *argv++; diff --git a/src/mkvpc-cl.opt b/src/mkvpc-cl.opt index 03611f0..740374e 100644 --- a/src/mkvpc-cl.opt +++ b/src/mkvpc-cl.opt @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2013-2015 Sergey Poznyakoff. + Copyright (C) 2013-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,7 +14,6 @@ You should have received a copy of the GNU General Public License along with Eclat. If not, see . */ -static int vpc_dry_run; static char *vpc_tenancy; ECLAT_CL_BEGIN([], diff --git a/src/util.c b/src/util.c index 7bf13ac..9f8533f 100644 --- a/src/util.c +++ b/src/util.c @@ -1,5 +1,5 @@ /* This file is part of Eclat. - Copyright (C) 2012-2015 Sergey Poznyakoff. + Copyright (C) 2012-2018 Sergey Poznyakoff. Eclat is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -213,79 +213,130 @@ describe_request_update(eclat_command_env_t *env, int argc, char **argv, void describe_request_create(eclat_command_env_t *env, int argc, char **argv, - const char *uparm) + const char *uparm) { describe_request_update(env, argc, argv, uparm, 1, NULL); } - + +unsigned long max_retry_sleep = 600; +unsigned long max_retry_time = 1800; + int -eclat_send_request(CURL *curl, struct ec2_request *req) +eclat_send_request(struct ec2_request *orig, struct grecs_node **ret_tree) { char *url; CURLcode res; - int rc = 0; + int ret = 0; struct curl_slist *headers = NULL; + struct eclat_io *io; + time_t endtime; + unsigned long tts, t; - /* Prepare the request */ - if (req->flags & EC2_RF_POST) { - eclat_request_finalize(req); - curl_easy_setopt(curl, CURLOPT_POST, 1); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req->postdata); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, - strlen(req->postdata)); + io = eclat_io_init(0); + if (!io) { + err("cannot initialize IO structure"); + return -1; } - eclat_request_sign(req, secret_key, signature_version); - url = eclat_request_to_url(req); - curl_easy_setopt(curl, CURLOPT_URL, url); - debug(ECLAT_DEBCAT_MAIN, 1, ("using URL: %s", url)); - free(url); - if (req->headers) { - struct grecs_list_entry *ep; - struct grecs_txtacc *acc; - int rc; - - acc = grecs_txtacc_create(); - - for (ep = req->headers->head; ep; ep = ep->next) { - struct ec2_param *p = ep->data; - char *str; - - grecs_txtacc_grow_string(acc, p->name); - grecs_txtacc_grow_char(acc, ':'); - grecs_txtacc_grow_string(acc, p->value); - grecs_txtacc_grow_char(acc, 0); - str = grecs_txtacc_finish(acc, 0); - debug(ECLAT_DEBCAT_MAIN, 1, ("HDR: %s", str)); - headers = curl_slist_append(headers, str); - grecs_txtacc_free_string(acc, str); + endtime = time(NULL) + max_retry_time; + tts = 1; + do { + struct grecs_node *node; + /* Prepare the request */ + struct ec2_request *req = eclat_request_dup(orig); + if (req->flags & EC2_RF_POST) { + eclat_request_finalize(orig); + curl_easy_setopt(io->curl, CURLOPT_POST, 1); + curl_easy_setopt(io->curl, CURLOPT_POSTFIELDS, + req->postdata); + curl_easy_setopt(io->curl, CURLOPT_POSTFIELDSIZE, + strlen(req->postdata)); } + eclat_request_sign(req, secret_key, signature_version); + url = eclat_request_to_url(req); + curl_easy_setopt(io->curl, CURLOPT_URL, url); + debug(ECLAT_DEBCAT_MAIN, 1, ("using URL: %s", url)); + free(url); + if (req->headers) { + struct grecs_list_entry *ep; + struct grecs_txtacc *acc; + int rc; + + acc = grecs_txtacc_create(); + + for (ep = req->headers->head; ep; ep = ep->next) { + struct ec2_param *p = ep->data; + char *str; + + grecs_txtacc_grow_string(acc, p->name); + grecs_txtacc_grow_char(acc, ':'); + grecs_txtacc_grow_string(acc, p->value); + grecs_txtacc_grow_char(acc, 0); + str = grecs_txtacc_finish(acc, 0); + debug(ECLAT_DEBCAT_MAIN, 1, ("HDR: %s", str)); + + headers = curl_slist_append(headers, str); + grecs_txtacc_free_string(acc, str); + } - rc = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - grecs_txtacc_free(acc); + rc = curl_easy_setopt(io->curl, CURLOPT_HTTPHEADER, + headers); + grecs_txtacc_free(acc); - if (rc) - die(EX_SOFTWARE, - "failed to add headers: %s", - curl_easy_strerror(rc)); - } + if (rc) + die(EX_SOFTWARE, + "failed to add headers: %s", + curl_easy_strerror(rc)); + } - if (req->flags & EC2_RF_POST) - debug(ECLAT_DEBCAT_MAIN, 1, ("DATA: %s", req->postdata)); + if (req->flags & EC2_RF_POST) + debug(ECLAT_DEBCAT_MAIN, 1, + ("DATA: %s", req->postdata)); - if (dry_run_mode) - debug(ECLAT_DEBCAT_MAIN, 1, ("not sending request")); - else { - res = curl_easy_perform(curl); + if (dry_run_mode) + debug(ECLAT_DEBCAT_MAIN, 1, ("not sending request")); + else { + res = curl_easy_perform(io->curl); + if (res == CURLE_OK) { + *ret_tree = eclat_io_finish(io); + } else { + err("CURL: %s", curl_easy_strerror(res)); + ret = 1; + } + } + + curl_slist_free_all(headers); + eclat_request_free(req); + + if (ret) + break; - if (res != CURLE_OK) { - err("CURL: %s", curl_easy_strerror(res)); - rc = 1; + node = grecs_find_node(*ret_tree, + ".Response.Errors.Error.Code"); + if (!node) + break; + if (node->type != grecs_node_stmt + || node->v.value->type == GRECS_TYPE_STRING) { + err("unexpectedly formatted error code"); + break; } - } - eclat_request_free(req); - curl_slist_free_all(headers); - return rc; + + if (strcmp(node->v.value->v.string, + "Client.RequestLimitExceeded")) + break; + + t = random() % tts; + sleep(t); + if (tts < max_retry_sleep) { + tts <<= 1; + if (tts == 0 || tts > max_retry_sleep) + tts = max_retry_sleep; + } + } while (time(NULL) < endtime); + + eclat_io_free(io); + + return ret; } int @@ -429,7 +480,6 @@ char *instance_store_credentials_path = "meta-data/iam/security-credentials"; static CURL * get_curl(struct grecs_txtacc *acc) { - CURLcode res; CURL *curl = instance_store_curl_new(acc); eclat_set_curl_trace(curl, debug_level(ECLAT_DEBCAT_CURL)); -- cgit v1.2.1