Mercurial > dovecot > core-2.2
changeset 17005:d2f2423615d7
http: Implemented delayed requests scheduling.
Requests can now be (re)submitted with a delay. The request is not sent
until the delay time expires. This facilitates handling the Retry-After
header in responses. This can either be performed automatically if the
indicated delay is not too long or explicitly by the code using lib-http.
author | Stephan Bosch <stephan@rename-it.nl> |
---|---|
date | Fri, 22 Nov 2013 22:12:08 +0200 |
parents | c215131c2c18 |
children | 75b6c3b8ee20 |
files | src/lib-http/http-client-connection.c src/lib-http/http-client-private.h src/lib-http/http-client-queue.c src/lib-http/http-client-request.c src/lib-http/http-client.c src/lib-http/http-client.h |
diffstat | 6 files changed, 174 insertions(+), 22 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-http/http-client-connection.c Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-connection.c Fri Nov 22 22:12:08 2013 +0200 @@ -595,19 +595,36 @@ conn->close_indicated = response.connection_close; if (!aborted) { + bool handled = FALSE; + + /* failed Expect: */ if (response.status == 417 && req->payload_sync) { /* drop Expect: continue */ req->payload_sync = FALSE; conn->output_locked = FALSE; conn->peer->no_payload_sync = TRUE; - http_client_request_retry_response(req, &response); - + if (http_client_request_try_retry(req)) + handled = TRUE; + /* redirection */ } else if (!req->client->set.no_auto_redirect && response.status / 100 == 3 && response.status != 304 && response.location != NULL) { - /* redirect */ - http_client_request_redirect(req, response.status, response.location); - } else { + /* redirect (possibly after delay) */ + if (http_client_request_delay_from_response(req, &response) >= 0) { + http_client_request_redirect + (req, response.status, response.location); + handled = TRUE; + } + /* service unavailable */ + } else if (response.status == 503) { + /* automatically retry after delay if indicated */ + if ( response.retry_after != (time_t)-1 && + http_client_request_delay_from_response(req, &response) > 0 && + http_client_request_try_retry(req)) + handled = TRUE; + } + + if (!handled) { /* response for application */ if (!http_client_connection_return_response(conn, req, &response)) return;
--- a/src/lib-http/http-client-private.h Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-private.h Fri Nov 22 22:12:08 2013 +0200 @@ -70,6 +70,8 @@ uoff_t payload_size, payload_offset; struct ostream *payload_output; + struct timeval release_time; + unsigned int attempts; unsigned int redirects; @@ -182,9 +184,9 @@ ARRAY_TYPE(http_client_peer) pending_peers; /* requests pending in queue to be picked up by connections */ - ARRAY_TYPE(http_client_request) request_queue; + ARRAY_TYPE(http_client_request) request_queue, delayed_request_queue; - struct timeout *to_connect; + struct timeout *to_connect, *to_delayed; }; struct http_client_host { @@ -229,6 +231,8 @@ void http_client_request_ref(struct http_client_request *req); void http_client_request_unref(struct http_client_request **_req); +int http_client_request_delay_from_response(struct http_client_request *req, + const struct http_response *response); enum http_response_payload_type http_client_request_get_payload_type(struct http_client_request *req); int http_client_request_send(struct http_client_request *req, @@ -243,8 +247,6 @@ void http_client_request_resubmit(struct http_client_request *req); void http_client_request_retry(struct http_client_request *req, unsigned int status, const char *error); -void http_client_request_retry_response(struct http_client_request *req, - struct http_response *response); void http_client_request_send_error(struct http_client_request *req, unsigned int status, const char *error); void http_client_request_error_delayed(struct http_client_request **_req);
--- a/src/lib-http/http-client-queue.c Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-queue.c Fri Nov 22 22:12:08 2013 +0200 @@ -5,10 +5,12 @@ #include "str.h" #include "hash.h" #include "array.h" +#include "bsearch-insert-pos.h" #include "llist.h" #include "ioloop.h" #include "istream.h" #include "ostream.h" +#include "time-util.h" #include "dns-lookup.h" #include "http-response-parser.h" @@ -41,6 +43,10 @@ * Queue */ +static void +http_client_queue_set_delay_timer(struct http_client_queue *queue, + struct timeval time); + static struct http_client_queue * http_client_queue_find(struct http_client_host *host, const struct http_client_peer_addr *addr) @@ -92,6 +98,7 @@ queue->name = name; queue->ips_connect_idx = 0; i_array_init(&queue->request_queue, 16); + i_array_init(&queue->delayed_request_queue, 4); array_append(&host->queues, &queue, 1); } @@ -106,8 +113,11 @@ if (array_is_created(&queue->pending_peers)) array_free(&queue->pending_peers); array_free(&queue->request_queue); + array_free(&queue->delayed_request_queue); if (queue->to_connect != NULL) timeout_remove(&queue->to_connect); + if (queue->to_delayed != NULL) + timeout_remove(&queue->to_delayed); i_free(queue->name); i_free(queue); } @@ -122,6 +132,12 @@ http_client_request_error(*req, status, error); } array_clear(&queue->request_queue); + + /* abort all delayed requests */ + array_foreach_modifiable(&queue->delayed_request_queue, req) { + http_client_request_error(*req, status, error); + } + array_clear(&queue->delayed_request_queue); } void @@ -309,10 +325,11 @@ return TRUE; } -void http_client_queue_submit_request(struct http_client_queue *queue, +static void http_client_queue_submit_now(struct http_client_queue *queue, struct http_client_request *req) { - req->queue = queue; + req->release_time.tv_sec = 0; + req->release_time.tv_usec = 0; if (req->urgent) array_insert(&queue->request_queue, 0, &req, 1); @@ -320,6 +337,84 @@ array_append(&queue->request_queue, &req, 1); } +static void +http_client_queue_delay_timeout(struct http_client_queue *queue) +{ + struct http_client_request *const *reqs; + unsigned int count, i, finished; + + io_loop_time_refresh(); + + finished = 0; + reqs = array_get(&queue->delayed_request_queue, &count); + for (i = 0; i < count; i++) { + if (timeval_cmp(&reqs[i]->release_time, &ioloop_timeval) > 0) { + break; + } + + http_client_queue_debug(queue, + "Activated delayed request %s%s", + http_client_request_label(reqs[i]), + (reqs[i]->urgent ? " (urgent)" : "")); + http_client_queue_submit_now(queue, reqs[i]); + finished++; + } + i_assert(finished > 0); + if (i < count) { + http_client_queue_set_delay_timer(queue, reqs[i]->release_time); + } + array_delete(&queue->delayed_request_queue, 0, finished); + + http_client_queue_connection_setup(queue); +} + +static void +http_client_queue_set_delay_timer(struct http_client_queue *queue, + struct timeval time) +{ + int usecs = timeval_diff_usecs(&time, &ioloop_timeval); + int msecs; + + /* round up to nearest microsecond */ + msecs = (usecs + 999) / 1000; + + /* set timer */ + if (queue->to_delayed != NULL) + timeout_remove(&queue->to_delayed); + queue->to_delayed = timeout_add + (msecs, http_client_queue_delay_timeout, queue); +} + +static int +http_client_queue_delayed_cmp(struct http_client_request *const *req1, + struct http_client_request *const *req2) +{ + return timeval_cmp(&(*req1)->release_time, &(*req2)->release_time); +} + +void http_client_queue_submit_request(struct http_client_queue *queue, + struct http_client_request *req) +{ + unsigned int insert_idx; + + req->queue = queue; + + if (req->release_time.tv_sec > 0) { + io_loop_time_refresh(); + + if (timeval_cmp(&req->release_time, &ioloop_timeval) > 0) { + (void)array_bsearch_insert_pos(&queue->delayed_request_queue, + &req, http_client_queue_delayed_cmp, &insert_idx); + array_insert(&queue->delayed_request_queue, insert_idx, &req, 1); + if (insert_idx == 0) + http_client_queue_set_delay_timer(queue, req->release_time); + return; + } + } + + http_client_queue_submit_now(queue, req); +} + struct http_client_request * http_client_queue_claim_request(struct http_client_queue *queue, const struct http_client_peer_addr *addr, bool no_urgent) @@ -372,4 +467,6 @@ { if (queue->to_connect != NULL) queue->to_connect = io_loop_move_timeout(&queue->to_connect); + if (queue->to_delayed != NULL) + queue->to_delayed = io_loop_move_timeout(&queue->to_delayed); }
--- a/src/lib-http/http-client-request.c Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-request.c Fri Nov 22 22:12:08 2013 +0200 @@ -269,6 +269,40 @@ req->payload_sync = TRUE; } +void http_client_request_delay_until(struct http_client_request *req, + time_t time) +{ + req->release_time.tv_sec = time; + req->release_time.tv_usec = 0; +} + +void http_client_request_delay(struct http_client_request *req, + time_t seconds) +{ + req->release_time = ioloop_timeval; + req->release_time.tv_sec += seconds; +} + +int http_client_request_delay_from_response(struct http_client_request *req, + const struct http_response *response) +{ + time_t retry_after = response->retry_after; + unsigned int max; + + if (retry_after == (time_t)-1) + return 0; /* no delay */ + if (retry_after < ioloop_time) + return 0; /* delay already expired */ + max = (req->client->set.max_auto_retry_delay == 0 ? + req->client->set.request_timeout_msecs / 1000 : + req->client->set.max_auto_retry_delay); + if ((retry_after - ioloop_time) > max) + return -1; /* delay too long */ + req->release_time.tv_sec = retry_after; + req->release_time.tv_usec = 0; + return 1; /* valid delay */ +} + enum http_request_state http_client_request_get_state(struct http_client_request *req) { @@ -902,7 +936,7 @@ req->conn = NULL; req->peer = NULL; req->state = HTTP_REQUEST_STATE_QUEUED; - http_client_queue_submit_request(req->queue, req); + http_client_host_submit_request(req->host, req); } void http_client_request_retry(struct http_client_request *req, @@ -912,16 +946,6 @@ http_client_request_error(req, status, error); } -void http_client_request_retry_response(struct http_client_request *req, - struct http_response *response) -{ - if (!http_client_request_try_retry(req)) { - i_assert(req->submitted || req->state >= HTTP_REQUEST_STATE_FINISHED); - (void)http_client_request_callback(req, response); - http_client_request_unref(&req); - } -} - bool http_client_request_try_retry(struct http_client_request *req) { /* limit the number of attempts for each request */
--- a/src/lib-http/http-client.c Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client.c Fri Nov 22 22:12:08 2013 +0200 @@ -127,6 +127,7 @@ client->set.request_timeout_msecs = set->request_timeout_msecs; client->set.connect_timeout_msecs = set->connect_timeout_msecs; client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs; + client->set.max_auto_retry_delay = set->max_auto_retry_delay; client->set.debug = set->debug; client->conn_list = http_client_connection_list_init();
--- a/src/lib-http/http-client.h Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client.h Fri Nov 22 22:12:08 2013 +0200 @@ -95,6 +95,12 @@ (default = 0; wait until current connection attempt finishes) */ unsigned int soft_connect_timeout_msecs; + /* maximum acceptable delay in seconds for automatically + retrying/redirecting requests. if a server sends a response with a + Retry-After header that causes a delay longer than this, the request + is not automatically retried and the response is returned */ + unsigned int max_auto_retry_delay; + bool debug; }; @@ -172,6 +178,11 @@ void http_client_request_set_payload(struct http_client_request *req, struct istream *input, bool sync); +void http_client_request_delay_until(struct http_client_request *req, + time_t time); +void http_client_request_delay(struct http_client_request *req, + time_t seconds); + enum http_request_state http_client_request_get_state(struct http_client_request *req); void http_client_request_submit(struct http_client_request *req);