--- a/src/lib-http/Makefile.am	Fri Nov 23 23:49:39 2012 +0200
+++ b/src/lib-http/Makefile.am	Sat Nov 24 00:30:14 2012 +0200
@@ -8,26 +8,51 @@
 
 libhttp_la_SOURCES = \
 	http-date.c \
-	http-url.c
+	http-url.c \
+	http-parser.c \
+	http-header-parser.c \
+	http-transfer-chunked.c \
+	http-response-parser.c \
+	http-client-request.c \
+	http-client-connection.c \
+	http-client-peer.c \
+	http-client-host.c \
+	http-client.c
 
 headers = \
 	http-date.h \
-	http-url.h
+	http-url.h \
+	http-parser.h \
+	http-header-parser.h \
+	http-transfer.h \
+	http-response-parser.h \
+	http-client-private.h \
+	http-client.h
 
 pkginc_libdir=$(pkgincludedir)
 pkginc_lib_HEADERS = $(headers)
 
 test_programs = \
 	test-http-date \
-	test-http-url
+	test-http-url \
+	test-http-header-parser \
+	test-http-transfer \
+	test-http-response-parser \
+	test-http-client
 
-noinst_PROGRAMS = $(test_programs)
+noinst_PROGRAMS = $(test_programs) test-http-responses
 
 test_libs = \
 	../lib-test/libtest.la \
-	../lib/liblib.la
+	../lib/liblib.la \
+	../lib/safe-memset.lo \
+	$(MODULE_LIBS)
 
-test_deps = $(noinst_LTLIBRARIES) $(test_libs)
+test_deps = \
+	$(noinst_LTLIBRARIES)
+	../lib-test/libtest.la \
+	../lib/liblib.la \
+	../lib/safe-memset.lo
 
 test_http_url_SOURCES = test-http-url.c
 test_http_url_LDADD = http-url.lo  $(test_libs)
@@ -37,6 +62,61 @@
 test_http_date_LDADD = http-date.lo  $(test_libs)
 test_http_date_DEPENDENCIES = $(test_deps)
 
+test_http_header_parser_SOURCES = test-http-header-parser.c
+test_http_header_parser_LDADD = http-parser.lo http-header-parser.lo $(test_libs)
+test_http_header_parser_DEPENDENCIES = $(test_deps)
+
+test_http_transfer_SOURCES = test-http-transfer.c
+test_http_transfer_LDADD = \
+	http-parser.lo \
+	http-header-parser.lo \
+	http-transfer-chunked.lo \
+	$(test_libs)
+test_http_transfer_DEPENDENCIES = $(test_deps)
+
+test_http_response_parser_SOURCES = test-http-response-parser.c
+test_http_response_parser_LDADD = \
+	http-date.lo \
+	http-parser.lo \
+	http-header-parser.lo \
+	http-transfer-chunked.lo \
+	http-response-parser.lo \
+	$(test_libs)
+test_http_response_parser_DEPENDENCIES = $(test_deps)
+
+test_http_client_SOURCES = test-http-client.c
+test_http_client_LDFLAGS = -export-dynamic
+test_http_client_LDADD = \
+	libhttp.la \
+	../lib-dns/libdns.la \
+	../lib-ssl-iostream/libssl_iostream.la \
+	$(test_libs)
+test_http_client_DEPENDENCIES = \
+	libhttp.la \
+	../lib-dns/libdns.la \
+	../lib-ssl-iostream/libssl_iostream.la \
+	$(test_deps)
+
+test_http_responses_SOURCES = test-http-responses.c
+test_http_responses_LDADD = \
+	http-date.lo \
+	http-parser.lo \
+	http-header-parser.lo \
+	http-transfer-chunked.lo \
+	http-response-parser.lo \
+	../lib/liblib.la \
+	../lib/safe-memset.lo \
+	$(MODULE_LIBS)
+test_http_responses_DEPENDENCIES = \
+	http-date.lo \
+	http-parser.lo \
+	http-header-parser.lo \
+	http-transfer-chunked.lo \
+	http-response-parser.lo \
+	$(noinst_LTLIBRARIES) \
+	../lib/liblib.la \
+	../lib/safe-memset.lo
+
 check: check-am check-test
 check-test: all-am
 	for bin in $(test_programs); do \

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-client-connection.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,788 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "iostream-rawlog.h"
+#include "iostream-ssl.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_connection_debug(struct http_client_connection *conn,
+	const char *format, ...) ATTR_FORMAT(2, 3);
+static inline void
+http_client_connection_error(struct http_client_connection *conn,
+	const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_connection_debug(struct http_client_connection *conn,
+	const char *format, ...)
+{
+	va_list args;
+
+	if (conn->client->set.debug) {
+
+		va_start(args, format);	
+		i_debug("http-client: conn %s: %s",
+			http_client_connection_label(conn),	t_strdup_vprintf(format, args));
+		va_end(args);
+	}
+}
+
+static inline void
+http_client_connection_error(struct http_client_connection *conn,
+	const char *format, ...)
+{
+	va_list args;
+
+	va_start(args, format);	
+	i_error("http-client: conn %s: %s",
+		http_client_connection_label(conn),	t_strdup_vprintf(format, args));
+	va_end(args);
+}
+
+
+/*
+ * Connection
+ */
+
+static void http_client_connection_input(struct connection *_conn);
+
+bool http_client_connection_is_ready(struct http_client_connection *conn)
+{
+	return (conn->connected && !conn->output_locked &&
+		array_count(&conn->request_wait_list) <
+			conn->client->set.max_pipelined_requests);
+}
+
+static void
+http_client_connection_retry_requests(struct http_client_connection *conn,
+	unsigned int status, const char *error)
+{
+	struct http_client_request **req;
+
+	array_foreach_modifiable(&conn->request_wait_list, req) {
+		http_client_request_retry(*req, status, error);
+	}	
+	array_clear(&conn->request_wait_list);
+}
+
+static void
+http_client_connection_server_close(struct http_client_connection **_conn)
+{
+	struct http_client_connection *conn = *_conn;
+	struct http_client_request **req;
+
+	conn->connected = FALSE;
+	conn->closing = TRUE;
+
+	http_client_connection_debug(conn,
+		"Server explicitly closed connection");
+
+	array_foreach_modifiable(&conn->request_wait_list, req) {
+		http_client_request_resubmit(*req);
+	}	
+	array_clear(&conn->request_wait_list);
+
+	if (conn->client->ioloop != NULL)
+		io_loop_stop(conn->client->ioloop);
+
+	http_client_connection_free(_conn);
+}
+
+static void
+http_client_connection_abort_temp_error(struct http_client_connection **_conn,
+	unsigned int status, const char *error)
+{
+	struct http_client_connection *conn = *_conn;
+
+	conn->connected = FALSE;
+	conn->closing = TRUE;
+	
+	http_client_connection_retry_requests(conn, status, error);
+	http_client_connection_free(_conn);
+}
+
+static void
+http_client_connection_abort_error(struct http_client_connection **_conn,
+	unsigned int status, const char *error)
+{
+	struct http_client_connection *conn = *_conn;
+	struct http_client_request **req;
+
+	conn->connected = FALSE;
+	conn->closing = TRUE;
+	
+	array_foreach_modifiable(&conn->request_wait_list, req) {
+		http_client_request_error(*req, status, error);
+	}	
+	array_clear(&conn->request_wait_list);
+	http_client_connection_free(_conn);
+}
+
+static void
+http_client_connection_idle_timeout(struct http_client_connection *conn)
+{
+	http_client_connection_debug(conn, "Idle connection timed out");
+
+	http_client_connection_free(&conn);
+}
+
+static void
+http_client_connection_check_idle(struct http_client_connection *conn)
+{
+	unsigned int timeout, count;
+
+	if (array_count(&conn->request_wait_list) == 0 &&
+		conn->incoming_payload == NULL &&
+		conn->client->set.max_idle_time_msecs > 0) {
+
+		if (conn->to_idle != NULL) {
+			/* timeout already set */
+			return;
+		}
+
+		http_client_connection_debug(conn, 
+			"No more requests queued; going idle");
+
+		if (conn->client->ioloop != NULL)
+			io_loop_stop(conn->client->ioloop);
+
+		count = array_count(&conn->peer->conns);
+		i_assert(count > 0);
+
+		/* set timeout for this connection */
+		if (count > conn->client->set.max_parallel_connections) {
+			/* instant death for (urgent) connections above limit */
+			timeout = 0;
+		} else {
+			/* kill duplicate connections quicker;
+				 linearly based on the number of connections */
+			timeout = (conn->client->set.max_parallel_connections - count) *
+				(conn->client->set.max_idle_time_msecs /
+					conn->client->set.max_parallel_connections);
+		}
+
+		conn->to_idle =
+			timeout_add(timeout, http_client_connection_idle_timeout, conn);
+
+	} else {
+		/* there should be no idle timeout */
+		i_assert(conn->to_idle == NULL);
+	}
+}
+
+static void
+http_client_connection_continue_timeout(struct http_client_connection *conn)
+{
+	struct http_client_request *const *req_idx;
+	struct http_client_request *req;
+
+	if (conn->to_response != NULL)
+		timeout_remove(&conn->to_response);
+	conn->peer->no_payload_sync = TRUE;
+
+	http_client_connection_debug(conn, 
+		"Expected 100-continue response timed out; sending payload anyway");
+
+	req_idx = array_idx(&conn->request_wait_list, 0);
+	req = req_idx[0];
+
+	// FIXME: we should mark this in the peer object for next requests
+	conn->payload_continue = TRUE;
+	if (http_client_request_send_more(req) < 0) {
+		http_client_connection_abort_temp_error(&conn,
+			HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Failed to send request");
+	}
+}
+
+bool http_client_connection_next_request(struct http_client_connection *conn)
+{
+	struct http_client_request *req = NULL;
+
+	if (!http_client_connection_is_ready(conn)) {
+		
+		http_client_connection_debug(conn, "Not ready for next request");
+		return FALSE;
+	}
+
+	/* claim request, but no urgent request can be second in line */
+	req = http_client_peer_claim_request(conn->peer,
+		array_count(&conn->request_wait_list) > 0); 
+	if (req == NULL) {
+		http_client_connection_check_idle(conn);
+		return FALSE;	
+	}
+
+	if (conn->to_idle != NULL)
+		timeout_remove(&conn->to_idle);
+
+	req->conn = conn;
+	conn->payload_continue = FALSE;
+	if (conn->peer->no_payload_sync)
+		req->payload_sync = FALSE;
+	array_append(&conn->request_wait_list, &req, 1);
+	http_client_request_ref(req);
+
+	if (http_client_request_send(req) < 0) {
+		http_client_connection_abort_temp_error(&conn,
+			HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
+			"Failed to send request");
+		return FALSE;
+	}
+
+	/* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21;
+			Section 6.1.2.1:
+
+	   Because of the presence of older implementations, the protocol allows
+	   ambiguous situations in which a client might send "Expect: 100-continue"
+	   without receiving either a 417 (Expectation Failed) or a 100 (Continue)
+	   status code.  Therefore, when a client sends this header field to an
+	   origin server (possibly via a proxy) from which it has never seen a 100
+	   (Continue) status code, the client SHOULD NOT wait for an indefinite
+	   period before sending the payload body.
+	 */
+	if (req->payload_sync) {
+		i_assert(req->input_size > 0);
+		i_assert(conn->to_response == NULL);
+		conn->to_response =	timeout_add(HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS,
+			http_client_connection_continue_timeout, conn);
+	}
+
+	return TRUE;
+}
+
+static void http_client_connection_destroy(struct connection *_conn)
+{
+	struct http_client_connection *conn =
+		(struct http_client_connection *)_conn;
+
+	conn->closing = TRUE;
+	conn->connected = FALSE;
+
+	switch (_conn->disconnect_reason) {
+	case CONNECTION_DISCONNECT_CONNECT_TIMEOUT:
+		http_client_peer_connection_failure(conn->peer);
+		break;
+	case CONNECTION_DISCONNECT_CONN_CLOSED:
+		/* retry pending requests if possible */
+		http_client_connection_retry_requests(conn,
+			HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+	default:
+		break;
+	}
+
+	http_client_connection_free(&conn);
+}
+
+static void http_client_payload_finished(struct http_client_connection *conn)
+{
+	timeout_remove(&conn->to_input);
+	conn->conn.io = io_add(conn->conn.fd_in, IO_READ,
+			       http_client_connection_input, &conn->conn);
+}
+
+static void http_client_payload_destroyed(struct http_client_connection *conn)
+{
+	i_assert(conn->incoming_payload != NULL);
+	i_assert(conn->pending_request != NULL);
+	i_assert(conn->conn.io == NULL);
+
+	http_client_connection_debug(conn, "Response payload stream destroyed");
+
+	/* caller is allowed to change the socket fd to blocking while reading
+	   the payload. make sure here that it's switched back. */
+	net_set_nonblock(conn->conn.fd_in, TRUE);
+
+	conn->incoming_payload = NULL;
+
+	http_client_request_finish(&conn->pending_request);
+	conn->pending_request = NULL;
+
+	if (conn->close_indicated) {
+		http_client_connection_server_close(&conn);
+		return;
+	}
+
+	/* input stream may have pending input. make sure input handler
+	   gets called (but don't do it directly, since we get get here
+	   somewhere from the API user's code, which we can't really know what
+	   state it is in). this call also triggers sending a new request if
+	   necessary. */
+	conn->to_input =
+		timeout_add_short(0, http_client_connection_input, &conn->conn);
+}
+
+static bool
+http_client_connection_return_response(struct http_client_connection *conn,
+	struct http_client_request *req, struct http_response *response)
+{
+	struct istream *payload;
+
+	i_assert(conn->incoming_payload == NULL);
+	i_assert(conn->pending_request == NULL);
+
+	req->state = HTTP_REQUEST_STATE_GOT_RESPONSE;
+
+	if (response->payload != NULL) {
+		/* wrap the stream to capture the destroy event without destroying the
+		   actual payload stream. */
+		conn->incoming_payload = response->payload =
+			i_stream_create_limit(response->payload, (uoff_t)-1);
+		i_stream_set_destroy_callback(response->payload,
+					      http_client_payload_destroyed,
+					      conn);
+		/* the callback may add its own I/O, so we need to remove
+		   our one before calling it */
+		io_remove(&conn->conn.io);
+	}
+
+	http_client_request_callback(req, response);
+
+	// FIXME: conn may be freed at this point..
+
+	if (response->payload != NULL) {
+		req->state = HTTP_REQUEST_STATE_PAYLOAD_IN;
+		payload = response->payload;
+		response->payload = NULL;
+		conn->pending_request = req;
+		i_stream_unref(&payload);
+		if (conn->to_input != NULL) {
+			/* already finished reading the payload */
+			http_client_payload_finished(conn);
+		}
+	} else {
+		http_client_request_finish(&req);
+	}
+
+	if (conn->incoming_payload == NULL) {
+		i_assert(conn->conn.io != NULL);
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static void http_client_connection_input(struct connection *_conn)
+{
+	struct http_client_connection *conn =
+		(struct http_client_connection *)_conn;
+	struct http_response *response;
+	struct http_client_request *const *req_idx;
+	struct http_client_request *req = NULL;
+	int finished = 0, ret;
+	const char *error;
+	bool no_payload = FALSE;
+
+	i_assert(conn->incoming_payload == NULL);
+
+	if (conn->to_input != NULL) {
+		/* We came here from a timeout added by
+		   http_client_payload_destroyed(). The IO couldn't be added
+		   back immediately in there, because the HTTP API user may
+		   still have had its own IO pointed to the same fd. It should
+		   be removed by now, so we can add it back. */
+		http_client_payload_finished(conn);
+		finished++;
+	}
+
+	/* get first waiting request */
+	if (array_count(&conn->request_wait_list) > 0) {
+		req_idx = array_idx(&conn->request_wait_list, 0);
+		req = req_idx[0];
+
+		/* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21
+		     Section 3.3.2:
+
+		   A server MAY send a Content-Length header field in a response to a
+		   HEAD request [...]
+		 */
+		no_payload = (strcmp(req->method, "HEAD") == 0);
+	}
+
+	// FIXME: handle somehow if server replies before request->input is at EOF
+	while ((ret=http_response_parse_next
+		(conn->http_parser, no_payload, &response, &error)) > 0) {
+		bool aborted;
+
+		if (req == NULL) {
+			/* server sent response without any requests in the wait list */
+			http_client_connection_error(conn, "Got unexpected input from server");
+			http_client_connection_free(&conn);
+			return;
+		}
+
+		/* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21
+		     Section 7.2:
+
+		   A client MUST be prepared to accept one or more 1xx status responses
+		   prior to a regular response, even if the client does not expect a 100
+		   (Continue) status message.  Unexpected 1xx status responses MAY be
+		   ignored by a user agent.
+		 */
+		if (req->payload_sync && response->status == 100) {
+			conn->payload_continue = TRUE;
+			if (conn->to_response != NULL)
+				timeout_remove(&conn->to_response);
+			http_client_connection_debug(conn,
+				"Got expected 100-continue response");
+			if (http_client_request_send_more(req) < 0) {
+				http_client_connection_abort_temp_error(&conn,
+					HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
+					"Failed to send request");
+			}
+			return;
+		} else if (response->status / 100 == 1) {
+			/* ignore them for now */
+			http_client_connection_debug(conn,
+				"Got unexpected %u response; ignoring", response->status);
+			continue;
+		} // FIXME: handle 417 Expectation Failed
+
+		http_client_connection_debug(conn,
+			"Got %u response for request %s",
+			response->status, http_client_request_label(req));
+
+		/* remove request from queue */
+		array_delete(&conn->request_wait_list, 0, 1);
+		aborted = (req->state == HTTP_REQUEST_STATE_ABORTED);
+		http_client_request_unref(&req);
+		
+		conn->close_indicated = response->connection_close;
+
+		if (!aborted) {
+			if (response->status / 100 == 3) {
+				/* redirect */
+				http_client_request_redirect(req, response->location);
+			} else {
+				/* response for application */
+				if (!http_client_connection_return_response(conn, req, response))
+					return;
+			}
+
+			finished++;
+		}
+
+		/* server closing connection? */
+		if (conn->close_indicated) {
+			http_client_connection_server_close(&conn);
+			return;
+		}
+
+		/* get next waiting request */
+		if (array_count(&conn->request_wait_list) > 0) {
+			req_idx = array_idx(&conn->request_wait_list, 0);
+			req = req_idx[0];
+			no_payload = (strcmp(req->method, "HEAD") == 0);
+		} else {
+			req = NULL;
+			no_payload = FALSE;
+		}
+	}
+
+	if (ret <= 0 &&
+		(conn->conn.input->eof || conn->conn.input->stream_errno != 0)) {
+		int stream_errno = conn->conn.input->stream_errno;
+		http_client_connection_debug(conn,
+			"Lost connection to server (error=%s)",
+			stream_errno != 0 ? strerror(stream_errno) : "EOF");
+		http_client_connection_abort_temp_error(&conn,
+			HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+		return;
+	}
+
+	if (ret < 0) {
+		http_client_connection_abort_error(&conn,
+			HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error);
+		return;
+	}
+
+	if (finished > 0) {
+		/* room for new requests */
+		http_client_peer_handle_requests(conn->peer);
+		http_client_connection_check_idle(conn);
+	}
+}
+
+static int http_client_connection_output(struct http_client_connection *conn)
+{
+	struct http_client_request *const *req_idx, *req;
+	struct ostream *output = conn->conn.output;
+	int ret;
+
+	if ((ret = o_stream_flush(output)) <= 0) {
+		if (ret < 0) {
+			http_client_connection_abort_temp_error(&conn,
+				HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+		}
+		return ret;
+	}
+
+	if (array_count(&conn->request_wait_list) > 0 && conn->output_locked) {
+		req_idx = array_idx(&conn->request_wait_list, 0);
+		req = req_idx[0];
+
+		if (!req->payload_sync || conn->payload_continue) {
+			if (http_client_request_send_more(req) < 0) {
+				http_client_connection_abort_temp_error(&conn,
+					HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+				return -1;
+			}
+			if (!conn->output_locked) {
+				/* room for new requests */
+				http_client_peer_handle_requests(conn->peer);
+				http_client_connection_check_idle(conn);
+			}
+		}
+	}
+	return 1;
+}
+
+static void 
+http_client_connection_ready(struct http_client_connection *conn)
+{
+	struct stat st;
+
+	conn->connected = TRUE;
+
+	if (*conn->client->set.rawlog_dir != '\0' &&
+		stat(conn->client->set.rawlog_dir, &st) == 0) {
+		iostream_rawlog_create(conn->client->set.rawlog_dir,
+				       &conn->conn.input, &conn->conn.output);
+	}
+
+	conn->http_parser = http_response_parser_init(conn->conn.input);
+	o_stream_set_flush_callback(conn->conn.output,
+    http_client_connection_output, conn);
+
+	/* we never pipeline before the first response */
+	(void)http_client_connection_next_request(conn);
+}
+
+#ifdef HTTP_BUILD_SSL
+static int http_client_connection_ssl_handshaked(void *context)
+{
+	struct http_client_connection *conn = context;
+
+	if (!conn->client->set.ssl_verify) {
+		/* skip certificate checks */
+		http_client_connection_debug(conn, "SSL handshake successful");
+		return 0;
+	} else if (!ssl_iostream_has_valid_client_cert(conn->ssl_iostream)) {
+		if (!ssl_iostream_has_broken_client_cert(conn->ssl_iostream)) {
+			http_client_connection_error(conn, "SSL certificate not received");
+		} else {
+			http_client_connection_error(conn, "Received invalid SSL certificate");
+		}
+	} else {
+		const char *host = http_client_peer_get_hostname(conn->peer);
+
+		i_assert(host != NULL);
+
+		if (ssl_iostream_cert_match_name(conn->ssl_iostream, host) < 0) {
+			http_client_connection_error(conn, 
+				"SSL certificate doesn't match host name");
+		} else {
+			http_client_connection_debug(conn, "SSL handshake successful");
+			return 0;
+		}
+	}
+	i_stream_close(conn->conn.input);
+	return -1;
+}
+
+static int 
+http_client_connection_ssl_init(struct http_client_connection *conn)
+{
+	struct ssl_iostream_settings ssl_set;
+	const char *source;
+
+	if (conn->peer->ssl_ctx == NULL) {
+		http_client_connection_error(conn, "No SSL context");
+		return -1;
+	}
+
+	memset(&ssl_set, 0, sizeof(ssl_set));
+	if (conn->client->set.ssl_verify) {
+		ssl_set.verbose_invalid_cert = TRUE;
+		ssl_set.verify_remote_cert = TRUE;
+		ssl_set.require_valid_cert = TRUE;
+	}
+
+	if (conn->client->set.debug)
+		http_client_connection_debug(conn, "Starting SSL handshake");
+
+	source = t_strdup_printf
+		("connection %s: ", http_client_connection_label(conn));
+	if (io_stream_create_ssl(conn->peer->ssl_ctx, source, &ssl_set,
+				 &conn->conn.input, &conn->conn.output, &conn->ssl_iostream) < 0) {
+		http_client_connection_error(conn, "Couldn't initialize SSL client");
+		return -1;
+	}
+	ssl_iostream_set_handshake_callback(conn->ssl_iostream,
+					    http_client_connection_ssl_handshaked, conn);
+	if (ssl_iostream_handshake(conn->ssl_iostream) < 0) {
+		http_client_connection_error(conn, "SSL handshake failed: %s",
+			ssl_iostream_get_last_error(conn->ssl_iostream));
+		return -1;
+	}
+
+	http_client_connection_ready(conn);
+	return 0;
+}
+#endif
+
+static void 
+http_client_connection_connected(struct connection *_conn, bool success)
+{
+	struct http_client_connection *conn =
+		(struct http_client_connection *)_conn;
+
+	if (!success) {
+		http_client_connection_error(conn, "Connect failed: %m");
+		http_client_peer_connection_failure(conn->peer);
+
+	} else {
+		http_client_connection_debug(conn, "Connected");
+#ifdef HTTP_BUILD_SSL
+		if (conn->peer->addr.ssl) {
+			if (http_client_connection_ssl_init(conn) < 0)
+				http_client_peer_connection_failure(conn->peer);
+			return;
+		}
+#endif
+		http_client_connection_ready(conn);
+	}
+}
+
+static const struct connection_settings http_client_connection_set = {
+	.input_max_size = (size_t)-1,
+	.output_max_size = (size_t)-1,
+	.client = TRUE
+};
+
+static const struct connection_vfuncs http_client_connection_vfuncs = {
+	.destroy = http_client_connection_destroy,
+	.input = http_client_connection_input,
+	.client_connected = http_client_connection_connected
+};
+
+struct connection_list *
+http_client_connection_list_init(void)
+{
+	return connection_list_init
+		(&http_client_connection_set, &http_client_connection_vfuncs);
+}
+
+static int http_client_connection_connect(struct http_client_connection *conn)
+{
+	if (conn->conn.fd_in == -1) {
+		if (connection_client_connect(&conn->conn) < 0) {
+			http_client_connection_error(conn, "Could not connect");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+struct http_client_connection *
+http_client_connection_create(struct http_client_peer *peer)
+{
+	struct http_client_connection *conn;
+	static unsigned int id = 0;
+
+	conn = i_new(struct http_client_connection, 1);
+	conn->client = peer->client;
+	conn->peer = peer;
+	i_array_init(&conn->request_wait_list, 16);
+
+	connection_init_client_ip
+		(peer->client->conn_list, &conn->conn, &peer->addr.ip, peer->addr.port);
+
+	if (http_client_connection_connect(conn) < 0) {
+		http_client_connection_free(&conn);
+		return NULL;
+	}
+
+	conn->id = id++;
+	array_append(&peer->conns, &conn, 1);
+
+	http_client_connection_debug(conn,
+		"Connection created (%d parallel connections exist)",
+		array_count(&peer->conns));
+	return conn;
+}
+
+void http_client_connection_free(struct http_client_connection **_conn)
+{
+	struct http_client_connection *conn = *_conn;
+	struct http_client_connection *const *conn_idx;
+	struct http_client_peer *peer = conn->peer;
+	struct http_client_request **req;
+
+	http_client_connection_debug(conn, "Connection destroy");
+
+	conn->connected = FALSE;
+
+#ifdef HTTP_BUILD_SSL
+	if (conn->ssl_iostream != NULL)
+		ssl_iostream_unref(&conn->ssl_iostream);
+#endif
+
+	connection_deinit(&conn->conn);
+
+	/* abort all pending requests */
+	array_foreach_modifiable(&conn->request_wait_list, req) {
+		http_client_request_error(*req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
+			"Aborting");
+		http_client_request_unref(req);
+	}
+	array_free(&conn->request_wait_list);
+
+	if (conn->to_input != NULL)
+		timeout_remove(&conn->to_input);
+	if (conn->to_idle != NULL)
+		timeout_remove(&conn->to_idle);
+	if (conn->to_response != NULL)
+		timeout_remove(&conn->to_response);
+	
+	/* remove this connection from the list */
+	array_foreach(&conn->peer->conns, conn_idx) {
+		if (*conn_idx == conn) {
+			array_delete(&conn->peer->conns,
+				array_foreach_idx(&conn->peer->conns, conn_idx), 1);
+			break;
+		}
+	}
+
+	if (conn->http_parser != NULL)
+		http_response_parser_deinit(&conn->http_parser);
+
+	i_free(conn);
+	*_conn = NULL;
+
+	http_client_peer_connection_lost(peer);
+}
+
+void http_client_connection_switch_ioloop(struct http_client_connection *conn)
+{
+	if (conn->to_input != NULL)
+		conn->to_input = io_loop_move_timeout(&conn->to_input);
+	if (conn->to_idle != NULL)
+		conn->to_idle = io_loop_move_timeout(&conn->to_idle);
+	if (conn->to_response != NULL)
+		conn->to_response = io_loop_move_timeout(&conn->to_response);
+	connection_switch_ioloop(&conn->conn);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-client-host.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,388 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "dns-lookup.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_host_debug(struct http_client_host *host,
+	const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_host_debug(struct http_client_host *host,
+	const char *format, ...)
+{
+	va_list args;
+
+	if (host->client->set.debug) {
+
+		va_start(args, format);	
+		i_debug("http-client: host %s: %s", 
+			host->name, t_strdup_vprintf(format, args));
+		va_end(args);
+	}
+}
+
+/*
+ * Host:port
+ */
+
+static struct http_client_host_port *
+http_client_host_port_find(struct http_client_host *host,
+	unsigned int port, bool ssl)
+{
+	struct http_client_host_port *hport;
+
+	array_foreach_modifiable(&host->ports, hport) {
+		if (hport->port == port && hport->ssl == ssl)
+			return hport;
+	}
+
+	return NULL;
+}
+
+static struct http_client_host_port *
+http_client_host_port_init(struct http_client_host *host,
+	unsigned int port, bool ssl)
+{
+	struct http_client_host_port *hport;
+
+	hport = http_client_host_port_find(host, port, ssl);
+	if (hport == NULL) {
+		hport = array_append_space(&host->ports);
+		hport->port = port;
+		hport->ssl = ssl;
+		hport->ips_connect_idx = 0;
+		i_array_init(&hport->request_queue, 16);
+		i_array_init(&hport->urgent_request_queue, 4);
+	}
+
+	return hport;
+}
+
+static void http_client_host_port_error(struct http_client_host_port *hport,
+	unsigned int status, const char *error)
+{
+	struct http_client_request **req;
+
+	/* abort all pending requests */
+	array_foreach_modifiable(&hport->request_queue, req) {
+		http_client_request_error(*req, status, error);
+	}
+	array_foreach_modifiable(&hport->urgent_request_queue, req) {
+		http_client_request_error(*req, status, error);
+	}
+	array_clear(&hport->request_queue);
+	array_clear(&hport->urgent_request_queue);
+}
+
+static void http_client_host_port_deinit(struct http_client_host_port *hport)
+{
+	http_client_host_port_error
+		(hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted");
+	array_free(&hport->request_queue);
+	array_free(&hport->urgent_request_queue);
+}
+
+static void
+http_client_host_port_drop_request(struct http_client_host_port *hport,
+	struct http_client_request *req)
+{
+	struct http_client_request **req_idx;
+	ARRAY_TYPE(http_client_request) *queue =
+		(req->urgent ? &hport->urgent_request_queue : &hport->request_queue);
+	unsigned int idx;
+
+	array_foreach_modifiable(queue, req_idx) {
+		if (*req_idx == req) {
+			idx = array_foreach_idx(queue, req_idx);
+			break;
+		}
+	}
+	
+	array_delete(queue, idx, 1);
+}
+
+/*
+ * Host
+ */
+
+static void
+http_client_host_connection_setup(struct http_client_host *host,
+	struct http_client_host_port *hport)
+{
+	struct http_client_peer *peer = NULL;
+	struct http_client_peer_addr addr;
+
+	while (hport->ips_connect_idx < host->ips_count) {
+		addr.ip = host->ips[hport->ips_connect_idx];
+		addr.port = hport->port;
+		addr.ssl = hport->ssl;
+
+		http_client_host_debug(host, "Setting up connection to %s:%u (ssl=%s)", 
+			net_ip2addr(&addr.ip), addr.port, (addr.ssl ? "yes" : "no"));
+
+		if ((peer=http_client_peer_get(host->client, &addr)) != NULL)
+			break;
+
+		hport->ips_connect_idx++;
+	}
+
+	if (peer == NULL) {
+		http_client_host_port_error
+			(hport, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, "Connection failed");
+		if (host->client->ioloop != NULL)
+			io_loop_stop(host->client->ioloop);
+		return;
+	}
+
+	http_client_peer_add_host(peer, host);
+}
+
+void http_client_host_connection_failure(struct http_client_host *host,
+	const struct http_client_peer_addr *addr)
+{
+	struct http_client_host_port *hport;
+
+	http_client_host_debug(host, "Failed to connect to %s:%u", 
+		net_ip2addr(&addr->ip), addr->port);
+
+	hport = http_client_host_port_find(host, addr->port, addr->ssl);
+	if (hport == NULL)
+		return;
+
+	hport->ips_connect_idx++;
+	http_client_host_connection_setup(host, hport);
+}
+
+static void
+http_client_host_lookup_failure(struct http_client_host *host)
+{
+	struct http_client_host_port *hport;
+
+	array_foreach_modifiable(&host->ports, hport) {
+		http_client_host_port_error(hport,
+			HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED, "Failed to lookup host");
+	}
+}
+
+static void
+http_client_host_dns_callback(const struct dns_lookup_result *result,
+			      struct http_client_host *host)
+{
+	struct http_client_host_port *hport;
+	unsigned int requests = 0;
+
+	host->dns_lookup = NULL;
+
+	if (result->ret != 0) {
+		i_error("http-client: dns_lookup(%s) failed: %s",
+			host->name, result->error);
+		http_client_host_lookup_failure(host);
+		return;
+	}
+
+	http_client_host_debug(host,
+		"DNS lookup successful; got %d IPs", result->ips_count);
+
+	i_assert(result->ips_count > 0);
+	host->ips_count = result->ips_count;
+	host->ips = i_new(struct ip_addr, host->ips_count);
+	memcpy(host->ips, result->ips, sizeof(*host->ips) * host->ips_count);
+
+	// FIXME: make DNS result expire 
+
+	/* make connections to requested ports */
+	array_foreach_modifiable(&host->ports, hport) {
+		unsigned int count = array_count(&hport->request_queue);
+		hport->ips_connect_idx = 0;
+		if (count > 0)
+			http_client_host_connection_setup(host, hport);
+		requests += count;
+	}
+
+	if (requests == 0 && host->client->ioloop != NULL)
+		io_loop_stop(host->client->ioloop);
+}
+
+static void http_client_host_lookup
+(struct http_client_host *host)
+{
+	struct http_client *client = host->client;
+	struct dns_lookup_settings dns_set;
+	struct ip_addr ip, *ips;
+	unsigned int ips_count;
+	int ret;
+
+	memset(&dns_set, 0, sizeof(dns_set));
+	dns_set.dns_client_socket_path =
+		client->set.dns_client_socket_path;
+	dns_set.timeout_msecs = HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS;
+
+	if (host->ips_count == 0 &&
+	    net_addr2ip(host->name, &ip) == 0) { // FIXME: remove this?
+		host->ips_count = 1;
+		host->ips = i_new(struct ip_addr, host->ips_count);
+		host->ips[0] = ip;
+	} else if (*dns_set.dns_client_socket_path == '\0') {
+		ret = net_gethostbyname(host->name,	&ips, &ips_count);
+		if (ret != 0) {
+			i_error("http-client: net_gethostbyname(%s) failed: %s",
+				host->name,	net_gethosterror(ret));
+			http_client_host_lookup_failure(host);
+			return;
+		}
+
+		http_client_host_debug(host,
+			"DNS lookup successful; got %d IPs", ips_count);
+
+		host->ips_count = ips_count;
+		host->ips = i_new(struct ip_addr, ips_count);
+		memcpy(host->ips, ips, ips_count * sizeof(*ips));
+	}
+
+	if (host->ips_count == 0) {
+		http_client_host_debug(host,
+			"Performing asynchronous DNS lookup");
+		(void)dns_lookup(host->name, &dns_set,
+				 http_client_host_dns_callback, host, &host->dns_lookup);
+	}
+}
+
+struct http_client_host *http_client_host_get
+(struct http_client *client, const char *hostname)
+{
+	struct http_client_host *host;
+
+	host = hash_table_lookup(client->hosts, hostname);
+	if (host == NULL) {
+		// FIXME: limit the maximum number of inactive cached hosts
+		host = i_new(struct http_client_host, 1);
+		host->client = client;
+		host->name = i_strdup(hostname);
+		i_array_init(&host->ports, 4);
+
+		hash_table_insert(client->hosts, hostname, host);
+
+		http_client_host_debug(host, "Host created");
+	}
+	return host;
+}
+
+void http_client_host_submit_request(struct http_client_host *host,
+	struct http_client_request *req)
+{
+	struct http_client_host_port *hport;
+
+	req->host = host;
+
+	if (host->ips_count == 0 && host->dns_lookup == NULL)	
+		http_client_host_lookup(host);
+	hport = http_client_host_port_init(host, req->port, req->ssl);
+	if (req->urgent)
+		array_append(&hport->urgent_request_queue, &req, 1);
+	else
+		array_append(&hport->request_queue, &req, 1);
+
+	if (host->ips_count == 0)
+		return;
+	
+	http_client_host_connection_setup(host, hport);
+}
+
+struct http_client_request *
+http_client_host_claim_request(struct http_client_host *host,
+	const struct http_client_peer_addr *addr, bool no_urgent)
+{
+	struct http_client_host_port *hport;
+	struct http_client_request *const *req_idx;
+	struct http_client_request *req;
+
+	hport = http_client_host_port_find(host, addr->port, addr->ssl);
+	if (hport == NULL)
+		return NULL;
+	if (!no_urgent && array_count(&hport->urgent_request_queue) > 0) {
+		req_idx = array_idx(&hport->urgent_request_queue, 0);
+		req = *req_idx;
+		array_delete(&hport->urgent_request_queue, 0, 1);
+	} else if (array_count(&hport->request_queue) > 0) {
+		req_idx = array_idx(&hport->request_queue, 0);
+		req = *req_idx;
+		array_delete(&hport->request_queue, 0, 1);
+	} else {
+		return NULL;
+	}
+
+	http_client_host_debug(host,
+		"Connection %s:%u claimed request %s %s",
+		net_ip2addr(&addr->ip), addr->port, http_client_request_label(req),
+		(req->urgent ? "(urgent)" : ""));
+
+	return req;
+}
+
+bool http_client_host_have_requests(struct http_client_host *host,
+	const struct http_client_peer_addr *addr, bool urgent)
+{
+	struct http_client_host_port *hport;
+
+	hport = http_client_host_port_find(host, addr->port, addr->ssl);
+	if (hport == NULL)
+		return FALSE;
+	
+	if (urgent)
+		return (array_count(&hport->urgent_request_queue) > 0);
+	return (array_count(&hport->request_queue) > 0);
+}
+
+void http_client_host_drop_request(struct http_client_host *host,
+	struct http_client_request *req)
+{
+	struct http_client_host_port *hport;
+
+	hport = http_client_host_port_find(host, req->port, req->ssl);
+	if (hport == NULL)
+		return;
+	
+	http_client_host_port_drop_request(hport, req);
+}
+
+void http_client_host_free(struct http_client_host **_host)
+{
+	struct http_client_host *host = *_host;
+	struct http_client_host_port *hport;
+
+	http_client_host_debug(host, "Host destroy");
+
+	if (host->dns_lookup != NULL)
+		dns_lookup_abort(&host->dns_lookup);
+
+	/* drop request queues */
+	array_foreach_modifiable(&host->ports, hport) {
+		http_client_host_port_deinit(hport);
+	}
+	array_free(&host->ports);
+
+	i_free(host->ips);
+	i_free(host->name);
+	i_free(host);
+}
+
+void http_client_host_switch_ioloop(struct http_client_host *host)
+{
+	if (host->dns_lookup != NULL)
+		dns_lookup_switch_ioloop(host->dns_lookup);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-client-peer.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,358 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "iostream-ssl.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_peer_debug(struct http_client_peer *peer,
+	const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_peer_debug(struct http_client_peer *peer,
+	const char *format, ...)
+{
+	va_list args;
+
+	if (peer->client->set.debug) {
+		va_start(args, format);	
+		i_debug("http-client: peer %s:%u: %s", 
+			net_ip2addr(&peer->addr.ip), peer->addr.port,
+			t_strdup_vprintf(format, args));
+		va_end(args);
+	}
+}
+
+static inline void ATTR_FORMAT(2, 3)
+http_client_peer_error(struct http_client_peer *peer,
+	const char *format, ...)
+{
+	va_list args;
+	va_start(args, format);	
+	i_error("http-client: peer %s:%u: %s", 
+		net_ip2addr(&peer->addr.ip), peer->addr.port,
+		t_strdup_vprintf(format, args));
+	va_end(args);
+}
+
+/*
+ * Peer address
+ */
+
+unsigned int http_client_peer_addr_hash
+(const struct http_client_peer_addr *peer)
+{
+	return net_ip_hash(&peer->ip) + peer->port + peer->ssl;
+}
+
+int http_client_peer_addr_cmp
+(const struct http_client_peer_addr *peer1,
+	const struct http_client_peer_addr *peer2)
+{
+	int ret;
+
+	if ((ret=net_ip_cmp(&peer1->ip, &peer2->ip)) != 0)
+		return ret;
+	if (peer1->port != peer2->port)
+		return (peer1->port > peer2->port ? 1 : -1);
+	if (peer1->ssl != peer2->ssl)
+		return (peer1->ssl > peer2->ssl ? 1 : -1);
+	return 0;
+}
+
+/*
+ * Peer
+ */
+
+const char *
+http_client_peer_get_hostname(struct http_client_peer *peer)
+{
+	struct http_client_host *const *host;
+
+	if (array_count(&peer->hosts) == 0)
+		return NULL;
+
+	/* just return name of initial host */
+	host = array_idx(&peer->hosts, 1);
+	return (*host)->name;
+}
+
+static int
+http_client_peer_connect(struct http_client_peer *peer)
+{
+	struct http_client_connection *conn;
+
+	conn = http_client_connection_create(peer);
+	if (conn == NULL) {
+		http_client_peer_debug(peer, "Failed to make new connection");
+		return -1;
+	}
+
+	return 0;
+}
+
+static bool
+http_client_peer_requests_pending(struct http_client_peer *peer, bool urgent)
+{
+	struct http_client_host *const *host;
+
+	array_foreach(&peer->hosts, host) {
+		if (http_client_host_have_requests(*host, &peer->addr, urgent))
+			return TRUE;
+	}
+
+	return FALSE;
+}
+
+static bool
+http_client_peer_next_request(struct http_client_peer *peer,
+	bool urgent)
+{
+	struct http_client_connection *const *conn_idx;
+	struct http_client_connection *conn = NULL;
+	unsigned int closing = 0, min_waiting = (unsigned int)-1;
+	
+	/* at this point we already know that a request for this peer is pending
+	 */
+
+	/* find the least busy connection */
+	array_foreach(&peer->conns, conn_idx) {
+		if (http_client_connection_is_ready(*conn_idx)) {
+			unsigned int waiting = array_count(&(*conn_idx)->request_wait_list);
+			if (waiting < min_waiting) {
+				min_waiting = waiting;
+				conn = *conn_idx;
+				if (min_waiting == 0)
+					break;
+			}
+		}
+		/* count the number of closing connections */
+		if ((*conn_idx)->closing)
+			closing++;
+	}
+
+	/* do we have an idle connection? */
+	if (conn != NULL && min_waiting == 0) {
+		/* yes */
+		return http_client_connection_next_request(conn);
+	}
+
+	/* no, but can we create a new connection? */		
+	if (!urgent && (array_count(&peer->conns) - closing) >=
+		peer->client->set.max_parallel_connections) {
+		/* no */
+		if (conn == NULL)
+			return FALSE;
+		/* pipeline it */
+		return http_client_connection_next_request(conn);
+	}
+
+	/* yes */
+	if (http_client_peer_connect(peer) < 0) {
+		/* connection failed */
+		if (conn == NULL)
+			return FALSE;
+		/* pipeline it on the least busy connection we found */
+		return http_client_connection_next_request(conn);
+	}
+
+	/* now we wait until it is connected */
+	return FALSE;
+}
+
+void http_client_peer_handle_requests(struct http_client_peer *peer)
+{
+	/* check urgent requests first */
+	while (http_client_peer_requests_pending(peer, TRUE)) {
+		if (!http_client_peer_next_request(peer, TRUE))
+			break;
+	}
+
+	/* check normal requests once we're done */
+	while (http_client_peer_requests_pending(peer, FALSE)) {
+		if (!http_client_peer_next_request(peer, FALSE))
+			break;
+	}
+}
+
+static struct http_client_peer *
+http_client_peer_create(struct http_client *client,
+			      const struct http_client_peer_addr *addr)
+{
+	struct http_client_peer *peer;
+#ifdef HTTP_BUILD_SSL
+	struct ssl_iostream_settings ssl_set;
+#endif
+
+	peer = i_new(struct http_client_peer, 1);
+	peer->client = client;
+	peer->addr = *addr;
+	i_array_init(&peer->hosts, 16);
+	i_array_init(&peer->conns, 16);
+
+	hash_table_insert
+		(client->peers, (const struct http_client_peer_addr *)&peer->addr, peer);
+
+	http_client_peer_debug(peer, "Peer created");
+
+#ifdef HTTP_BUILD_SSL
+	if (peer->addr.ssl && peer->ssl_ctx == NULL) {
+		const char *source;
+		memset(&ssl_set, 0, sizeof(ssl_set));
+		ssl_set.ca_dir = peer->client->set.ssl_ca_dir;
+		ssl_set.verify_remote_cert = TRUE;
+		ssl_set.crypto_device = peer->client->set.ssl_crypto_device;
+
+		source = t_strdup_printf("http-client: peer %s:%u",
+			net_ip2addr(&peer->addr.ip), peer->addr.port);
+		if (ssl_iostream_context_init_client
+			(source, &ssl_set, &peer->ssl_ctx) < 0) {
+			http_client_peer_error(peer, "Couldn't initialize SSL context");
+			http_client_peer_free(&peer);
+			return NULL;
+		}
+	}
+#else
+	if (peer->addr.ssl) {
+		http_client_peer_error(peer, "HTTPS is not supported");
+		http_client_peer_free(&peer);
+		return NULL;
+	}
+#endif
+
+	if (http_client_peer_connect(peer) < 0) {
+		http_client_peer_free(&peer);
+		return NULL;
+	}
+
+	return peer;
+}
+
+void http_client_peer_free(struct http_client_peer **_peer)
+{
+	struct http_client_peer *peer = *_peer;
+	struct http_client_connection **conn;
+	ARRAY_TYPE(http_client_connection) conns;
+
+	if (peer->destroyed)
+		return;
+	peer->destroyed = TRUE;
+
+	http_client_peer_debug(peer, "Peer destroy");
+
+	/* make a copy of the connection array; freed connections modify it */
+	t_array_init(&conns, array_count(&peer->conns));
+	array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
+
+	array_foreach_modifiable(&conns, conn) {
+		http_client_connection_free(conn);
+	}
+
+	i_assert(array_count(&peer->conns) == 0);
+	array_free(&peer->conns);
+	array_free(&peer->hosts);
+
+#ifdef HTTP_BUILD_SSL
+	if (peer->ssl_ctx != NULL)
+		ssl_iostream_context_deinit(&peer->ssl_ctx);
+#endif
+
+	hash_table_remove
+		(peer->client->peers, (const struct http_client_peer_addr *)&peer->addr);
+
+	i_free(peer);
+	*_peer = NULL;
+}
+
+struct http_client_peer *
+http_client_peer_get(struct http_client *client,
+			   const struct http_client_peer_addr *addr)
+{
+	struct http_client_peer *peer;
+
+	peer = hash_table_lookup(client->peers, addr);
+	if (peer == NULL)
+		peer = http_client_peer_create(client, addr);
+
+	return peer;
+}
+
+void http_client_peer_add_host(struct http_client_peer *peer,
+			   struct http_client_host *host)
+{
+	struct http_client_host *const *host_idx;
+	bool exists = FALSE;
+
+	array_foreach(&peer->hosts, host_idx) {
+		if (*host_idx == host) {
+			exists = TRUE;
+			break;
+		}
+	}
+
+	if (!exists)
+		array_append(&peer->hosts, &host, 1);
+	if (exists || array_count(&peer->hosts) > 1)
+		(void)http_client_peer_next_request(peer, FALSE);
+}
+
+struct http_client_request *
+http_client_peer_claim_request(struct http_client_peer *peer, bool no_urgent)
+{
+	struct http_client_host *const *host_idx;
+	struct http_client_request *req;
+
+	array_foreach(&peer->hosts, host_idx) {
+		if ((req=http_client_host_claim_request
+			(*host_idx, &peer->addr, no_urgent)) != NULL) {
+			req->peer = peer;
+			return req;
+		}
+	}
+
+	return NULL;
+}
+
+void http_client_peer_connection_failure(struct http_client_peer *peer)
+{
+	struct http_client_host *const *host;
+
+	http_client_peer_debug(peer, "Failed to make connection");
+
+	if (array_count(&peer->conns) == 1) {
+		array_foreach(&peer->hosts, host) {
+			http_client_host_connection_failure(*host, &peer->addr);
+		}
+	}
+}
+
+void http_client_peer_connection_lost(struct http_client_peer *peer)
+{
+	if (peer->destroyed)
+		return;
+
+	http_client_peer_debug(peer, "Lost connection (%d connections left)",
+		array_count(&peer->conns));
+
+	if (array_count(&peer->conns) == 0) {
+		if (http_client_peer_requests_pending(peer, TRUE))
+			(void)http_client_peer_next_request(peer, TRUE);
+		else if (http_client_peer_requests_pending(peer, FALSE))
+			(void)http_client_peer_next_request(peer, FALSE);
+		else
+			http_client_peer_free(&peer);
+	}
+}
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-client-private.h	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,250 @@
+#ifndef HTTP_CLIENT_PRIVATE_H
+#define HTTP_CLIENT_PRIVATE_H
+
+#include "connection.h"
+
+#include "http-client.h"
+
+#define HTTP_BUILD_SSL
+
+#define HTTP_DEFAULT_PORT 80
+#define HTTPS_DEFAULT_PORT 443
+
+#define HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS (1000*30)
+#define HTTP_CLIENT_CONNECT_TIMEOUT_MSECS (1000*30)
+#define HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS (1000*60*5)
+#define HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS (1000*2)
+
+struct http_client_host;
+struct http_client_host_port;
+struct http_client_peer;
+struct http_client_connection;
+
+ARRAY_DEFINE_TYPE(http_client_host, struct http_client_host *);
+ARRAY_DEFINE_TYPE(http_client_host_port, struct http_client_host_port);
+ARRAY_DEFINE_TYPE(http_client_connection, struct http_client_connection *);
+ARRAY_DEFINE_TYPE(http_client_request, struct http_client_request *);
+
+HASH_TABLE_DEFINE_TYPE(http_client_host, const char *,
+	struct http_client_host *);
+HASH_TABLE_DEFINE_TYPE(http_client_peer, const struct http_client_peer_addr *,
+	struct http_client_peer *);
+
+enum http_request_state {
+	HTTP_REQUEST_STATE_NEW = 0,
+	HTTP_REQUEST_STATE_QUEUED,
+	HTTP_REQUEST_STATE_PAYLOAD_OUT,
+	HTTP_REQUEST_STATE_WAITING,
+	HTTP_REQUEST_STATE_GOT_RESPONSE,
+	HTTP_REQUEST_STATE_PAYLOAD_IN,
+	HTTP_REQUEST_STATE_FINISHED,
+	HTTP_REQUEST_STATE_ABORTED
+};
+
+struct http_client_request {
+	pool_t pool;
+	unsigned int refcount;
+
+	struct http_client_request *prev, *next;
+
+	const char *method, *hostname, *target;
+	unsigned int port;
+
+	struct http_client *client;
+	struct http_client_host *host;
+	struct http_client_peer *peer;
+	struct http_client_connection *conn;
+
+	string_t *headers;
+	struct istream *input;
+	uoff_t input_size;
+
+	unsigned int attempts;
+	unsigned int redirects;
+
+	http_client_request_callback_t *callback;
+	void *context;
+
+	enum http_request_state state;
+
+	unsigned int payload_sync:1;
+	unsigned int ssl:1;
+	unsigned int urgent:1;
+};
+
+struct http_client_host_port {
+	unsigned int port;
+
+	/* current index in host->ips */
+	unsigned int ips_connect_idx;
+
+	/* requests pending in queue to be picked up by connections */
+	ARRAY_TYPE(http_client_request) request_queue;
+
+	/* urgent request queue */
+	ARRAY_TYPE(http_client_request) urgent_request_queue;
+
+	unsigned int ssl:1;
+};
+
+struct http_client_host {
+	struct http_client *client;
+	char *name;
+
+	/* the ip addresses DNS returned for this host */
+	unsigned int ips_count;
+	struct ip_addr *ips;
+
+	/* requests are managed on a per-port basis */
+	ARRAY_TYPE(http_client_host_port) ports;
+
+	/* active DNS lookup */
+	struct dns_lookup *dns_lookup;
+};
+
+struct http_client_peer_addr {
+	struct ip_addr ip;
+	unsigned int port;
+	unsigned int ssl:1;              /* https */
+};
+
+struct http_client_peer {
+	struct http_client_peer_addr addr;
+	struct http_client *client;
+
+	/* hosts served through this peer */
+	ARRAY_TYPE(http_client_host) hosts;
+
+	/* active connections to this peer */
+	ARRAY_TYPE(http_client_connection) conns;
+
+	/* ssl */
+#ifdef HTTP_BUILD_SSL
+	struct ssl_iostream_context *ssl_ctx;
+#endif
+
+	unsigned int destroyed:1;        /* peer is being destroyed */
+	unsigned int no_payload_sync:1;  /* expect: 100-continue failed before */
+};
+
+struct http_client_connection {
+	struct connection conn;
+	struct http_client_peer *peer;
+	struct http_client *client;
+
+	const char *label;
+
+	unsigned int id; // DEBUG: identify parallel connections
+
+	/* ssl */
+#ifdef HTTP_BUILD_SSL
+	struct ssl_iostream *ssl_iostream;
+#endif
+
+	struct http_response_parser *http_parser;
+	struct timeout *to_input, *to_idle, *to_response;
+
+	struct http_client_request *pending_request;
+	struct istream *incoming_payload;
+
+	/* requests that have been sent, waiting for response */
+	ARRAY_TYPE(http_client_request) request_wait_list;
+
+	unsigned int connected:1;           /* connection is connected */
+	unsigned int closing:1;
+	unsigned int close_indicated:1;
+	unsigned int output_locked:1;       /* output is locked; no pipelining */
+	unsigned int payload_continue:1;    /* received 100-continue for current
+	                                        request */
+};
+
+struct http_client {
+	pool_t pool;
+
+	struct http_client_settings set;
+
+	struct ioloop *ioloop;
+
+	struct connection_list *conn_list;
+
+	HASH_TABLE_TYPE(http_client_host) hosts;
+	HASH_TABLE_TYPE(http_client_peer) peers;
+	unsigned int pending_requests;
+};
+
+static inline const char *
+http_client_request_label(struct http_client_request *req)
+{
+	return t_strdup_printf("[%s http%s://%s:%d%s]",
+		req->method, req->ssl ? "s" : "", req->hostname, req->port, req->target);
+}
+
+static inline const char *
+http_client_connection_label(struct http_client_connection *conn)
+{
+	return t_strdup_printf("%s:%u [%d]",
+		net_ip2addr(&conn->conn.ip), conn->conn.port, conn->id);
+}
+
+void http_client_request_ref(struct http_client_request *req);
+void http_client_request_unref(struct http_client_request **_req);
+int http_client_request_send(struct http_client_request *req);
+int http_client_request_send_more(struct http_client_request *req);
+void http_client_request_callback(struct http_client_request *req,
+	struct http_response *response);
+void http_client_request_resubmit(struct http_client_request *req);
+void http_client_request_retry(struct http_client_request *req,
+	unsigned int status, const char *error);
+void http_client_request_error(struct http_client_request *req,
+	unsigned int status, const char *error);
+void http_client_request_redirect(struct http_client_request *req,
+	const char *location);
+void http_client_request_finish(struct http_client_request **_req);
+
+struct connection_list *http_client_connection_list_init(void);
+
+struct http_client_connection *
+	http_client_connection_create(struct http_client_peer *peer);
+void http_client_connection_free(struct http_client_connection **_conn);
+bool http_client_connection_is_ready(struct http_client_connection *conn);
+bool http_client_connection_next_request(struct http_client_connection *conn);
+void http_client_connection_switch_ioloop(struct http_client_connection *conn);
+
+unsigned int http_client_peer_addr_hash
+	(const struct http_client_peer_addr *peer) ATTR_PURE;
+int http_client_peer_addr_cmp
+	(const struct http_client_peer_addr *peer1,
+		const struct http_client_peer_addr *peer2) ATTR_PURE;
+
+const char *
+	http_client_peer_get_hostname(struct http_client_peer *peer);
+struct http_client_peer *
+	http_client_peer_get(struct http_client *client,
+		const struct http_client_peer_addr *addr);
+void http_client_peer_free(struct http_client_peer **_peer);
+void http_client_peer_add_host(struct http_client_peer *peer,
+	struct http_client_host *host);
+struct http_client_request *
+	http_client_peer_claim_request(struct http_client_peer *peer,
+		bool no_urgent);
+void http_client_peer_handle_requests(struct http_client_peer *peer);
+void http_client_peer_connection_failure(struct http_client_peer *peer);
+void http_client_peer_connection_lost(struct http_client_peer *peer);
+
+struct http_client_host *
+	http_client_host_get(struct http_client *client, const char *hostname);
+void http_client_host_free(struct http_client_host **_host);
+void http_client_host_submit_request(struct http_client_host *host,
+	struct http_client_request *req);
+struct http_client_request *
+http_client_host_claim_request(struct http_client_host *host,
+	const struct http_client_peer_addr *addr, bool no_urgent);
+void http_client_host_connection_failure(struct http_client_host *host,
+	const struct http_client_peer_addr *addr);
+bool http_client_host_have_requests(struct http_client_host *host,
+	const struct http_client_peer_addr *addr, bool urgent);
+void http_client_host_drop_request(struct http_client_host *host,
+	struct http_client_request *req);
+void http_client_host_switch_ioloop(struct http_client_host *host);
+
+#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-client-request.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,386 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "http-url.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_request_debug(struct http_client_request *req,
+	const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_request_debug(struct http_client_request *req,
+	const char *format, ...)
+{
+	va_list args;
+
+	if (req->client->set.debug) {
+		va_start(args, format);	
+		i_debug("http-client: request %s: %s",
+			http_client_request_label(req), t_strdup_vprintf(format, args));
+		va_end(args);
+	}
+}
+
+/*
+ * Request
+ */
+
+#undef http_client_request
+struct http_client_request *
+http_client_request(struct http_client *client,
+		    const char *method, const char *host, const char *target,
+		    http_client_request_callback_t *callback, void *context)
+{
+	pool_t pool;
+	struct http_client_request *req;
+
+	pool = pool_alloconly_create("http client request", 2048);
+	req = p_new(pool, struct http_client_request, 1);
+	req->pool = pool;
+	req->refcount = 1;
+	req->client = client;
+	req->method = p_strdup(pool, method);
+	req->hostname = p_strdup(pool, host);
+	req->port = HTTP_DEFAULT_PORT;
+	req->target = p_strdup(pool, target);
+	req->callback = callback;
+	req->context = context;
+	req->headers = str_new(default_pool, 256);
+
+	req->state = HTTP_REQUEST_STATE_NEW;
+	return req;
+}
+
+void http_client_request_ref(struct http_client_request *req)
+{
+	req->refcount++;
+}
+
+void http_client_request_unref(struct http_client_request **_req)
+{
+	struct http_client_request *req = *_req;
+	struct http_client *client = req->client;
+
+	i_assert(req->refcount > 0);
+
+	if (--req->refcount > 0)
+		return;
+
+	/* only decrease pending request counter if this request was submitted */
+	if (req->state > HTTP_REQUEST_STATE_NEW)
+		req->client->pending_requests--;
+
+	http_client_request_debug(req, "Destroy (requests left=%d)",
+		client->pending_requests);
+
+	if (req->input != NULL)
+		i_stream_unref(&req->input);
+	str_free(&req->headers);
+	pool_unref(&req->pool);
+	*_req = NULL;
+}
+
+void http_client_request_set_port(struct http_client_request *req,
+	unsigned int port)
+{
+	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+	req->port = port;
+}
+
+void http_client_request_set_ssl(struct http_client_request *req,
+	bool ssl)
+{
+	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+	if (ssl) {
+		if (!req->ssl && req->port == HTTP_DEFAULT_PORT)
+			req->port = HTTPS_DEFAULT_PORT;
+	} else {
+		if (req->ssl && req->port == HTTPS_DEFAULT_PORT)
+			req->port = HTTP_DEFAULT_PORT;
+	}
+	req->ssl = ssl;
+}
+
+void http_client_request_set_urgent(struct http_client_request *req)
+{
+	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+	req->urgent = TRUE;
+}
+
+void http_client_request_add_header(struct http_client_request *req,
+				    const char *key, const char *value)
+{
+	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+	str_printfa(req->headers, "%s: %s\r\n", key, value);
+}
+
+void http_client_request_set_payload(struct http_client_request *req,
+				     struct istream *input, bool sync)
+{
+	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+	i_assert(req->input == NULL);
+
+	i_stream_ref(input);
+	req->input = input;
+	if (i_stream_get_size(input, TRUE, &req->input_size) <= 0)
+		i_unreached(); //FIXME	
+
+	/* prepare request payload sync using 100 Continue response from server */
+	if (req->input_size > 0 && sync) {
+		req->payload_sync = TRUE;
+	}
+}
+
+void http_client_request_submit(struct http_client_request *req)
+{
+	struct http_client_host *host;
+
+	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+	http_client_request_debug(req, "Submitted");
+	
+	host = http_client_host_get(req->client, req->hostname);
+	http_client_host_submit_request(host, req);
+	req->state = HTTP_REQUEST_STATE_QUEUED;
+	req->client->pending_requests++;
+}
+
+int http_client_request_send_more(struct http_client_request *req)
+{
+	struct http_client_connection *conn = req->conn;
+	struct ostream *output = conn->conn.output;
+	int ret = 0;
+
+	i_assert(req->input != NULL);
+
+	o_stream_set_max_buffer_size(output, 0);
+	if (o_stream_send_istream(output, req->input) < 0)
+		ret = -1;
+	o_stream_set_max_buffer_size(output, (size_t)-1);
+
+	if (req->input->eof) {
+		if (req->input->v_offset != req->input_size) {
+			i_error("stream input size changed"); //FIXME
+			return -1;
+		}
+		i_stream_unref(&req->input);
+		req->state = HTTP_REQUEST_STATE_WAITING;
+		conn->output_locked = FALSE;
+		http_client_request_debug(req, "Sent all payload");
+
+	} else {
+		conn->output_locked = TRUE;
+		o_stream_set_flush_pending(output, TRUE);
+		http_client_request_debug(req, "Partially sent payload");
+	}
+	return ret;
+}
+
+int http_client_request_send(struct http_client_request *req)
+{
+	struct http_client_connection *conn = req->conn;
+	struct ostream *output = conn->conn.output;
+	string_t *rtext = t_str_new(256);
+	struct const_iovec iov[3];
+	int ret = 0;
+
+	i_assert(!req->conn->output_locked);
+
+	str_append(rtext, req->method);
+	str_append(rtext, " ");
+	str_append(rtext, req->target);
+	str_append(rtext, " HTTP/1.1\r\n");
+	str_append(rtext, "Host: ");
+	str_append(rtext, req->hostname);
+	if ((!req->ssl &&req->port != HTTP_DEFAULT_PORT) ||
+		(req->ssl && req->port != HTTPS_DEFAULT_PORT)) {
+		str_printfa(rtext, ":%u", req->port);
+	}
+	str_append(rtext, "\r\n");
+	if (req->payload_sync) {
+		str_append(rtext, "Expect: 100-continue\r\n");
+	}
+	if (req->input_size != 0) {
+		str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n",
+			    req->input_size);
+	}
+
+	iov[0].iov_base = str_data(rtext);
+	iov[0].iov_len = str_len(rtext);
+	iov[1].iov_base = str_data(req->headers);
+	iov[1].iov_len = str_len(req->headers);
+	iov[2].iov_base = "\r\n";
+	iov[2].iov_len = 2;
+
+	req->state = HTTP_REQUEST_STATE_PAYLOAD_OUT;
+	o_stream_cork(output);
+	if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0)
+		ret = -1;
+
+	http_client_request_debug(req, "Sent");
+
+	if (ret >= 0 && req->input_size != 0) {
+		if (!req->payload_sync) {
+			if (http_client_request_send_more(req) < 0)
+				ret = -1;
+		} else {
+			http_client_request_debug(req, "Waiting for 100-continue");
+		}
+	} else {
+		req->state = HTTP_REQUEST_STATE_WAITING;
+		conn->output_locked = FALSE;
+	}
+	o_stream_uncork(output);
+	return ret;
+}
+
+void http_client_request_callback(struct http_client_request *req,
+			     struct http_response *response)
+{
+	http_client_request_callback_t *callback = req->callback;
+
+	req->state = HTTP_REQUEST_STATE_GOT_RESPONSE;
+
+	req->callback = NULL;
+	if (callback != NULL)
+		callback(response, req->context);
+}
+
+static void
+http_client_request_send_error(struct http_client_request *req,
+			       unsigned int status, const char *error)
+{
+	http_client_request_callback_t *callback;
+
+	req->state = HTTP_REQUEST_STATE_ABORTED;
+
+	callback = req->callback;
+	req->callback = NULL;
+	if (callback != NULL) {
+		struct http_response response;
+
+		memset(&response, 0, sizeof(response));
+		response.status = status;
+		response.reason = error;
+		(void)callback(&response, req->context);
+	}
+}
+
+void http_client_request_error(struct http_client_request *req,
+	unsigned int status, const char *error)
+{
+	http_client_request_send_error(req, status, error);
+	http_client_request_unref(&req);
+}
+
+void http_client_request_abort(struct http_client_request **_req)
+{
+	struct http_client_request *req = *_req;
+
+	if (req->state >= HTTP_REQUEST_STATE_FINISHED)
+		return;
+	req->callback = NULL;
+	req->state = HTTP_REQUEST_STATE_ABORTED;
+	if (req->host != NULL)
+		http_client_host_drop_request(req->host, req);
+	http_client_request_unref(_req);
+}
+
+void http_client_request_finish(struct http_client_request **_req)
+{
+	struct http_client_request *req = *_req;
+
+	if (req->state >= HTTP_REQUEST_STATE_FINISHED)
+		return;
+
+	http_client_request_debug(req, "Finished");
+
+	req->callback = NULL;
+	req->state = HTTP_REQUEST_STATE_FINISHED;
+	http_client_request_unref(_req);
+}
+
+void http_client_request_redirect(struct http_client_request *req,
+	const char *location)
+{
+	struct http_url *url;
+	const char *error;
+	unsigned int newport;
+
+	/* parse URL */
+	if (http_url_parse(location, NULL, 0, &url, &error) < 0) {
+		http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+			t_strdup_printf("Invalid redirect location: %s", error));
+		return;
+	}
+
+	if (++req->redirects > req->client->set.max_redirects) {
+		if (req->client->set.max_redirects > 0) {
+			http_client_request_error(req,
+				HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+				t_strdup_printf("Redirected more than %d times",
+					req->client->set.max_redirects));
+		} else {
+			http_client_request_error(req,
+				HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+					"Redirect refused");
+		}
+		return;
+	}
+
+	newport = (url->have_port ? url->port : (url->have_ssl ? 443 : 80));
+
+	http_client_request_debug(req, "Redirecting to http://%s:%u%s",
+		url->host_name, newport, url->path);
+
+	// FIXME: handle literal IP specially (avoid duplicate parsing)
+	req->host = NULL;
+	req->conn = NULL;
+	req->hostname = p_strdup(req->pool, url->host_name);
+	req->port = newport;
+	req->target = p_strdup(req->pool, url->path);
+	req->ssl = url->have_ssl;
+
+	/* resubmit */
+	req->client->pending_requests--;
+	req->state = HTTP_REQUEST_STATE_NEW;
+	http_client_request_submit(req);
+}
+
+void http_client_request_resubmit(struct http_client_request *req)
+{
+	http_client_request_debug(req, "Resubmitting request");
+
+	req->conn = NULL;
+	req->peer = NULL;
+	req->state = HTTP_REQUEST_STATE_QUEUED;
+	http_client_host_submit_request(req->host, req);
+}
+
+void http_client_request_retry(struct http_client_request *req,
+	unsigned int status, const char *error)
+{
+	/* limit the number of attempts for each request */
+	if (++req->attempts >= req->client->set.max_attempts) {
+		/* return error */
+		http_client_request_error(req, status, error);
+		return;
+	}
+
+	http_client_request_debug(req, "Retrying (attempts=%d)", req->attempts);
+
+	/* resubmit */
+	http_client_request_resubmit(req);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-client.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,182 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "connection.h"
+#include "dns-lookup.h"
+#include "iostream-rawlog.h"
+#include "iostream-ssl.h"
+#include "http-url.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+#define HTTP_DEFAULT_PORT 80
+#define HTTPS_DEFAULT_PORT 443
+
+/* FIXME: This implementation not yet finished. The essence works: it is
+   possible to submit requests through the client. Responses are dumped to
+   stdout
+
+    Structure so far:
+    Client - Acts much like a browser; it is not dedicated to a single host.
+             Client can accept requests to different hosts, which can be served
+             at different IPs. Redirects can be handled in the background by
+             making a new connection. Connections to new hosts are created once
+             needed for servicing a request.
+    Requests - Semantics are similar to imapc commands. Create a request, 
+               optionally modify some aspects of it and finally submit it.
+    Hosts - We maintain a 'cache' of hosts for which we have looked up IPs.
+            Requests are first queued in the host struct on a per-port basis.
+    Peers - Group connections to the same ip/port (== peer_addr).
+    Connections - Actual connections to a server. Once a connection is ready to
+                  handle requests, it claims a request from a host object. One
+                  connection hand service multiple hosts and one host can have
+                  multiple associated connections, possibly to different ips and
+                  ports.
+
+  TODO: lots of cleanup, authentication, ssl, timeouts,	 rawlog etc.
+ */
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_debug(struct http_client *client,
+	const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_debug(struct http_client *client,
+	const char *format, ...)
+{
+	va_list args;
+
+	va_start(args, format);	
+	if (client->set.debug)
+		i_debug("http-client: %s", t_strdup_vprintf(format, args));
+	va_end(args);
+}
+
+/*
+ * Client
+ */
+
+struct http_client *http_client_init(const struct http_client_settings *set)
+{
+	struct http_client *client;
+	pool_t pool;
+
+	pool = pool_alloconly_create("http client", 1024);
+	client = p_new(pool, struct http_client, 1);
+	client->pool = pool;
+	client->set.dns_client_socket_path =
+		p_strdup(pool, set->dns_client_socket_path);
+	client->set.rawlog_dir = p_strdup(pool, set->rawlog_dir);
+	client->set.ssl_ca_dir = p_strdup(pool, set->ssl_ca_dir);
+	client->set.max_idle_time_msecs = set->max_idle_time_msecs;
+	client->set.max_parallel_connections =
+		(set->max_parallel_connections > 0 ? set->max_parallel_connections : 1);
+	client->set.max_pipelined_requests =
+		(set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
+	client->set.max_attempts = set->max_attempts;
+	client->set.max_redirects = set->max_redirects;
+	client->set.debug = set->debug;
+
+	client->conn_list = http_client_connection_list_init();
+
+	hash_table_create(&client->hosts, default_pool, 0, str_hash, strcmp);
+	hash_table_create(&client->peers, default_pool, 0,
+		http_client_peer_addr_hash, http_client_peer_addr_cmp);
+
+	return client;
+}
+
+void http_client_deinit(struct http_client **_client)
+{
+	struct http_client *client = *_client;
+	struct hash_iterate_context *iter;
+	const char *hostname;
+	struct http_client_host *host;
+	const struct http_client_peer_addr *addr;
+	struct http_client_peer *peer;
+
+	/* free peers */
+	iter = hash_table_iterate_init(client->peers);
+	while (hash_table_iterate(iter, client->peers, &addr, &peer)) {
+		http_client_peer_free(&peer);
+	}
+	hash_table_iterate_deinit(&iter);
+	hash_table_destroy(&client->peers);
+
+	/* free hosts */
+	iter = hash_table_iterate_init(client->hosts);
+	while (hash_table_iterate(iter, client->hosts, &hostname, &host)) {
+		http_client_host_free(&host);
+	}
+	hash_table_iterate_deinit(&iter);
+	hash_table_destroy(&client->hosts);
+
+	connection_list_deinit(&client->conn_list);
+	pool_unref(&client->pool);
+	*_client = NULL;
+}
+
+void http_client_switch_ioloop(struct http_client *client)
+{
+	struct connection *_conn = client->conn_list->connections;
+
+	/* move connections */
+	/* FIXME: we wouldn't necessarily need to switch all of them
+	   immediately, only those that have requests now. but also connections
+	   that get new requests before ioloop is switched again.. */
+	for (; _conn != NULL; _conn = _conn->next) {
+		struct http_client_connection *conn =
+			(struct http_client_connection *)_conn;
+
+		http_client_connection_switch_ioloop(conn);
+	}
+
+	/* move dns lookups */
+	if (client->set.dns_client_socket_path != '\0') {
+		struct hash_iterate_context *iter;
+		struct http_client_host *host;
+		const char *hostname;
+	
+		iter = hash_table_iterate_init(client->hosts);
+		while (hash_table_iterate(iter, client->hosts, &hostname, &host)) {
+			http_client_host_switch_ioloop(host);
+		}
+		hash_table_iterate_deinit(&iter);
+	}
+}
+
+void http_client_wait(struct http_client *client)
+{
+	struct ioloop *prev_ioloop = current_ioloop;
+
+	i_assert(client->ioloop == NULL);
+
+	client->ioloop = io_loop_create();
+	http_client_switch_ioloop(client);
+
+	do {
+		http_client_debug(client,
+			"Waiting for %d requests to finish", client->pending_requests);
+		io_loop_run(client->ioloop);
+	} while (client->pending_requests > 0);
+
+	http_client_debug(client, "All requests finished");
+
+	current_ioloop = prev_ioloop;
+	http_client_switch_ioloop(client);
+	current_ioloop = client->ioloop;
+	io_loop_destroy(&client->ioloop);
+}
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-client.h	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,83 @@
+#ifndef HTTP_CLIENT_H
+#define HTTP_CLIENT_H
+
+struct http_response;
+
+struct http_client;
+struct http_client_request;
+
+enum http_client_request_error {
+	HTTP_CLIENT_REQUEST_ERROR_ABORTED = 9000,
+	HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED,
+	HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED,
+	HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+	HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
+	HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE,
+	HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT,
+};
+
+struct http_client_settings {
+	const char *dns_client_socket_path;
+
+	const char *ssl_ca_dir;
+	const char *ssl_crypto_device;
+	bool ssl_allow_invalid_cert;
+	bool ssl_verify;
+
+	const char *rawlog_dir;
+
+	unsigned int max_idle_time_msecs;
+
+	/* maximum number of parallel connections per peer (default = 1) */
+	unsigned int max_parallel_connections;
+
+	/* maximum number of pipelined requests per connection (default = 1) */
+	unsigned int max_pipelined_requests;
+
+	/* maximum number of redirects for a request
+	   (default = 0; redirects refused) 
+   */
+	unsigned int max_redirects;
+
+	/* maximum number of attempts for a request */
+	unsigned int max_attempts;
+
+	bool debug;
+};
+
+typedef void
+http_client_request_callback_t(const struct http_response *response,
+			       void *context);
+
+struct http_client *http_client_init(const struct http_client_settings *set);
+void http_client_deinit(struct http_client **_client);
+
+struct http_client_request *
+http_client_request(struct http_client *client,
+		    const char *method, const char *host, const char *target,
+		    http_client_request_callback_t *callback, void *context);
+#define http_client_request(client, method, host, target, callback, context) \
+	http_client_request(client, method, host, target + \
+		CALLBACK_TYPECHECK(callback, void (*)( \
+			const struct http_response *response, typeof(context))), \
+		(http_client_request_callback_t *)callback, context)
+
+void http_client_request_set_port(struct http_client_request *req,
+	unsigned int port);
+void http_client_request_set_ssl(struct http_client_request *req,
+	bool ssl);
+void http_client_request_set_urgent(struct http_client_request *req);
+
+void http_client_request_add_header(struct http_client_request *req,
+				    const char *key, const char *value);
+void http_client_request_set_payload(struct http_client_request *req,
+				     struct istream *input, bool sync);
+void http_client_request_submit(struct http_client_request *req);
+void http_client_request_abort(struct http_client_request **req);
+
+void http_client_switch_ioloop(struct http_client *client);
+
+/* blocks until all currently submitted requests are handled */
+void http_client_wait(struct http_client *client);
+
+#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-header-parser.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,282 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "istream.h"
+#include "str.h"
+#include "str-sanitize.h"
+#include "http-parser.h"
+
+#include "http-header-parser.h"
+
+enum http_header_parse_state {
+	HTTP_HEADER_PARSE_STATE_INIT = 0,
+	HTTP_HEADER_PARSE_STATE_NAME,
+	HTTP_HEADER_PARSE_STATE_COLON,
+	HTTP_HEADER_PARSE_STATE_OWS,
+	HTTP_HEADER_PARSE_STATE_CONTENT,
+	HTTP_HEADER_PARSE_STATE_CR,
+	HTTP_HEADER_PARSE_STATE_LF,
+	HTTP_HEADER_PARSE_STATE_NEW_LINE,
+	HTTP_HEADER_PARSE_STATE_LAST_LINE,
+	HTTP_HEADER_PARSE_STATE_EOH
+};
+
+struct http_header_parser {
+	struct istream *input;
+
+	const unsigned char *begin, *cur, *end;
+	const char *error;
+
+	string_t *name;
+	buffer_t *value_buf;
+
+	enum http_header_parse_state state;
+};
+
+// FIXME(Stephan): Add support for limiting maximum header size.
+
+struct http_header_parser *http_header_parser_init(struct istream *input)
+{
+	struct http_header_parser *parser;
+
+	parser = i_new(struct http_header_parser, 1);
+	parser->input = input;
+	parser->name = str_new(default_pool, 128);
+	parser->value_buf = buffer_create_dynamic(default_pool, 4096);
+
+	return parser;
+}
+
+void http_header_parser_deinit(struct http_header_parser **_parser)
+{
+	struct http_header_parser *parser = *_parser;
+
+	//i_stream_skip(ctx->input, ctx->skip);
+	buffer_free(&parser->value_buf);
+	str_free(&parser->name);
+	i_free(parser);
+	*_parser = NULL;
+}
+
+void http_header_parser_reset(struct http_header_parser *parser)
+{
+	parser->state = HTTP_HEADER_PARSE_STATE_INIT;
+}
+
+static int http_header_parse_name(struct http_header_parser *parser)
+{
+	const unsigned char *first = parser->cur;
+
+	/* field-name     = token
+	   token          = 1*tchar
+	 */
+	while (parser->cur < parser->end && http_char_is_token(*parser->cur))
+		parser->cur++;
+
+	str_append_n(parser->name, first, parser->cur-first);
+
+	if (parser->cur == parser->end)
+		return 0;
+	if (str_len(parser->name) == 0) {
+		parser->error = "Empty header field name";
+		return -1;
+	}
+	return 1;
+}
+
+static int http_header_parse_ows(struct http_header_parser *parser)
+{
+	/* OWS            = *( SP / HTAB )
+	                  ; "optional" whitespace
+	 */
+	while (parser->cur < parser->end &&
+		(*parser->cur == ' ' || *parser->cur == '\t'))
+		parser->cur++;
+	return (parser->cur == parser->end ? 0 : 1);
+}
+
+static int http_header_parse_content(struct http_header_parser *parser)
+{
+	const unsigned char *first = parser->cur;
+
+	/* field-content  = *( HTAB / SP / VCHAR / obs-text )
+	 */
+	while (parser->cur < parser->end && http_char_is_text(*parser->cur))
+		parser->cur++;
+
+	buffer_append(parser->value_buf, first, parser->cur-first);
+
+	if (parser->cur == parser->end)
+		return 0;
+	return 1;
+}
+
+static inline const char *_chr_sanitize(unsigned char c)
+{
+	if (c >= 0x20 && c < 0x7F)
+		return t_strdup_printf("'%c'", c);
+	return t_strdup_printf("0x%02x", c);
+}
+
+static int http_header_parse(struct http_header_parser *parser)
+{
+	int ret;
+
+	/* 'header'       = *( header-field CRLF ) CRLF
+	   header-field   = field-name ":" OWS field-value BWS
+	   field-name     = token
+	   field-value    = *( field-content / obs-fold )
+	   field-content  = *( HTAB / SP / VCHAR / obs-text )
+	   obs-fold       = CRLF ( SP / HTAB )
+	                  ; obsolete line folding
+	                  ; see Section 3.2.2
+	 */
+
+	for (;;) {
+		switch (parser->state) {
+		case HTTP_HEADER_PARSE_STATE_INIT:
+			buffer_set_used_size(parser->value_buf, 0);
+			str_truncate(parser->name, 0);
+			parser->state = HTTP_HEADER_PARSE_STATE_NAME;
+			/* fall through */
+		case HTTP_HEADER_PARSE_STATE_NAME:
+			if (http_char_is_token(*parser->cur)) {
+				if ((ret=http_header_parse_name(parser)) <= 0)
+					return ret;
+			} else if (str_len(parser->name) == 0) {
+				parser->state = HTTP_HEADER_PARSE_STATE_LAST_LINE;
+				break;
+			}
+			parser->state = HTTP_HEADER_PARSE_STATE_COLON;
+			/* fall through */
+		case HTTP_HEADER_PARSE_STATE_COLON:
+			if (*parser->cur != ':') {
+				parser->error = t_strdup_printf
+					("Expected ':' after header field name '%s', but found %s",
+						str_sanitize(str_c(parser->name),64),
+						_chr_sanitize(*parser->cur));
+				return -1;
+			}
+			parser->cur++;
+			if (str_len(parser->name) == 0) {
+				parser->error = "Empty header field name";
+				return -1;
+			}
+			parser->state = HTTP_HEADER_PARSE_STATE_OWS;
+			/* fall through */
+		case HTTP_HEADER_PARSE_STATE_OWS:
+			if ((ret=http_header_parse_ows(parser)) <= 0)
+				return ret;
+			parser->state = HTTP_HEADER_PARSE_STATE_CONTENT;
+			/* fall through */
+		case HTTP_HEADER_PARSE_STATE_CONTENT:
+			if ((ret=http_header_parse_content(parser)) <= 0)
+				return ret;
+			parser->state = HTTP_HEADER_PARSE_STATE_CR;
+			/* fall through */
+		case HTTP_HEADER_PARSE_STATE_CR:
+			if (*parser->cur == '\r') {
+				parser->cur++;
+			}
+			parser->state = HTTP_HEADER_PARSE_STATE_LF;
+			if (parser->cur == parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_HEADER_PARSE_STATE_LF:
+			if (*parser->cur != '\n') {
+				parser->error = t_strdup_printf
+					("Expected line end after header field '%s', but found %s",
+						str_sanitize(str_c(parser->name),64),
+						_chr_sanitize(*parser->cur));
+				return -1;
+			}
+			parser->cur++;
+			parser->state = HTTP_HEADER_PARSE_STATE_NEW_LINE;
+			if (parser->cur == parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_HEADER_PARSE_STATE_NEW_LINE:
+			if (*parser->cur == ' ' || *parser->cur == '\t') {
+				/* obs-fold */
+				buffer_append_c(parser->value_buf, ' ');
+				parser->state = HTTP_HEADER_PARSE_STATE_OWS;
+				break;
+			}
+			parser->state = HTTP_HEADER_PARSE_STATE_NAME;
+			return 1;
+		case HTTP_HEADER_PARSE_STATE_LAST_LINE:
+			if (*parser->cur == '\r') {
+				/* last CRLF */
+				parser->cur++;
+				parser->state = HTTP_HEADER_PARSE_STATE_EOH;
+				if (parser->cur == parser->end)
+					return 0;
+				break;
+			} else if (*parser->cur == '\n') {
+				/* header fully parsed */
+				parser->cur++;
+				parser->state = HTTP_HEADER_PARSE_STATE_EOH;
+				return 1;
+			}
+			parser->error = t_strdup_printf
+				("Expected CRLF or header field name, but found %s",
+					_chr_sanitize(*parser->cur));
+			return -1;
+		case HTTP_HEADER_PARSE_STATE_EOH:
+			if (*parser->cur != '\n') {
+				parser->error = t_strdup_printf
+					("Expected LF after CR at end of header, but found %s",
+						_chr_sanitize(*parser->cur));
+				return -1;
+			}
+			/* header fully parsed */
+			parser->cur++;
+			return 1;
+
+		default:
+			i_unreached();
+		}
+	}
+
+	i_unreached();
+	return -1;
+}
+
+int http_header_parse_next_field(struct http_header_parser *parser,
+	const char **name_r, const unsigned char **data_r, size_t *size_r,
+	const char **error_r)
+{
+	size_t size;
+	int ret;
+
+	while ((ret=i_stream_read_data
+		(parser->input, &parser->begin, &size, 0)) > 0) {
+		parser->cur = parser->begin;
+		parser->end = parser->cur + size;
+
+		if ((ret=http_header_parse(parser)) < 0) {
+			*error_r = parser->error;
+			return -1;
+		}
+
+		i_stream_skip(parser->input, parser->cur - parser->begin);
+
+		if (ret == 1) {
+			if (parser->state != HTTP_HEADER_PARSE_STATE_EOH) {
+				*name_r = str_c(parser->name);
+				*data_r = buffer_get_data(parser->value_buf, size_r);
+				parser->state = HTTP_HEADER_PARSE_STATE_INIT;
+			} else {
+				*name_r = NULL;
+				*data_r = NULL;
+			}
+			return 1;
+		}
+	}
+
+	i_assert(ret != -2);
+	if (ret < 0)
+		*error_r = "Stream error";
+	return ret;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-header-parser.h	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,15 @@
+#ifndef HTTP_HEADER_PARSER_H
+#define HTTP_HEADER_PARSER_H
+
+struct http_header_parser;
+
+struct http_header_parser *http_header_parser_init(struct istream *input);
+void http_header_parser_deinit(struct http_header_parser **_parser);
+
+void http_header_parser_reset(struct http_header_parser *parser);
+
+int http_header_parse_next_field(struct http_header_parser *parser,
+	const char **name_r, const unsigned char **data_r, size_t *size_r,
+	const char **error_r);
+
+#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-parser.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,132 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "strescape.h"
+#include "http-url.h"
+
+#include "http-parser.h"
+
+/*
+ Character definitions:
+
+ tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+                / DIGIT / ALPHA
+                ; any VCHAR, except special
+ special        = "(" / ")" / "<" / ">" / "@" / ","
+                / ";" / ":" / "\" / DQUOTE / "/" / "["
+                / "]" / "?" / "=" / "{" / "}"
+ qdtext         = OWS / %x21 / %x23-5B / %x5D-7E / obs-text
+ qdtext-nf      = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text
+ ctext          = OWS / %x21-27 / %x2A-5B / %x5D-7E / obs-text 
+ obs-text       = %x80-FF
+ OWS            = *( SP / HTAB )
+
+ Mapping
+ (1<<0) => tchar
+ (1<<1) => special
+ (1<<2) => %x21 / %x2A-5B / %x5D-7E
+ (1<<3) => %x23-29
+ (1<<4) => %x22-27
+ (1<<5) => HTAB / SP / obs-text
+ */
+
+const unsigned char _http_token_char_mask  = (1<<0);
+const unsigned char _http_value_char_mask  = (1<<0)|(1<<1);
+const unsigned char _http_text_char_mask   = (1<<0)|(1<<1)|(1<<5);
+const unsigned char _http_qdtext_char_mask = (1<<2)|(1<<3)|(1<<5);
+const unsigned char _http_ctext_char_mask  = (1<<2)|(1<<4)|(1<<5);
+
+const unsigned char _http_char_lookup[256] = {
+	 0,  0,  0,  0,  0,  0,  0,  0,  0, 32,  0,  0,  0,  0,  0,  0, // 00
+	 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, // 10
+	32, 21, 18, 25, 25, 25, 25, 25, 10, 10,  5,  5,  6,  5,  5,  6, // 20
+	 5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  6,  6,  6,  6,  6,  6, // 30
+	 6,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5, // 40
+	 5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  6,  2,  6,  5,  5, // 50
+	 5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5, // 60
+	 5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  6,  5,  6,  5,  0, // 70
+
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // 80
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // 90
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // A0
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // B0
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // C0
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // D0
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // E0
+	32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // F0
+};
+
+/*
+ * HTTP value parsing
+ */
+
+void http_parser_init(struct http_parser *parser,
+			const unsigned char *data, size_t size)
+{
+	memset(parser, 0, sizeof(*parser));
+	parser->begin = data;
+	parser->cur = data;
+	parser->end = data + size;
+}
+
+void http_parse_ows(struct http_parser *parser)
+{
+	/* OWS            = *( SP / HTAB ) */
+	if (parser->cur >= parser->end)
+		return;
+	while (parser->cur < parser->end &&
+		(parser->cur[0] == ' ' || parser->cur[0] == '\t')) {
+		parser->cur++;
+	}
+}
+
+int http_parse_token(struct http_parser *parser, const char **token_r)
+{
+	const unsigned char *first;
+
+	/* token          = 1*tchar */
+
+	if (parser->cur >= parser->end || !http_char_is_token(*parser->cur))
+		return 0;
+
+	first = parser->cur++;
+	while (parser->cur < parser->end && http_char_is_token(*parser->cur))
+		parser->cur++;
+
+	*token_r = t_strndup(first, parser->cur - first);
+	return 1;
+}
+
+int http_parse_token_list_next(struct http_parser *parser,
+	const char **token_r)
+{
+	/* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21;
+	     Appendix B:
+
+	   For compatibility with legacy list rules, recipients SHOULD accept
+	   empty list elements.  In other words, consumers would follow the list
+	   productions:
+
+	   #element => [ ( "," / element ) *( OWS "," [ OWS element ] ) ]
+	   1#element => *( "," OWS ) element *( OWS "," [ OWS element ] )
+	*/
+
+	for (;;) {	
+		if (http_parse_token(parser, token_r) > 0)
+			break;
+		http_parse_ows(parser);
+		if (parser->cur >= parser->end || parser->cur[0] != ',')
+			return 0;
+		parser->cur++;
+		http_parse_ows(parser);
+	}
+
+	return 1;
+}
+
+
+
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-parser.h	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,53 @@
+#ifndef HTTP_PARSER_H
+#define HTTP_PARSER_H
+
+/*
+ * Character definitions
+ */
+
+extern const unsigned char _http_token_char_mask;
+extern const unsigned char _http_value_char_mask;
+extern const unsigned char _http_text_char_mask;
+extern const unsigned char _http_qdtext_char_mask;
+extern const unsigned char _http_ctext_char_mask;
+
+extern const unsigned char _http_char_lookup[256];
+
+static inline bool http_char_is_token(unsigned char ch) {
+	return (_http_char_lookup[ch] & _http_token_char_mask) != 0;
+}
+
+static inline bool http_char_is_value(unsigned char ch) {
+	return (_http_char_lookup[ch] & _http_value_char_mask) != 0;
+}
+
+static inline bool http_char_is_text(unsigned char ch) {
+	return (_http_char_lookup[ch] & _http_text_char_mask) != 0;
+}
+
+static inline bool http_char_is_qdtext(unsigned char ch) {
+	return (_http_char_lookup[ch] & _http_qdtext_char_mask) != 0;
+}
+
+static inline bool http_char_is_ctext(unsigned char ch) {
+	return (_http_char_lookup[ch] & _http_ctext_char_mask) != 0;
+}
+
+/*
+ * HTTP value parsing
+ */
+
+struct http_parser {
+	const unsigned char *begin, *cur, *end;
+};
+
+void http_parser_init(struct http_parser *parser,
+			const unsigned char *data, size_t size);
+
+void http_parse_ows(struct http_parser *parser);
+
+int http_parse_token(struct http_parser *parser, const char **token_r);
+int http_parse_token_list_next(struct http_parser *parser,
+	const char **token_r);
+
+#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-response-parser.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,503 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "strfuncs.h"
+#include "istream.h"
+#include "http-parser.h"
+#include "http-date.h"
+#include "http-header-parser.h"
+#include "http-transfer.h"
+
+#include "http-response-parser.h"
+
+#include <ctype.h>
+
+enum http_response_parser_state {
+	HTTP_RESPONSE_PARSE_STATE_INIT = 0,
+	HTTP_RESPONSE_PARSE_STATE_VERSION,
+	HTTP_RESPONSE_PARSE_STATE_SP1,
+	HTTP_RESPONSE_PARSE_STATE_STATUS,
+	HTTP_RESPONSE_PARSE_STATE_SP2,
+	HTTP_RESPONSE_PARSE_STATE_REASON,
+	HTTP_RESPONSE_PARSE_STATE_CR,
+	HTTP_RESPONSE_PARSE_STATE_LF,
+	HTTP_RESPONSE_PARSE_STATE_HEADER
+};
+
+struct http_response_parser {
+	struct istream *input;
+
+	const unsigned char *begin, *cur, *end;
+	const char *error;
+
+	string_t *strbuf;
+
+	enum http_response_parser_state state;
+	struct http_header_parser *header_parser;
+
+	uoff_t content_length;
+	const char *transfer_encoding;
+	struct istream *payload;
+
+	struct http_response *response;
+	pool_t response_pool;
+};
+
+struct http_response_parser *http_response_parser_init(struct istream *input)
+{
+	struct http_response_parser *parser;
+
+	parser = i_new(struct http_response_parser, 1);
+	parser->input = input;
+	parser->strbuf = str_new(default_pool, 128);
+	return parser;
+}
+
+void http_response_parser_deinit(struct http_response_parser **_parser)
+{
+	struct http_response_parser *parser = *_parser;
+
+	str_free(&parser->strbuf);
+	if (parser->header_parser != NULL)
+		http_header_parser_deinit(&parser->header_parser);
+	if (parser->response_pool != NULL)
+		pool_unref(&parser->response_pool);
+	if (parser->payload != NULL)
+		i_stream_unref(&parser->payload);
+	i_free(parser);
+}
+
+static void
+http_response_parser_restart(struct http_response_parser *parser)
+{
+	i_assert(parser->payload == NULL);
+	parser->content_length = 0;
+	parser->transfer_encoding = NULL;
+	str_truncate(parser->strbuf, 0);
+	if (parser->response_pool != NULL)
+		pool_unref(&parser->response_pool);
+	parser->response_pool = pool_alloconly_create("http_response", 2048);
+	parser->response = p_new(parser->response_pool, struct http_response, 1);
+	parser->response->date = (time_t)-1;
+	p_array_init(&parser->response->headers, parser->response_pool, 32);
+}
+
+static int http_response_parse_version(struct http_response_parser *parser)
+{
+	const unsigned char *first = parser->cur;
+	const char *p;
+
+	/* HTTP-version  = HTTP-name "/" DIGIT "." DIGIT
+	   HTTP-name     = %x48.54.54.50 ; "HTTP", case-sensitive
+	 */
+	while (parser->cur < parser->end && http_char_is_value(*parser->cur))
+		parser->cur++;
+
+	if (str_len(parser->strbuf) + (parser->cur-first) > 8)
+		return -1;
+
+	str_append_n(parser->strbuf, first, parser->cur-first);
+	if (parser->cur == parser->end)
+		return 0;
+
+	if (str_len(parser->strbuf) != 8)
+		return -1;
+	if (strncmp(str_c(parser->strbuf), "HTTP/",5) != 0)
+		return -1;
+	p = str_c(parser->strbuf) + 5;
+	if (!i_isdigit(*p))
+		return -1;
+	parser->response->version_major = *p - '0';
+	p++;
+	if (*(p++) != '.')
+		return -1;
+	if (!i_isdigit(*p))
+		return -1;
+	parser->response->version_minor = *p - '0';
+	return 1;
+}
+
+static int http_response_parse_status(struct http_response_parser *parser)
+{
+	const unsigned char *first = parser->cur;
+	const char *p;
+
+	/* status-code   = 3DIGIT
+	 */
+	while (parser->cur < parser->end && i_isdigit(*parser->cur))
+		parser->cur++;
+
+	if (str_len(parser->strbuf) + (parser->cur-first) > 3)
+		return -1;
+
+	str_append_n(parser->strbuf, first, parser->cur-first);
+	if (parser->cur == parser->end)
+		return 0;
+	if (str_len(parser->strbuf) != 3)
+		return -1;
+	p = str_c(parser->strbuf);
+	parser->response->status =
+		(p[0] - '0')*100 + (p[1] - '0')*10 + (p[2] - '0');
+	return 1;
+}
+
+static int http_response_parse_reason(struct http_response_parser *parser)
+{
+	const unsigned char *first = parser->cur;
+
+	/* reason-phrase = *( HTAB / SP / VCHAR / obs-text )
+	 */
+	while (parser->cur < parser->end && http_char_is_text(*parser->cur))
+		parser->cur++;
+
+	str_append_n(parser->strbuf, first, parser->cur-first);
+
+	if (parser->cur == parser->end)
+		return 0;
+	parser->response->reason =
+		p_strdup(parser->response_pool, str_c(parser->strbuf));
+	return 1;
+}
+
+static inline const char *_chr_sanitize(unsigned char c)
+{
+	if (c >= 0x20 && c < 0x7F)
+		return t_strdup_printf("'%c'", c);
+	return t_strdup_printf("0x%02x", c);
+}
+
+static int http_response_parse(struct http_response_parser *parser)
+{
+	int ret;
+
+	/* status-line   = HTTP-version SP status-code SP reason-phrase CRLF
+	   status-code   = 3DIGIT
+	   reason-phrase = *( HTAB / SP / VCHAR / obs-text )
+	 */
+
+	for (;;) {
+		switch (parser->state) {
+		case HTTP_RESPONSE_PARSE_STATE_INIT:
+			http_response_parser_restart(parser);
+			parser->state = HTTP_RESPONSE_PARSE_STATE_VERSION;
+			/* fall through */
+		case HTTP_RESPONSE_PARSE_STATE_VERSION:
+			if ((ret=http_response_parse_version(parser)) <= 0) {
+				if (ret < 0)
+					parser->error = "Invalid HTTP version in response";
+				return ret;
+			}
+			str_truncate(parser->strbuf, 0);
+			parser->state = HTTP_RESPONSE_PARSE_STATE_SP1;
+			if (parser->cur == parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_RESPONSE_PARSE_STATE_SP1:
+			if (*parser->cur != ' ') {
+				parser->error = t_strdup_printf
+					("Expected ' ' after response version, but found %s",
+						_chr_sanitize(*parser->cur));
+				return -1;
+			}
+			parser->cur++;
+			str_truncate(parser->strbuf, 0);
+			parser->state = HTTP_RESPONSE_PARSE_STATE_STATUS;
+			if (parser->cur >= parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_RESPONSE_PARSE_STATE_STATUS:
+			if ((ret=http_response_parse_status(parser)) <= 0) {
+				if (ret < 0)
+					parser->error = "Invalid HTTP status code in response";
+				return ret;
+			}
+			str_truncate(parser->strbuf, 0);
+			parser->state = HTTP_RESPONSE_PARSE_STATE_SP2;
+			if (parser->cur == parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_RESPONSE_PARSE_STATE_SP2:
+			if (*parser->cur != ' ') {
+				parser->error = t_strdup_printf
+					("Expected ' ' after response status code, but found %s",
+						_chr_sanitize(*parser->cur));
+				return -1;
+			}
+			parser->cur++;
+			str_truncate(parser->strbuf, 0);
+			parser->state = HTTP_RESPONSE_PARSE_STATE_REASON;
+			if (parser->cur >= parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_RESPONSE_PARSE_STATE_REASON:
+			if ((ret=http_response_parse_reason(parser)) <= 0)
+				return ret;
+			str_truncate(parser->strbuf, 0);
+			parser->state = HTTP_RESPONSE_PARSE_STATE_CR;
+			if (parser->cur == parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_RESPONSE_PARSE_STATE_CR:
+			if (*parser->cur == '\r')
+				parser->cur++;
+			parser->state = HTTP_RESPONSE_PARSE_STATE_LF;
+			if (parser->cur == parser->end)
+				return 0;
+			/* fall through */
+		case HTTP_RESPONSE_PARSE_STATE_LF:
+			if (*parser->cur != '\n') {
+				parser->error = t_strdup_printf
+					("Expected line end after response, but found %s",
+						_chr_sanitize(*parser->cur));
+				return -1;
+			}
+			parser->cur++;
+			parser->state = HTTP_RESPONSE_PARSE_STATE_HEADER;
+			return 1;
+		case HTTP_RESPONSE_PARSE_STATE_HEADER:
+		default:
+			i_unreached();
+		}
+	}
+
+	i_unreached();
+	return -1;
+}
+
+static int http_response_parse_status_line(struct http_response_parser *parser)
+{
+	size_t size;
+	int ret;
+
+	while ((ret = i_stream_read_data(parser->input,
+					 &parser->begin, &size, 0)) > 0) {
+		parser->cur = parser->begin;
+		parser->end = parser->cur + size;
+
+		if ((ret = http_response_parse(parser)) < 0)
+			return -1;
+
+		i_stream_skip(parser->input, parser->cur - parser->begin);
+		if (ret > 0)
+			return 1;
+	}
+
+	i_assert(ret != -2);
+	if (ret < 0) {
+		if (parser->input->eof && parser->state == HTTP_RESPONSE_PARSE_STATE_INIT)
+			return 0;
+		parser->error = "Stream error";
+		return -1;
+	}
+	return 0;
+}
+
+static int
+http_response_parse_header(struct http_response_parser *parser,
+			   const char *name, const unsigned char *data, size_t size)
+{
+	struct http_response_header *hdr;
+	struct http_parser hparser;
+
+	hdr = array_append_space(&parser->response->headers);
+	hdr->key = p_strdup(parser->response_pool, name);
+	hdr->value = p_strndup(parser->response_pool, data, size);
+	hdr->size = size;
+
+	switch (name[0]) {
+	case 'C': case 'c':
+		if (strcasecmp(name, "Connection") == 0) {
+			const char *option;
+
+			/* Connection        = 1#connection-option
+				 connection-option = token
+			*/
+			http_parser_init(&hparser, data, size);
+			for (;;) {
+				if (http_parse_token_list_next(&hparser, &option) <= 0)
+					break;
+				if (strcasecmp(option, "close") == 0) {
+					parser->response->connection_close = TRUE;
+					break; // not interested in any other options
+				}
+			}
+			return 1;
+		}
+		if (strcasecmp(name, "Content-Length") == 0) {
+			/* Content-Length = 1*DIGIT */
+			if (str_to_uoff(hdr->value, &parser->content_length) < 0) {
+				parser->error = "Invalid Content-Length header";
+				return -1;
+			}
+			return 1;
+		}
+		break;
+	case 'D': case 'd':
+		if (strcasecmp(name, "Date") == 0) {
+			/* Date = HTTP-date */
+			(void)http_date_parse(data, size, &parser->response->date);
+			return 1;
+		}
+		break;
+	case 'L': case 'l':
+		if (strcasecmp(name, "Location") == 0) {
+			/* Location = URI-reference (not parsed here) */
+			parser->response->location =
+				p_strndup(parser->response_pool, data, size);
+			return 1;
+		}
+		break;
+	case 'T': case 't':
+		if (strcasecmp(name, "Transfer-Encoding") == 0) {
+			/* Transfer-Encoding = 1#transfer-coding */
+			parser->transfer_encoding = hdr->value;
+			return 1;
+		}
+		break;
+	default:
+		break;
+	}
+	return 1;
+}
+
+int http_response_parse_next(struct http_response_parser *parser,
+			     bool no_payload, struct http_response **response_r,
+			     const char **error_r)
+{
+	struct http_parser hparser;
+	const char *field_name, *error;
+	const unsigned char *field_data;
+	size_t field_size;
+	int ret;
+
+	/* make sure we finished streaming payload from previous response
+	   before we continue. */
+	if (parser->payload != NULL) {
+		struct istream *payload = parser->payload;
+
+		i_assert(parser->state == HTTP_RESPONSE_PARSE_STATE_INIT);
+
+		if (!payload->eof) {
+			while ((ret=i_stream_read(payload)) > 0) {
+				i_stream_skip(payload, i_stream_get_data_size(payload));
+			}
+			if (ret == 0)
+				return 0;
+			if (ret < 0 && !payload->eof) {
+				*error_r = "Stream error while skipping payload";
+				return -1;
+			}
+		}
+
+		if (payload->eof)	{
+			i_stream_unref(&parser->payload);
+			parser->payload = NULL;
+		}
+	}
+
+	/* HTTP-message   = start-line
+	                   *( header-field CRLF )
+	                    CRLF
+	                    [ message-body ]
+	 */
+
+	/* start-line */
+	if (parser->state != HTTP_RESPONSE_PARSE_STATE_HEADER) {
+		if ((ret=http_response_parse_status_line(parser)) <= 0) {
+			*error_r = parser->error;
+			return ret;
+		}
+	} 
+
+	/* *( header-field CRLF ) CRLF */
+	if (parser->header_parser == NULL)
+		parser->header_parser = http_header_parser_init(parser->input);
+	else
+		http_header_parser_reset(parser->header_parser);
+
+	while ((ret=http_header_parse_next_field
+		(parser->header_parser, &field_name, &field_data, &field_size, &error)) > 0) {
+		if (field_name == NULL) break;
+		if ((ret=http_response_parse_header
+		     (parser, field_name, field_data, field_size)) < 0) {
+			*error_r = parser->error;
+			return -1;
+		}
+	}
+
+	if (ret <= 0) {
+		if (ret < 0)
+			*error_r = t_strdup_printf("Failed to parse response header: %s", error);
+		return ret;
+	}
+
+	/* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21
+	     Section 3.3.2:
+
+	   A server MUST NOT send a Content-Length header field in any response
+	   with a status code of 1xx (Informational) or 204 (No Content). [...]
+	 */
+	if ((parser->response->status / 100 == 1 ||
+		parser->response->status == 204) && parser->content_length > 0) {
+		*error_r = t_strdup_printf(
+			"Unexpected Content-Length header field for %u response "
+			"(length=%"PRIuUOFF_T")", parser->response->status,
+			parser->content_length);
+		return -1;
+	}
+
+	/* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21
+	     Section 3.3.3:
+
+	   Any response to a HEAD request and any response with a 1xx
+	   (Informational), 204 (No Content), or 304 (Not Modified) status
+	   code is always terminated by the first empty line after the
+	   header fields, regardless of the header fields present in the
+	   message, and thus cannot contain a message body.
+	 */
+	if (parser->response->status / 100 == 1 || parser->response->status == 204
+		|| parser->response->status == 304) { // HEAD is handled in caller
+		no_payload = TRUE;
+	}
+
+	if (!no_payload) {
+		/* [ message-body ] */
+		if (parser->content_length > 0) {
+			/* Got explicit message size from Content-Length: header */
+			parser->payload = parser->response->payload =
+				i_stream_create_limit(parser->input, parser->content_length);
+		} else if (parser->transfer_encoding != NULL) {
+			const char *tenc;
+
+			/* Transfer-Encoding = 1#transfer-coding
+			   transfer-coding    = "chunked" / "compress" / "deflate" / "gzip"
+			                      / transfer-extension       ;  [FIXME]
+			   transfer-extension = token *( OWS ";" OWS transfer-parameter )
+			*/
+			// FIXME: parse this directly when the header is parsed
+			http_parser_init(&hparser,
+				(const unsigned char *)parser->transfer_encoding,
+				strlen(parser->transfer_encoding));
+			for (;;) {
+				if (http_parse_token_list_next(&hparser, &tenc) <= 0)
+					break;
+				if (strcasecmp(tenc, "chunked") == 0) {
+					parser->payload =  parser->response->payload =
+						http_transfer_chunked_istream_create(parser->input);
+					break; // FIXME
+				} else {
+					*error_r = t_strdup_printf(
+						"Unkown Transfer-Encoding `%s' for %u response",
+						tenc, parser->response->status);
+					return -1;
+				}
+			}
+		}
+	}
+
+	parser->state = HTTP_RESPONSE_PARSE_STATE_INIT;
+	*response_r = parser->response;
+	return 1;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-response-parser.h	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,37 @@
+#ifndef HTTP_RESPONSE_PARSER_H
+#define HTTP_RESPONSE_PARSER_H
+
+struct http_response_header {
+	const char *key;
+	const char *value;
+	size_t size;
+};
+
+struct http_response {
+	unsigned char version_major;
+	unsigned char version_minor;
+
+	unsigned int status;
+
+	const char *reason;
+	const char *location;
+
+	time_t date;
+	struct istream *payload;
+
+	ARRAY(struct http_response_header) headers;
+
+	unsigned int connection_close:1;
+};
+
+struct http_response_parser;
+
+struct http_response_parser *
+http_response_parser_init(struct istream *input);
+void http_response_parser_deinit(struct http_response_parser **_parser);
+
+int http_response_parse_next(struct http_response_parser *parser,
+			     bool no_payload, struct http_response **response_r,
+			     const char **error_r);
+
+#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-http/http-transfer-chunked.c	Sat Nov 24 00:30:14 2012 +0200
@@ -0,0 +1,479 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream-private.h"
+#include "ostream-private.h"
+#include "http-parser.h"
+#include "http-header-parser.h"
+
+#include "http-transfer.h"
+
+/*
+ * Chunked input stream
+ */
+
+enum http_transfer_chunked_parse_state {
+	HTTP_CHUNKED_PARSE_STATE_INIT,
+	HTTP_CHUNKED_PARSE_STATE_SIZE,
+	HTTP_CHUNKED_PARSE_STATE_EXT,
+	HTTP_CHUNKED_PARSE_STATE_EXT_NAME,
+	HTTP_CHUNKED_PARSE_STATE_EXT_EQ,
+	HTTP_CHUNKED_PARSE_STATE_EXT_VALUE,
+	HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING,
+	HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE,
+	HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN,
+	HTTP_CHUNKED_PARSE_STATE_CR,
+	HTTP_CHUNKED_PARSE_STATE_LF,
+	HTTP_CHUNKED_PARSE_STATE_DATA,
+	HTTP_CHUNKED_PARSE_STATE_DATA_READY,
+	HTTP_CHUNKED_PARSE_STATE_DATA_CR,
+	HTTP_CHUNKED_PARSE_STATE_DATA_LF,
+	HTTP_CHUNKED_PARSE_STATE_TRAILER,
+	HTTP_CHUNKED_PARSE_STATE_FINISHED,
+};
+
+struct http_transfer_chunked_istream {
+	struct istream_private istream;
+	struct stat statbuf;
+
+	const unsigned char *begin, *cur, *end;
+	enum http_transfer_chunked_parse_state state;
+	unsigned int parsed_chars;
+
+	uoff_t chunk_size, chunk_v_offset, chunk_pos;
+	const char *error;
+
+	struct http_header_parser *header_parser;
+
+	unsigned int finished:1;
+};
+
+/* Chunk parser */
+
+static inline const char *_chr_sanitize(unsigned char c)
+{
+	if (c >= 0x20 && c < 0x7F)
+		return t_strdup_printf("'%c'", c);
+	return t_strdup_printf("0x%02x", c);
+}
+
+static int http_transfer_chunked_parse_size
+(struct http_transfer_chunked_istream *tcstream)
+{
+	uoff_t size = 0, prev;
+
+	/* chunk-size     = 1*HEXDIG */
+
+	while (tcstream->cur < tcstream->end) {
+		prev = tcstream->chunk_size;
+
+		if (*tcstream->cur >= '0' && *tcstream->cur <= '9')
+			size = *tcstream->cur-'0';
+		else if (*tcstream->cur >= 'A' && *tcstream->cur <= 'F')
+			size = *tcstream->cur-'A' + 10;
+		else if (*tcstream->cur >= 'a' && *tcstream->cur <= 'f')
+			size = *tcstream->cur-'a' + 10;
+		else {
+			if (tcstream->parsed_chars == 0) {
+				tcstream->error = t_strdup_printf(
+					"Expected chunk size digit, but found %s",
+					_chr_sanitize(*tcstream->cur));
+				return -1;
+			}
+			tcstream->parsed_chars = 0;
+			return 1;
+		}
+		tcstream->chunk_size <<= 4;
+		tcstream->chunk_size += size;
+		if (tcstream->chunk_size < prev) {
+			tcstream->error = "Chunk size exceeds integer limit";
+			return -1;
+		}
+		tcstream->parsed_chars++;
+		tcstream->cur++;
+	}
+
+	return 0;
+}
+
+static int http_transfer_chunked_skip_token
+(struct http_transfer_chunked_istream *tcstream)
+{
+	const unsigned char *first = tcstream->cur;
+
+	/* token          = 1*tchar */
+	while (tcstream->cur < tcstream->end && http_char_is_token(*tcstream->cur))
+		tcstream->cur++;
+
+	tcstream->parsed_chars += (tcstream->cur-first);
+	if (tcstream->cur == tcstream->end)
+		return 0;
+	if (tcstream->parsed_chars == 0)
+		return -1;
+	return 1;
+}
+
+static int http_transfer_chunked_skip_qdtext
+(struct http_transfer_chunked_istream *tcstream)
+{
+	/* qdtext-nf      = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text */
+	while (tcstream->cur < tcstream->end && http_char_is_qdtext(*tcstream->cur))
+		tcstream->cur++;
+	if (tcstream->cur == tcstream->end)
+		return 0;
+	return 1;
+}
+
+static int
+http_transfer_chunked_parse(struct http_transfer_chunked_istream *tcstream)
+{
+	int ret;
+
+	/* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21;
+		   Section 4.1:
+
+		 chunked-body   = *chunk
+		                  last-chunk
+		                  trailer-part
+		                  CRLF
+
+		 chunk          = chunk-size [ chunk-ext ] CRLF
+		                  chunk-data CRLF
+		 chunk-size     = 1*HEXDIG
+		 last-chunk     = 1*("0") [ chunk-ext ] CRLF
+
+		 chunk-ext      = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
+		 chunk-ext-name = token
+		 chunk-ext-val  = token / quoted-str-nf
+		 chunk-data     = 1*OCTET ; a sequence of chunk-size octets
+		 trailer-part   = *( header-field CRLF )
+
+		 quoted-str-nf  = DQUOTE *( qdtext-nf / quoted-pair ) DQUOTE
+		                ; like quoted-string, but disallowing line folding
+		 qdtext-nf      = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text
+	   quoted-pair    = "\" ( HTAB / SP / VCHAR / obs-text )
+	*/
+
+	for (;;) {
+		switch (tcstream->state) {
+		case HTTP_CHUNKED_PARSE_STATE_INIT:
+			tcstream->chunk_size = 0;
+			tcstream->chunk_pos = 0;
+			tcstream->parsed_chars = 0;
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_SIZE;
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_SIZE:
+			if ((ret=http_transfer_chunked_parse_size(tcstream)) <= 0)
+				return ret;
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_EXT:
+			if (*tcstream->cur != ';') {
+				tcstream->state = HTTP_CHUNKED_PARSE_STATE_CR;
+				break;
+			}
+			/* chunk-ext */
+			tcstream->cur++;
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_NAME;
+			if (tcstream->cur >= tcstream->end)
+				return 0;
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_EXT_NAME:
+			/* chunk-ext-name = token */
+			if ((ret=http_transfer_chunked_skip_token(tcstream)) <= 0) {
+				if (ret < 0)
+					tcstream->error = "Invalid chunked extension name";
+				return ret;
+			}
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_EQ;
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_EXT_EQ:
+			if (*tcstream->cur != '=') {
+				tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+				break;
+			}
+			tcstream->cur++;
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE;
+			if (tcstream->cur >= tcstream->end)
+				return 0;
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE:
+			/* chunk-ext-val  = token / quoted-str-nf */
+			if (*tcstream->cur != '"') {
+				tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN;
+				break;
+			} 
+			tcstream->cur++;
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING;
+			if (tcstream->cur >= tcstream->end)
+				return 0;
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING:
+			for (;;) {
+				if (*tcstream->cur == '"') {
+					tcstream->cur++;
+					tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+					if (tcstream->cur >= tcstream->end)
+						return 0;
+					break;
+				} else if ((ret=http_transfer_chunked_skip_qdtext(tcstream)) <= 0) {
+					if (ret < 0)
+						tcstream->error = "Invalid chunked extension value";
+					return ret;	
+				} else if (*tcstream->cur == '\\') {
+					tcstream->cur++;
+					tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE;
+					if (tcstream->cur >= tcstream->end)						
+						return 0;
+					break;
+				} else {
+					tcstream->error = t_strdup_printf(
+						"Invalid character %s in chunked extension value string",
+						_chr_sanitize(*tcstream->cur));
+					return -1;
+				}
+			}
+			break;
+		case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE:
+			/* ( HTAB / SP / VCHAR / obs-text ) */
+			if (!http_char_is_text(*tcstream->cur)) {
+				tcstream->error = t_strdup_printf(
+					"Escaped invalid character %s in chunked extension value string",
+					_chr_sanitize(*tcstream->cur));
+				return -1;
+			}
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING;
+			if (tcstream->cur >= tcstream->end)						
+				return 0;
+			break;
+		case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN:
+			if ((ret=http_transfer_chunked_skip_token(tcstream)) <= 0) {
+				if (ret < 0)
+					tcstream->error = "Invalid chunked extension value";
+				return ret;
+			}
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+			break;
+		case HTTP_CHUNKED_PARSE_STATE_CR:
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_LF;
+			if (*tcstream->cur == '\r') {
+				tcstream->cur++;
+				if (tcstream->cur >= tcstream->end)						
+					return 0;
+			}
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_LF:
+			if (*tcstream->cur != '\n') {
+				tcstream->error = t_strdup_printf(
+					"Expected new line after chunk size, but found %s",
+					_chr_sanitize(*tcstream->cur));
+				return -1;
+			}
+			tcstream->cur++;
+			if (tcstream->chunk_size > 0)
+				tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA;
+			else
+				tcstream->state = HTTP_CHUNKED_PARSE_STATE_TRAILER;
+			return 1;
+		case HTTP_CHUNKED_PARSE_STATE_DATA_READY:
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_DATA_CR:
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_LF;
+			if (*tcstream->cur == '\r') {
+				tcstream->cur++;
+				if (tcstream->cur >= tcstream->end)						
+					return 0;
+			}
+			/* fall through */
+		case HTTP_CHUNKED_PARSE_STATE_DATA_LF:
+			if (*tcstream->cur != '\n') {
+				tcstream->error = t_strdup_printf(
+					"Expected new line after chunk data, but found %s",
+					_chr_sanitize(*tcstream->cur));
+				return -1;
+			}
+			tcstream->cur++;
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_INIT;
+			break;
+		default:
+			i_unreached();
+		}
+	}
+
+	i_unreached();
+	return -1;
+}
+
+static int http_transfer_chunked_parse_next(
+	struct http_transfer_chunked_istream *tcstream)
+{
+	struct istream *input = tcstream->istream.parent;
+	size_t size;
+	int ret;
+
+	while ((ret=i_stream_read_data
+		(input, &tcstream->begin, &size, 0)) > 0) {
+		tcstream->cur = tcstream->begin;
+		tcstream->end = tcstream->cur + size;
+
+		if ((ret=http_transfer_chunked_parse(tcstream)) < 0)
+			return -1;
+
+		i_stream_skip(input, tcstream->cur - tcstream->begin);
+
+		if (ret > 0) {
+			if (tcstream->state == HTTP_CHUNKED_PARSE_STATE_DATA)
+				tcstream->chunk_v_offset = input->v_offset;
+			return ret;
+		}
+	}
+
+	i_assert(ret != -2);
+	return ret;
+}
+
+static ssize_t
+http_transfer_chunked_istream_read_data(
+	struct http_transfer_chunked_istream *tcstream)
+{
+	struct istream_private *stream = &tcstream->istream;
+	const unsigned char *data;
+	size_t size, avail;
+	ssize_t ret = 0;
+
+	if (tcstream->chunk_pos >= tcstream->chunk_size) {
+		tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_READY;
+		return 0;
+	}
+
+	// FIXME: is this even necessary?
+	i_stream_seek(stream->parent, tcstream->chunk_v_offset + tcstream->chunk_pos);
+
+	/* read from parent if necessary */
+	data = i_stream_get_data(stream->parent, &size);
+	if (size == 0) {
+		ret = i_stream_read(stream->parent);
+		if (ret <= 0 && (ret != -2 || stream->skip == 0)) {
+			if ( stream->parent->eof && stream->parent->stream_errno == 0 ) {
+				/* unexpected EOF */
+				tcstream->error = "Unexpected end of payload";
+				stream->istream.stream_errno = EIO;
+			} else {
+				/* parent stream error */
+				tcstream->error = "Stream error";
+				stream->istream.stream_errno = stream->parent->stream_errno;
+			}
+			return ret;
+		}
+		data = i_stream_get_data(stream->parent, &size);
+		i_assert(size != 0);
+	}
+
+	size = size > (tcstream->chunk_size - tcstream->chunk_pos) ? 
+		(tcstream->chunk_size - tcstream->chunk_pos) : size;
+
+	/* Allocate buffer space */
+	if (!i_stream_try_alloc(stream, size, &avail))
+		return -2;
+
+	/* Copy payload */
+	size = size > avail ? avail : size;
+	memcpy(&stream->w_buffer[stream->pos], data, size);
+
+	i_stream_skip(stream->parent, size);
+
+	tcstream->chunk_pos += size;
+	if (tcstream->chunk_pos >= tcstream->chunk_size)
+		tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_READY;
+
+	if ( ret < 0 ) {
+		stream->pos = stream->pos+size;
+		return ret;
+	}
+
+	ret = size;
+	stream->pos = stream->pos+size;
+	return ret;
+}
+
+static int http_transfer_chunked_parse_trailer(
+	struct http_transfer_chunked_istream *tcstream)
+{
+	const char *field_name, *error;
+	const unsigned char *field_data;
+	size_t field_size;
+	int ret;
+
+	if (tcstream->header_parser == NULL) {
+		tcstream->header_parser = http_header_parser_init(tcstream->istream.parent);
+	}
+
+	while ((ret=http_header_parse_next_field(tcstream->header_parser,
+		&field_name, &field_data, &field_size, &error)) > 0) {
+		if (field_name == NULL) break;
+	}
+
+	if (ret <= 0) {
+		if (ret < 0) {
+			tcstream->error = t_strdup_printf
+				("Failed to parse chunked trailer: %s", error);
+		}
+		return ret;
+	}
+	return 1;
+}
+
+static ssize_t
+http_transfer_chunked_istream_read(struct istream_private *stream)
+{
+	struct http_transfer_chunked_istream *tcstream =
+		(struct http_transfer_chunked_istream *)stream;
+	ssize_t ret = 0;
+
+	for (;;) {
+		switch (tcstream->state) {
+		case HTTP_CHUNKED_PARSE_STATE_FINISHED:
+			tcstream->istream.istream.eof = TRUE;
+			return -1;
+		case 	HTTP_CHUNKED_PARSE_STATE_DATA:
+			if ((ret=http_transfer_chunked_istream_read_data(tcstream)) != 0)
+				return ret;
+			if (tcstream->state != HTTP_CHUNKED_PARSE_STATE_DATA_READY)
+				return 0;
+			break;
+		case HTTP_CHUNKED_PARSE_STATE_TRAILER:
+			if ((ret=http_transfer_chunked_parse_trailer(tcstream)) <= 0)
+				return ret;
+			tcstream->state = HTTP_CHUNKED_PARSE_STATE_FINISHED;
+			tcstream->istream.istream.eof = TRUE;
+			return -1;
+		default:
+			if ((ret=http_transfer_chunked_parse_next(tcstream)) <= 0)
+				return ret;
+		}
+	}
+
+	return -1;
+}
+
+struct istream *
+http_transfer_chunked_istream_create(struct istream *input)
+{
+	struct http_transfer_chunked_istream *tcstream;
+
+	tcstream = i_new(struct http_transfer_chunked_istream, 1);
+
+	tcstream->istream.max_buffer_size =
+		input->real_stream->max_buffer_size;
+
+	tcstream->istream.read = http_transfer_chunked_istream_read;
+
+	tcstream->istream.istream.readable_fd = FALSE;
+	tcstream->istream.istream.blocking = input->blocking;
+	tcstream->istream.istream.seekable = FALSE;
+	return i_stream_create(&tcstream->istream, input, i_stream_get_fd(input));
+}
+
+
+
+