Mercurial > dovecot > core-2.2

changeset 16855:c3884a6acde6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lib-http: Added support for creating CONNECT tunnels through HTTP.
author Stephan Bosch <stephan@rename-it.nl>
date Sat, 12 Oct 2013 11:05:08 +0300
parents 462ae2cb8094
children b9498573f0d0
files src/lib-http/http-client-connection.c src/lib-http/http-client-host.c src/lib-http/http-client-peer.c src/lib-http/http-client-private.h src/lib-http/http-client-request.c src/lib-http/http-client.h src/lib-http/http-response-parser.c src/lib-http/http-response-parser.h src/lib-http/http-response.h
diffstat 9 files changed, 319 insertions(+), 89 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-http/http-client-connection.c	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-client-connection.c	Sat Oct 12 11:05:08 2013 +0300
@@ -57,7 +57,7 @@
 bool http_client_connection_is_ready(struct http_client_connection *conn)
 {
 	return (conn->connected && !conn->output_locked &&
-		!conn->close_indicated &&
+		!conn->close_indicated && !conn->tunneling &&
 		http_client_connection_count_pending(conn) <
 			conn->client->set.max_pipelined_requests);
 }
@@ -173,7 +173,8 @@
 {
 	unsigned int timeout, count;
 
-	if (array_count(&conn->request_wait_list) == 0 &&
+	if (array_is_created(&conn->request_wait_list) &&
+		array_count(&conn->request_wait_list) == 0 &&
 		conn->incoming_payload == NULL &&
 		conn->client->set.max_idle_time_msecs > 0) {
 
@@ -302,6 +303,9 @@
 		return -1;
 	}
 
+	if (req->connect_tunnel)
+		conn->tunneling = TRUE;
+
 	/* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21;
 			Section 6.1.2.1:
 
@@ -388,6 +392,7 @@
 {
 	struct http_client_connection *conn = req->conn;
 
+	i_assert(conn != NULL);
 	i_assert(conn->pending_request == req);
 	i_assert(conn->incoming_payload != NULL);
 	i_assert(conn->conn.io == NULL);
@@ -479,7 +484,8 @@
 	}
 
 	if (conn->incoming_payload == NULL) {
-		i_assert(conn->conn.io != NULL);
+		i_assert(conn->conn.io != NULL ||
+			conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW);
 		return TRUE;
 	}
 
@@ -495,7 +501,7 @@
 	struct http_client_request *req = NULL;
 	int finished = 0, ret;
 	const char *error;
-	bool no_payload = FALSE;
+	enum http_response_payload_type payload_type;
 
 	i_assert(conn->incoming_payload == NULL);
 
@@ -516,18 +522,16 @@
 		req_idx = array_idx(&conn->request_wait_list, 0);
 		req = req_idx[0];
 
-		/* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21
-		     Section 3.3.2:
-
-		   A server MAY send a Content-Length header field in a response to a
-		   HEAD request [...]
-		 */
-		no_payload = (strcmp(req->method, "HEAD") == 0);
+		/* determine whether to expect a response payload */
+		payload_type = http_client_request_get_payload_type(req);
+	} else {
+		req = NULL;
+		payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
 	}
 
 	// FIXME: handle somehow if server replies before request->input is at EOF
 	while ((ret=http_response_parse_next
-		(conn->http_parser, no_payload, &response, &error)) > 0) {
+		(conn->http_parser, payload_type, &response, &error)) > 0) {
 		bool aborted;
 
 		if (req == NULL) {
@@ -621,13 +625,15 @@
 		if (array_count(&conn->request_wait_list) > 0) {
 			req_idx = array_idx(&conn->request_wait_list, 0);
 			req = req_idx[0];
-			no_payload = (strcmp(req->method, "HEAD") == 0);
+
+			/* determine whether to expect a response payload */
+			payload_type = http_client_request_get_payload_type(req);
 		} else {
 			/* no more requests waiting for the connection */
 			if (conn->to_requests != NULL)
 				timeout_remove(&conn->to_requests);
 			req = NULL;
-			no_payload = FALSE;
+			payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
 		}
 	}
 
@@ -702,6 +708,33 @@
 	return 1;
 }
 
+void
+http_client_connection_start_tunnel(struct http_client_connection **_conn,
+	struct http_client_tunnel *tunnel)
+{
+	struct http_client_connection *conn = *_conn;
+
+	i_assert(conn->tunneling);
+
+	/* claim connection streams */
+	memset(tunnel, 0, sizeof(*tunnel));
+	tunnel->input = conn->conn.input;
+	tunnel->output = conn->conn.output;
+	tunnel->fd_in = conn->conn.fd_in;
+	tunnel->fd_out = conn->conn.fd_out;
+
+	/* detach from connection */
+	conn->conn.input = NULL;
+	conn->conn.output = NULL;
+	conn->conn.fd_in = -1;
+	conn->conn.fd_out = -1;
+	conn->closing = TRUE;
+	conn->connected = FALSE;
+	connection_disconnect(&conn->conn);
+
+	http_client_connection_unref(_conn);
+}
+
 static void 
 http_client_connection_ready(struct http_client_connection *conn)
 {
@@ -725,6 +758,34 @@
 				       &conn->conn.input, &conn->conn.output);
 	}
 
+	/* direct tunneling connections handle connect requests just by providing a
+	   raw connection */
+	if (conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW) {
+		struct http_client_request *req;
+		
+		req = http_client_peer_claim_request(conn->peer, FALSE);
+		if (req != NULL) {
+			struct http_response response;
+
+			http_client_request_ref(req);
+			req->conn = conn;
+			conn->tunneling = TRUE;
+
+			memset(&response, 0, sizeof(response));
+			response.status = 200;
+			response.reason = "OK";
+
+			(void)http_client_connection_return_response(conn, req, &response);
+			http_client_request_unref(&req);
+			return;
+		} 
+		
+		http_client_connection_debug(conn,
+			"No raw connect requests pending; closing useless connection");
+		http_client_connection_unref(&conn);
+		return;
+	}
+
 	/* start protocol I/O */
 	conn->http_parser = http_response_parser_init
 		(conn->conn.input, &conn->client->set.response_hdr_limits);
@@ -881,13 +942,27 @@
 	struct http_client_connection *conn;
 	static unsigned int id = 0;
 	const struct http_client_peer_addr *addr = &peer->addr;
+	const char *conn_type = "UNKNOWN";
+
+	switch (peer->addr.type) {
+	case HTTP_CLIENT_PEER_ADDR_HTTP:
+		conn_type = "HTTP";
+		break;
+	case HTTP_CLIENT_PEER_ADDR_HTTPS:
+		conn_type = "HTTPS";
+		break;
+	case HTTP_CLIENT_PEER_ADDR_RAW:
+		conn_type = "Raw";
+		break;
+	}
 
 	conn = i_new(struct http_client_connection, 1);
 	conn->refcount = 1;
 	conn->client = peer->client;
 	conn->id = id++;
 	conn->peer = peer;
-	i_array_init(&conn->request_wait_list, 16);
+	if (peer->addr.type != HTTP_CLIENT_PEER_ADDR_RAW)
+		i_array_init(&conn->request_wait_list, 16);
 
 	connection_init_client_ip
 		(peer->client->conn_list, &conn->conn, &addr->ip, addr->port);
@@ -896,8 +971,9 @@
 	array_append(&peer->conns, &conn, 1);
 
 	http_client_connection_debug(conn,
-		"Connection created (%d parallel connections exist)%s",
-		array_count(&peer->conns), (conn->to_input == NULL ? "" : " [broken]"));
+		"%s connection created (%d parallel connections exist)%s",
+		conn_type, array_count(&peer->conns),
+		(conn->to_input == NULL ? "" : " [broken]"));
 	return conn;
 }
 
@@ -910,6 +986,7 @@
 {
 	struct http_client_connection *conn = *_conn;
 	struct http_client_connection *const *conn_idx;
+	ARRAY_TYPE(http_client_connection) *conn_arr;
 	struct http_client_peer *peer = conn->peer;
 	struct http_client_request **req;
 
@@ -931,17 +1008,19 @@
 
 	connection_disconnect(&conn->conn);
 
-	/* abort all pending requests */
-	array_foreach_modifiable(&conn->request_wait_list, req) {
-		i_assert((*req)->submitted);
-		http_client_request_error(*req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
-			"Aborting");
+	if (array_is_created(&conn->request_wait_list)) {
+		/* abort all pending requests */
+		array_foreach_modifiable(&conn->request_wait_list, req) {
+			i_assert((*req)->submitted);
+			http_client_request_error(*req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
+				"Aborting");
+		}
+		array_free(&conn->request_wait_list);
 	}
 	if (conn->pending_request != NULL) {
 		http_client_request_error(conn->pending_request,
 			HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborting");
 	}
-	array_free(&conn->request_wait_list);
 
 	if (conn->http_parser != NULL)
 		http_response_parser_deinit(&conn->http_parser);
@@ -964,10 +1043,10 @@
 		timeout_remove(&conn->to_response);
 	
 	/* remove this connection from the list */
-	array_foreach(&conn->peer->conns, conn_idx) {
+	conn_arr = &conn->peer->conns;
+	array_foreach(conn_arr, conn_idx) {
 		if (*conn_idx == conn) {
-			array_delete(&conn->peer->conns,
-				array_foreach_idx(&conn->peer->conns, conn_idx), 1);
+			array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1);
 			break;
 		}
 	}
--- a/src/lib-http/http-client-host.c	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-client-host.c	Sat Oct 12 11:05:08 2013 +0300
@@ -46,13 +46,13 @@
 
 static struct http_client_host_port *
 http_client_host_port_find(struct http_client_host *host,
-	in_port_t port, const char *https_name)
+	const struct http_client_peer_addr *addr)
 {
 	struct http_client_host_port *hport;
 
 	array_foreach_modifiable(&host->ports, hport) {
-		if (hport->addr.port == port &&
-		    null_strcmp(hport->addr.https_name, https_name) == 0)
+		if (hport->addr.type == addr->type && hport->addr.port == addr->port &&
+		    null_strcmp(hport->addr.https_name, addr->https_name) == 0)
 			return hport;
 	}
 
@@ -61,16 +61,17 @@
 
 static struct http_client_host_port *
 http_client_host_port_init(struct http_client_host *host,
-	in_port_t port, const char *https_name)
+	const struct http_client_peer_addr *addr)
 {
 	struct http_client_host_port *hport;
 
-	hport = http_client_host_port_find(host, port, https_name);
+	hport = http_client_host_port_find(host, addr);
 	if (hport == NULL) {
 		hport = array_append_space(&host->ports);
 		hport->host = host;
-		hport->addr.port = port;
-		hport->addr.https_name = i_strdup(https_name);
+		hport->addr = *addr;
+		hport->https_name = i_strdup(addr->https_name);
+		hport->addr.https_name = hport->https_name;
 		hport->ips_connect_idx = 0;
 		i_array_init(&hport->request_queue, 16);
 	}
@@ -94,7 +95,7 @@
 {
 	http_client_host_port_error
 		(hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted");
-	i_free(hport->addr.https_name);
+	i_free(hport->https_name);
 	if (array_is_created(&hport->pending_peers))
 		array_free(&hport->pending_peers);
 	array_free(&hport->request_queue);
@@ -104,13 +105,12 @@
 http_client_host_port_drop_request(struct http_client_host_port *hport,
 	struct http_client_request *req)
 {
+	ARRAY_TYPE(http_client_request) *req_arr = &hport->request_queue;
 	struct http_client_request **req_idx;
-	unsigned int idx;
 
-	array_foreach_modifiable(&hport->request_queue, req_idx) {
+	array_foreach_modifiable(req_arr, req_idx) {
 		if (*req_idx == req) {
-			idx = array_foreach_idx(&hport->request_queue, req_idx);
-			array_delete(&hport->request_queue, idx, 1);
+			array_delete(req_arr, array_foreach_idx(req_arr, req_idx), 1);
 			break;
 		}
 	}
@@ -309,7 +309,7 @@
 	http_client_host_debug(host, "Successfully connected to %s",
 		http_client_peer_addr2str(addr));
 
-	hport = http_client_host_port_find(host, addr->port, addr->https_name);
+	hport = http_client_host_port_find(host, addr);
 	if (hport == NULL)
 		return;
 
@@ -324,7 +324,7 @@
 	http_client_host_debug(host, "Failed to connect to %s: %s",
 		http_client_peer_addr2str(addr), reason);
 
-	hport = http_client_host_port_find(host, addr->port, addr->https_name);
+	hport = http_client_host_port_find(host, addr);
 	if (hport == NULL)
 		return;
 
@@ -455,8 +455,7 @@
 {
 	struct http_client_host_port *hport;
 	const struct http_url *host_url = req->host_url;
-	const char *https_name = http_client_request_https_name(req);
-	in_port_t port = http_client_request_port(req);
+	struct http_client_peer_addr addr;
 	const char *error;
 
 	req->host = host;
@@ -469,8 +468,10 @@
 		}
 	}
 
+	http_client_request_get_peer_addr(req, &addr);
+
 	/* add request to host (grouped by tcp port) */
-	hport = http_client_host_port_init(host, port, https_name);
+	hport = http_client_host_port_init(host, &addr);
 	if (req->urgent)
 		array_insert(&hport->request_queue, 0, &req, 1);
 	else
@@ -496,7 +497,7 @@
 	struct http_client_request *req;
 	unsigned int i, count;
 
-	hport = http_client_host_port_find(host, addr->port, addr->https_name);
+	hport = http_client_host_port_find(host, addr);
 	if (hport == NULL)
 		return NULL;
 
@@ -530,13 +531,17 @@
 
 	*num_urgent_r = 0;
 
-	hport = http_client_host_port_find(host, addr->port, addr->https_name);
+	hport = http_client_host_port_find(host, addr);
 	if (hport == NULL)
 		return 0;
 
 	requests = array_get(&hport->request_queue, &count);
-	for (i = 0; i < count && requests[i]->urgent; i++)
-		(*num_urgent_r)++;
+	for (i = 0; i < count; i++) {
+		if (requests[i]->urgent)
+			(*num_urgent_r)++;
+		else
+			break;
+	}
 	return count;
 }
 
@@ -544,10 +549,11 @@
 	struct http_client_request *req)
 {
 	struct http_client_host_port *hport;
-	const char *https_name = http_client_request_https_name(req);
-	in_port_t port = http_client_request_port(req);
+	struct http_client_peer_addr addr;
 
-	hport = http_client_host_port_find(host, port, https_name);
+	http_client_request_get_peer_addr(req, &addr);
+
+	hport = http_client_host_port_find(host, &addr);
 	if (hport == NULL)
 		return;
 
--- a/src/lib-http/http-client-peer.c	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-client-peer.c	Sat Oct 12 11:05:08 2013 +0300
@@ -42,8 +42,17 @@
 unsigned int http_client_peer_addr_hash
 (const struct http_client_peer_addr *peer)
 {
-	return net_ip_hash(&peer->ip) + peer->port +
-		(peer->https_name == NULL ? 0 : str_hash(peer->https_name));
+	switch (peer->type) {
+	case HTTP_CLIENT_PEER_ADDR_RAW:
+		return net_ip_hash(&peer->ip) + peer->port + 1;
+	case HTTP_CLIENT_PEER_ADDR_HTTP:
+		return net_ip_hash(&peer->ip) + peer->port;
+	case HTTP_CLIENT_PEER_ADDR_HTTPS:
+		return net_ip_hash(&peer->ip) + peer->port +
+			(peer->https_name == NULL ? 0 : str_hash(peer->https_name));
+	}
+	i_unreached();
+	return 0;
 }
 
 int http_client_peer_addr_cmp
@@ -52,10 +61,14 @@
 {
 	int ret;
 
+	if (peer1->type != peer2->type)
+		return (peer1->type > peer2->type ? 1 : -1);
 	if ((ret=net_ip_cmp(&peer1->ip, &peer2->ip)) != 0)
 		return ret;
 	if (peer1->port != peer2->port)
 		return (peer1->port > peer2->port ? 1 : -1);
+	if (peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS)
+		return 0;
 	return null_strcmp(peer1->https_name, peer2->https_name);
 }
 
@@ -69,7 +82,8 @@
 	unsigned int i;
 
 	for (i = 0; i < count; i++) {
-		http_client_peer_debug(peer, "Making new connection %u of %u", i+1, count);
+		http_client_peer_debug(peer,
+			"Making new connection %u of %u", i+1, count);
 		(void)http_client_connection_create(peer);
 	}
 }
@@ -339,7 +353,8 @@
 	peer = i_new(struct http_client_peer, 1);
 	peer->client = client;
 	peer->addr = *addr;
-	peer->addr.https_name = i_strdup(addr->https_name);
+	peer->https_name = i_strdup(addr->https_name);
+	peer->addr.https_name = peer->https_name;
 	i_array_init(&peer->hosts, 16);
 	i_array_init(&peer->conns, 16);
 
@@ -348,7 +363,6 @@
 	DLLIST_PREPEND(&client->peers_list, peer);
 
 	http_client_peer_debug(peer, "Peer created");
-	http_client_peer_connect(peer, 1);
 	return peer;
 }
 
@@ -370,7 +384,6 @@
 	/* make a copy of the connection array; freed connections modify it */
 	t_array_init(&conns, array_count(&peer->conns));
 	array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
-
 	array_foreach_modifiable(&conns, conn) {
 		http_client_connection_unref(conn);
 	}
@@ -383,7 +396,7 @@
 		(peer->client->peers, (const struct http_client_peer_addr *)&peer->addr);
 	DLLIST_REMOVE(&peer->client->peers_list, peer);
 
-	i_free(peer->addr.https_name);
+	i_free(peer->https_name);
 	i_free(peer);
 	*_peer = NULL;
 }
--- a/src/lib-http/http-client-private.h	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-client-private.h	Sat Oct 12 11:05:08 2013 +0300
@@ -14,6 +14,8 @@
 #define HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS (1000*60*5)
 #define HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS (1000*2)
 
+enum http_response_payload_type;
+
 struct http_client_host;
 struct http_client_host_port;
 struct http_client_peer;
@@ -30,8 +32,15 @@
 HASH_TABLE_DEFINE_TYPE(http_client_peer, const struct http_client_peer_addr *,
 	struct http_client_peer *);
 
+enum http_client_peer_addr_type {
+	HTTP_CLIENT_PEER_ADDR_HTTP = 0,
+	HTTP_CLIENT_PEER_ADDR_HTTPS,
+	HTTP_CLIENT_PEER_ADDR_RAW
+};
+
 struct http_client_peer_addr {
-	char *https_name; /* TLS SNI */
+	enum http_client_peer_addr_type type;
+	const char *https_name; /* TLS SNI */
 	struct ip_addr ip;
 	in_port_t port;
 };
@@ -86,12 +95,15 @@
 	unsigned int payload_wait:1;
 	unsigned int urgent:1;
 	unsigned int submitted:1;
+	unsigned int connect_tunnel:1;
+	unsigned int connect_direct:1;
 };
 
 struct http_client_host_port {
 	struct http_client_host *host;
 
 	struct http_client_peer_addr addr;
+	char *https_name;
 
 	/* current index in host->ips */
 	unsigned int ips_connect_idx;
@@ -132,6 +144,8 @@
 
 struct http_client_peer {
 	struct http_client_peer_addr addr;
+	char *https_name;
+
 	struct http_client *client;
 	struct http_client_peer *prev, *next;
 
@@ -178,6 +192,8 @@
 	ARRAY_TYPE(http_client_request) request_wait_list;
 
 	unsigned int connected:1;           /* connection is connected */
+	unsigned int tunneling:1;          /* last sent request turns this
+	                                      connection into tunnel */
 	unsigned int connect_succeeded:1;
 	unsigned int closing:1;
 	unsigned int close_indicated:1;
@@ -221,21 +237,24 @@
 	return req->label;
 }
 
-static inline in_port_t
-http_client_request_port(const struct http_client_request *req)
+static inline void
+http_client_request_get_peer_addr(const struct http_client_request *req,
+	struct http_client_peer_addr *addr)
 {
 	const struct http_url *host_url = req->host_url;
-
-	return (host_url->have_port ? host_url->port :
-		(host_url->have_ssl ? HTTPS_DEFAULT_PORT : HTTP_DEFAULT_PORT));
-}
-
-static inline const char *
-http_client_request_https_name(const struct http_client_request *req)
-{
-	const struct http_url *host_url = req->host_url;
-
-	return (host_url->have_ssl ? host_url->host_name : NULL);
+	
+	memset(addr, 0, sizeof(*addr));
+	if (req->connect_direct) {
+		addr->type = HTTP_CLIENT_PEER_ADDR_RAW;
+		addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT);
+	} else if (host_url->have_ssl) {
+		addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS;
+		addr->https_name = host_url->host_name;
+ 		addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT);
+	} else {
+		addr->type = HTTP_CLIENT_PEER_ADDR_HTTP;
+		addr->port = (host_url->have_port ? host_url->port : HTTP_DEFAULT_PORT);
+	}
 }
 
 static inline const char *
@@ -255,12 +274,17 @@
 
 void http_client_request_ref(struct http_client_request *req);
 void http_client_request_unref(struct http_client_request **_req);
+enum http_response_payload_type
+http_client_request_get_payload_type(struct http_client_request *req);
 int http_client_request_send(struct http_client_request *req,
 			    const char **error_r);
 int http_client_request_send_more(struct http_client_request *req,
 				  const char **error_r);
 bool http_client_request_callback(struct http_client_request *req,
 	struct http_response *response);
+void http_client_request_connect_callback(struct http_client_request *req,
+			     const struct http_client_tunnel *tunnel,
+			     struct http_response *response);
 void http_client_request_resubmit(struct http_client_request *req);
 void http_client_request_retry(struct http_client_request *req,
 	unsigned int status, const char *error);
@@ -288,6 +312,8 @@
 int http_client_connection_next_request(struct http_client_connection *conn);
 void http_client_connection_check_idle(struct http_client_connection *conn);
 void http_client_connection_switch_ioloop(struct http_client_connection *conn);
+void http_client_connection_start_tunnel(struct http_client_connection **_conn,
+	struct http_client_tunnel *tunnel);
 
 unsigned int http_client_peer_addr_hash
 	(const struct http_client_peer_addr *peer) ATTR_PURE;
@@ -325,12 +351,18 @@
 struct http_client_request *
 http_client_host_claim_request(struct http_client_host *host,
 	const struct http_client_peer_addr *addr, bool no_urgent);
+struct http_client_request *
+http_client_host_claim_connect_request(struct http_client_host *host,
+	const struct http_client_peer_addr *addr);
 void http_client_host_connection_success(struct http_client_host *host,
 	const struct http_client_peer_addr *addr);
 void http_client_host_connection_failure(struct http_client_host *host,
 	const struct http_client_peer_addr *addr, const char *reason);
 unsigned int http_client_host_requests_pending(struct http_client_host *host,
 	const struct http_client_peer_addr *addr, unsigned int *num_urgent_r);
+unsigned int
+http_client_host_connect_requests_pending(struct http_client_host *host,
+	const struct http_client_peer_addr *addr);
 void http_client_host_drop_request(struct http_client_host *host,
 	struct http_client_request *req);
 void http_client_host_switch_ioloop(struct http_client_host *host);
--- a/src/lib-http/http-client-request.c	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-client-request.c	Sat Oct 12 11:05:08 2013 +0300
@@ -84,7 +84,6 @@
 	req = http_client_request_new(client, method, callback, context);
 	req->origin_url.host_name = p_strdup(req->pool, host);
 	req->target = (target == NULL ? "/" : p_strdup(req->pool, target));
-	req->headers = str_new(default_pool, 256);
 	return req;
 }
 
@@ -99,7 +98,24 @@
 	req = http_client_request_new(client, method, callback, context);
 	http_url_copy_authority(req->pool, &req->origin_url, target_url);
 	req->target = p_strdup(req->pool, http_url_create_target(target_url));
-	req->headers = str_new(default_pool, 256);
+	return req;
+}
+
+#undef http_client_request_connect
+struct http_client_request *
+http_client_request_connect(struct http_client *client,
+		    const char *host, in_port_t port,
+		    http_client_request_callback_t *callback,
+				void *context)
+{
+	struct http_client_request *req;
+
+	req = http_client_request_new(client, "CONNECT", callback, context);
+	req->origin_url.host_name = p_strdup(req->pool, host);
+	req->origin_url.port = port;
+	req->origin_url.have_port = TRUE;
+	req->connect_tunnel = TRUE;
+	req->target = "";
 	return req;
 }
 
@@ -139,7 +155,8 @@
 		i_stream_unref(&req->payload_input);
 	if (req->payload_output != NULL)
 		o_stream_unref(&req->payload_output);
-	str_free(&req->headers);
+	if (req->headers != NULL)
+		str_free(&req->headers);
 	pool_unref(&req->pool);
 	*_req = NULL;
 }
@@ -241,17 +258,45 @@
 	return req->state;
 }
 
+enum http_response_payload_type
+http_client_request_get_payload_type(struct http_client_request *req)
+{
+	/* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23
+	     Section 3.3:
+
+	   The presence of a message body in a response depends on both the
+	   request method to which it is responding and the response status code.
+	   Responses to the HEAD request method never include a message body
+	   because the associated response header fields, if present, indicate only
+	   what their values would have been if the request method had been GET
+	   2xx (Successful) responses to CONNECT switch to tunnel mode instead of
+	   having a message body (Section 4.3.6 of [Part2]).
+	 */
+	if (strcmp(req->method, "HEAD") == 0)
+		return HTTP_RESPONSE_PAYLOAD_TYPE_NOT_PRESENT;
+	if (strcmp(req->method, "CONNECT") == 0)
+		return HTTP_RESPONSE_PAYLOAD_TYPE_ONLY_UNSUCCESSFUL;
+	return HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
+}
+
 static void http_client_request_do_submit(struct http_client_request *req)
 {
 	struct http_client *client = req->client;
 	struct http_client_host *host;
 	const struct http_url *proxy_url = client->set.proxy_url;
-	const char *target;
+	const char *authority, *target;
 
 	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
 
-	target = t_strconcat
-		(http_url_create_host(&req->origin_url), req->target, NULL);
+	authority = http_url_create_authority(&req->origin_url);
+	if (req->connect_tunnel) {
+		/* connect requests require authority form for request target */
+		target = authority;
+	} else {
+		/* absolute target url */
+		target = t_strconcat
+			(http_url_create_host(&req->origin_url), req->target, NULL);
+	}
 
 	/* determine what host to contact to submit this request */
 	if (proxy_url != NULL) {
@@ -265,16 +310,23 @@
 		req->date = ioloop_time;
 	
 	/* prepare value for Host header */
-	req->authority =
-		p_strdup(req->pool, http_url_create_authority(req->host_url));
+	req->authority = p_strdup(req->pool, authority);
 
 	/* debug label */
 	req->label = p_strdup_printf(req->pool, "[%s %s]", req->method, target);
 
-	/* request target needs to be made absolute url for proxy requests */
-	if (proxy_url != NULL)
+	/* update request target */
+	if (req->connect_tunnel || proxy_url != NULL)
 		req->target = p_strdup(req->pool, target);
 
+	if (proxy_url == NULL) {
+		/* if we don't have a proxy, CONNECT requests are handled by creating
+		   the requested connection directly */
+		req->connect_direct = req->connect_tunnel;
+		if (req->connect_direct)
+			req->urgent = TRUE;
+	}
+
 	host = http_client_host_get(req->client, req->host_url->host_name);
 	req->state = HTTP_REQUEST_STATE_QUEUED;
 
@@ -542,8 +594,13 @@
 	iov[0].iov_base = str_data(rtext);
 	iov[0].iov_len = str_len(rtext);	
 	/* explicit headers */
-	iov[1].iov_base = str_data(req->headers);
-	iov[1].iov_len = str_len(req->headers);
+	if (req->headers != NULL) {
+		iov[1].iov_base = str_data(req->headers);
+		iov[1].iov_len = str_len(req->headers);
+	} else {
+		iov[1].iov_base = "";
+		iov[1].iov_len = 0;
+	}
 	/* end of header */
 	iov[2].iov_base = "\r\n";
 	iov[2].iov_len = 2;
@@ -876,3 +933,11 @@
 	req->destroy_callback = callback;
 	req->destroy_context = context;
 }
+
+void http_client_request_start_tunnel(struct http_client_request *req,
+	struct http_client_tunnel *tunnel)
+{
+	i_assert(req->state == HTTP_REQUEST_STATE_GOT_RESPONSE);
+
+	http_client_connection_start_tunnel(&req->conn, tunnel);
+}
--- a/src/lib-http/http-client.h	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-client.h	Sat Oct 12 11:05:08 2013 +0300
@@ -89,6 +89,12 @@
 	bool debug;
 };
 
+struct http_client_tunnel {
+	int fd_in, fd_out;
+	struct istream *input;
+	struct ostream *output;
+};
+
 typedef void
 http_client_request_callback_t(const struct http_response *response,
 			       void *context);
@@ -96,6 +102,7 @@
 struct http_client *http_client_init(const struct http_client_settings *set);
 void http_client_deinit(struct http_client **_client);
 
+/* create new HTTP request */
 struct http_client_request *
 http_client_request(struct http_client *client,
 		    const char *method, const char *host, const char *target,
@@ -116,6 +123,22 @@
 			const struct http_response *response, typeof(context))), \
 		(http_client_request_callback_t *)callback, context)
 
+/* create new HTTP CONNECT request. If this HTTP is configured to use a proxy,
+   a CONNECT request will be submitted at that proxy, otherwise the connection
+   is created directly. Call http_client_request_start_tunnel() to
+   to take over the connection.
+ */
+struct http_client_request *
+http_client_request_connect(struct http_client *client,
+		    const char *host, in_port_t port,
+		    http_client_request_callback_t *callback,
+		    void *context);
+#define http_client_request_connect(client, host, port, callback, context) \
+	http_client_request_connect(client, host, port + \
+		CALLBACK_TYPECHECK(callback, void (*)( \
+			const struct http_response *response, typeof(context))), \
+		(http_client_request_callback_t *)callback, context)
+
 void http_client_request_set_port(struct http_client_request *req,
 	in_port_t port);
 void http_client_request_set_ssl(struct http_client_request *req,
@@ -134,6 +157,7 @@
 http_client_request_get_state(struct http_client_request *req);
 void http_client_request_submit(struct http_client_request *req);
 bool http_client_request_try_retry(struct http_client_request *req);
+
 void http_client_request_abort(struct http_client_request **req);
 
 /* Call the specified callback when HTTP request is destroyed. */
@@ -148,6 +172,9 @@
 	const unsigned char *data, size_t size);
 int http_client_request_finish_payload(struct http_client_request **req);
 
+void http_client_request_start_tunnel(struct http_client_request *req,
+	struct http_client_tunnel *tunnel);
+
 void http_client_switch_ioloop(struct http_client *client);
 
 /* blocks until all currently submitted requests are handled */
--- a/src/lib-http/http-response-parser.c	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-response-parser.c	Sat Oct 12 11:05:08 2013 +0300
@@ -233,8 +233,8 @@
 }
 
 int http_response_parse_next(struct http_response_parser *parser,
-			     bool no_payload, struct http_response *response,
-			     const char **error_r)
+			     enum http_response_payload_type payload_type,
+			     struct http_response *response, const char **error_r)
 {
 	int ret;
 
@@ -287,10 +287,12 @@
 	 */
 	if (parser->response_status / 100 == 1 || parser->response_status == 204
 		|| parser->response_status == 304) { // HEAD is handled in caller
-		no_payload = TRUE;
+		payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_NOT_PRESENT;
 	}
 
-	if (!no_payload) {
+	if ((payload_type == HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED) ||
+		(payload_type == HTTP_RESPONSE_PAYLOAD_TYPE_ONLY_UNSUCCESSFUL &&
+			parser->response_status / 100 != 2)) {
 		/* [ message-body ] */
 		if (http_message_parse_body(&parser->parser, FALSE) < 0) {
 			*error_r = parser->parser.error;
--- a/src/lib-http/http-response-parser.h	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-response-parser.h	Sat Oct 12 11:05:08 2013 +0300
@@ -12,7 +12,7 @@
 void http_response_parser_deinit(struct http_response_parser **_parser);
 
 int http_response_parse_next(struct http_response_parser *parser,
-			     bool no_payload, struct http_response *response,
-			     const char **error_r);
+			     enum http_response_payload_type payload_type,
+			     struct http_response *response, const char **error_r);
 
 #endif
--- a/src/lib-http/http-response.h	Sat Oct 12 11:00:15 2013 +0300
+++ b/src/lib-http/http-response.h	Sat Oct 12 11:05:08 2013 +0300
@@ -7,6 +7,12 @@
 
 #define http_response_header http_header_field /* FIXME: remove in v2.3 */
 
+enum http_response_payload_type {
+	HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED,
+	HTTP_RESPONSE_PAYLOAD_TYPE_NOT_PRESENT,
+	HTTP_RESPONSE_PAYLOAD_TYPE_ONLY_UNSUCCESSFUL
+};
+
 struct http_response {
 	unsigned char version_major;
 	unsigned char version_minor;