Mercurial > dovecot > core-2.2
changeset 18952:4d7a83ddb644
lib-master: Added support for HAProxy protocol.
Patch by Stephan Bosch - with some small changes.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Tue, 18 Aug 2015 20:39:06 +0300 |
parents | 52368e60177c |
children | f73ed907fe64 |
files | src/lib-master/Makefile.am src/lib-master/master-service-haproxy.c src/lib-master/master-service-private.h src/lib-master/master-service-settings.c src/lib-master/master-service-settings.h src/lib-master/master-service.c src/lib-master/service-settings.h src/master/master-settings.c src/master/service-process.c |
diffstat | 9 files changed, 510 insertions(+), 3 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-master/Makefile.am Tue Aug 18 20:23:45 2015 +0300 +++ b/src/lib-master/Makefile.am Tue Aug 18 20:39:06 2015 +0300 @@ -21,6 +21,7 @@ master-login.c \ master-login-auth.c \ master-service.c \ + master-service-haproxy.c \ master-service-settings.c \ master-service-settings-cache.c \ master-service-ssl.c \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-master/master-service-haproxy.c Tue Aug 18 20:39:06 2015 +0300 @@ -0,0 +1,476 @@ +/* Copyright (c) 2013-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "llist.h" +#include "ioloop.h" +#include "str-sanitize.h" +#include "master-service-private.h" +#include "master-service-settings.h" + +#define HAPROXY_V1_MAX_HEADER_SIZE (108) + +enum { + HAPROXY_CMD_LOCAL = 0x00, + HAPROXY_CMD_PROXY = 0x01 +}; + +enum { + HAPROXY_AF_UNSPEC = 0x00, + HAPROXY_AF_INET = 0x01, + HAPROXY_AF_INET6 = 0x02, + HAPROXY_AF_UNIX = 0x03 +}; + +enum { + HAPROXY_SOCK_UNSPEC = 0x00, + HAPROXY_SOCK_STREAM = 0x01, + HAPROXY_SOCK_DGRAM = 0x02 +}; + +static const char haproxy_v2sig[12] = + "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A"; + +struct haproxy_header_v2 { + uint8_t sig[12]; + uint8_t ver_cmd; + uint8_t fam; + uint16_t len; +}; + +struct haproxy_data_v2 { + union { + struct { /* for TCP/UDP over IPv4, len = 12 */ + uint32_t src_addr; + uint32_t dst_addr; + uint16_t src_port; + uint16_t dst_port; + } ip4; + struct { /* for TCP/UDP over IPv6, len = 36 */ + uint8_t src_addr[16]; + uint8_t dst_addr[16]; + uint16_t src_port; + uint16_t dst_port; + } ip6; + struct { /* for AF_UNIX sockets, len = 216 */ + uint8_t src_addr[108]; + uint8_t dst_addr[108]; + } unx; + } addr; +}; + +struct master_service_haproxy_conn { + struct master_service_connection conn; + + struct master_service_haproxy_conn *prev, *next; + + struct master_service *service; + + struct io *io; + struct timeout *to; +}; + +static void +master_service_haproxy_conn_free(struct master_service_haproxy_conn *hpconn) +{ + struct master_service *service = hpconn->service; + + DLLIST_REMOVE(&service->haproxy_conns, hpconn); + + if (hpconn->io != NULL) + io_remove(&hpconn->io); + if (hpconn->to != NULL) + timeout_remove(&hpconn->to); + i_free(hpconn); +} + +static void +master_service_haproxy_conn_failure(struct master_service_haproxy_conn *hpconn) +{ + struct master_service *service = hpconn->service; + struct master_service_connection conn = hpconn->conn; + + master_service_haproxy_conn_free(hpconn); + master_service_client_connection_handled(service, &conn); +} + +static void +master_service_haproxy_conn_success(struct master_service_haproxy_conn *hpconn) +{ + struct master_service *service = hpconn->service; + struct master_service_connection conn = hpconn->conn; + + master_service_haproxy_conn_free(hpconn); + master_service_client_connection_callback(service, &conn); +} + +static void +master_service_haproxy_timeout(struct master_service_haproxy_conn *hpconn) +{ + i_error("haproxy: Client timed out (rip=%s)", + net_ip2addr(&hpconn->conn.remote_ip)); + master_service_haproxy_conn_failure(hpconn); +} + +static int +master_service_haproxy_read(struct master_service_haproxy_conn *hpconn) +{ + static union { + unsigned char v1_data[HAPROXY_V1_MAX_HEADER_SIZE]; + struct { + const struct haproxy_header_v2 hdr; + const struct haproxy_data_v2 data; + } v2; + } buf; + struct ip_addr *real_remote_ip = &hpconn->conn.remote_ip; + int fd = hpconn->conn.fd; + struct ip_addr local_ip, remote_ip; + unsigned int local_port, remote_port; + size_t size; + ssize_t ret; + + /* the protocol specification explicitly states that the protocol header + must be sent as one TCP frame, meaning that we will get it in full + with the first recv() call. + FIXME: still, it would be cleaner to allow reading it incrementally. + */ + do { + ret = recv(fd, &buf, sizeof(buf), MSG_PEEK); + } while (ret < 0 && errno == EINTR); + + if (ret < 0 && errno == EAGAIN) + return 0; + if (ret <= 0) { + i_info("haproxy: Client disconnected (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + + /* don't update true connection data until we succeed */ + local_ip = hpconn->conn.local_ip; + remote_ip = hpconn->conn.remote_ip; + local_port = hpconn->conn.local_port; + remote_port = hpconn->conn.remote_port; + + /* protocol version 2 */ + if (ret >= (ssize_t)sizeof(buf.v2.hdr) && + memcmp(buf.v2.hdr.sig, haproxy_v2sig, + sizeof(buf.v2.hdr.sig)) == 0) { + const struct haproxy_header_v2 *hdr = &buf.v2.hdr; + const struct haproxy_data_v2 *data = &buf.v2.data; + size_t hdr_len; + + if ((hdr->ver_cmd & 0xf0) != 0x20) { + i_error("haproxy: Client disconnected: " + "Unsupported protocol version (version=%02x, rip=%s)", + (hdr->ver_cmd & 0xf0) >> 4, + net_ip2addr(real_remote_ip)); + return -1; + } + + hdr_len = ntohs(hdr->len); + size = sizeof(*hdr) + hdr_len; + if (ret < (ssize_t)size) { + i_error("haproxy(v2): Client disconnected: " + "Protocol payload length does not match header " + "(got=%"PRIuSIZE_T", expect=%"PRIuSIZE_T", rip=%s)", + (size_t)ret, size, net_ip2addr(real_remote_ip)); + return -1; + } + + switch (hdr->ver_cmd & 0x0f) { + case HAPROXY_CMD_LOCAL: + /* keep local connection address for LOCAL */ + /*i_debug("haproxy(v2): Local connection (rip=%s)", + net_ip2addr(real_remote_ip));*/ + break; + case HAPROXY_CMD_PROXY: + if ((hdr->fam & 0x0f) != HAPROXY_SOCK_STREAM) { + /* UDP makes no sense currently */ + i_error("haproxy(v2): Client disconnected: " + "Not using TCP (type=%02x, rip=%s)", + (hdr->fam & 0x0f), net_ip2addr(real_remote_ip)); + return -1; + } + switch ((hdr->fam & 0xf0) >> 4) { + case HAPROXY_AF_INET: + /* IPv4 */ + if (hdr_len < sizeof(data->addr.ip4)) { + i_error("haproxy(v2): Client disconnected: " + "IPv4 data is incomplete (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + local_ip.family = AF_INET; + local_ip.u.ip4.s_addr = data->addr.ip4.dst_addr; + local_port = ntohs(data->addr.ip4.dst_port); + remote_ip.family = AF_INET; + remote_ip.u.ip4.s_addr = data->addr.ip4.src_addr; + remote_port = ntohs(data->addr.ip4.src_port); + break; + case HAPROXY_AF_INET6: + /* IPv6 */ + if (hdr_len < sizeof(data->addr.ip6)) { + i_error("haproxy(v2): Client disconnected: " + "IPv6 data is incomplete (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + local_ip.family = AF_INET6; + memcpy(&local_ip.u.ip6.s6_addr, data->addr.ip6.dst_addr, 16); + local_port = ntohs(data->addr.ip6.dst_port); + remote_ip.family = AF_INET6; + memcpy(&remote_ip.u.ip6.s6_addr, data->addr.ip6.src_addr, 16); + remote_port = ntohs(data->addr.ip6.src_port); + break; + case HAPROXY_AF_UNSPEC: + case HAPROXY_AF_UNIX: + /* unsupported; ignored */ + i_error("haproxy(v2): Unsupported address family " + "(family=%02x, rip=%s)", (hdr->fam & 0xf0) >> 4, + net_ip2addr(real_remote_ip)); + break; + default: + /* unsupported; error */ + i_error("haproxy(v2): Client disconnected: " + "Unknown address family " + "(family=%02x, rip=%s)", (hdr->fam & 0xf0) >> 4, + net_ip2addr(real_remote_ip)); + return -1; + } + break; + default: + i_error("haproxy(v2): Client disconnected: " + "Invalid command (cmd=%02x, rip=%s)", + (hdr->ver_cmd & 0x0f), + net_ip2addr(real_remote_ip)); + return -1; /* not a supported command */ + } + + // FIXME: TLV vectors are ignored + // (useful to see whether proxied client is using SSL) + + /* protocol version 1 (soon obsolete) */ + } else if (ret >= 8 && memcmp(buf.v1_data, "PROXY", 5) == 0) { + unsigned char *data = buf.v1_data, *end; + const char *const *fields; + unsigned int family = 0; + + /* find end of header line */ + end = memchr(data, '\r', ret - 1); + if (end == NULL || end[1] != '\n') + return -1; + *end = '\0'; + size = end + 2 - data; + + /* magic */ + fields = t_strsplit((char *)data, " "); + i_assert(strcmp(*fields, "PROXY") == 0); + fields++; + + /* protocol */ + if (*fields == NULL) { + i_error("haproxy(v1): Client disconnected: " + "Field for proxied protocol is missing " + "(rip=%s)", net_ip2addr(real_remote_ip)); + return -1; + } + if (strcmp(*fields, "TCP4") == 0) { + family = AF_INET; + } else if (strcmp(*fields, "TCP6") == 0) { + family = AF_INET6; + } else if (strcmp(*fields, "UNKNOWN") == 0) { + family = 0; + } else { + i_error("haproxy(v1): Client disconnected: " + "Unknown proxied protocol " + "(protocol=`%s', rip=%s)", str_sanitize(*fields, 64), + net_ip2addr(real_remote_ip)); + return -1; + } + fields++; + + if (family != 0) { + /* remote address */ + if (*fields == NULL) { + i_error("haproxy(v1): Client disconnected: " + "Field for proxied remote address is missing " + "(rip=%s)", net_ip2addr(real_remote_ip)); + return -1; + } + if (net_addr2ip(*fields, &remote_ip) < 0 || + remote_ip.family != family) { + i_error("haproxy(v1): Client disconnected: " + "Proxied remote address is invalid " + "(address=`%s', rip=%s)", str_sanitize(*fields, 64), + net_ip2addr(real_remote_ip)); + return -1; + } + fields++; + + /* local address */ + if (*fields == NULL) { + i_error("haproxy(v1): Client disconnected: " + "Field for proxied local address is missing " + "(rip=%s)", net_ip2addr(real_remote_ip)); + return -1; + } + if (net_addr2ip(*fields, &local_ip) < 0 || + local_ip.family != family) { + i_error("haproxy(v1): Client disconnected: " + "Proxied local address is invalid " + "(address=`%s', rip=%s)", str_sanitize(*fields, 64), + net_ip2addr(real_remote_ip)); + return -1; + } + fields++; + + /* remote port */ + if (*fields == NULL) { + i_error("haproxy(v1): Client disconnected: " + "Field for proxied local port is missing " + "(rip=%s)", net_ip2addr(real_remote_ip)); + return -1; + } + if (str_to_uint(*fields, &remote_port) < 0 || + remote_port > 65535) { + i_error("haproxy(v1): Client disconnected: " + "Proxied remote port is invalid " + "(port=`%s', rip=%s)", str_sanitize(*fields, 64), + net_ip2addr(real_remote_ip)); + return -1; + } + fields++; + + /* local port */ + if (*fields == NULL) { + i_error("haproxy(v1): Client disconnected: " + "Field for proxied local port is missing " + "(rip=%s)", net_ip2addr(real_remote_ip)); + return -1; + } + if (str_to_uint(*fields, &local_port) < 0 || + local_port > 65535) { + i_error("haproxy(v1): Client disconnected: " + "Proxied local port is invalid " + "(port=`%s', rip=%s)", str_sanitize(*fields, 64), + net_ip2addr(real_remote_ip)); + return -1; + } + fields++; + + if (*fields != NULL) { + i_error("haproxy(v1): Client disconnected: " + "Header line has spurius extra field " + "(field=`%s', rip=%s)", str_sanitize(*fields, 64), + net_ip2addr(real_remote_ip)); + return -1; + } + } + + /* invalid protocol */ + } else { + i_error("haproxy: Client disconnected: " + "No valid proxy header found (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + + /* remove proxy protocol header from socket buffer */ + i_assert(size <= sizeof(buf)); + do { + ret = recv(fd, &buf, size, 0); + } while (ret == -1 && errno == EINTR); + + if (ret <= 0) { + i_info("haproxy: Client disconnected (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + if (ret != (ssize_t)size) { + /* not supposed to happen */ + i_error("haproxy: Client disconencted: " + "Failed to read full header (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + + /* assign data from proxy */ + hpconn->conn.local_ip = local_ip; + hpconn->conn.remote_ip = remote_ip; + hpconn->conn.local_port = local_port; + hpconn->conn.remote_port = remote_port; + return 1; +} + +static void +master_service_haproxy_input(struct master_service_haproxy_conn *hpconn) +{ + int ret; + + if ((ret = master_service_haproxy_read(hpconn)) <= 0) { + if (ret < 0) + master_service_haproxy_conn_failure(hpconn); + } else { + master_service_haproxy_conn_success(hpconn); + } +} + +static bool +master_service_haproxy_conn_is_trusted(struct master_service *service, + struct master_service_connection *conn) +{ + const char *const *net; + struct ip_addr net_ip; + unsigned int bits; + + if (service->set->haproxy_trusted_networks == NULL) + return FALSE; + + net = t_strsplit_spaces(service->set->haproxy_trusted_networks, ", "); + for (; *net != NULL; net++) { + if (net_parse_range(*net, &net_ip, &bits) < 0) { + i_error("haproxy_trusted_networks: " + "Invalid network '%s'", *net); + break; + } + + if (net_is_in_network(&conn->real_remote_ip, &net_ip, bits)) + return TRUE; + } + return FALSE; +} + +void master_service_haproxy_new(struct master_service *service, + struct master_service_connection *conn) +{ + struct master_service_haproxy_conn *hpconn; + + if (!master_service_haproxy_conn_is_trusted(service, conn)) { + i_warning("haproxy: Client not trusted (rip=%s)", + net_ip2addr(&conn->real_remote_ip)); + master_service_client_connection_handled(service, conn); + return; + } + + hpconn = i_new(struct master_service_haproxy_conn, 1); + hpconn->conn = *conn; + hpconn->service = service; + DLLIST_PREPEND(&service->haproxy_conns, hpconn); + + hpconn->io = io_add(conn->fd, IO_READ, + master_service_haproxy_input, hpconn); + hpconn->to = timeout_add(service->set->haproxy_timeout*1000, + master_service_haproxy_timeout, hpconn); +} + +void master_service_haproxy_abort(struct master_service *service) +{ + while (service->haproxy_conns != NULL) { + int fd = service->haproxy_conns->conn.fd; + + if (close(fd) < 0) + i_error("haproxy: close(service connection) failed: %m"); + master_service_haproxy_conn_free(service->haproxy_conns); + } +} +
--- a/src/lib-master/master-service-private.h Tue Aug 18 20:23:45 2015 +0300 +++ b/src/lib-master/master-service-private.h Tue Aug 18 20:39:06 2015 +0300 @@ -4,12 +4,15 @@ #include "master-interface.h" #include "master-service.h" +struct master_service_haproxy_conn; + struct master_service_listener { struct master_service *service; char *name; /* settings */ bool ssl; + bool haproxy; /* state */ int fd; @@ -65,6 +68,8 @@ struct ssl_iostream_context *ssl_ctx; time_t ssl_params_last_refresh; + struct master_service_haproxy_conn *haproxy_conns; + unsigned int killed:1; unsigned int stopping:1; unsigned int keep_environment:1; @@ -90,4 +95,8 @@ void master_service_client_connection_callback(struct master_service *service, struct master_service_connection *conn); +void master_service_haproxy_new(struct master_service *service, + struct master_service_connection *conn); +void master_service_haproxy_abort(struct master_service *service); + #endif
--- a/src/lib-master/master-service-settings.c Tue Aug 18 20:23:45 2015 +0300 +++ b/src/lib-master/master-service-settings.c Tue Aug 18 20:39:06 2015 +0300 @@ -46,6 +46,9 @@ DEF(SET_BOOL, shutdown_clients), DEF(SET_BOOL, verbose_proctitle), + DEF(SET_STR, haproxy_trusted_networks), + DEF(SET_TIME, haproxy_timeout), + SETTING_DEFINE_LIST_END }; @@ -60,7 +63,10 @@ .config_cache_size = 1024*1024, .version_ignore = FALSE, .shutdown_clients = TRUE, - .verbose_proctitle = FALSE + .verbose_proctitle = FALSE, + + .haproxy_trusted_networks = "", + .haproxy_timeout = 3 }; const struct setting_parser_info master_service_setting_parser_info = {
--- a/src/lib-master/master-service-settings.h Tue Aug 18 20:23:45 2015 +0300 +++ b/src/lib-master/master-service-settings.h Tue Aug 18 20:39:06 2015 +0300 @@ -19,6 +19,9 @@ bool version_ignore; bool shutdown_clients; bool verbose_proctitle; + + const char *haproxy_trusted_networks; + unsigned int haproxy_timeout; }; struct master_service_settings_input {
--- a/src/lib-master/master-service.c Tue Aug 18 20:23:45 2015 +0300 +++ b/src/lib-master/master-service.c Tue Aug 18 20:39:06 2015 +0300 @@ -137,6 +137,8 @@ if (strcmp(*settings, "ssl") == 0) { l->ssl = TRUE; have_ssl_sockets = TRUE; + } else if (strcmp(*settings, "haproxy") == 0) { + l->haproxy = TRUE; } settings++; } @@ -845,6 +847,8 @@ *_service = NULL; + master_service_haproxy_abort(service); + master_service_io_listeners_remove(service); master_service_ssl_ctx_deinit(service); @@ -944,7 +948,10 @@ net_set_nonblock(conn.fd, TRUE); master_service_client_connection_created(service); - master_service_client_connection_callback(service, &conn); + if (l->haproxy) + master_service_haproxy_new(service, &conn); + else + master_service_client_connection_callback(service, &conn); } void master_service_io_listeners_add(struct master_service *service)
--- a/src/lib-master/service-settings.h Tue Aug 18 20:23:45 2015 +0300 +++ b/src/lib-master/service-settings.h Tue Aug 18 20:39:06 2015 +0300 @@ -32,6 +32,7 @@ unsigned int port; bool ssl; bool reuse_port; + bool haproxy; }; ARRAY_DEFINE_TYPE(inet_listener_settings, struct inet_listener_settings *);
--- a/src/master/master-settings.c Tue Aug 18 20:23:45 2015 +0300 +++ b/src/master/master-settings.c Tue Aug 18 20:39:06 2015 +0300 @@ -65,6 +65,7 @@ DEF(SET_UINT, port), DEF(SET_BOOL, ssl), DEF(SET_BOOL, reuse_port), + DEF(SET_BOOL, haproxy), SETTING_DEFINE_LIST_END }; @@ -74,7 +75,8 @@ .address = "", .port = 0, .ssl = FALSE, - .reuse_port = FALSE + .reuse_port = FALSE, + .haproxy = FALSE }; static const struct setting_parser_info inet_listener_setting_parser_info = {
--- a/src/master/service-process.c Tue Aug 18 20:23:45 2015 +0300 +++ b/src/master/service-process.c Tue Aug 18 20:39:06 2015 +0300 @@ -103,6 +103,8 @@ if (listeners[i]->type == SERVICE_LISTENER_INET) { if (listeners[i]->set.inetset.set->ssl) str_append(listener_settings, "\tssl"); + if (listeners[i]->set.inetset.set->haproxy) + str_append(listener_settings, "\thaproxy"); } dup2_append(&dups, listeners[i]->fd, fd++);