changeset 17104:fb4a0a84da50

lib-compression: Added initial support for LZ4 There's no standard file format for LZ4, so we created our own. The code has had only minimal testing currently, so there may be bugs.
author Timo Sirainen <tss@iki.fi>
date Wed, 15 Jan 2014 00:57:59 +0200
parents 3f3c9f93a0b3
children ccb3535bf650
files configure.ac src/lib-compression/Makefile.am src/lib-compression/compression.c src/lib-compression/iostream-lz4.h src/lib-compression/istream-lz4.c src/lib-compression/istream-zlib.h src/lib-compression/ostream-lz4.c src/lib-compression/ostream-zlib.h
diffstat 8 files changed, 584 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/configure.ac	Wed Jan 15 00:28:35 2014 +0200
+++ b/configure.ac	Wed Jan 15 00:57:59 2014 +0200
@@ -184,6 +184,11 @@
   TEST_WITH(lzma, $withval),
   want_lzma=auto)
 
+AC_ARG_WITH(lz4,
+AS_HELP_STRING([--with-lz4], [Build with LZ4 compression support]),
+  TEST_WITH(lz4, $withval),
+  want_lz4=auto)
+
 AC_ARG_WITH(libcap,
 AS_HELP_STRING([--with-libcap], [Build with libcap support (Dropping capabilities).]),
   TEST_WITH(libcap, $withval),
@@ -2691,6 +2696,27 @@
   ])
 fi
 AC_SUBST(COMPRESS_LIBS)
+
+if test "$want_lz4" != "no"; then
+  AC_CHECK_HEADER(lz4.h, [
+    AC_CHECK_LIB(lz4, LZ4_compress, [
+      have_lz4=yes
+      have_compress_lib=yes
+      AC_DEFINE(HAVE_LZ4,, Define if you have lz4 library)
+      COMPRESS_LIBS="$COMPRESS_LIBS -llz4"
+    ], [
+      if test "$want_lz4" = "yes"; then
+	AC_ERROR([Can't build with lz4 support: liblz4 not found])
+      fi
+    ])
+  ], [
+    if test "$want_lz4" = "yes"; then
+      AC_ERROR([Can't build with lz4 support: lz4.h not found])
+    fi
+  ])
+fi
+AC_SUBST(COMPRESS_LIBS)
+
 AM_CONDITIONAL(BUILD_ZLIB_PLUGIN, test "$have_compress_lib" = "yes")
 
 RPCGEN=${RPCGEN-rpcgen}
--- a/src/lib-compression/Makefile.am	Wed Jan 15 00:28:35 2014 +0200
+++ b/src/lib-compression/Makefile.am	Wed Jan 15 00:57:59 2014 +0200
@@ -7,9 +7,11 @@
 libcompression_la_SOURCES = \
 	compression.c \
 	istream-lzma.c \
+	istream-lz4.c \
 	istream-zlib.c \
 	istream-bzlib.c \
 	ostream-lzma.c \
+	ostream-lz4.c \
 	ostream-zlib.c \
 	ostream-bzlib.c
 libcompression_la_LIBADD = \
--- a/src/lib-compression/compression.c	Wed Jan 15 00:28:35 2014 +0200
+++ b/src/lib-compression/compression.c	Wed Jan 15 00:57:59 2014 +0200
@@ -4,6 +4,7 @@
 #include "istream.h"
 #include "istream-zlib.h"
 #include "ostream-zlib.h"
+#include "iostream-lz4.h"
 #include "compression.h"
 
 #ifndef HAVE_ZLIB
@@ -20,6 +21,10 @@
 #  define i_stream_create_lzma NULL
 #  define o_stream_create_lzma NULL
 #endif
+#ifndef HAVE_LZ4
+#  define i_stream_create_lz4 NULL
+#  define o_stream_create_lz4 NULL
+#endif
 
 static bool is_compressed_zlib(struct istream *input)
 {
@@ -63,6 +68,17 @@
 	return memcmp(data, "\xfd\x37\x7a\x58\x5a", 6) == 0;
 }
 
+static bool is_compressed_lz4(struct istream *input)
+{
+	const unsigned char *data;
+	size_t size;
+
+	if (i_stream_read_data(input, &data, &size, 6 - 1) <= 0)
+		return FALSE;
+	/* there is no standard LZ4 header, so we've created our own */
+	return memcmp(data, IOSTREAM_LZ4_MAGIC, IOSTREAM_LZ4_MAGIC_LEN) == 0;
+}
+
 const struct compression_handler *compression_lookup_handler(const char *name)
 {
 	unsigned int i;
@@ -113,5 +129,7 @@
 	  i_stream_create_deflate, o_stream_create_deflate },
 	{ "xz", ".xz", is_compressed_xz,
 	  i_stream_create_lzma, o_stream_create_lzma },
+	{ "lz4", ".lz4", is_compressed_lz4,
+	  i_stream_create_lz4, o_stream_create_lz4 },
 	{ NULL, NULL, NULL, NULL, NULL }
 };
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-compression/iostream-lz4.h	Wed Jan 15 00:57:59 2014 +0200
@@ -0,0 +1,30 @@
+#ifndef IOSTREAM_LZ4_H
+#define IOSTREAM_LZ4_H
+
+/*
+   Dovecot's LZ4 compressed files contain:
+
+   IOSTREAM_LZ4_HEADER
+   n x (4 byte big-endian: compressed chunk length, compressed chunk)
+*/
+
+#define IOSTREAM_LZ4_MAGIC "Dovecot-LZ4\x0d\x2a\x9b\xc5"
+#define IOSTREAM_LZ4_MAGIC_LEN (sizeof(IOSTREAM_LZ4_MAGIC)-1)
+
+struct iostream_lz4_header {
+	unsigned char magic[IOSTREAM_LZ4_MAGIC_LEN];
+	/* OSTREAM_LZ4_CHUNK_SIZE in big-endian */
+	unsigned char max_uncompressed_chunk_size[4];
+};
+
+/* How large chunks we're buffering into memory before compressing them */
+#define OSTREAM_LZ4_CHUNK_SIZE (1024*64)
+/* How large chunks we allow in input data before returning a failure.
+   This must be at least OSTREAM_LZ4_CHUNK_SIZE, but for future compatibility
+   should be somewhat higher (but not too high to avoid wasting memory for
+   corrupted files). */
+#define ISTREAM_LZ4_CHUNK_SIZE (1024*1024)
+
+#define IOSTREAM_LZ4_CHUNK_PREFIX_LEN 4 /* big-endian size of chunk */
+
+#endif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-compression/istream-lz4.c	Wed Jan 15 00:57:59 2014 +0200
@@ -0,0 +1,318 @@
+/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZ4
+
+#include "buffer.h"
+#include "istream-private.h"
+#include "istream-zlib.h"
+#include "iostream-lz4.h"
+#include <lz4.h>
+
+struct lz4_istream {
+	struct istream_private istream;
+
+	uoff_t stream_size;
+	struct stat last_parent_statbuf;
+
+	buffer_t *chunk_buf;
+	uint32_t chunk_size, chunk_left, max_uncompressed_chunk_size;
+
+	unsigned int log_errors:1;
+	unsigned int marked:1;
+	unsigned int header_read:1;
+};
+
+static void i_stream_lz4_close(struct iostream_private *stream,
+			       bool close_parent)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *)stream;
+
+	buffer_free(&zstream->chunk_buf);
+	if (close_parent)
+		i_stream_close(zstream->istream.parent);
+}
+
+static void lz4_read_error(struct lz4_istream *zstream, const char *error)
+{
+	io_stream_set_error(&zstream->istream.iostream,
+			    "lz4.read(%s): %s at %"PRIuUOFF_T,
+			    i_stream_get_name(&zstream->istream.istream), error,
+			    zstream->istream.abs_start_offset +
+			    zstream->istream.istream.v_offset);
+	if (zstream->log_errors)
+		i_error("%s", zstream->istream.iostream.error);
+}
+
+static int i_stream_lz4_read_header(struct lz4_istream *zstream)
+{
+	const struct iostream_lz4_header *hdr;
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	ret = i_stream_read_data(zstream->istream.parent, &data, &size,
+				 sizeof(*hdr)-1);
+	if (ret < 0) {
+		zstream->istream.istream.stream_errno =
+			zstream->istream.parent->stream_errno;
+		return ret;
+	}
+	if (ret == 0 && !zstream->istream.istream.eof)
+		return 0;
+	hdr = (const void *)data;
+	if (ret == 0 || memcmp(hdr->magic, IOSTREAM_LZ4_MAGIC,
+			       IOSTREAM_LZ4_MAGIC_LEN) != 0) {
+		lz4_read_error(zstream, "wrong magic in header (not lz4 file?)");
+		zstream->istream.istream.stream_errno = EINVAL;
+		return -1;
+	}
+	zstream->max_uncompressed_chunk_size =
+		((uint32_t)hdr->max_uncompressed_chunk_size[0] << 24) |
+		(hdr->max_uncompressed_chunk_size[1] << 16) |
+		(hdr->max_uncompressed_chunk_size[2] << 8) |
+		hdr->max_uncompressed_chunk_size[3];
+	if (zstream->max_uncompressed_chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
+		lz4_read_error(zstream, t_strdup_printf(
+			"lz4 max chunk size too large (%u > %u)",
+			zstream->max_uncompressed_chunk_size,
+			ISTREAM_LZ4_CHUNK_SIZE));
+		zstream->istream.istream.stream_errno = EINVAL;
+		return -1;
+	}
+	i_stream_skip(zstream->istream.parent, sizeof(*hdr));
+	return 1;
+}
+
+static ssize_t i_stream_lz4_read(struct istream_private *stream)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *)stream;
+	const unsigned char *data;
+	size_t size, max_size;
+	int ret;
+
+	if (!zstream->header_read) {
+		if ((ret = i_stream_lz4_read_header(zstream)) <= 0)
+			return ret;
+		zstream->header_read = TRUE;
+	}
+
+	if (zstream->chunk_left == 0) {
+		ret = i_stream_read_data(stream->parent, &data, &size,
+					 IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+		if (ret < 0) {
+			stream->istream.stream_errno =
+				stream->parent->stream_errno;
+			if (stream->istream.stream_errno != 0) {
+				stream->istream.eof = TRUE;
+				zstream->stream_size = stream->istream.v_offset +
+					stream->pos - stream->skip;
+			}
+			return ret;
+		}
+		if (ret == 0 && !stream->istream.eof)
+			return 0;
+		zstream->chunk_size = zstream->chunk_left =
+			((uint32_t)data[0] << 24) |
+			(data[1] << 16) | (data[2] << 8) | data[3];
+		if (zstream->chunk_size == 0 ||
+		    zstream->chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
+			lz4_read_error(zstream, t_strdup_printf(
+				"invalid lz4 chunk size: %u", zstream->chunk_size));
+			stream->istream.stream_errno = EINVAL;
+			return -1;
+		}
+		i_stream_skip(stream->parent, IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+		buffer_set_used_size(zstream->chunk_buf, 0);
+	}
+
+	/* read the whole compressed chunk into memory */
+	while (zstream->chunk_left > 0 &&
+	       (ret = i_stream_read_data(zstream->istream.parent,
+					 &data, &size, 0)) > 0) {
+		if (size > zstream->chunk_left)
+			size = zstream->chunk_left;
+		buffer_append(zstream->chunk_buf, data, size);
+		i_stream_skip(zstream->istream.parent, size);
+		zstream->chunk_left -= size;
+	}
+	if (zstream->chunk_left > 0) {
+		if (ret == -1 && zstream->istream.parent->stream_errno == 0) {
+			lz4_read_error(zstream, "truncated lz4 chunk");
+			stream->istream.stream_errno = EINVAL;
+			return -1;
+		}
+		zstream->istream.istream.stream_errno =
+			zstream->istream.parent->stream_errno;
+		return ret;
+	}
+	/* if we already have max_buffer_size amount of data, fail here */
+	i_stream_compress(stream);
+	if (stream->pos >= stream->max_buffer_size)
+		return -2;
+	/* allocate enough space for the old data and the new
+	   decompressed chunk. we don't know the original compressed size,
+	   so just allocate the max amount of memory. */
+	max_size = stream->pos + zstream->max_uncompressed_chunk_size;
+	if (stream->buffer_size < max_size) {
+		stream->w_buffer = i_realloc(stream->w_buffer,
+					     stream->buffer_size, max_size);
+		stream->buffer_size = max_size;
+		stream->buffer = stream->w_buffer;
+	}
+	ret = LZ4_decompress_safe(zstream->chunk_buf->data,
+				  (void *)(stream->w_buffer + stream->pos),
+				  zstream->chunk_buf->used,
+				  stream->buffer_size - stream->pos);
+	i_assert(ret <= (int)zstream->max_uncompressed_chunk_size);
+	if (ret < 0) {
+		lz4_read_error(zstream, "corrupted lz4 chunk");
+		stream->istream.stream_errno = EINVAL;
+		return -1;
+	}
+	i_assert(ret > 0);
+	stream->pos += ret;
+	i_assert(stream->pos <= stream->buffer_size);
+	return ret;
+}
+
+static void i_stream_lz4_reset(struct lz4_istream *zstream)
+{
+	struct istream_private *stream = &zstream->istream;
+
+	i_stream_seek(stream->parent, stream->parent_start_offset);
+	zstream->header_read = FALSE;
+	zstream->chunk_size = zstream->chunk_left = 0;
+
+	stream->parent_expected_offset = stream->parent_start_offset;
+	stream->skip = stream->pos = 0;
+	stream->istream.v_offset = 0;
+}
+
+static void
+i_stream_lz4_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *) stream;
+	uoff_t start_offset = stream->istream.v_offset - stream->skip;
+
+	if (v_offset < start_offset) {
+		/* have to seek backwards */
+		i_stream_lz4_reset(zstream);
+		start_offset = 0;
+	}
+
+	if (v_offset <= start_offset + stream->pos) {
+		/* seeking backwards within what's already cached */
+		stream->skip = v_offset - start_offset;
+		stream->istream.v_offset = v_offset;
+		stream->pos = stream->skip;
+	} else {
+		/* read and cache forward */
+		do {
+			size_t avail = stream->pos - stream->skip;
+
+			if (stream->istream.v_offset + avail >= v_offset) {
+				i_stream_skip(&stream->istream,
+					      v_offset -
+					      stream->istream.v_offset);
+				break;
+			}
+
+			i_stream_skip(&stream->istream, avail);
+		} while (i_stream_read(&stream->istream) >= 0);
+
+		if (stream->istream.v_offset != v_offset) {
+			/* some failure, we've broken it */
+			if (stream->istream.stream_errno != 0) {
+				i_error("lz4_istream.seek(%s) failed: %s",
+					i_stream_get_name(&stream->istream),
+					strerror(stream->istream.stream_errno));
+				i_stream_close(&stream->istream);
+			} else {
+				/* unexpected EOF. allow it since we may just
+				   want to check if there's anything.. */
+				i_assert(stream->istream.eof);
+			}
+		}
+	}
+
+	if (mark)
+		zstream->marked = TRUE;
+}
+
+static int
+i_stream_lz4_stat(struct istream_private *stream, bool exact)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *) stream;
+	const struct stat *st;
+	size_t size;
+
+	if (i_stream_stat(stream->parent, exact, &st) < 0)
+		return -1;
+	stream->statbuf = *st;
+
+	/* when exact=FALSE always return the parent stat's size, even if we
+	   know the exact value. this is necessary because otherwise e.g. mbox
+	   code can see two different values and think that a compressed mbox
+	   file keeps changing. */
+	if (!exact)
+		return 0;
+
+	if (zstream->stream_size == (uoff_t)-1) {
+		uoff_t old_offset = stream->istream.v_offset;
+
+		do {
+			size = i_stream_get_data_size(&stream->istream);
+			i_stream_skip(&stream->istream, size);
+		} while (i_stream_read(&stream->istream) > 0);
+
+		i_stream_seek(&stream->istream, old_offset);
+		if (zstream->stream_size == (uoff_t)-1)
+			return -1;
+	}
+	stream->statbuf.st_size = zstream->stream_size;
+	return 0;
+}
+
+static void i_stream_lz4_sync(struct istream_private *stream)
+{
+	struct lz4_istream *zstream = (struct lz4_istream *) stream;
+	const struct stat *st;
+
+	if (i_stream_stat(stream->parent, FALSE, &st) < 0) {
+		if (memcmp(&zstream->last_parent_statbuf,
+			   st, sizeof(*st)) == 0) {
+			/* a compressed file doesn't change unexpectedly,
+			   don't clear our caches unnecessarily */
+			return;
+		}
+		zstream->last_parent_statbuf = *st;
+	}
+	i_stream_lz4_reset(zstream);
+}
+
+struct istream *i_stream_create_lz4(struct istream *input, bool log_errors)
+{
+	struct lz4_istream *zstream;
+
+	zstream = i_new(struct lz4_istream, 1);
+	zstream->stream_size = (uoff_t)-1;
+	zstream->log_errors = log_errors;
+
+	zstream->istream.iostream.close = i_stream_lz4_close;
+	zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+	zstream->istream.read = i_stream_lz4_read;
+	zstream->istream.seek = i_stream_lz4_seek;
+	zstream->istream.stat = i_stream_lz4_stat;
+	zstream->istream.sync = i_stream_lz4_sync;
+
+	zstream->istream.istream.readable_fd = FALSE;
+	zstream->istream.istream.blocking = input->blocking;
+	zstream->istream.istream.seekable = input->seekable;
+	zstream->chunk_buf = buffer_create_dynamic(default_pool, 1024);
+
+	return i_stream_create(&zstream->istream, input,
+			       i_stream_get_fd(input));
+}
+#endif
--- a/src/lib-compression/istream-zlib.h	Wed Jan 15 00:28:35 2014 +0200
+++ b/src/lib-compression/istream-zlib.h	Wed Jan 15 00:57:59 2014 +0200
@@ -5,5 +5,6 @@
 struct istream *i_stream_create_deflate(struct istream *input, bool log_errors);
 struct istream *i_stream_create_bz2(struct istream *input, bool log_errors);
 struct istream *i_stream_create_lzma(struct istream *input, bool log_errors);
+struct istream *i_stream_create_lz4(struct istream *input, bool log_errors);
 
 #endif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-compression/ostream-lz4.c	Wed Jan 15 00:57:59 2014 +0200
@@ -0,0 +1,188 @@
+/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZ4
+
+#include "ostream-private.h"
+#include "ostream-zlib.h"
+#include "iostream-lz4.h"
+#include <lz4.h>
+
+#define CHUNK_SIZE OSTREAM_LZ4_CHUNK_SIZE
+
+struct lz4_ostream {
+	struct ostream_private ostream;
+
+	unsigned char compressbuf[CHUNK_SIZE];
+	unsigned int compressbuf_offset;
+
+	/* chunk size, followed by compressed data */
+	unsigned char outbuf[IOSTREAM_LZ4_CHUNK_PREFIX_LEN + CHUNK_SIZE];
+	unsigned int outbuf_offset, outbuf_used;
+};
+
+static void o_stream_lz4_close(struct iostream_private *stream,
+			       bool close_parent)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+
+	(void)o_stream_flush(&zstream->ostream.ostream);
+	if (close_parent)
+		o_stream_close(zstream->ostream.parent);
+}
+
+static int o_stream_lz4_send_outbuf(struct lz4_ostream *zstream)
+{
+	ssize_t ret;
+	size_t size;
+
+	if (zstream->outbuf_used == 0)
+		return 1;
+
+	size = zstream->outbuf_used - zstream->outbuf_offset;
+	i_assert(size > 0);
+	ret = o_stream_send(zstream->ostream.parent,
+			    zstream->outbuf + zstream->outbuf_offset, size);
+	if (ret < 0) {
+		o_stream_copy_error_from_parent(&zstream->ostream);
+		return -1;
+	}
+	if ((size_t)ret != size) {
+		zstream->outbuf_offset += ret;
+		return 0;
+	}
+	zstream->outbuf_offset = 0;
+	zstream->outbuf_used = 0;
+	return 1;
+}
+
+static int o_stream_lz4_compress(struct lz4_ostream *zstream)
+{
+	uint32_t chunk_size;
+	int ret;
+
+	if (zstream->compressbuf_offset == 0)
+		return 1;
+	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0)
+		return ret;
+
+	i_assert(zstream->outbuf_offset == 0);
+	i_assert(zstream->outbuf_used == 0);
+
+	zstream->outbuf_used = IOSTREAM_LZ4_CHUNK_PREFIX_LEN +
+		LZ4_compress((void *)zstream->compressbuf,
+			     (void *)(zstream->outbuf + IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
+			     zstream->compressbuf_offset);
+	i_assert(zstream->outbuf_used > IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+	chunk_size = zstream->outbuf_used - IOSTREAM_LZ4_CHUNK_PREFIX_LEN;
+	zstream->outbuf[0] = (chunk_size & 0xff000000) >> 24;
+	zstream->outbuf[1] = (chunk_size & 0x00ff0000) >> 16;
+	zstream->outbuf[2] = (chunk_size & 0x0000ff00) >> 8;
+	zstream->outbuf[3] = (chunk_size & 0x000000ff);
+	zstream->compressbuf_offset = 0;
+	return 1;
+}
+
+static ssize_t
+o_stream_lz4_send_chunk(struct lz4_ostream *zstream,
+			const void *data, size_t size)
+{
+	size_t max_size;
+	ssize_t added_bytes = 0;
+	int ret;
+
+	i_assert(zstream->outbuf_used == 0);
+
+	do {
+		max_size = I_MIN(size, sizeof(zstream->compressbuf) -
+				 zstream->compressbuf_offset);
+		memcpy(zstream->compressbuf + zstream->compressbuf_offset,
+		       data, max_size);
+		zstream->compressbuf_offset += max_size;
+
+		data = CONST_PTR_OFFSET(data, max_size);
+		size -= max_size;
+		added_bytes += max_size;
+
+		if (zstream->compressbuf_offset == sizeof(zstream->compressbuf)) {
+			ret = o_stream_lz4_compress(zstream);
+			if (ret <= 0)
+				return added_bytes != 0 ? added_bytes : ret;
+		}
+	} while (size > 0);
+
+	return added_bytes;
+}
+
+static int o_stream_lz4_flush(struct ostream_private *stream)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+	int ret;
+
+	if (o_stream_lz4_compress(zstream) < 0)
+		return -1;
+	if (o_stream_lz4_send_outbuf(zstream) < 0)
+		return -1;
+
+	ret = o_stream_flush(stream->parent);
+	if (ret < 0)
+		o_stream_copy_error_from_parent(stream);
+	return ret;
+}
+
+static ssize_t
+o_stream_lz4_sendv(struct ostream_private *stream,
+		    const struct const_iovec *iov, unsigned int iov_count)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+	ssize_t ret, bytes = 0;
+	unsigned int i;
+
+	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0) {
+		/* error / we still couldn't flush existing data to
+		   parent stream. */
+		return ret;
+	}
+
+	for (i = 0; i < iov_count; i++) {
+		ret = o_stream_lz4_send_chunk(zstream, iov[i].iov_base,
+					      iov[i].iov_len);
+		if (ret < 0)
+			return -1;
+		bytes += ret;
+		if ((size_t)ret != iov[i].iov_len)
+			break;
+	}
+	stream->ostream.offset += bytes;
+	return bytes;
+}
+
+struct ostream *o_stream_create_lz4(struct ostream *output, int level)
+{
+	struct iostream_lz4_header *hdr;
+	struct lz4_ostream *zstream;
+
+	i_assert(level >= 1 && level <= 9);
+
+	zstream = i_new(struct lz4_ostream, 1);
+	zstream->ostream.sendv = o_stream_lz4_sendv;
+	zstream->ostream.flush = o_stream_lz4_flush;
+	zstream->ostream.iostream.close = o_stream_lz4_close;
+
+	i_assert(sizeof(zstream->outbuf) >= sizeof(*hdr));
+	hdr = (void *)zstream->outbuf;
+	memcpy(hdr->magic, IOSTREAM_LZ4_MAGIC, sizeof(hdr->magic));
+	hdr->max_uncompressed_chunk_size[0] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0xff000000) >> 24;
+	hdr->max_uncompressed_chunk_size[1] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x00ff0000) >> 16;
+	hdr->max_uncompressed_chunk_size[2] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x0000ff00) >> 8;
+	hdr->max_uncompressed_chunk_size[3] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x000000ff);
+	zstream->outbuf_used = sizeof(*hdr);
+	return o_stream_create(&zstream->ostream, output,
+			       o_stream_get_fd(output));
+}
+#endif
--- a/src/lib-compression/ostream-zlib.h	Wed Jan 15 00:28:35 2014 +0200
+++ b/src/lib-compression/ostream-zlib.h	Wed Jan 15 00:57:59 2014 +0200
@@ -5,5 +5,6 @@
 struct ostream *o_stream_create_deflate(struct ostream *output, int level);
 struct ostream *o_stream_create_bz2(struct ostream *output, int level);
 struct ostream *o_stream_create_lzma(struct ostream *output, int level);
+struct ostream *o_stream_create_lz4(struct ostream *output, int level);
 
 #endif