Mercurial > dovecot > core-2.2

annotate src/lib-compression/ostream-lz4.c @ 22325:e01bc3015b2f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lib-index: Check .log.2 rotation only when syncing Instead of also whenever appending transactions to .log file. This shouldn't change the behavior much, and it's needed for the following change to work correctly.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Tue, 11 Jul 2017 15:33:56 +0300
parents 2e2563132d5f
children cb108f786fb4
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
21390
2e2563132d5f Updated copyright notices to include the year 2017.
Stephan Bosch <stephan.bosch@dovecot.fi>
parents: 21359
diff changeset
1 /* Copyright (c) 2013-2017 Dovecot authors, see the included COPYING file */
17104
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
2
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
3 #include "lib.h"
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
4
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
5 #ifdef HAVE_LZ4
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
6
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
7 #include "ostream-private.h"
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
8 #include "ostream-zlib.h"
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
9 #include "iostream-lz4.h"
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
10 #include <lz4.h>
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
11
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
12 #define CHUNK_SIZE OSTREAM_LZ4_CHUNK_SIZE
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
13
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
14 struct lz4_ostream {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
15 struct ostream_private ostream;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
16
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
17 unsigned char compressbuf[CHUNK_SIZE];
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
18 unsigned int compressbuf_offset;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
19
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
20 /* chunk size, followed by compressed data */
17131
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
21 unsigned char outbuf[IOSTREAM_LZ4_CHUNK_PREFIX_LEN +
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
22 LZ4_COMPRESSBOUND(CHUNK_SIZE)];
17104
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
23 unsigned int outbuf_offset, outbuf_used;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
24 };
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
25
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
26 static void o_stream_lz4_close(struct iostream_private *stream,
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
27 bool close_parent)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
28 {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
29 struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
30
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
31 (void)o_stream_flush(&zstream->ostream.ostream);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
32 if (close_parent)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
33 o_stream_close(zstream->ostream.parent);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
34 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
35
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
36 static int o_stream_lz4_send_outbuf(struct lz4_ostream *zstream)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
37 {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
38 ssize_t ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
39 size_t size;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
40
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
41 if (zstream->outbuf_used == 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
42 return 1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
43
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
44 size = zstream->outbuf_used - zstream->outbuf_offset;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
45 i_assert(size > 0);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
46 ret = o_stream_send(zstream->ostream.parent,
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
47 zstream->outbuf + zstream->outbuf_offset, size);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
48 if (ret < 0) {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
49 o_stream_copy_error_from_parent(&zstream->ostream);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
50 return -1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
51 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
52 if ((size_t)ret != size) {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
53 zstream->outbuf_offset += ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
54 return 0;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
55 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
56 zstream->outbuf_offset = 0;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
57 zstream->outbuf_used = 0;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
58 return 1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
59 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
60
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
61 static int o_stream_lz4_compress(struct lz4_ostream *zstream)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
62 {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
63 uint32_t chunk_size;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
64 int ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
65
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
66 if (zstream->compressbuf_offset == 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
67 return 1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
68 if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
69 return ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
70
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
71 i_assert(zstream->outbuf_offset == 0);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
72 i_assert(zstream->outbuf_used == 0);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
73
21359
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
74 #if defined(HAVE_LZ4_COMPRESS_DEFAULT)
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
75 int max_dest_size = LZ4_compressBound(zstream->compressbuf_offset);
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
76 i_assert(max_dest_size >= 0);
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
77 if (max_dest_size == 0) {
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
78 io_stream_set_error(&zstream->ostream.iostream,
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
79 "lz4-compress: input size %u too large (> %u)",
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
80 zstream->compressbuf_offset, LZ4_MAX_INPUT_SIZE);
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
81 zstream->ostream.ostream.stream_errno = EINVAL;
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
82 return -1;
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
83 }
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
84 ret = LZ4_compress_default((void *)zstream->compressbuf,
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
85 (void *)(zstream->outbuf +
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
86 IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
87 zstream->compressbuf_offset,
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
88 max_dest_size);
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
89 #else
17131
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
90 ret = LZ4_compress((void *)zstream->compressbuf,
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
91 (void *)(zstream->outbuf +
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
92 IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
93 zstream->compressbuf_offset);
21359
3ebc6467c1a9 lib-compression: use LZ4_compress_default if can
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 19552
diff changeset
94 #endif /* defined(HAVE_LZ4_COMPRESS_DEFAULT) */
17131
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
95 i_assert(ret > 0 && (unsigned int)ret <= sizeof(zstream->outbuf) -
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
96 IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
e30597858a66 lib-compression: Add assert for LZ4_compress return value.
Teemu Huovila <teemu.huovila@dovecot.fi>
parents: 17119
diff changeset
97 zstream->outbuf_used = IOSTREAM_LZ4_CHUNK_PREFIX_LEN + ret;
17104
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
98 chunk_size = zstream->outbuf_used - IOSTREAM_LZ4_CHUNK_PREFIX_LEN;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
99 zstream->outbuf[0] = (chunk_size & 0xff000000) >> 24;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
100 zstream->outbuf[1] = (chunk_size & 0x00ff0000) >> 16;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
101 zstream->outbuf[2] = (chunk_size & 0x0000ff00) >> 8;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
102 zstream->outbuf[3] = (chunk_size & 0x000000ff);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
103 zstream->compressbuf_offset = 0;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
104 return 1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
105 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
106
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
107 static ssize_t
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
108 o_stream_lz4_send_chunk(struct lz4_ostream *zstream,
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
109 const void *data, size_t size)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
110 {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
111 size_t max_size;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
112 ssize_t added_bytes = 0;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
113 int ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
114
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
115 i_assert(zstream->outbuf_used == 0);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
116
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
117 do {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
118 max_size = I_MIN(size, sizeof(zstream->compressbuf) -
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
119 zstream->compressbuf_offset);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
120 memcpy(zstream->compressbuf + zstream->compressbuf_offset,
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
121 data, max_size);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
122 zstream->compressbuf_offset += max_size;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
123
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
124 data = CONST_PTR_OFFSET(data, max_size);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
125 size -= max_size;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
126 added_bytes += max_size;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
127
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
128 if (zstream->compressbuf_offset == sizeof(zstream->compressbuf)) {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
129 ret = o_stream_lz4_compress(zstream);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
130 if (ret <= 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
131 return added_bytes != 0 ? added_bytes : ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
132 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
133 } while (size > 0);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
134
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
135 return added_bytes;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
136 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
137
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
138 static int o_stream_lz4_flush(struct ostream_private *stream)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
139 {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
140 struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
141 int ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
142
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
143 if (o_stream_lz4_compress(zstream) < 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
144 return -1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
145 if (o_stream_lz4_send_outbuf(zstream) < 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
146 return -1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
147
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
148 ret = o_stream_flush(stream->parent);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
149 if (ret < 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
150 o_stream_copy_error_from_parent(stream);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
151 return ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
152 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
153
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
154 static ssize_t
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
155 o_stream_lz4_sendv(struct ostream_private *stream,
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
156 const struct const_iovec *iov, unsigned int iov_count)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
157 {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
158 struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
159 ssize_t ret, bytes = 0;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
160 unsigned int i;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
161
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
162 if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0) {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
163 /* error / we still couldn't flush existing data to
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
164 parent stream. */
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
165 return ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
166 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
167
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
168 for (i = 0; i < iov_count; i++) {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
169 ret = o_stream_lz4_send_chunk(zstream, iov[i].iov_base,
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
170 iov[i].iov_len);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
171 if (ret < 0)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
172 return -1;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
173 bytes += ret;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
174 if ((size_t)ret != iov[i].iov_len)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
175 break;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
176 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
177 stream->ostream.offset += bytes;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
178 return bytes;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
179 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
180
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
181 struct ostream *o_stream_create_lz4(struct ostream *output, int level)
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
182 {
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
183 struct iostream_lz4_header *hdr;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
184 struct lz4_ostream *zstream;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
185
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
186 i_assert(level >= 1 && level <= 9);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
187
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
188 zstream = i_new(struct lz4_ostream, 1);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
189 zstream->ostream.sendv = o_stream_lz4_sendv;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
190 zstream->ostream.flush = o_stream_lz4_flush;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
191 zstream->ostream.iostream.close = o_stream_lz4_close;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
192
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
193 i_assert(sizeof(zstream->outbuf) >= sizeof(*hdr));
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
194 hdr = (void *)zstream->outbuf;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
195 memcpy(hdr->magic, IOSTREAM_LZ4_MAGIC, sizeof(hdr->magic));
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
196 hdr->max_uncompressed_chunk_size[0] =
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
197 (OSTREAM_LZ4_CHUNK_SIZE & 0xff000000) >> 24;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
198 hdr->max_uncompressed_chunk_size[1] =
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
199 (OSTREAM_LZ4_CHUNK_SIZE & 0x00ff0000) >> 16;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
200 hdr->max_uncompressed_chunk_size[2] =
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
201 (OSTREAM_LZ4_CHUNK_SIZE & 0x0000ff00) >> 8;
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
202 hdr->max_uncompressed_chunk_size[3] =
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
203 (OSTREAM_LZ4_CHUNK_SIZE & 0x000000ff);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
204 zstream->outbuf_used = sizeof(*hdr);
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
205 return o_stream_create(&zstream->ostream, output,
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
206 o_stream_get_fd(output));
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
207 }
fb4a0a84da50 lib-compression: Added initial support for LZ4
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
208 #endif