Mercurial > dovecot > core-2.2
changeset 16052:0e5a359b7b7f
lib-storage: Mailbox attributes can now be accessed via istreams.
The idea is to use istreams for larger values.
| author | Timo Sirainen <tss@iki.fi> |
|---|---|
| date | Tue, 19 Mar 2013 19:05:27 +0200 |
| parents | 46b223892373 |
| children | 7a334ebc0145 |
| files | src/doveadm/dsync/dsync-brain-mails.c src/doveadm/dsync/dsync-ibc-stream.c src/doveadm/dsync/dsync-mailbox-export.c src/doveadm/dsync/dsync-mailbox-import.c src/doveadm/dsync/dsync-mailbox.c src/doveadm/dsync/dsync-mailbox.h src/lib-storage/index/index-attribute.c src/lib-storage/mail-storage.c src/lib-storage/mail-storage.h src/plugins/acl/acl-attributes.c |
| diffstat | 10 files changed, 469 insertions(+), 212 deletions(-) [+] |
line wrap: on
line diff
--- a/src/doveadm/dsync/dsync-brain-mails.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Tue Mar 19 19:05:27 2013 +0200 @@ -63,6 +63,7 @@ static bool dsync_brain_recv_mailbox_attribute(struct dsync_brain *brain) { const struct dsync_mailbox_attribute *attr; + struct istream *input; enum dsync_ibc_recv_ret ret; if ((ret = dsync_ibc_recv_mailbox_attribute(brain->ibc, &attr)) == 0) @@ -73,6 +74,9 @@ } if (dsync_mailbox_import_attribute(brain->box_importer, attr) < 0) brain->failed = TRUE; + input = attr->value_stream; + if (input != NULL) + i_stream_unref(&input); return TRUE; }
--- a/src/doveadm/dsync/dsync-ibc-stream.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Tue Mar 19 19:05:27 2013 +0200 @@ -99,7 +99,7 @@ { .name = "mailbox_attribute", .chr = 'A', .required_keys = "type key", - .optional_keys = "value deleted last_change modseq" + .optional_keys = "value stream deleted last_change modseq" }, { .name = "mail_change", .chr = 'C', @@ -142,9 +142,10 @@ pool_t ret_pool; struct dsync_deserializer_decoder *cur_decoder; - struct istream *mail_output, *mail_input; + struct istream *value_output, *value_input; struct dsync_mail *cur_mail; - char mail_output_last; + struct dsync_mailbox_attribute *cur_attr; + char value_output_last; unsigned int version_received:1; unsigned int handshake_received:1; @@ -163,21 +164,21 @@ static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc) { do { - i_stream_skip(ibc->mail_input, - i_stream_get_data_size(ibc->mail_input)); - } while (i_stream_read(ibc->mail_input) > 0); - if (ibc->mail_input->eof) { - if (ibc->mail_input->stream_errno != 0) { - errno = ibc->mail_input->stream_errno; + i_stream_skip(ibc->value_input, + i_stream_get_data_size(ibc->value_input)); + } while (i_stream_read(ibc->value_input) > 0); + if (ibc->value_input->eof) { + if (ibc->value_input->stream_errno != 0) { + errno = ibc->value_input->stream_errno; i_error("dsync(%s): read() failed: %m", ibc->name); dsync_ibc_stream_stop(ibc); return -1; } /* finished reading the mail stream */ - i_assert(ibc->mail_input->eof); - i_stream_seek(ibc->mail_input, 0); + i_assert(ibc->value_input->eof); + i_stream_seek(ibc->value_input, 0); ibc->has_pending_data = TRUE; - ibc->mail_input = NULL; + ibc->value_input = NULL; return 1; } return 0; @@ -185,7 +186,7 @@ static void dsync_ibc_stream_input(struct dsync_ibc_stream *ibc) { - if (ibc->mail_input != NULL) { + if (ibc->value_input != NULL) { if (dsync_ibc_stream_read_mail_stream(ibc) == 0) return; } @@ -194,19 +195,19 @@ o_stream_uncork(ibc->output); } -static int dsync_ibc_stream_send_mail_stream(struct dsync_ibc_stream *ibc) +static int dsync_ibc_stream_send_value_stream(struct dsync_ibc_stream *ibc) { const unsigned char *data; unsigned char add; size_t i, size; int ret; - while ((ret = i_stream_read_data(ibc->mail_output, + while ((ret = i_stream_read_data(ibc->value_output, &data, &size, 0)) > 0) { add = '\0'; for (i = 0; i < size; i++) { if (data[i] == '.' && - ((i == 0 && ibc->mail_output_last == '\n') || + ((i == 0 && ibc->value_output_last == '\n') || (i > 0 && data[i-1] == '\n'))) { /* escape the dot */ add = '.'; @@ -216,8 +217,8 @@ if (i > 0) { o_stream_nsend(ibc->output, data, i); - ibc->mail_output_last = data[i-1]; - i_stream_skip(ibc->mail_output, i); + ibc->value_output_last = data[i-1]; + i_stream_skip(ibc->value_output, i); } if (o_stream_get_buffer_used_size(ibc->output) >= 4096) { @@ -234,14 +235,14 @@ if (add != '\0') { o_stream_nsend(ibc->output, &add, 1); - ibc->mail_output_last = add; + ibc->value_output_last = add; } } i_assert(ret == -1); - if (ibc->mail_output->stream_errno != 0) { + if (ibc->value_output->stream_errno != 0) { i_error("dsync(%s): read(%s) failed: %m", - ibc->name, i_stream_get_name(ibc->mail_output)); + ibc->name, i_stream_get_name(ibc->value_output)); dsync_ibc_stream_stop(ibc); return -1; } @@ -249,7 +250,7 @@ /* finished sending the stream. use "CRLF." instead of "LF." just in case we're sending binary data that ends with CR. */ o_stream_nsend_str(ibc->output, "\r\n.\r\n"); - i_stream_unref(&ibc->mail_output); + i_stream_unref(&ibc->value_output); return 1; } @@ -261,8 +262,8 @@ o_stream_cork(ibc->output); if ((ret = o_stream_flush(output)) < 0) ret = 1; - else if (ibc->mail_output != NULL) { - if (dsync_ibc_stream_send_mail_stream(ibc) < 0) + else if (ibc->value_output != NULL) { + if (dsync_ibc_stream_send_value_stream(ibc) < 0) ret = 1; } timeout_reset(ibc->to); @@ -320,8 +321,8 @@ if (ibc->cur_decoder != NULL) dsync_deserializer_decode_finish(&ibc->cur_decoder); - if (ibc->mail_output != NULL) - i_stream_unref(&ibc->mail_output); + if (ibc->value_output != NULL) + i_stream_unref(&ibc->value_output); else { /* notify remote that we're closing. this is mainly to avoid "read() failed: EOF" errors on failing dsyncs */ @@ -399,10 +400,50 @@ dsync_ibc_stream_send_string(struct dsync_ibc_stream *ibc, const string_t *str) { - i_assert(ibc->mail_output == NULL); + i_assert(ibc->value_output == NULL); o_stream_nsend(ibc->output, str_data(str), str_len(str)); } +static int seekable_fd_callback(const char **path_r, void *context) +{ + struct dsync_ibc_stream *ibc = context; + string_t *path; + int fd; + + path = t_str_new(128); + str_append(path, ibc->temp_path_prefix); + fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_error("unlink(%s) failed: %m", str_c(path)); + i_close_fd(&fd); + return -1; + } + + *path_r = str_c(path); + return fd; +} + +static struct istream * +dsync_ibc_stream_input_stream(struct dsync_ibc_stream *ibc) +{ + struct istream *inputs[2]; + + inputs[0] = i_stream_create_dot(ibc->input, FALSE); + inputs[1] = NULL; + ibc->value_input = i_stream_create_seekable(inputs, MAIL_READ_FULL_BLOCK_SIZE, + seekable_fd_callback, ibc); + i_stream_unref(&inputs[0]); + + return ibc->value_input; +} + static int dsync_ibc_check_missing_deserializers(struct dsync_ibc_stream *ibc) { @@ -483,7 +524,7 @@ const char *line, *error; unsigned int i; - i_assert(ibc->mail_input == NULL); + i_assert(ibc->value_input == NULL); timeout_reset(ibc->to); @@ -659,7 +700,7 @@ { struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; - i_assert(ibc->mail_output == NULL); + i_assert(ibc->value_output == NULL); o_stream_nsend_str(ibc->output, END_OF_LIST_LINE"\n"); } @@ -1218,6 +1259,8 @@ dsync_serializer_encode_add(encoder, "key", attr->key); if (attr->value != NULL) dsync_serializer_encode_add(encoder, "value", attr->value); + else if (attr->value_stream != NULL) + dsync_serializer_encode_add(encoder, "stream", ""); if (attr->deleted) dsync_serializer_encode_add(encoder, "deleted", ""); @@ -1232,6 +1275,13 @@ dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); + + if (attr->value_stream != NULL) { + ibc->value_output_last = '\0'; + ibc->value_output = attr->value_stream; + i_stream_ref(ibc->value_output); + (void)dsync_ibc_stream_send_value_stream(ibc); + } } static enum dsync_ibc_recv_ret @@ -1248,6 +1298,13 @@ if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES) return DSYNC_IBC_RECV_RET_FINISHED; + if (ibc->cur_attr != NULL) { + /* finished reading the stream, return the mail now */ + *attr_r = ibc->cur_attr; + ibc->cur_attr = NULL; + return DSYNC_IBC_RECV_RET_OK; + } + p_clear(pool); attr = p_new(pool, struct dsync_mailbox_attribute, 1); @@ -1271,7 +1328,15 @@ value = dsync_deserializer_decode_get(decoder, "key"); attr->key = p_strdup(pool, value); - if (dsync_deserializer_decode_try(decoder, "value", &value)) + if (dsync_deserializer_decode_try(decoder, "stream", &value)) { + attr->value_stream = dsync_ibc_stream_input_stream(ibc); + if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) { + ibc->cur_attr = attr; + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + /* already finished reading the stream */ + i_assert(ibc->value_input == NULL); + } else if (dsync_deserializer_decode_try(decoder, "value", &value)) attr->value = p_strdup(pool, value); if (dsync_deserializer_decode_try(decoder, "deleted", &value)) attr->deleted = TRUE; @@ -1512,7 +1577,7 @@ struct dsync_serializer_encoder *encoder; string_t *str = t_str_new(128); - i_assert(ibc->mail_output == NULL); + i_assert(ibc->value_output == NULL); str_append_c(str, items[ITEM_MAIL].chr); encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAIL]); @@ -1539,39 +1604,13 @@ dsync_ibc_stream_send_string(ibc, str); if (mail->input != NULL) { - ibc->mail_output_last = '\0'; - ibc->mail_output = mail->input; - i_stream_ref(ibc->mail_output); - (void)dsync_ibc_stream_send_mail_stream(ibc); + ibc->value_output_last = '\0'; + ibc->value_output = mail->input; + i_stream_ref(ibc->value_output); + (void)dsync_ibc_stream_send_value_stream(ibc); } } -static int seekable_fd_callback(const char **path_r, void *context) -{ - struct dsync_ibc_stream *ibc = context; - string_t *path; - int fd; - - path = t_str_new(128); - str_append(path, ibc->temp_path_prefix); - fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); - if (fd == -1) { - i_error("safe_mkstemp(%s) failed: %m", str_c(path)); - return -1; - } - - /* we just want the fd, unlink it */ - if (unlink(str_c(path)) < 0) { - /* shouldn't happen.. */ - i_error("unlink(%s) failed: %m", str_c(path)); - i_close_fd(&fd); - return -1; - } - - *path_r = str_c(path); - return fd; -} - static enum dsync_ibc_recv_ret dsync_ibc_stream_recv_mail(struct dsync_ibc *_ibc, struct dsync_mail **mail_r) { @@ -1579,11 +1618,10 @@ pool_t pool = ibc->ret_pool; struct dsync_deserializer_decoder *decoder; struct dsync_mail *mail; - struct istream *inputs[2]; const char *value; enum dsync_ibc_recv_ret ret; - if (ibc->mail_input != NULL) { + if (ibc->value_input != NULL) { /* wait until the mail's stream has been read */ return DSYNC_IBC_RECV_RET_TRYAGAIN; } @@ -1621,19 +1659,13 @@ return DSYNC_IBC_RECV_RET_TRYAGAIN; } if (dsync_deserializer_decode_try(decoder, "stream", &value)) { - inputs[0] = i_stream_create_dot(ibc->input, FALSE); - inputs[1] = NULL; - mail->input = i_stream_create_seekable(inputs, - MAIL_READ_FULL_BLOCK_SIZE, seekable_fd_callback, ibc); - i_stream_unref(&inputs[0]); - - ibc->mail_input = mail->input; + mail->input = dsync_ibc_stream_input_stream(ibc); if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) { ibc->cur_mail = mail; return DSYNC_IBC_RECV_RET_TRYAGAIN; } /* already finished reading the stream */ - i_assert(ibc->mail_input == NULL); + i_assert(ibc->value_input == NULL); } *mail_r = mail; @@ -1644,8 +1676,8 @@ { struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; - if (ibc->mail_output != NULL) { - i_stream_unref(&ibc->mail_output); + if (ibc->value_output != NULL) { + i_stream_unref(&ibc->value_output); dsync_ibc_stream_stop(ibc); } } @@ -1655,7 +1687,7 @@ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; size_t bytes; - if (ibc->mail_output != NULL) + if (ibc->value_output != NULL) return TRUE; bytes = o_stream_get_buffer_used_size(ibc->output);
--- a/src/doveadm/dsync/dsync-mailbox-export.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Tue Mar 19 19:05:27 2013 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "hash.h" +#include "istream.h" #include "mail-index-modseq.h" #include "mail-storage-private.h" #include "mail-search-build.h" @@ -42,8 +43,10 @@ unsigned int change_idx; uint32_t highest_changed_uid; - ARRAY(struct dsync_mailbox_attribute) attr_changes; - unsigned int attr_change_idx; + struct mailbox_attribute_iter *attr_iter; + struct hash_iterate_context *attr_change_iter; + enum mail_attribute_type attr_type; + struct dsync_mailbox_attribute attr; struct dsync_mail_change change; struct dsync_mail dsync_mail; @@ -400,98 +403,12 @@ } static void -dsync_mailbox_export_attr_changes(struct dsync_mailbox_exporter *exporter, - enum mail_attribute_type type) +dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter, + enum mail_attribute_type type) { - HASH_TABLE_TYPE(dsync_attr_change) attr_changes; - struct mailbox_attribute_iter *iter; - struct dsync_mailbox_attribute lookup_attr, *attr; - struct dsync_mailbox_attribute *attr_change; - const char *key; - struct mail_attribute_value value; - bool export_all_attrs; - - export_all_attrs = exporter->return_all_mails || - exporter->last_common_uid == 0; - attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan); - lookup_attr.type = type; - - iter = mailbox_attribute_iter_init(exporter->box, type, ""); - while ((key = mailbox_attribute_iter_next(iter)) != NULL) { - lookup_attr.key = key; - attr_change = hash_table_lookup(attr_changes, &lookup_attr); - if (attr_change == NULL && !export_all_attrs) - continue; - - if (mailbox_attribute_get(exporter->trans, type, key, &value) < 0) { - exporter->error = p_strdup_printf(exporter->pool, - "Mailbox attribute %s lookup failed: %s", key, - mailbox_get_last_error(exporter->box, NULL)); - break; - } - if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) { - /* readonly attributes can't be changed, - no point in exporting them */ - continue; - } - - attr = array_append_space(&exporter->attr_changes); - attr->type = type; - attr->value = p_strdup(exporter->pool, value.value); - attr->last_change = value.last_change; - if (attr_change != NULL) { - i_assert(!attr_change->exported); - attr_change->exported = TRUE; - attr->key = attr_change->key; - attr->deleted = attr_change->deleted && - attr->value == NULL; - attr->modseq = attr_change->modseq; - } else { - attr->key = p_strdup(exporter->pool, key); - } - } - if (mailbox_attribute_iter_deinit(&iter) < 0) { - exporter->error = p_strdup_printf(exporter->pool, - "Mailbox attribute iteration failed: %s", - mailbox_get_last_error(exporter->box, NULL)); - } -} - -static void -dsync_mailbox_export_nonexistent_attrs(struct dsync_mailbox_exporter *exporter) -{ - HASH_TABLE_TYPE(dsync_attr_change) attr_changes; - struct hash_iterate_context *iter; - struct dsync_mailbox_attribute *attr; - struct mail_attribute_value value; - - attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan); - - iter = hash_table_iterate_init(attr_changes); - while (hash_table_iterate(iter, attr_changes, &attr, &attr)) { - if (attr->exported || !attr->deleted) - continue; - - /* lookup the value mainly to get its last_change value. */ - if (mailbox_attribute_get(exporter->trans, attr->type, - attr->key, &value) < 0) { - exporter->error = p_strdup_printf(exporter->pool, - "Mailbox attribute %s lookup failed: %s", attr->key, - mailbox_get_last_error(exporter->box, NULL)); - break; - } - if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) - continue; - - attr->last_change = value.last_change; - if (value.value != NULL) { - attr->value = p_strdup(exporter->pool, value.value); - attr->deleted = FALSE; - } - attr->exported = TRUE; - array_append(&exporter->attr_changes, attr, 1); - } - hash_table_iterate_deinit(&iter); + exporter->attr_iter = + mailbox_attribute_iter_init(exporter->box, type, ""); + exporter->attr_type = type; } static void @@ -556,26 +473,149 @@ /* get the changes sorted by UID */ dsync_mailbox_export_sort_changes(exporter); - p_array_init(&exporter->attr_changes, pool, 16); - dsync_mailbox_export_attr_changes(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE); - dsync_mailbox_export_attr_changes(exporter, MAIL_ATTRIBUTE_TYPE_SHARED); - dsync_mailbox_export_nonexistent_attrs(exporter); + dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE); return exporter; } +static int +dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter) +{ + HASH_TABLE_TYPE(dsync_attr_change) attr_changes; + struct dsync_mailbox_attribute *attr; + struct mail_attribute_value value; + + attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan); + + while (hash_table_iterate(exporter->attr_change_iter, attr_changes, + &attr, &attr)) { + if (attr->exported || !attr->deleted) + continue; + + /* lookup the value mainly to get its last_change value. */ + if (mailbox_attribute_get_stream(exporter->trans, attr->type, + attr->key, &value) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute %s lookup failed: %s", attr->key, + mailbox_get_last_error(exporter->box, NULL)); + break; + } + if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) { + if (value.value_stream != NULL) + i_stream_unref(&value.value_stream); + continue; + } + + attr->last_change = value.last_change; + if (value.value != NULL || value.value_stream != NULL) { + attr->value = p_strdup(exporter->pool, value.value); + attr->value_stream = value.value_stream; + attr->deleted = FALSE; + } + attr->exported = TRUE; + exporter->attr = *attr; + return 1; + } + hash_table_iterate_deinit(&exporter->attr_change_iter); + return 0; +} + +static int +dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter) +{ + HASH_TABLE_TYPE(dsync_attr_change) attr_changes; + struct dsync_mailbox_attribute lookup_attr, *attr; + struct dsync_mailbox_attribute *attr_change; + const char *key; + struct mail_attribute_value value; + bool export_all_attrs; + + export_all_attrs = exporter->return_all_mails || + exporter->last_common_uid == 0; + attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan); + lookup_attr.type = exporter->attr_type; + + /* note that the order of processing may be important for some + attributes. for example sieve can't set a script active until it's + first been created */ + while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) { + lookup_attr.key = key; + attr_change = hash_table_lookup(attr_changes, &lookup_attr); + if (attr_change == NULL && !export_all_attrs) + continue; + + if (mailbox_attribute_get_stream(exporter->trans, + exporter->attr_type, key, + &value) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute %s lookup failed: %s", key, + mailbox_get_last_error(exporter->box, NULL)); + return -1; + } + if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) { + /* readonly attributes can't be changed, + no point in exporting them */ + if (value.value_stream != NULL) + i_stream_unref(&value.value_stream); + continue; + } + if (value.value == NULL && value.value_stream == NULL && + (attr_change == NULL || !attr_change->deleted)) { + /* the attribute was just deleted? + skip for this sync. */ + continue; + } + + attr = &exporter->attr; + memset(attr, 0, sizeof(*attr)); + attr->type = exporter->attr_type; + attr->value = p_strdup(exporter->pool, value.value); + attr->value_stream = value.value_stream; + attr->last_change = value.last_change; + if (attr_change != NULL) { + i_assert(!attr_change->exported); + attr_change->exported = TRUE; + attr->key = attr_change->key; + attr->deleted = attr_change->deleted && + !DSYNC_ATTR_HAS_VALUE(attr); + attr->modseq = attr_change->modseq; + } else { + attr->key = p_strdup(exporter->pool, key); + } + return 1; + } + if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute iteration failed: %s", + mailbox_get_last_error(exporter->box, NULL)); + return -1; + } + if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) { + /* export shared attributes */ + dsync_mailbox_export_attr_init(exporter, + MAIL_ATTRIBUTE_TYPE_SHARED); + return dsync_mailbox_export_iter_next_attr(exporter); + } + exporter->attr_change_iter = hash_table_iterate_init(attr_changes); + return dsync_mailbox_export_iter_next_nonexistent_attr(exporter); +} + const struct dsync_mailbox_attribute * dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter) { - const struct dsync_mailbox_attribute *changes; - unsigned int count; - if (exporter->error != NULL) return NULL; - changes = array_get(&exporter->attr_changes, &count); - if (exporter->attr_change_idx == count) - return NULL; - return &changes[exporter->attr_change_idx++]; + if (exporter->attr.value_stream != NULL) + i_stream_unref(&exporter->attr.value_stream); + + if (exporter->attr_iter != NULL) { + if (dsync_mailbox_export_iter_next_attr(exporter) <= 0) + return NULL; + } else { + if (dsync_mailbox_export_iter_next_nonexistent_attr(exporter) <= 0) + return NULL; + } + return &exporter->attr; } const struct dsync_mail_change * @@ -800,6 +840,8 @@ dsync_mailbox_export_body_search_deinit(exporter); (void)mailbox_transaction_commit(&exporter->trans); + if (exporter->attr.value_stream != NULL) + i_stream_unref(&exporter->attr.value_stream); hash_table_destroy(&exporter->export_guids); *error_r = t_strdup(exporter->error);
--- a/src/doveadm/dsync/dsync-mailbox-import.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Tue Mar 19 19:05:27 2013 +0200 @@ -205,7 +205,7 @@ static int dsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer, enum mail_attribute_type type, const char *key, - const struct dsync_mailbox_attribute **attr_r) + struct dsync_mailbox_attribute **attr_r) { struct dsync_mailbox_attribute lookup_attr, *attr; const struct dsync_mailbox_attribute *attr_change; @@ -213,7 +213,7 @@ *attr_r = NULL; - if (mailbox_attribute_get(importer->trans, type, key, &value) < 0) { + if (mailbox_attribute_get_stream(importer->trans, type, key, &value) < 0) { i_error("Mailbox %s: Failed to get attribute %s: %s", mailbox_get_vname(importer->box), key, mailbox_get_last_error(importer->box, NULL)); @@ -226,7 +226,8 @@ attr_change = hash_table_lookup(importer->local_attr_changes, &lookup_attr); - if (attr_change == NULL && value.value == NULL) { + if (attr_change == NULL && + value.value == NULL && value.value_stream == NULL) { /* we have no knowledge of this attribute */ return 0; } @@ -234,33 +235,118 @@ attr->type = type; attr->key = key; attr->value = value.value; + attr->value_stream = value.value_stream; attr->last_change = value.last_change; if (attr_change != NULL) { - attr->deleted = attr_change->deleted && attr->value == NULL; + attr->deleted = attr_change->deleted && + !DSYNC_ATTR_HAS_VALUE(attr); attr->modseq = attr_change->modseq; } *attr_r = attr; return 0; } +static int +dsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r) +{ + const unsigned char *data1, *data2; + size_t size1, size2, size; + + for (;;) { + (void)i_stream_read_data(input1, &data1, &size1, 0); + (void)i_stream_read_data(input2, &data2, &size2, 0); + + if (size1 == 0 || size2 == 0) + break; + size = I_MIN(size1, size2); + *cmp_r = memcmp(data1, data2, size); + if (*cmp_r != 0) + return 0; + i_stream_skip(input1, size); + i_stream_skip(input2, size); + } + if (input1->stream_errno != 0) { + errno = input1->stream_errno; + i_error("read(%s) failed: %m", i_stream_get_name(input1)); + return -1; + } + if (input2->stream_errno != 0) { + errno = input2->stream_errno; + i_error("read(%s) failed: %m", i_stream_get_name(input2)); + return -1; + } + if (size1 == 0 && size2 == 0) + *cmp_r = 0; + else + *cmp_r = size1 == 0 ? -1 : 1; + return 0; +} + +static int +dsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1, + const struct dsync_mailbox_attribute *attr2, + int *cmp_r) +{ + struct istream *input1, *input2; + int ret; + + if (attr1->value != NULL && attr2->value != NULL) { + *cmp_r = strcmp(attr1->value, attr2->value); + return 0; + } + /* at least one of them is a stream. make both of them streams. */ + input1 = attr1->value_stream != NULL ? attr1->value_stream : + i_stream_create_from_data(attr1->value, strlen(attr1->value)); + input2 = attr2->value_stream != NULL ? attr2->value_stream : + i_stream_create_from_data(attr2->value, strlen(attr2->value)); + i_stream_seek(input1, 0); + i_stream_seek(input2, 0); + ret = dsync_istreams_cmp(input1, input2, cmp_r); + if (attr1->value_stream == NULL) + i_stream_unref(&input1); + if (attr2->value_stream == NULL) + i_stream_unref(&input2); + return ret; +} + +static int +dsync_attributes_cmp(const struct dsync_mailbox_attribute *attr, + const struct dsync_mailbox_attribute *local_attr, + int *cmp_r) +{ + if (DSYNC_ATTR_HAS_VALUE(attr) && + !DSYNC_ATTR_HAS_VALUE(local_attr)) { + /* remote has a value and local doesn't -> use it */ + return 1; + } else if (!DSYNC_ATTR_HAS_VALUE(attr) && + DSYNC_ATTR_HAS_VALUE(local_attr)) { + /* remote doesn't have a value, bt local does -> skip */ + return -1; + } + + return dsync_attributes_cmp_values(attr, local_attr, cmp_r); +} + int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, const struct dsync_mailbox_attribute *attr) { - const struct dsync_mailbox_attribute *local_attr; + struct dsync_mailbox_attribute *local_attr; struct mail_attribute_value value; - int ret; + int ret, cmp; + bool ignore = FALSE; - i_assert(attr->value != NULL || attr->deleted); + i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted); if (dsync_mailbox_import_lookup_attr(importer, attr->type, attr->key, &local_attr) < 0) return -1; + if (attr->deleted && + (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) { + /* attribute doesn't exist on either side -> ignore */ + return 0; + } if (local_attr == NULL) { /* we haven't seen this locally -> use whatever remote has */ - } else if (null_strcmp(attr->value, local_attr->value) == 0) { - /* the values are identical anyway -> we can skip them. - FIXME: but should timestamp/modseq still be updated?.. */ - return 0; } else if (local_attr->modseq <= importer->last_common_modseq && attr->modseq > importer->last_common_modseq && importer->last_common_modseq > 0) { @@ -271,37 +357,46 @@ importer->last_common_modseq > 0) { /* we're doing incremental syncing, and we can see that the attribute was changed locally, but not remotely -> ignore */ - return 0; + ignore = TRUE; } else if (attr->last_change > local_attr->last_change) { /* remote has a newer timestamp -> use it */ } else if (attr->last_change < local_attr->last_change) { /* remote has an older timestamp -> ignore */ - return 0; + ignore = TRUE; } else { /* the timestamps are the same. now we're down to guessing - the right answer. first try to use modseqs, but if even they - are the same, fallback to just picking one based on the + the right answer, unless the values are actually equal, + so check that first. next try to use modseqs, but if even + they are the same, fallback to just picking one based on the value. */ + if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) { + importer->failed = TRUE; + return -1; + } + if (cmp == 0) { + /* identical scripts */ + return 0; + } + if (attr->modseq > local_attr->modseq) { /* remote has a higher modseq -> use it */ } else if (attr->modseq < local_attr->modseq) { /* remote has an older modseq -> ignore */ - return 0; - } else if (attr->value != NULL && local_attr->value == NULL) { - /* remote has a value and local doesn't -> use it */ - } else if (attr->value == NULL && local_attr->value != NULL) { - /* remote doesn't have a value, bt local does -> skip */ - return 0; + ignore = TRUE; } else { - /* now we have absolutely no reasonable guesses left. - just pick one of them that are used. */ - if (strcmp(attr->value, local_attr->value) < 0) - return 0; + if (cmp < 0) + ignore = TRUE; } } + if (ignore) { + if (local_attr->value_stream != NULL) + i_stream_unref(&local_attr->value_stream); + return 0; + } memset(&value, 0, sizeof(value)); value.value = attr->value; + value.value_stream = attr->value_stream; value.last_change = attr->last_change; ret = mailbox_attribute_set(importer->trans, attr->type, attr->key, &value); @@ -311,6 +406,8 @@ mailbox_get_last_error(importer->box, NULL)); importer->failed = TRUE; } + if (local_attr != NULL && local_attr->value_stream != NULL) + i_stream_unref(&local_attr->value_stream); return ret; }
--- a/src/doveadm/dsync/dsync-mailbox.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox.c Tue Mar 19 19:05:27 2013 +0200 @@ -10,6 +10,10 @@ dest_r->type = src->type; dest_r->key = p_strdup(pool, src->key); dest_r->value = p_strdup(pool, src->value); + if (src->value_stream != NULL) { + dest_r->value_stream = src->value_stream; + i_stream_ref(dest_r->value_stream); + } dest_r->deleted = src->deleted; dest_r->last_change = src->last_change;
--- a/src/doveadm/dsync/dsync-mailbox.h Tue Mar 19 18:52:39 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox.h Tue Mar 19 19:05:27 2013 +0200 @@ -18,7 +18,9 @@ struct dsync_mailbox_attribute { enum mail_attribute_type type; const char *key; - const char *value; /* NULL = not looked up yet / deleted */ + /* if both values are NULL = not looked up yet / deleted */ + const char *value; + struct istream *value_stream; time_t last_change; /* 0 = unknown */ uint64_t modseq; /* 0 = unknown */ @@ -26,6 +28,8 @@ bool deleted; /* attribute is known to have been deleted */ bool exported; /* internally used by exporting */ }; +#define DSYNC_ATTR_HAS_VALUE(attr) \ + ((attr)->value != NULL || (attr)->value_stream != NULL) void dsync_mailbox_attribute_dup(pool_t pool, const struct dsync_mailbox_attribute *src,
--- a/src/lib-storage/index/index-attribute.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/lib-storage/index/index-attribute.c Tue Mar 19 19:05:27 2013 +0200 @@ -179,6 +179,7 @@ struct dict_transaction_context *dtrans; const char *mailbox_prefix; bool pvt = type == MAIL_ATTRIBUTE_TYPE_PRIVATE; + int ret = 0; if (strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) == 0) { @@ -194,16 +195,20 @@ T_BEGIN { const char *prefixed_key = key_get_prefixed(type, mailbox_prefix, key); + const char *value_str; - if (value->value != NULL) { - dict_set(dtrans, prefixed_key, value->value); + if (mailbox_attribute_value_to_string(t->box->storage, value, + &value_str) < 0) { + ret = -1; + } else if (value_str != NULL) { + dict_set(dtrans, prefixed_key, value_str); mail_index_attribute_set(t->itrans, pvt, key); } else { dict_unset(dtrans, prefixed_key); mail_index_attribute_unset(t->itrans, pvt, key); } } T_END; - return 0; + return ret; } int index_storage_attribute_get(struct mailbox_transaction_context *t,
--- a/src/lib-storage/mail-storage.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/lib-storage/mail-storage.c Tue Mar 19 19:05:27 2013 +0200 @@ -4,6 +4,7 @@ #include "ioloop.h" #include "array.h" #include "llist.h" +#include "str.h" #include "unichar.h" #include "istream.h" #include "eacces-error.h" @@ -1537,11 +1538,64 @@ return t->box->v.attribute_set(t, type, key, &value); } +int mailbox_attribute_value_to_string(struct mail_storage *storage, + const struct mail_attribute_value *value, + const char **str_r) +{ + string_t *str; + const unsigned char *data; + size_t size; + + if (value->value_stream == NULL) { + *str_r = value->value; + return 0; + } + str = t_str_new(128); + i_stream_seek(value->value_stream, 0); + while (i_stream_read_data(value->value_stream, &data, &size, 0) > 0) { + if (memchr(data, '\0', size) != NULL) { + mail_storage_set_error(storage, MAIL_ERROR_PARAMS, + "Attribute string value has NULs"); + return -1; + } + str_append_n(str, data, size); + i_stream_skip(value->value_stream, size); + } + if (value->value_stream->stream_errno != 0) { + mail_storage_set_critical(storage, "read(%s) failed: %m", + i_stream_get_name(value->value_stream)); + return -1; + } + i_assert(value->value_stream->eof); + *str_r = str_c(str); + return 0; +} + int mailbox_attribute_get(struct mailbox_transaction_context *t, enum mail_attribute_type type, const char *key, struct mail_attribute_value *value_r) { - return t->box->v.attribute_get(t, type, key, value_r); + int ret; + + memset(value_r, 0, sizeof(*value_r)); + if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0) + return ret; + i_assert(value_r->value != NULL); + return 1; +} + +int mailbox_attribute_get_stream(struct mailbox_transaction_context *t, + enum mail_attribute_type type, const char *key, + struct mail_attribute_value *value_r) +{ + int ret; + + memset(value_r, 0, sizeof(*value_r)); + value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_INT_STREAMS; + if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0) + return ret; + i_assert(value_r->value != NULL || value_r->value_stream != NULL); + return 1; } struct mailbox_attribute_iter *
--- a/src/lib-storage/mail-storage.h Tue Mar 19 18:52:39 2013 +0200 +++ b/src/lib-storage/mail-storage.h Tue Mar 19 19:05:27 2013 +0200 @@ -216,12 +216,18 @@ MAIL_ATTRIBUTE_TYPE_SHARED }; enum mail_attribute_value_flags { - MAIL_ATTRIBUTE_VALUE_FLAG_READONLY = 0x01 + MAIL_ATTRIBUTE_VALUE_FLAG_READONLY = 0x01, + MAIL_ATTRIBUTE_VALUE_FLAG_INT_STREAMS = 0x02 }; struct mail_attribute_value { - /* The actual value */ + /* mailbox_attribute_set() can set either value or value_stream. + mailbox_attribute_get() returns only values, but + mailbox_attribute_get_stream() may return either value or + value_stream. The caller must unreference the returned streams. */ const char *value; + struct istream *value_stream; + /* Last time the attribute was changed (0 = unknown). This may be returned even for values that don't exist anymore. */ time_t last_change; @@ -564,6 +570,11 @@ int mailbox_attribute_get(struct mailbox_transaction_context *t, enum mail_attribute_type type, const char *key, struct mail_attribute_value *value_r); +/* Same as mailbox_attribute_get(), but the returned value may be either an + input stream or a string. */ +int mailbox_attribute_get_stream(struct mailbox_transaction_context *t, + enum mail_attribute_type type, const char *key, + struct mail_attribute_value *value_r); /* Iterate through mailbox attributes of the given type. The prefix can be used to restrict what attributes are returned. */
--- a/src/plugins/acl/acl-attributes.c Tue Mar 19 18:52:39 2013 +0200 +++ b/src/plugins/acl/acl-attributes.c Tue Mar 19 19:05:27 2013 +0200 @@ -21,7 +21,7 @@ acl_attribute_update_acl(struct mailbox_transaction_context *t, const char *key, const struct mail_attribute_value *value) { - const char *id, *const *rights, *error; + const char *value_str, *id, *const *rights, *error; struct acl_rights_update update; /* for now allow only admin (=dsync) to update ACLs this way. @@ -35,12 +35,16 @@ return -1; } + if (mailbox_attribute_value_to_string(t->box->storage, value, + &value_str) < 0) + return -1; + memset(&update, 0, sizeof(update)); update.modify_mode = ACL_MODIFY_MODE_REPLACE; update.neg_modify_mode = ACL_MODIFY_MODE_REPLACE; update.last_change = value->last_change; id = key + strlen(MAILBOX_ATTRIBUTE_PREFIX_ACL); - rights = value->value == NULL ? NULL : t_strsplit(value->value, " "); + rights = value_str == NULL ? NULL : t_strsplit(value_str, " "); if (acl_rights_update_import(&update, id, rights, &error) < 0) { mail_storage_set_error(t->box->storage, MAIL_ERROR_PARAMS, error); return -1;