Mercurial > dovecot > core-2.2
changeset 15956:d3815278ed19
Merged changes from v2.1 tree.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 25 Feb 2013 17:29:28 +0200 |
parents | 539baf0adc2e (current diff) fa9387588430 (diff) |
children | 508d46f85815 |
files | src/imap/cmd-list.c src/indexer/indexer-worker.c src/lib-dict/dict-client.c src/lib-dict/dict-sql.c src/lib-index/mail-transaction-log-file.c src/lib-lda/duplicate.c src/lib-storage/index/dbox-common/dbox-file-fix.c src/lib-storage/index/dbox-common/dbox-file.h src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c src/lib-storage/index/index-status.c src/lib-storage/index/index-storage.h src/lib-storage/index/index-sync.c src/lib-storage/index/maildir/maildir-sync.c src/lib-storage/mail-storage.c src/lib/unichar.c src/plugins/fts-solr/fts-backend-solr.c src/plugins/fts/fts-build-mail.c src/plugins/quota/Makefile.am |
diffstat | 18 files changed, 95 insertions(+), 33 deletions(-) [+] |
line wrap: on
line diff
--- a/src/imap/cmd-list.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/imap/cmd-list.c Mon Feb 25 17:29:28 2013 +0200 @@ -397,7 +397,7 @@ return TRUE; } if (imap_utf7_to_utf8(pattern, str) == 0) - pattern = t_strdup(str_c(str)); + pattern = p_strdup(cmd->pool, str_c(str)); array_append(&patterns, &pattern, 1); str_truncate(str, 0); } @@ -408,7 +408,7 @@ return TRUE; } if (imap_utf7_to_utf8(pattern, str) == 0) - pattern = str_c(str); + pattern = p_strdup(cmd->pool, str_c(str)); p_array_init(&patterns, cmd->pool, 1); array_append(&patterns, &pattern, 1);
--- a/src/indexer/indexer-worker.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/indexer/indexer-worker.c Mon Feb 25 17:29:28 2013 +0200 @@ -45,6 +45,8 @@ int main(int argc, char *argv[]) { + enum master_service_flags service_flags = + MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN; enum mail_storage_service_flags storage_service_flags = MAIL_STORAGE_SERVICE_FLAG_DISALLOW_ROOT | MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP | @@ -52,7 +54,7 @@ MAIL_STORAGE_SERVICE_FLAG_NO_IDLE_TIMEOUT; int c; - master_service = master_service_init("indexer-worker", 0, + master_service = master_service_init("indexer-worker", service_flags, &argc, &argv, "D"); while ((c = master_getopt(master_service)) > 0) { switch (c) {
--- a/src/lib-dict/dict-client.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-dict/dict-client.c Mon Feb 25 17:29:28 2013 +0200 @@ -262,15 +262,17 @@ i_error("dict-client: Unknown transaction id %u", id); return; } - if (ctx->callback != NULL) - ctx->callback(ret, ctx->context); - DLLIST_REMOVE(&dict->transactions, ctx); - i_free(ctx); - + /* the callback may call the dict code again, so remove this + transaction before calling it */ i_assert(dict->async_commits > 0); if (--dict->async_commits == 0) io_remove(&dict->io); + DLLIST_REMOVE(&dict->transactions, ctx); + + if (ctx->callback != NULL) + ctx->callback(ret, ctx->context); + i_free(ctx); } static ssize_t client_dict_read_timeout(struct client_dict *dict)
--- a/src/lib-dict/dict-sql.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-dict/dict-sql.c Mon Feb 25 17:29:28 2013 +0200 @@ -353,6 +353,9 @@ if (map == NULL) return FALSE; + if (ctx->result != NULL) + sql_result_unref(ctx->result); + T_BEGIN { string_t *query = t_str_new(256);
--- a/src/lib-index/mail-transaction-log-file.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-index/mail-transaction-log-file.c Mon Feb 25 17:29:28 2013 +0200 @@ -1721,7 +1721,7 @@ } if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) { - if (start_offset < file->buffer_offset) { + if (start_offset < file->buffer_offset || file->buffer == NULL) { /* we had moved the log to memory but failed to read the beginning of the log file */ mail_index_set_error(index, @@ -1729,7 +1729,6 @@ file->filepath); return 0; } - i_assert(file->buffer != NULL); return log_file_map_check_offsets(file, start_offset, end_offset); }
--- a/src/lib-lda/duplicate.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-lda/duplicate.c Mon Feb 25 17:29:28 2013 +0200 @@ -209,9 +209,16 @@ file->path = p_strdup(pool, ctx->path); file->new_fd = file_dotlock_open(&ctx->dotlock_set, file->path, 0, &file->dotlock); - if (file->new_fd == -1) - i_error("file_dotlock_create(%s) failed: %m", file->path); + if (file->new_fd != -1) + ; + else if (errno != EAGAIN) + i_error("file_dotlock_open(%s) failed: %m", file->path); + else { + i_error("Creating lock file for %s timed out in %u secs", + file->path, ctx->dotlock_set.timeout); + } hash_table_create(&file->hash, pool, 0, duplicate_hash, duplicate_cmp); + (void)duplicate_read(file); return file; }
--- a/src/lib-storage/index/dbox-common/dbox-file-fix.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-file-fix.c Mon Feb 25 17:29:28 2013 +0200 @@ -299,7 +299,7 @@ { struct ostream *output; const char *dir, *p, *temp_path, *broken_path; - bool deleted; + bool deleted, have_messages; int fd, ret; i_assert(dbox_file_is_open(file)); @@ -318,6 +318,7 @@ ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output); if (ret < 0) o_stream_ignore_last_errors(output); + have_messages = output->offset > file->file_header_size; o_stream_unref(&output); if (close(fd) < 0) { mail_storage_set_critical(&file->storage->storage, @@ -343,6 +344,15 @@ i_warning("dbox: Copy of the broken file saved to %s", broken_path); } + if (!have_messages) { + /* the resulting file has no messages. just delete the file. */ + dbox_file_close(file); + if (unlink(temp_path) < 0) + i_error("unlink(%s) failed: %m", temp_path); + if (unlink(file->cur_path) < 0) + i_error("unlink(%s) failed: %m", file->cur_path); + return 0; + } if (rename(temp_path, file->cur_path) < 0) { mail_storage_set_critical(&file->storage->storage, "rename(%s, %s) failed: %m", @@ -358,5 +368,5 @@ file->cur_path); return -1; } - return 0; + return 1; }
--- a/src/lib-storage/index/dbox-common/dbox-file.h Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-file.h Mon Feb 25 17:29:28 2013 +0200 @@ -190,7 +190,8 @@ /* Fix a broken dbox file by rename()ing over it with a fixed file. Everything before start_offset is assumed to be valid and is simply copied. The file - is reopened afterwards. Returns 0 if ok, -1 if I/O error. */ + is reopened afterwards. Returns 1 if ok, 0 if the resulting file has no + mails and was deleted, -1 if I/O error. */ int dbox_file_fix(struct dbox_file *file, uoff_t start_offset); /* Delete the given dbox file. Returns 1 if deleted, 0 if file wasn't found or -1 if error. */
--- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Mon Feb 25 17:29:28 2013 +0200 @@ -153,9 +153,11 @@ /* use existing file header if it was ok */ prev_offset = offset; } - if (dbox_file_fix(file, prev_offset) < 0) { - ret = -1; + if ((ret = dbox_file_fix(file, prev_offset)) < 0) break; + if (ret == 0) { + /* file was deleted */ + return 1; } fixed = TRUE; if (!first) {
--- a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c Mon Feb 25 17:29:28 2013 +0200 @@ -39,7 +39,7 @@ ret = dbox_file_seek(file, 0); } if (ret == 0) { - if ((ret = dbox_file_fix(file, 0)) == 0) + if ((ret = dbox_file_fix(file, 0)) > 0) ret = dbox_file_seek(file, 0); }
--- a/src/lib-storage/index/index-status.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/index-status.c Mon Feb 25 17:29:28 2013 +0200 @@ -113,9 +113,16 @@ hdr = mail_index_get_header(box->view); status_r->messages = hdr->messages_count; if ((items & STATUS_RECENT) != 0) { - /* make sure recent count is set, in case syncing hasn't - been done yet */ - index_sync_update_recent_count(box); + if ((box->flags & MAILBOX_FLAG_DROP_RECENT) != 0) { + /* recent flags are set and dropped by the previous + sync while index was locked. if we updated the + recent flags here we'd have a race condition. */ + i_assert(box->synced); + } else { + /* make sure recent count is set, in case we haven't + synced yet */ + index_sync_update_recent_count(box); + } status_r->recent = index_mailbox_get_recent_count(box); i_assert(status_r->recent <= status_r->messages); }
--- a/src/lib-storage/index/index-storage.h Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/index-storage.h Mon Feb 25 17:29:28 2013 +0200 @@ -37,7 +37,7 @@ struct mail_cache_field *cache_fields; ARRAY_TYPE(seq_range) recent_flags; - uint32_t recent_flags_prev_uid; + uint32_t recent_flags_prev_uid, recent_flags_last_check_nextuid; uint32_t recent_flags_count; uint32_t vsize_hdr_ext_id;
--- a/src/lib-storage/index/index-sync.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/index-sync.c Mon Feb 25 17:29:28 2013 +0200 @@ -367,7 +367,9 @@ uint32_t seq1, seq2; hdr = mail_index_get_header(box->view); - if (hdr->first_recent_uid > ibox->recent_flags_prev_uid) { + if (hdr->first_recent_uid > ibox->recent_flags_prev_uid || + hdr->next_uid > ibox->recent_flags_last_check_nextuid) { + ibox->recent_flags_last_check_nextuid = hdr->next_uid; if (mail_index_lookup_seq_range(box->view, hdr->first_recent_uid, hdr->next_uid,
--- a/src/lib-storage/index/maildir/maildir-sync.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/index/maildir/maildir-sync.c Mon Feb 25 17:29:28 2013 +0200 @@ -288,6 +288,7 @@ const char *fname1, *path1, *path2; const char *new_fname, *new_path; struct stat st1, st2; + uoff_t size; fname1 = maildir_uidlist_sync_get_full_filename(ctx->uidlist_sync_ctx, fname2); @@ -330,6 +331,16 @@ } new_fname = maildir_filename_generate(); + /* preserve S= and W= sizes if they're available. + (S=size is required for zlib plugin to work) */ + if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_FILE_SIZE, &size)) { + new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T, + new_fname, MAILDIR_EXTRA_FILE_SIZE, size); + } + if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) { + new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T, + new_fname, MAILDIR_EXTRA_VIRTUAL_SIZE, size); + } new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box), "/new/", new_fname, NULL);
--- a/src/lib-storage/mail-storage.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/lib-storage/mail-storage.c Mon Feb 25 17:29:28 2013 +0200 @@ -1036,6 +1036,13 @@ { time_t mtime; + if ((box->flags & MAILBOX_FLAG_READONLY) != 0) { + /* most importantly we don't do this because we want to avoid + a loop: mdbox storage rebuild -> mailbox_open() -> + mailbox_mark_index_deleted() -> mailbox_sync() -> + mdbox storage rebuild. */ + return FALSE; + } if (mail_index_get_modification_time(box->index, &mtime) < 0) return FALSE; if (mtime + MAILBOX_DELETE_RETRY_SECS > time(NULL))
--- a/src/plugins/fts-solr/fts-backend-solr.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Mon Feb 25 17:29:28 2013 +0200 @@ -18,9 +18,14 @@ #define SOLR_CMDBUF_SIZE (1024*64) #define SOLR_CMDBUF_FLUSH_SIZE (SOLR_CMDBUF_SIZE-128) -#define SOLR_BUFFER_WARN_SIZE (1024*1024) #define SOLR_MAX_MULTI_ROWS 100000 +/* If header is larger than this, truncate it. */ +#define SOLR_HEADER_MAX_SIZE (1024*1024) +/* If SOLR_HEADER_MAX_SIZE was already reached, write still to individual + header fields as long as they're smaller than this */ +#define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024 + struct solr_fts_backend { struct fts_backend backend; }; @@ -43,12 +48,12 @@ ARRAY(struct solr_fts_field) fields; uint32_t last_indexed_uid; - uint32_t size_warned_uid; unsigned int last_indexed_uid_set:1; unsigned int body_open:1; unsigned int documents_added:1; unsigned int expunges:1; + unsigned int truncate_header:1; }; static bool is_valid_xml_char(unichar_t chr) @@ -446,6 +451,7 @@ fts_backend_solr_doc_close(ctx); } ctx->prev_uid = uid; + ctx->truncate_header = FALSE; fts_backend_solr_doc_open(ctx, uid); } @@ -532,8 +538,11 @@ } xml_encode_data(ctx->cmd, data, size); } else { - xml_encode_data(ctx->cur_value, data, size); - if (ctx->cur_value2 != NULL) + if (!ctx->truncate_header) + xml_encode_data(ctx->cur_value, data, size); + if (ctx->cur_value2 != NULL && + (!ctx->truncate_header || + str_len(ctx->cur_value2) < SOLR_HEADER_LINE_MAX_TRUNC_SIZE)) xml_encode_data(ctx->cur_value2, data, size); } @@ -542,15 +551,15 @@ str_len(ctx->cmd)); str_truncate(ctx->cmd, 0); } - if (str_len(ctx->cur_value) >= SOLR_BUFFER_WARN_SIZE && - ctx->size_warned_uid != ctx->prev_uid) { + if (!ctx->truncate_header && + str_len(ctx->cur_value) >= SOLR_HEADER_MAX_SIZE) { /* a large header */ i_assert(ctx->cur_value != ctx->cmd); - ctx->size_warned_uid = ctx->prev_uid; - i_warning("fts-solr(%s): Mailbox %s UID=%u header size is huge", + i_warning("fts-solr(%s): Mailbox %s UID=%u header size is huge, truncating", ctx->cur_box->storage->user->username, mailbox_get_vname(ctx->cur_box), ctx->prev_uid); + ctx->truncate_header = TRUE; } return 0; }
--- a/src/plugins/fts/fts-build-mail.c Mon Feb 25 17:25:10 2013 +0200 +++ b/src/plugins/fts/fts-build-mail.c Mon Feb 25 17:29:28 2013 +0200 @@ -170,6 +170,7 @@ strncmp(content_type, "message/", 8) == 0) { /* text body parts */ key.type = FTS_BACKEND_BUILD_KEY_BODY_PART; + ctx->body_parser = fts_parser_text_init(); } else { /* possibly binary */ if ((ctx->update_ctx->backend->flags & @@ -178,8 +179,6 @@ *binary_body_r = TRUE; key.type = FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY; } - if (ctx->body_parser == NULL) - ctx->body_parser = fts_parser_text_init(); key.body_content_type = content_type; key.body_content_disposition = ctx->content_disposition; return fts_backend_update_set_build_key(ctx->update_ctx, &key);