Mercurial > dovecot > core-2.2
changeset 9977:0bb321c347ae HEAD
Split dbox (single-dbox) and mdbox (multi-dbox) into separate storage backends.
This cleans up the code, makes it faster and also fixes some bugs.
Super-fast maildir migration code was also dropped, at least for now.
line wrap: on
line diff
--- a/configure.in Tue Oct 06 13:24:01 2009 -0400 +++ b/configure.in Tue Oct 06 19:22:42 2009 -0400 @@ -216,12 +216,12 @@ AC_ARG_WITH(storages, [ --with-storages Build with specified mail storage formats - (maildir mbox dbox cydir)], [ + (maildir mbox dbox mdbox cydir)], [ if test "$withval" = "yes" || test "$withval" = "no"; then AC_MSG_ERROR([--with-storages needs storage list as parameter]) fi mail_storages="shared `echo "$withval"|sed 's/,/ /g'`" ], - mail_storages="shared maildir mbox dbox cydir") + mail_storages="shared maildir mbox dbox mdbox cydir") AC_SUBST(mail_storages) AC_ARG_WITH(sql-drivers, @@ -2273,7 +2273,9 @@ maildir_libs='$(top_builddir)/src/lib-storage/index/maildir/libstorage_maildir.la' mbox_libs='$(top_builddir)/src/lib-storage/index/mbox/libstorage_mbox.la' -dbox_libs='$(top_builddir)/src/lib-storage/index/dbox/libstorage_dbox.la' +dbox_common_libs='$(top_builddir)/src/lib-storage/index/dbox-common/libstorage_dbox_common.la' +dbox_libs='$(top_builddir)/src/lib-storage/index/dbox-single/libstorage_dbox_single.la' +mdbox_libs='$(top_builddir)/src/lib-storage/index/dbox-multi/libstorage_dbox_multi.la' cydir_libs='$(top_builddir)/src/lib-storage/index/cydir/libstorage_cydir.la' raw_libs='$(top_builddir)/src/lib-storage/index/raw/libstorage_raw.la' shared_libs='$(top_builddir)/src/lib-storage/index/shared/libstorage_shared.la' @@ -2287,6 +2289,10 @@ mail_storages=`(for i in $mail_storages; do echo $i; done)|sort|uniq|xargs echo` for storage in $mail_storages; do LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS `eval echo \\$${storage}_libs`" + if test $storage = dbox || test $storage = mdbox; then + LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS $dbox_common_libs" + dbox_common_libs="" + fi done AC_SUBST(LINKED_STORAGE_LIBS) AC_DEFINE_UNQUOTED(MAIL_STORAGES, "$mail_storages", List of compiled in mail storages) @@ -2480,7 +2486,9 @@ src/lib-storage/index/Makefile src/lib-storage/index/maildir/Makefile src/lib-storage/index/mbox/Makefile -src/lib-storage/index/dbox/Makefile +src/lib-storage/index/dbox-common/Makefile +src/lib-storage/index/dbox-multi/Makefile +src/lib-storage/index/dbox-single/Makefile src/lib-storage/index/cydir/Makefile src/lib-storage/index/raw/Makefile src/lib-storage/index/shared/Makefile
--- a/src/lib-storage/index/Makefile.am Tue Oct 06 13:24:01 2009 -0400 +++ b/src/lib-storage/index/Makefile.am Tue Oct 06 19:22:42 2009 -0400 @@ -1,4 +1,4 @@ -SUBDIRS = maildir mbox dbox cydir raw shared +SUBDIRS = maildir mbox dbox-common dbox-multi dbox-single cydir raw shared noinst_LTLIBRARIES = libstorage_index.la
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/Makefile.am Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,32 @@ +noinst_LTLIBRARIES = libstorage_dbox_common.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_dbox_common_la_SOURCES = \ + dbox-file.c \ + dbox-file-fix.c \ + dbox-mail.c \ + dbox-save.c \ + dbox-storage.c \ + dbox-sync-rebuild.c + +headers = \ + dbox-file.h \ + dbox-mail.h \ + dbox-save.h \ + dbox-storage.h \ + dbox-sync-rebuild.h + +if INSTALL_HEADERS + pkginc_libdir=$(pkgincludedir) + pkginc_lib_HEADERS = $(headers) +else + noinst_HEADERS = $(headers) +endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-file-fix.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,354 @@ +/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "ostream.h" +#include "str-find.h" +#include "hex-binary.h" +#include "message-size.h" +#include "dbox-storage.h" +#include "dbox-file.h" + +#include <stdio.h> + +#define DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX ".broken" + +static int +dbox_file_find_next_magic(struct dbox_file *file, uoff_t *offset_r, bool *pre_r) +{ + struct istream *input = file->input; + struct str_find_context *pre_ctx, *post_ctx; + uoff_t orig_offset, pre_offset, post_offset; + const unsigned char *data; + size_t size; + int ret; + + *pre_r = FALSE; + + pre_ctx = str_find_init(default_pool, "\n"DBOX_MAGIC_PRE); + post_ctx = str_find_init(default_pool, DBOX_MAGIC_POST); + + /* \n isn't part of the DBOX_MAGIC_PRE, but it always preceds it. + assume that at this point we've already just read the \n. when + scanning for it later we'll need to find the \n though. */ + str_find_more(pre_ctx, (const unsigned char *)"\n", 1); + + orig_offset = input->v_offset; + while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { + pre_offset = (uoff_t)-1; + post_offset = (uoff_t)-1; + if (str_find_more(pre_ctx, data, size)) { + pre_offset = input->v_offset + + str_find_get_match_end_pos(pre_ctx) - + (strlen(DBOX_MAGIC_PRE) + 1); + *pre_r = TRUE; + } + if (str_find_more(post_ctx, data, size)) { + post_offset = input->v_offset + + str_find_get_match_end_pos(post_ctx) - + strlen(DBOX_MAGIC_POST); + if (pre_offset == (uoff_t)-1 || + post_offset < pre_offset) { + pre_offset = post_offset; + *pre_r = FALSE; + } + } + + if (pre_offset != (uoff_t)-1) { + if (*pre_r) { + /* LF isn't part of the magic */ + pre_offset++; + } + *offset_r = pre_offset; + break; + } + i_stream_skip(input, size); + } + if (ret <= 0) { + i_assert(ret == -1); + if (input->stream_errno != 0) + dbox_file_set_syscall_error(file, "read()"); + else { + ret = 0; + *offset_r = input->v_offset; + } + } + i_stream_seek(input, orig_offset); + str_find_deinit(&pre_ctx); + str_find_deinit(&post_ctx); + return ret; +} + +static int +stream_copy(struct dbox_file *file, struct ostream *output, + const char *path, uoff_t count) +{ + struct istream *input; + off_t bytes; + + input = i_stream_create_limit(file->input, count); + bytes = o_stream_send_istream(output, input); + i_stream_unref(&input); + + if (bytes < 0) { + mail_storage_set_critical(&file->storage->storage, + "o_stream_send_istream(%s, %s) failed: %m", + file->cur_path, path); + return -1; + } + if ((uoff_t)bytes != count) { + mail_storage_set_critical(&file->storage->storage, + "o_stream_send_istream(%s) copied only %" + PRIuUOFF_T" of %"PRIuUOFF_T" bytes", + path, bytes, count); + return -1; + } + return 0; +} + +static void dbox_file_skip_broken_header(struct dbox_file *file) +{ + const unsigned int magic_len = strlen(DBOX_MAGIC_PRE); + const unsigned char *data; + size_t i, size; + + /* if there's LF close to our position, assume that the header ends + there. */ + data = i_stream_get_data(file->input, &size); + if (size > file->msg_header_size + 16) + size = file->msg_header_size + 16; + for (i = 0; i < size; i++) { + if (data[i] == '\n') { + i_stream_skip(file->input, i); + return; + } + } + + /* skip at least the magic bytes if possible */ + if (size > magic_len && memcmp(data, DBOX_MAGIC_PRE, magic_len) == 0) + i_stream_skip(file->input, magic_len); +} + +static void +dbox_file_copy_metadata(struct dbox_file *file, struct ostream *output, + bool *have_guid_r) +{ + const char *line; + uoff_t prev_offset = file->input->v_offset; + + *have_guid_r = FALSE; + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + return; + } + if (*line < 32) { + /* broken - possibly a new pre-magic block */ + i_stream_seek(file->input, prev_offset); + return; + } + if (*line == DBOX_METADATA_VIRTUAL_SIZE) { + /* it may be wrong - recreate it */ + continue; + } + if (*line == DBOX_METADATA_GUID) + *have_guid_r = TRUE; + o_stream_send_str(output, line); + o_stream_send_str(output, "\n"); + } +} + +static int +dbox_file_fix_write_stream(struct dbox_file *file, uoff_t start_offset, + const char *temp_path, struct ostream *output) +{ + struct dbox_message_header msg_hdr; + uoff_t offset, msg_size, hdr_offset, body_offset; + bool pre, write_header, have_guid; + struct message_size body; + struct istream *body_input; + uint8_t guid_128[MAIL_GUID_128_SIZE]; + int ret; + + i_stream_seek(file->input, 0); + if (start_offset > 0) { + /* copy the valid data */ + if (stream_copy(file, output, temp_path, start_offset) < 0) + return -1; + } else { + /* the file header is broken. recreate it */ + if (dbox_file_header_write(file, output) < 0) { + dbox_file_set_syscall_error(file, "write()"); + return -1; + } + } + + while ((ret = dbox_file_find_next_magic(file, &offset, &pre)) > 0) { + msg_size = offset - file->input->v_offset; + if (msg_size < 256 && pre) { + /* probably some garbage or some broken headers. + we most likely don't miss anything by skipping + over this data. */ + i_stream_skip(file->input, msg_size); + hdr_offset = file->input->v_offset; + ret = dbox_file_read_mail_header(file, &msg_size); + if (ret <= 0) { + if (ret < 0) + return -1; + dbox_file_skip_broken_header(file); + body_offset = file->input->v_offset; + msg_size = (uoff_t)-1; + } else { + i_stream_skip(file->input, + file->msg_header_size); + body_offset = file->input->v_offset; + i_stream_skip(file->input, msg_size); + } + + ret = dbox_file_find_next_magic(file, &offset, &pre); + if (ret <= 0) + break; + + if (!pre && msg_size == offset - body_offset) { + /* msg header ok, copy it */ + i_stream_seek(file->input, hdr_offset); + if (stream_copy(file, output, temp_path, + file->msg_header_size) < 0) + return -1; + write_header = FALSE; + } else { + /* msg header is broken. write our own. */ + i_stream_seek(file->input, body_offset); + if (msg_size != (uoff_t)-1) { + /* previous magic find might have + skipped too much. seek back and + make sure */ + ret = dbox_file_find_next_magic(file, &offset, &pre); + if (ret <= 0) + break; + } + + write_header = TRUE; + msg_size = offset - body_offset; + } + } else { + /* treat this data as a separate message. */ + write_header = TRUE; + body_offset = file->input->v_offset; + } + /* write msg header */ + if (write_header) { + dbox_msg_header_fill(&msg_hdr, msg_size); + (void)o_stream_send(output, &msg_hdr, sizeof(msg_hdr)); + } + /* write msg body */ + i_assert(file->input->v_offset == body_offset); + if (stream_copy(file, output, temp_path, msg_size) < 0) + return -1; + i_assert(file->input->v_offset == offset); + + /* get message body size */ + i_stream_seek(file->input, body_offset); + body_input = i_stream_create_limit(file->input, msg_size); + ret = message_get_body_size(body_input, &body, NULL); + i_stream_unref(&body_input); + if (ret < 0) { + errno = output->stream_errno; + mail_storage_set_critical(&file->storage->storage, + "read(%s) failed: %m", file->cur_path); + return -1; + } + + /* write msg metadata. */ + i_assert(file->input->v_offset == offset); + ret = dbox_file_metadata_skip_header(file); + if (ret < 0) + return -1; + o_stream_send_str(output, DBOX_MAGIC_POST); + if (ret == 0) + have_guid = FALSE; + else + dbox_file_copy_metadata(file, output, &have_guid); + if (!have_guid) { + mail_generate_guid_128(guid_128); + o_stream_send_str(output, + t_strdup_printf("%c%s\n", DBOX_METADATA_GUID, + binary_to_hex(guid_128, sizeof(guid_128)))); + } + o_stream_send_str(output, + t_strdup_printf("%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, + (unsigned long long)body.virtual_size)); + o_stream_send_str(output, "\n"); + if (output->stream_errno != 0) { + errno = output->stream_errno; + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %m", temp_path); + return -1; + } + } + return 0; +} + +int dbox_file_fix(struct dbox_file *file, uoff_t start_offset) +{ + struct ostream *output; + const char *dir, *fname, *temp_path, *broken_path; + bool deleted; + int fd, ret; + + i_assert(dbox_file_is_open(file)); + + fname = strrchr(file->cur_path, '/'); + i_assert(fname != NULL); + dir = t_strdup_until(file->cur_path, fname); + fname++; + + temp_path = t_strdup_printf("%s/%s", dir, dbox_generate_tmp_filename()); + fd = file->storage->v.file_create_fd(file, temp_path, FALSE); + if (fd == -1) + return -1; + + output = o_stream_create_fd_file(fd, 0, FALSE); + ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output); + o_stream_unref(&output); + if (close(fd) < 0) { + mail_storage_set_critical(&file->storage->storage, + "close(%s) failed: %m", temp_path); + ret = -1; + } + if (ret < 0) { + if (unlink(temp_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "unlink(%s) failed: %m", temp_path); + } + return -1; + } + /* keep a copy of the original file in case someone wants to look + at it */ + broken_path = t_strconcat(file->cur_path, + DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX, NULL); + if (link(file->cur_path, broken_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "link(%s, %s) failed: %m", + file->cur_path, broken_path); + } else { + i_warning("dbox: Copy of the broken file saved to %s", + broken_path); + } + if (rename(temp_path, file->cur_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "rename(%s, %s) failed: %m", + temp_path, file->cur_path); + return -1; + } + + /* file was successfully recreated - reopen it */ + dbox_file_close(file); + if (dbox_file_open(file, &deleted) <= 0) { + mail_storage_set_critical(&file->storage->storage, + "dbox_file_fix(%s): reopening file failed", + file->cur_path); + return -1; + } + return 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,755 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "hex-dec.h" +#include "hex-binary.h" +#include "hostpid.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "mkdir-parents.h" +#include "fdatasync-path.h" +#include "eacces-error.h" +#include "str.h" +#include "dbox-storage.h" +#include "dbox-file.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <ctype.h> +#include <fcntl.h> + +#define DBOX_READ_BLOCK_SIZE 4096 + +const char *dbox_generate_tmp_filename(void) +{ + static unsigned int create_count = 0; + + return t_strdup_printf("temp.%lu.P%sQ%uM%u.%s", + (unsigned long)ioloop_timeval.tv_sec, my_pid, + create_count++, + (unsigned int)ioloop_timeval.tv_usec, + my_hostname); +} + +void dbox_file_set_syscall_error(struct dbox_file *file, const char *function) +{ + mail_storage_set_critical(&file->storage->storage, + "%s failed for file %s: %m", + function, file->cur_path); +} + +void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) +{ + va_list args; + + file->storage->files_corrupted = TRUE; + + va_start(args, reason); + mail_storage_set_critical(&file->storage->storage, + "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s", + file->cur_path, file->input == NULL ? 0 : file->input->v_offset, + t_strdup_vprintf(reason, args)); + va_end(args); +} + +void dbox_file_init(struct dbox_file *file) +{ + file->refcount = 1; + file->fd = -1; + file->cur_offset = (uoff_t)-1; + file->cur_path = file->primary_path; +} + +void dbox_file_free(struct dbox_file *file) +{ + i_assert(file->refcount == 0); + + if (file->metadata_pool != NULL) + pool_unref(&file->metadata_pool); + dbox_file_close(file); + i_free(file->primary_path); + i_free(file->alt_path); + i_free(file); +} + +void dbox_file_unref(struct dbox_file **_file) +{ + struct dbox_file *file = *_file; + + *_file = NULL; + + i_assert(file->refcount > 0); + if (--file->refcount == 0) + file->storage->v.file_unrefed(file); +} + +static int dbox_file_parse_header(struct dbox_file *file, const char *line) +{ + const char *const *tmp, *value; + unsigned int pos; + enum dbox_header_key key; + + file->file_version = *line - '0'; + if (!i_isdigit(line[0]) || line[1] != ' ' || + (file->file_version != 1 && file->file_version != DBOX_VERSION)) { + dbox_file_set_corrupted(file, "Invalid dbox version"); + return -1; + } + line += 2; + pos = 2; + + file->msg_header_size = 0; + + for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) { + key = **tmp; + value = *tmp + 1; + + switch (key) { + case DBOX_HEADER_OLDV1_APPEND_OFFSET: + break; + case DBOX_HEADER_MSG_HEADER_SIZE: + file->msg_header_size = strtoul(value, NULL, 16); + break; + case DBOX_HEADER_CREATE_STAMP: + file->create_time = strtoul(value, NULL, 16); + break; + } + pos += strlen(value) + 2; + } + + if (file->msg_header_size == 0) { + dbox_file_set_corrupted(file, "Missing message header size"); + return -1; + } + return 0; +} + +static int dbox_file_read_header(struct dbox_file *file) +{ + const char *line; + unsigned int hdr_size; + int ret; + + i_stream_seek(file->input, 0); + line = i_stream_read_next_line(file->input); + if (line == NULL) { + if (file->input->stream_errno == 0) { + dbox_file_set_corrupted(file, + "EOF while reading file header"); + return 0; + } + + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + hdr_size = file->input->v_offset; + T_BEGIN { + ret = dbox_file_parse_header(file, line) < 0 ? 0 : 1; + } T_END; + if (ret > 0) + file->file_header_size = hdr_size; + return ret; +} + +static int dbox_file_open_fd(struct dbox_file *file) +{ + const char *path; + bool alt = FALSE; + + /* try the primary path first */ + path = file->primary_path; + while ((file->fd = open(path, O_RDWR)) == -1) { + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "open(%s) failed: %m", path); + return -1; + } + + if (file->alt_path == NULL || alt) { + /* not found */ + return 0; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + file->cur_path = path; + return 1; +} + +int dbox_file_open(struct dbox_file *file, bool *deleted_r) +{ + int ret; + + *deleted_r = FALSE; + if (file->input != NULL) + return 1; + + if (file->fd == -1) { + T_BEGIN { + ret = dbox_file_open_fd(file); + } T_END; + if (ret <= 0) { + if (ret < 0) + return -1; + *deleted_r = TRUE; + return 1; + } + } + + file->input = i_stream_create_fd(file->fd, 0, FALSE); + i_stream_set_init_buffer_size(file->input, DBOX_READ_BLOCK_SIZE); + return dbox_file_read_header(file); +} + +int dbox_file_header_write(struct dbox_file *file, struct ostream *output) +{ + string_t *hdr; + + hdr = t_str_new(128); + str_printfa(hdr, "%u %c%x %c%x\n", DBOX_VERSION, + DBOX_HEADER_MSG_HEADER_SIZE, + (unsigned int)sizeof(struct dbox_message_header), + DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time); + + file->file_version = DBOX_VERSION; + file->file_header_size = str_len(hdr); + file->msg_header_size = sizeof(struct dbox_message_header); + return o_stream_send(output, str_data(hdr), str_len(hdr)); +} + +void dbox_file_close(struct dbox_file *file) +{ + dbox_file_unlock(file); + if (file->input != NULL) + i_stream_unref(&file->input); + if (file->fd != -1) { + if (close(file->fd) < 0) + dbox_file_set_syscall_error(file, "close()"); + file->fd = -1; + } + file->cur_offset = (uoff_t)-1; +} + +int dbox_file_try_lock(struct dbox_file *file) +{ + int ret; + + i_assert(file->fd != -1); + + ret = file_try_lock(file->fd, file->cur_path, F_WRLCK, + FILE_LOCK_METHOD_FCNTL, &file->lock); + if (ret < 0) { + mail_storage_set_critical(&file->storage->storage, + "file_try_lock(%s) failed: %m", file->cur_path); + } + return ret; +} + +void dbox_file_unlock(struct dbox_file *file) +{ + i_assert(!file->appending); + + if (file->lock != NULL) + file_unlock(&file->lock); + if (file->input != NULL) + i_stream_sync(file->input); +} + +int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) +{ + struct dbox_message_header hdr; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_data(file->input, &data, &size, + file->msg_header_size - 1); + if (ret <= 0) { + if (file->input->stream_errno == 0) { + /* EOF, broken offset or file truncated */ + dbox_file_set_corrupted(file, "EOF reading msg header " + "(got %"PRIuSIZE_T"/%u bytes)", + size, file->msg_header_size); + return 0; + } + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size)); + if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) { + /* probably broken offset */ + dbox_file_set_corrupted(file, "msg header has bad magic value"); + return 0; + } + + if (data[file->msg_header_size-1] != '\n') { + dbox_file_set_corrupted(file, "msg header doesn't end with LF"); + return 0; + } + + *physical_size_r = hex2dec(hdr.message_size_hex, + sizeof(hdr.message_size_hex)); + return 1; +} + +int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, + uoff_t *physical_size_r, + struct istream **stream_r) +{ + uoff_t size; + int ret; + + i_assert(file->input != NULL); + + if (offset == 0) + offset = file->file_header_size; + + if (offset != file->cur_offset) { + i_stream_seek(file->input, offset); + ret = dbox_file_read_mail_header(file, &size); + if (ret <= 0) + return ret; + file->cur_offset = offset; + file->cur_physical_size = size; + } + i_stream_seek(file->input, offset + file->msg_header_size); + if (stream_r != NULL) { + *stream_r = i_stream_create_limit(file->input, + file->cur_physical_size); + } + *physical_size_r = file->cur_physical_size; + return 1; +} + +static int +dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset) +{ + const char *line; + int ret; + + i_stream_seek(file->input, *offset); + if ((ret = dbox_file_metadata_skip_header(file)) <= 0) + return ret; + + /* skip over the actual metadata */ + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + break; + } + } + *offset = file->input->v_offset; + return 1; +} + +void dbox_file_seek_rewind(struct dbox_file *file) +{ + file->cur_offset = (uoff_t)-1; +} + +int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r) +{ + uoff_t offset, size; + int ret; + + if (file->cur_offset == (uoff_t)-1) { + /* first mail. we may not have read the file at all yet, + so set the offset afterwards. */ + offset = 0; + } else { + offset = file->cur_offset + file->msg_header_size + + file->cur_physical_size; + if ((ret = dbox_file_seek_next_at_metadata(file, &offset)) <= 0) { + *offset_r = file->cur_offset; + return ret; + } + } + *offset_r = offset; + + if (file->input != NULL && i_stream_is_eof(file->input)) { + *last_r = TRUE; + return 0; + } + *last_r = FALSE; + + ret = dbox_file_get_mail_stream(file, offset, &size, NULL); + if (*offset_r == 0) + *offset_r = file->file_header_size; + return ret; +} + +struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file) +{ + struct dbox_file_append_context *ctx; + + i_assert(!file->appending); + + file->appending = TRUE; + + ctx = i_new(struct dbox_file_append_context, 1); + ctx->file = file; + if (file->fd != -1) { + ctx->output = o_stream_create_fd_file(file->fd, 0, FALSE); + o_stream_cork(ctx->output); + } + return ctx; +} + +int dbox_file_append_commit(struct dbox_file_append_context **_ctx) +{ + struct dbox_file_append_context *ctx = *_ctx; + int ret; + + i_assert(ctx->file->appending); + + *_ctx = NULL; + + ret = dbox_file_append_flush(ctx); + o_stream_unref(&ctx->output); + ctx->file->appending = FALSE; + i_free(ctx); + return 0; +} + +void dbox_file_append_rollback(struct dbox_file_append_context **_ctx) +{ + struct dbox_file_append_context *ctx = *_ctx; + struct dbox_file *file = ctx->file; + bool close_file = FALSE; + + i_assert(ctx->file->appending); + + *_ctx = NULL; + if (ctx->first_append_offset == 0) { + /* nothing changed */ + } else if (ctx->first_append_offset == file->file_header_size) { + /* rollbacking everything */ + if (unlink(file->cur_path) < 0) + dbox_file_set_syscall_error(file, "unlink()"); + close_file = TRUE; + } else { + /* truncating only some mails */ + o_stream_close(ctx->output); + if (ftruncate(file->fd, ctx->first_append_offset) < 0) + dbox_file_set_syscall_error(file, "ftruncate()"); + } + if (ctx->output != NULL) + o_stream_unref(&ctx->output); + i_free(ctx); + + if (close_file) + dbox_file_close(file); + file->appending = FALSE; +} + +int dbox_file_append_flush(struct dbox_file_append_context *ctx) +{ + if (ctx->last_flush_offset == ctx->output->offset) + return 0; + + if (o_stream_flush(ctx->output) < 0) { + dbox_file_set_syscall_error(ctx->file, "write()"); + return -1; + } + + if (!ctx->file->storage->storage.set->fsync_disable) { + if (fdatasync(ctx->file->fd) < 0) { + dbox_file_set_syscall_error(ctx->file, "fdatasync()"); + return -1; + } + } + ctx->last_flush_offset = ctx->output->offset; + return 0; +} + +int dbox_file_get_append_stream(struct dbox_file_append_context *ctx, + struct ostream **output_r) +{ + struct dbox_file *file = ctx->file; + struct stat st; + + if (ctx->output == NULL) { + /* file creation had failed */ + return -1; + } + + if (file->file_version == 0) { + /* newly created file, write the file header */ + if (dbox_file_header_write(file, ctx->output) < 0) { + dbox_file_set_syscall_error(file, "write()"); + return -1; + } + *output_r = ctx->output; + return 1; + } + + /* file has existing mails */ + if (file->file_version != DBOX_VERSION || + file->msg_header_size != sizeof(struct dbox_message_header)) { + /* created by an incompatible version, can't append */ + return 0; + } + + if (ctx->output->offset == 0) { + /* first append to existing file. seek to eof first. */ + if (fstat(file->fd, &st) < 0) { + dbox_file_set_syscall_error(file, "fstat()"); + return -1; + } + o_stream_seek(ctx->output, st.st_size); + } + *output_r = ctx->output; + return 1; +} + +int dbox_file_metadata_skip_header(struct dbox_file *file) +{ + struct dbox_metadata_header metadata_hdr; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_data(file->input, &data, &size, + sizeof(metadata_hdr) - 1); + if (ret <= 0) { + if (file->input->stream_errno == 0) { + /* EOF, broken offset */ + dbox_file_set_corrupted(file, + "Unexpected EOF while reading metadata header"); + return 0; + } + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + memcpy(&metadata_hdr, data, sizeof(metadata_hdr)); + if (memcmp(metadata_hdr.magic_post, DBOX_MAGIC_POST, + sizeof(metadata_hdr.magic_post)) != 0) { + /* probably broken offset */ + dbox_file_set_corrupted(file, + "metadata header has bad magic value"); + return 0; + } + i_stream_skip(file->input, sizeof(metadata_hdr)); + return 1; +} + +static int +dbox_file_metadata_read_at(struct dbox_file *file, uoff_t metadata_offset) +{ + const char *line; + int ret; + + if (file->metadata_pool != NULL) + p_clear(file->metadata_pool); + else { + file->metadata_pool = + pool_alloconly_create("dbox metadata", 1024); + } + p_array_init(&file->metadata, file->metadata_pool, 16); + + i_stream_seek(file->input, metadata_offset); + if ((ret = dbox_file_metadata_skip_header(file)) <= 0) + return ret; + + ret = 0; + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + ret = 1; + break; + } + line = p_strdup(file->metadata_pool, line); + array_append(&file->metadata, &line, 1); + } + if (ret == 0) + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return ret; +} + +int dbox_file_metadata_read(struct dbox_file *file) +{ + uoff_t metadata_offset; + int ret; + + i_assert(file->cur_offset != (uoff_t)-1); + + if (file->metadata_read_offset == file->cur_offset) + return 1; + + metadata_offset = file->cur_offset + file->msg_header_size + + file->cur_physical_size; + ret = dbox_file_metadata_read_at(file, metadata_offset); + if (ret <= 0) + return ret; + + file->metadata_read_offset = file->cur_offset; + return 1; +} + +const char *dbox_file_metadata_get(struct dbox_file *file, + enum dbox_metadata_key key) +{ + const char *const *metadata; + unsigned int i, count; + + metadata = array_get(&file->metadata, &count); + for (i = 0; i < count; i++) { + if (*metadata[i] == (char)key) + return metadata[i] + 1; + } + return NULL; +} + +int dbox_file_move(struct dbox_file *file, bool alt_path) +{ + struct ostream *output; + const char *dest_dir, *temp_path, *dest_path, *p; + struct stat st; + bool deleted; + int out_fd, ret = 0; + + i_assert(file->input != NULL); + i_assert(file->lock != NULL); + + if (dbox_file_is_in_alt(file) == alt_path) + return 0; + + if (stat(file->cur_path, &st) < 0 && errno == ENOENT) { + /* already expunged/moved by another session */ + dbox_file_unlock(file); + return 0; + } + + dest_path = !alt_path ? file->primary_path : file->alt_path; + p = strrchr(dest_path, '/'); + i_assert(p != NULL); + dest_dir = t_strdup_until(dest_path, p); + temp_path = t_strdup_printf("%s/%s", dest_dir, + dbox_generate_tmp_filename()); + + /* first copy the file. make sure to catch every possible error + since we really don't want to break the file. */ + out_fd = file->storage->v.file_create_fd(file, temp_path, TRUE); + if (out_fd == -1) + return -1; + + output = o_stream_create_fd_file(out_fd, 0, FALSE); + i_stream_seek(file->input, 0); + while ((ret = o_stream_send_istream(output, file->input)) > 0) ; + if (ret == 0) + ret = o_stream_flush(output); + if (output->stream_errno != 0) { + errno = output->stream_errno; + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %m", temp_path); + ret = -1; + } else if (file->input->stream_errno != 0) { + errno = file->input->stream_errno; + dbox_file_set_syscall_error(file, "ftruncate()"); + ret = -1; + } else if (ret < 0) { + mail_storage_set_critical(&file->storage->storage, + "o_stream_send_istream(%s, %s) " + "failed with unknown error", + temp_path, file->cur_path); + } + o_stream_unref(&output); + + if (!file->storage->storage.set->fsync_disable && ret == 0) { + if (fsync(out_fd) < 0) { + mail_storage_set_critical(&file->storage->storage, + "fsync(%s) failed: %m", temp_path); + ret = -1; + } + } + if (close(out_fd) < 0) { + mail_storage_set_critical(&file->storage->storage, + "close(%s) failed: %m", temp_path); + ret = -1; + } + if (ret < 0) { + (void)unlink(temp_path); + return -1; + } + + /* the temp file was successfully written. rename it now to the + destination file. the destination shouldn't exist, but if it does + its contents should be the same (except for maybe older metadata) */ + if (rename(temp_path, dest_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "rename(%s, %s) failed: %m", temp_path, dest_path); + (void)unlink(temp_path); + return -1; + } + if (!file->storage->storage.set->fsync_disable) { + if (fdatasync_path(dest_dir) < 0) { + mail_storage_set_critical(&file->storage->storage, + "fdatasync(%s) failed: %m", dest_dir); + (void)unlink(dest_path); + return -1; + } + } + if (unlink(file->cur_path) < 0) { + dbox_file_set_syscall_error(file, "unlink()"); + if (errno == EACCES) { + /* configuration problem? revert the write */ + (void)unlink(dest_path); + } + /* who knows what happened to the file. keep both just to be + sure both won't get deleted. */ + return -1; + } + + /* file was successfully moved - reopen it */ + dbox_file_close(file); + if (dbox_file_open(file, &deleted) <= 0) { + mail_storage_set_critical(&file->storage->storage, + "dbox_file_move(%s): reopening file failed", dest_path); + return -1; + } + return 0; +} + +void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, + uoff_t message_size) +{ + memset(dbox_msg_hdr, ' ', sizeof(*dbox_msg_hdr)); + memcpy(dbox_msg_hdr->magic_pre, DBOX_MAGIC_PRE, + sizeof(dbox_msg_hdr->magic_pre)); + dbox_msg_hdr->type = DBOX_MESSAGE_TYPE_NORMAL; + dec2hex(dbox_msg_hdr->message_size_hex, message_size, + sizeof(dbox_msg_hdr->message_size_hex)); + dbox_msg_hdr->save_lf = '\n'; +} + +int dbox_file_unlink(struct dbox_file *file) +{ + const char *path; + bool alt = FALSE; + + path = file->primary_path; + while (unlink(path) < 0) { + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "unlink(%s) failed: %m", path); + return -1; + } + if (file->alt_path == NULL || alt) { + /* not found */ + i_warning("dbox: File unexpectedly lost: %s", + file->primary_path); + return 0; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + return 1; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-file.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,194 @@ +#ifndef DBOX_FILE_H +#define DBOX_FILE_H + +/* The file begins with a header followed by zero or more messages: + + <dbox message header> + <LF> + <message body> + <metadata> + + Metadata block begins with DBOX_MAGIC_POST, followed by zero or more lines + in format <key character><value><LF>. The block ends with an empty line. + Unknown metadata should be ignored, but preserved when copying. + + There should be no duplicates for the current metadata, but future + extensions may need them so they should be preserved. +*/ +#define DBOX_VERSION 2 +#define DBOX_MAGIC_PRE "\001\002" +#define DBOX_MAGIC_POST "\n\001\003\n" + +struct dbox_file; + +enum dbox_header_key { + /* Must be sizeof(struct dbox_message_header) when appending (hex) */ + DBOX_HEADER_MSG_HEADER_SIZE = 'M', + /* Creation UNIX timestamp (hex) */ + DBOX_HEADER_CREATE_STAMP = 'C', + + /* metadata used by old Dovecot versions */ + DBOX_HEADER_OLDV1_APPEND_OFFSET = 'A' +}; + +enum dbox_metadata_key { + /* Globally unique identifier for the message. Preserved when + copying. */ + DBOX_METADATA_GUID = 'G', + /* POP3 UIDL overriding the default format */ + DBOX_METADATA_POP3_UIDL = 'P', + /* Received UNIX timestamp in hex */ + DBOX_METADATA_RECEIVED_TIME = 'R', + /* Saved UNIX timestamp in hex */ + DBOX_METADATA_SAVE_TIME = 'S', + /* Virtual message size in hex (line feeds counted as CRLF) */ + DBOX_METADATA_VIRTUAL_SIZE = 'V', + /* Pointer to external message data. Format is: + 1*(<start offset> <byte count> <ref>) */ + DBOX_METADATA_EXT_REF = 'X', + /* Mailbox name where this message was originally saved to. + When rebuild finds a message whose mailbox is unknown, it's + placed to this mailbox. */ + DBOX_METADATA_ORIG_MAILBOX = 'B', + + /* metadata used by old Dovecot versions */ + DBOX_METADATA_OLDV1_EXPUNGED = 'E', + DBOX_METADATA_OLDV1_FLAGS = 'F', + DBOX_METADATA_OLDV1_KEYWORDS = 'K', + DBOX_METADATA_OLDV1_SPACE = ' ' +}; + +enum dbox_message_type { + /* Normal message */ + DBOX_MESSAGE_TYPE_NORMAL = 'N', + /* Parts of the message exists outside the following data. + See the metadata for how to find them. */ + DBOX_MESSAGE_TYPE_EXT_REFS = 'E' +}; + +struct dbox_message_header { + unsigned char magic_pre[2]; + unsigned char type; + unsigned char space1; + unsigned char oldv1_uid_hex[8]; + unsigned char space2; + unsigned char message_size_hex[16]; + /* <space reserved for future extensions, LF is always last> */ + unsigned char save_lf; +}; + +struct dbox_metadata_header { + unsigned char magic_post[sizeof(DBOX_MAGIC_POST)-1]; +}; + +struct dbox_file { + struct dbox_storage *storage; + int refcount; + + time_t create_time; + unsigned int file_version; + unsigned int file_header_size; + unsigned int msg_header_size; + + const char *cur_path; + char *primary_path, *alt_path; + int fd; + struct istream *input; + struct file_lock *lock; + + uoff_t cur_offset; + uoff_t cur_physical_size; + + /* Metadata for the currently seeked metadata block. */ + pool_t metadata_pool; + ARRAY_DEFINE(metadata, const char *); + uoff_t metadata_read_offset; + + unsigned int appending:1; + unsigned int deleted:1; + unsigned int corrupted:1; +}; + +struct dbox_file_append_context { + struct dbox_file *file; + + uoff_t first_append_offset, last_flush_offset; + struct ostream *output; +}; + +#define dbox_file_is_open(file) ((file)->fd != -1) +#define dbox_file_is_in_alt(file) ((file)->cur_path == (file)->alt_path) + +void dbox_file_init(struct dbox_file *file); +void dbox_file_unref(struct dbox_file **file); + +/* Open the file. Returns 1 if ok, 0 if file header is corrupted, -1 if error. + If file is deleted, deleted_r=TRUE and 1 is returned. */ +int dbox_file_open(struct dbox_file *file, bool *deleted_r); +/* Close the file handle from the file, but don't free it. */ +void dbox_file_close(struct dbox_file *file); + +/* Try to lock the dbox file. Returns 1 if ok, 0 if already locked by someone + else, -1 if error. */ +int dbox_file_try_lock(struct dbox_file *file); +void dbox_file_unlock(struct dbox_file *file); + +/* Seek to given offset in file and return the message's input stream + and physical size. Returns 1 if ok/expunged, 0 if file/offset is corrupted, + -1 if I/O error. */ +int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, + uoff_t *physical_size_r, + struct istream **input_r); +/* Start seeking at the beginning of the file. */ +void dbox_file_seek_rewind(struct dbox_file *file); +/* Seek to next message after current one. If there are no more messages, + returns 0 and last_r is set to TRUE. Returns 1 if ok, 0 if file is + corrupted, -1 if I/O error. */ +int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r); + +/* Start appending to dbox file */ +struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file); +/* Finish writing appended mails. */ +int dbox_file_append_commit(struct dbox_file_append_context **ctx); +/* Truncate appended mails. */ +void dbox_file_append_rollback(struct dbox_file_append_context **ctx); +/* Get output stream for appending a new message. Returns 1 if ok, 0 if file + can't be appended to (old file version or corruption) or -1 if error. */ +int dbox_file_get_append_stream(struct dbox_file_append_context *ctx, + struct ostream **output_r); +/* Flush output buffer. */ +int dbox_file_append_flush(struct dbox_file_append_context *ctx); + +/* Read current message's metadata. Returns 1 if ok, 0 if metadata is + corrupted, -1 if I/O error. */ +int dbox_file_metadata_read(struct dbox_file *file); +/* Return wanted metadata value, or NULL if not found. */ +const char *dbox_file_metadata_get(struct dbox_file *file, + enum dbox_metadata_key key); + +/* Move the file to alt path or back. */ +int dbox_file_move(struct dbox_file *file, bool alt_path); +/* Fix a broken dbox file by rename()ing over it with a fixed file. Everything + before start_offset is assumed to be valid and is simply copied. The file + is reopened afterwards. Returns 0 if ok, -1 if I/O error. */ +int dbox_file_fix(struct dbox_file *file, uoff_t start_offset); +/* Delete the given dbox file. Returns 1 if deleted, 0 if file wasn't found + or -1 if error. */ +int dbox_file_unlink(struct dbox_file *file); + +/* Fill dbox_message_header with given size. */ +void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, + uoff_t message_size); + +void dbox_file_set_syscall_error(struct dbox_file *file, const char *function); +void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) + ATTR_FORMAT(2, 3); + +/* private: */ +const char *dbox_generate_tmp_filename(void); +void dbox_file_free(struct dbox_file *file); +int dbox_file_header_write(struct dbox_file *file, struct ostream *output); +int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r); +int dbox_file_metadata_skip_header(struct dbox_file *file); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-mail.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,230 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "str.h" +#include "index-storage.h" +#include "index-mail.h" +#include "dbox-storage.h" +#include "dbox-file.h" +#include "dbox-mail.h" + +#include <stdlib.h> + +struct mail * +dbox_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct dbox_mail *mail; + pool_t pool; + + pool = pool_alloconly_create("mail", 1024); + mail = p_new(pool, struct dbox_mail, 1); + mail->imail.mail.pool = pool; + + index_mail_init(&mail->imail, t, wanted_fields, wanted_headers); + return &mail->imail.mail.mail; +} + +void dbox_mail_close(struct mail *_mail) +{ + struct dbox_mail *mail = (struct dbox_mail *)_mail; + + if (mail->open_file != NULL) + dbox_file_unref(&mail->open_file); + index_mail_close(_mail); +} + +int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r) +{ + struct dbox_storage *storage = + (struct dbox_storage *)mail->imail.mail.mail.box->storage; + uoff_t offset, size; + + if (storage->v.mail_open(mail, &offset, file_r) < 0) + return -1; + + if (dbox_file_get_mail_stream(*file_r, offset, &size, NULL) <= 0) + return -1; + if (dbox_file_metadata_read(*file_r) <= 0) + return -1; + return 0; +} + +int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = (struct index_mail *)_mail; + struct index_mail_data *data = &mail->data; + struct istream *input; + + if (index_mail_get_physical_size(_mail, size_r) == 0) + return 0; + + if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + return -1; + + i_assert(data->physical_size != (uoff_t)-1); + *size_r = data->physical_size; + return 0; +} + +int dbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) +{ + struct dbox_mail *mail = (struct dbox_mail *)_mail; + struct index_mail_data *data = &mail->imail.data; + struct dbox_file *file; + const char *value; + + if (index_mail_get_cached_virtual_size(&mail->imail, size_r)) + return 0; + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + value = dbox_file_metadata_get(file, DBOX_METADATA_VIRTUAL_SIZE); + if (value == NULL) + return index_mail_get_virtual_size(_mail, size_r); + + data->virtual_size = strtoul(value, NULL, 16); + *size_r = data->virtual_size; + return 0; +} + +int dbox_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct dbox_mail *mail = (struct dbox_mail *)_mail; + struct index_mail_data *data = &mail->imail.data; + struct dbox_file *file; + const char *value; + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + value = dbox_file_metadata_get(file, DBOX_METADATA_RECEIVED_TIME); + data->received_date = value == NULL ? 0 : strtoul(value, NULL, 16); + *date_r = data->received_date; + return 0; +} + +int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct dbox_mail *mail = (struct dbox_mail *)_mail; + struct index_mail_data *data = &mail->imail.data; + struct dbox_file *file; + struct stat st; + const char *value; + + if (index_mail_get_save_date(_mail, date_r) == 0) + return 0; + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + value = dbox_file_metadata_get(file, DBOX_METADATA_SAVE_TIME); + data->save_date = value == NULL ? 0 : strtoul(value, NULL, 16); + + if (data->save_date == 0) { + /* missing / corrupted save time - use the file's ctime */ + i_assert(dbox_file_is_open(file)); + mail->imail.mail.stats_fstat_lookup_count++; + if (fstat(file->fd, &st) < 0) { + mail_storage_set_critical(_mail->box->storage, + "fstat(%s) failed: %m", file->cur_path); + return -1; + } + data->save_date = st.st_ctime; + } + *date_r = data->save_date; + return 0; +} + +static int +dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key, + enum index_cache_field cache_field, + const char **value_r) +{ + struct index_mail *imail = &mail->imail; + const struct mail_cache_field *cache_fields = imail->ibox->cache_fields; + struct dbox_file *file; + const char *value; + string_t *str; + + str = str_new(imail->data_pool, 64); + if (mail_cache_lookup_field(imail->trans->cache_view, str, + imail->mail.mail.seq, + cache_fields[cache_field].idx) > 0) { + *value_r = str_c(str); + return 0; + } + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + value = dbox_file_metadata_get(file, key); + if (value == NULL) + value = ""; + index_mail_cache_add_idx(imail, cache_fields[cache_field].idx, + value, strlen(value)+1); + *value_r = value; + return 0; +} + +int dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct dbox_mail *mail = (struct dbox_mail *)_mail; + + /* keep the UIDL in cache file, otherwise POP3 would open all + mail files and read the metadata. same for GUIDs if they're + used. */ + switch (field) { + case MAIL_FETCH_UIDL_BACKEND: + return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, + MAIL_CACHE_POP3_UIDL, value_r); + case MAIL_FETCH_GUID: + return dbox_get_cached_metadata(mail, DBOX_METADATA_GUID, + MAIL_CACHE_GUID, value_r); + default: + break; + } + + return index_mail_get_special(_mail, field, value_r); +} + +int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) +{ + struct dbox_storage *storage = + (struct dbox_storage *)_mail->box->storage; + struct dbox_mail *mail = (struct dbox_mail *)_mail; + struct index_mail_data *data = &mail->imail.data; + struct istream *input; + uoff_t offset, size; + int ret; + + if (data->stream == NULL) { + if (storage->v.mail_open(mail, &offset, &mail->open_file) < 0) + return -1; + + ret = dbox_file_get_mail_stream(mail->open_file, offset, + &size, &input); + if (ret <= 0) { + if (ret < 0) + return -1; + dbox_file_set_corrupted(mail->open_file, + "uid=%u points to broken data at offset=" + "%"PRIuUOFF_T, _mail->uid, offset); + return -1; + } + data->physical_size = size; + data->stream = input; + } + + return index_mail_init_stream(&mail->imail, hdr_size, body_size, + stream_r); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-mail.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,29 @@ +#ifndef DBOX_MAIL_H +#define DBOX_MAIL_H + +struct dbox_mail { + struct index_mail imail; + + struct dbox_file *open_file; + uoff_t offset; +}; + +struct mail * +dbox_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); +void dbox_mail_close(struct mail *mail); + +int dbox_mail_get_physical_size(struct mail *mail, uoff_t *size_r); +int dbox_mail_get_virtual_size(struct mail *mail, uoff_t *size_r); +int dbox_mail_get_received_date(struct mail *mail, time_t *date_r); +int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r); +int dbox_mail_get_special(struct mail *mail, enum mail_fetch_field field, + const char **value_r); +int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r); + +int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-save.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,138 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "str.h" +#include "hex-binary.h" +#include "index-mail.h" +#include "dbox-file.h" +#include "dbox-save.h" + +void dbox_save_add_to_index(struct dbox_save_context *ctx) +{ + enum mail_flags save_flags; + + save_flags = ctx->ctx.flags & ~MAIL_RECENT; + mail_index_append(ctx->trans, ctx->ctx.uid, &ctx->seq); + mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE, + save_flags); + if (ctx->ctx.keywords != NULL) { + mail_index_update_keywords(ctx->trans, ctx->seq, + MODIFY_REPLACE, ctx->ctx.keywords); + } + if (ctx->ctx.min_modseq != 0) { + mail_index_update_modseq(ctx->trans, ctx->seq, + ctx->ctx.min_modseq); + } +} + +void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input) +{ + struct mail_save_context *_ctx = &ctx->ctx; + struct dbox_message_header dbox_msg_hdr; + struct istream *crlf_input; + + dbox_save_add_to_index(ctx); + + if (_ctx->dest_mail == NULL) { + if (ctx->mail == NULL) + ctx->mail = mail_alloc(_ctx->transaction, 0, NULL); + _ctx->dest_mail = ctx->mail; + } + mail_set_seq(_ctx->dest_mail, ctx->seq); + + crlf_input = i_stream_create_lf(input); + ctx->input = index_mail_cache_parse_init(_ctx->dest_mail, crlf_input); + i_stream_unref(&crlf_input); + + /* write a dummy header. it'll get rewritten when we're finished */ + memset(&dbox_msg_hdr, 0, sizeof(dbox_msg_hdr)); + o_stream_cork(ctx->cur_output); + if (o_stream_send(ctx->cur_output, &dbox_msg_hdr, + sizeof(dbox_msg_hdr)) < 0) { + mail_storage_set_critical(_ctx->transaction->box->storage, + "o_stream_send(%s) failed: %m", + ctx->cur_file->cur_path); + ctx->failed = TRUE; + } + + if (_ctx->received_date == (time_t)-1) + _ctx->received_date = ioloop_time; +} + +int dbox_save_continue(struct mail_save_context *_ctx) +{ + struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; + struct mail_storage *storage = _ctx->transaction->box->storage; + + if (ctx->failed) + return -1; + + do { + if (o_stream_send_istream(ctx->cur_output, ctx->input) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_storage_set_critical(storage, + "o_stream_send_istream(%s) failed: %m", + ctx->cur_file->cur_path); + } + ctx->failed = TRUE; + return -1; + } + index_mail_cache_parse_continue(_ctx->dest_mail); + + /* both tee input readers may consume data from our primary + input stream. we'll have to make sure we don't return with + one of the streams still having data in them. */ + } while (i_stream_read(ctx->input) > 0); + return 0; +} + +void dbox_save_write_metadata(struct mail_save_context *ctx, + struct ostream *output, + const char *orig_mailbox_name, + uint8_t guid_128[MAIL_GUID_128_SIZE]) +{ + struct dbox_metadata_header metadata_hdr; + const char *guid; + string_t *str; + uoff_t vsize; + + memset(&metadata_hdr, 0, sizeof(metadata_hdr)); + memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST, + sizeof(metadata_hdr.magic_post)); + o_stream_send(output, &metadata_hdr, sizeof(metadata_hdr)); + + str = t_str_new(256); + str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME, + (unsigned long)ctx->received_date); + str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME, + (unsigned long)ioloop_time); + if (mail_get_virtual_size(ctx->dest_mail, &vsize) < 0) + i_unreached(); + str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, + (unsigned long long)vsize); + + guid = ctx->guid; + if (guid != NULL) + mail_generate_guid_128_hash(guid, guid_128); + else { + mail_generate_guid_128(guid_128); + guid = binary_to_hex(guid_128, MAIL_GUID_128_SIZE); + } + str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid); + + if (orig_mailbox_name != NULL && + strchr(orig_mailbox_name, '\r') == NULL && + strchr(orig_mailbox_name, '\n') == NULL) { + /* save the original mailbox name so if mailbox indexes get + corrupted we can place at least some (hopefully most) of + the messages to correct mailboxes. */ + str_printfa(str, "%c%s\n", DBOX_METADATA_ORIG_MAILBOX, + orig_mailbox_name); + } + + str_append_c(str, '\n'); + o_stream_send(output, str_data(str), str_len(str)); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-save.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,30 @@ +#ifndef DBOX_SAVE_H +#define DBOX_SAVE_H + +struct dbox_save_context { + struct mail_save_context ctx; + struct mail_index_transaction *trans; + + /* updated for each appended mail: */ + uint32_t seq; + struct istream *input; + struct mail *mail; + + struct dbox_file *cur_file; + struct ostream *cur_output; + + unsigned int failed:1; + unsigned int finished:1; +}; + +void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input); +int dbox_save_continue(struct mail_save_context *_ctx); + +void dbox_save_write_metadata(struct mail_save_context *ctx, + struct ostream *output, + const char *orig_mailbox_name, + uint8_t guid_128_r[MAIL_GUID_128_SIZE]); + +void dbox_save_add_to_index(struct dbox_save_context *ctx); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,459 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "randgen.h" +#include "hex-binary.h" +#include "mkdir-parents.h" +#include "unlink-directory.h" +#include "unlink-old-files.h" +#include "mailbox-uidvalidity.h" +#include "mailbox-list-private.h" +#include "index-storage.h" +#include "dbox-storage.h" + +#include <stdio.h> + +void dbox_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_FS; + if (set->subscription_fname == NULL) + set->subscription_fname = DBOX_SUBSCRIPTION_FILE_NAME; + if (set->dir_guid_fname == NULL) + set->dir_guid_fname = DBOX_DIR_GUID_FILE_NAME; + if (set->maildir_name == NULL) + set->maildir_name = DBOX_MAILDIR_NAME; + if (set->mailbox_dir_name == NULL) + set->mailbox_dir_name = DBOX_MAILBOX_DIR_NAME; +} + +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_path(list, NULL, + MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"DBOX_UIDVALIDITY_FILE_NAME, NULL); + return mailbox_uidvalidity_next(path); +} + +void dbox_notify_changes(struct mailbox *box) +{ + struct index_mailbox *ibox = (struct index_mailbox *)box; + const char *path; + + if (box->notify_callback == NULL) + index_mailbox_check_remove_all(ibox); + else { + path = t_strdup_printf("%s/"DBOX_INDEX_PREFIX".log", + ibox->box.path); + index_mailbox_check_add(ibox, path); + } +} + +static bool +dbox_cleanup_if_exists(struct mailbox_list *list, const char *path) +{ + struct stat st; + + if (stat(path, &st) < 0) + return FALSE; + + /* check once in a while if there are temp files to clean up */ + if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { + /* there haven't been any changes to this directory since we + last checked it. */ + } else if (st.st_atime < ioloop_time - DBOX_TMP_SCAN_SECS) { + /* time to scan */ + const char *prefix = + mailbox_list_get_global_temp_prefix(list); + + (void)unlink_old_files(path, prefix, + ioloop_time - DBOX_TMP_DELETE_SECS); + } + return TRUE; +} + +int dbox_mailbox_open(struct mailbox *box) +{ + struct dbox_storage *storage = (struct dbox_storage *)box->storage; + + if (box->input != NULL) { + mail_storage_set_critical(box->storage, + "dbox doesn't support streamed mailboxes"); + return -1; + } + + if (dbox_cleanup_if_exists(box->list, box->path)) { + return index_storage_mailbox_open(box); + } else if (errno == ENOENT) { + if (strcmp(box->name, "INBOX") == 0 && + (box->list->ns->flags & NAMESPACE_FLAG_INBOX) != 0) { + /* INBOX always exists, create it */ + if (storage->v.mailbox_create_indexes(box, NULL) < 0) + return -1; + return box->opened ? 0 : + index_storage_mailbox_open(box); + } + + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->name)); + return -1; + } else if (errno == EACCES) { + mail_storage_set_critical(box->storage, "%s", + mail_error_eacces_msg("stat", box->path)); + return -1; + } else { + mail_storage_set_critical(box->storage, + "stat(%s) failed: %m", box->path); + return -1; + } +} + +static const char * +dbox_get_alt_path(struct mailbox_list *list, const char *path) +{ + struct mail_storage *storage = list->ns->storage; + const char *root; + unsigned int len; + + if (list->set.alt_dir == NULL || + (storage->class_flags & MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT) != 0) + return NULL; + + root = mailbox_list_get_path(list, NULL, MAILBOX_LIST_PATH_TYPE_DIR); + len = strlen(root); + if (strncmp(path, root, len) != 0 && path[len] == '/') { + /* can't determine the alt path - shouldn't happen */ + return NULL; + } + return t_strconcat(list->set.alt_dir, path + len, NULL); +} + +int dbox_mailbox_create(struct mailbox *box, + const struct mailbox_update *update, bool directory) +{ + struct dbox_storage *storage = (struct dbox_storage *)box->storage; + const char *path, *alt_path, *origin; + struct stat st; + + path = mailbox_list_get_path(box->list, box->name, + directory ? MAILBOX_LIST_PATH_TYPE_DIR : + MAILBOX_LIST_PATH_TYPE_MAILBOX); + if (stat(path, &st) == 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } + + if (directory) { + mode_t mode; + gid_t gid; + + mailbox_list_get_dir_permissions(box->list, NULL, &mode, + &gid, &origin); + if (mkdir_parents_chgrp(path, mode, gid, origin) == 0) + return 0; + else if (errno == EEXIST) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + } else if (!mail_storage_set_error_from_errno(box->storage)) { + mail_storage_set_critical(box->storage, + "mkdir(%s) failed: %m", path); + } + return -1; + } + + /* make sure the alt path doesn't exist yet. it shouldn't (except with + race conditions with RENAME/DELETE), but if something crashed and + left it lying around we don't want to start overwriting files in + it. */ + alt_path = dbox_get_alt_path(box->list, path); + if (alt_path != NULL && stat(alt_path, &st) == 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } + + return storage->v.mailbox_create_indexes(box, update); +} + +int dbox_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx ATTR_UNUSED, + const char *dir, const char *fname, + const char *mailbox_name ATTR_UNUSED, + enum mailbox_list_file_type type, + enum mailbox_info_flags *flags) +{ + const char *path, *maildir_path; + struct stat st, st2; + int ret = 1; + + /* try to avoid stat() with these checks */ + if (type != MAILBOX_LIST_FILE_TYPE_DIR && + type != MAILBOX_LIST_FILE_TYPE_SYMLINK && + type != MAILBOX_LIST_FILE_TYPE_UNKNOWN) { + /* it's a file */ + *flags |= MAILBOX_NOSELECT | MAILBOX_NOINFERIORS; + return 0; + } + + /* need to stat() then */ + path = t_strconcat(dir, "/", fname, NULL); + if (stat(path, &st) == 0) { + if (!S_ISDIR(st.st_mode)) { + /* non-directory */ + *flags |= MAILBOX_NOSELECT | MAILBOX_NOINFERIORS; + ret = 0; + } else if (st.st_nlink == 2) { + /* no subdirectories */ + *flags |= MAILBOX_NOCHILDREN; + } else if (*ctx->list->set.maildir_name != '\0') { + /* default configuration: we have one directory + containing the mailboxes. if there are 3 links, + either this is a selectable mailbox without children + or non-selectable mailbox with children */ + if (st.st_nlink > 3) + *flags |= MAILBOX_CHILDREN; + } else { + /* non-default configuration: all subdirectories are + child mailboxes. */ + if (st.st_nlink > 2) + *flags |= MAILBOX_CHILDREN; + } + } else if (errno == ENOENT) { + /* doesn't exist - probably a non-existing subscribed mailbox */ + *flags |= MAILBOX_NONEXISTENT; + } else { + /* non-selectable. probably either access denied, or symlink + destination not found. don't bother logging errors. */ + *flags |= MAILBOX_NOSELECT; + } + if ((*flags & (MAILBOX_NOSELECT | MAILBOX_NONEXISTENT)) == 0) { + /* make sure it's a selectable mailbox */ + maildir_path = t_strconcat(path, "/", + ctx->list->set.maildir_name, NULL); + if (stat(maildir_path, &st2) < 0 || !S_ISDIR(st2.st_mode)) + *flags |= MAILBOX_NOSELECT; + if (st.st_nlink == 3 && *ctx->list->set.maildir_name != '\0') { + /* now we know what link count 3 means. */ + if ((*flags & MAILBOX_NOSELECT) != 0) + *flags |= MAILBOX_CHILDREN; + else + *flags |= MAILBOX_NOCHILDREN; + } + } + return ret; +} + +static int +dbox_list_rename_get_alt_paths(struct mailbox_list *oldlist, + const char *oldname, + struct mailbox_list *newlist, + const char *newname, + enum mailbox_list_path_type path_type, + const char **oldpath_r, const char **newpath_r) +{ + const char *path; + + path = mailbox_list_get_path(oldlist, oldname, path_type); + *oldpath_r = dbox_get_alt_path(oldlist, path); + if (*oldpath_r == NULL) + return 0; + + path = mailbox_list_get_path(newlist, newname, path_type); + *newpath_r = dbox_get_alt_path(newlist, path); + if (*newpath_r == NULL) { + /* destination dbox storage doesn't have alt-path defined. + we can't do the rename easily. */ + mailbox_list_set_error(oldlist, MAIL_ERROR_NOTPOSSIBLE, + "Can't rename mailboxes across specified storages."); + return -1; + } + return 1; +} + +int dbox_list_rename_mailbox_pre(struct mailbox_list *oldlist, + const char *oldname, + struct mailbox_list *newlist, + const char *newname) +{ + const char *alt_oldpath, *alt_newpath; + struct stat st; + int ret; + + ret = dbox_list_rename_get_alt_paths(oldlist, oldname, newlist, newname, + MAILBOX_LIST_PATH_TYPE_DIR, + &alt_oldpath, &alt_newpath); + if (ret <= 0) + return ret; + + if (stat(alt_newpath, &st) == 0) { + /* race condition or a directory left there lying around? + safest to just report error. */ + mailbox_list_set_error(oldlist, MAIL_ERROR_EXISTS, + "Target mailbox already exists"); + return -1; + } else if (errno != ENOENT) { + mailbox_list_set_critical(oldlist, "stat(%s) failed: %m", + alt_newpath); + return -1; + } + return 0; +} + +int dbox_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname, + struct mailbox_list *newlist, const char *newname, + bool rename_children) +{ + enum mailbox_list_path_type path_type; + const char *alt_oldpath, *alt_newpath, *path; + int ret; + + path_type = rename_children ? MAILBOX_LIST_PATH_TYPE_DIR : + MAILBOX_LIST_PATH_TYPE_MAILBOX; + ret = dbox_list_rename_get_alt_paths(oldlist, oldname, newlist, newname, + path_type, &alt_oldpath, + &alt_newpath); + if (ret <= 0) + return ret; + + if (rename(alt_oldpath, alt_newpath) == 0) { + /* ok */ + if (!rename_children) { + path = mailbox_list_get_path(oldlist, oldname, + MAILBOX_LIST_PATH_TYPE_DIR); + if (rmdir(path) < 0 && + errno != ENOENT && errno != ENOTEMPTY) { + mailbox_list_set_critical(oldlist, + "rmdir(%s) failed: %m", path); + } + } + } else if (errno != ENOENT) { + /* renaming is done already, so just log the error */ + mailbox_list_set_critical(oldlist, "rename(%s, %s) failed: %m", + alt_oldpath, alt_newpath); + } + return 0; +} + +static const char *dbox_get_trash_dest(const char *trash_dir) +{ + const char *path; + unsigned char randbuf[16]; + struct stat st; + + do { + random_fill_weak(randbuf, sizeof(randbuf)); + path = t_strconcat(trash_dir, "/", + binary_to_hex(randbuf, sizeof(randbuf)), NULL); + } while (lstat(path, &st) == 0); + return path; +} + +int dbox_list_delete_mailbox1(struct mailbox_list *list, const char *name, + const char **trash_dest_r) +{ + struct stat st; + const char *path, *trash_dir, *trash_dest; + int ret; + + path = mailbox_list_get_path(list, name, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + trash_dir = mailbox_list_get_path(list, NULL, + MAILBOX_LIST_PATH_TYPE_DIR); + trash_dir = t_strconcat(trash_dir, "/"DBOX_TRASH_DIR_NAME, NULL); + trash_dest = *trash_dest_r = dbox_get_trash_dest(trash_dir); + + /* first try renaming the actual mailbox to trash directory */ + ret = rename(path, trash_dest); + if (ret < 0 && errno == ENOENT) { + /* either source mailbox doesn't exist or trash directory + doesn't exist. try creating the trash and retrying. */ + const char *origin; + mode_t mode; + gid_t gid; + + mailbox_list_get_dir_permissions(list, NULL, &mode, + &gid, &origin); + if (mkdir_parents_chgrp(trash_dir, mode, gid, origin) < 0 && + errno != EEXIST) { + mailbox_list_set_critical(list, + "mkdir(%s) failed: %m", trash_dir); + return -1; + } + ret = rename(path, trash_dest); + } + if (ret == 0) + return 1; + else if (errno != ENOENT) { + mailbox_list_set_critical(list, "stat(%s) failed: %m", path); + return -1; + } else { + /* mailbox not found - what about the directory? */ + path = mailbox_list_get_path(list, name, + MAILBOX_LIST_PATH_TYPE_DIR); + if (stat(path, &st) == 0) { + /* delete the directory */ + } else if (errno == ENOENT) { + mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(name)); + return -1; + } else if (!mailbox_list_set_error_from_errno(list)) { + mailbox_list_set_critical(list, "stat(%s) failed: %m", + path); + return -1; + } + return 0; + } +} + +int dbox_list_delete_mailbox2(struct mailbox_list *list, const char *name, + int ret, const char *trash_dest) +{ + const char *path, *alt_path; + bool deleted = FALSE; + + path = mailbox_list_get_path(list, name, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + if (ret > 0) { + if (unlink_directory(trash_dest, TRUE) < 0) { + mailbox_list_set_critical(list, + "unlink_directory(%s) failed: %m", trash_dest); + ret = -1; + } + /* if there's an alt path, delete it too */ + alt_path = dbox_get_alt_path(list, path); + if (alt_path != NULL) { + if (unlink_directory(alt_path, TRUE) < 0) { + mailbox_list_set_critical(list, + "unlink_directory(%s) failed: %m", alt_path); + ret = -1; + } + } + /* try to delete the parent directory also */ + deleted = TRUE; + path = mailbox_list_get_path(list, name, + MAILBOX_LIST_PATH_TYPE_DIR); + } + + alt_path = dbox_get_alt_path(list, path); + if (alt_path != NULL) + (void)rmdir(alt_path); + + if (rmdir(path) == 0) + return ret; + else if (errno == ENOTEMPTY) { + if (deleted) + return ret; + mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND, + t_strdup_printf("Directory %s isn't empty, " + "can't delete it.", name)); + } else if (!mailbox_list_set_error_from_errno(list)) { + mailbox_list_set_critical(list, "rmdir() failed for %s: %m", + path); + } + return -1; +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-storage.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,71 @@ +#ifndef DBOX_STORAGE_H +#define DBOX_STORAGE_H + +#include "mail-storage-private.h" + +struct dbox_file; +struct dbox_mail; + +#define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions" +#define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity" +#define DBOX_INDEX_PREFIX "dovecot.index" +#define DBOX_DIR_GUID_FILE_NAME "dbox-GUID" + +#define DBOX_MAILBOX_DIR_NAME "mailboxes" +#define DBOX_TRASH_DIR_NAME "trash" +#define DBOX_MAILDIR_NAME "dbox-Mails" + +/* How often to scan for stale temp files (based on dir's atime) */ +#define DBOX_TMP_SCAN_SECS (8*60*60) +/* Delete temp files having ctime older than this. */ +#define DBOX_TMP_DELETE_SECS (36*60*60) + +struct dbox_storage_vfuncs { + /* dbox file has zero references now. it should be either freed or + left open in case it's accessed again soon */ + void (*file_unrefed)(struct dbox_file *file); + /* create a new file using the same permissions as file. + if parents=TRUE, create the directory if necessary */ + int (*file_create_fd)(struct dbox_file *file, const char *path, + bool parents); + /* open the mail and return its file/offset */ + int (*mail_open)(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r); + /* create/update mailbox indexes */ + int (*mailbox_create_indexes)(struct mailbox *box, + const struct mailbox_update *update); +}; + +struct dbox_storage { + struct mail_storage storage; + struct dbox_storage_vfuncs v; + + unsigned int files_corrupted:1; +}; + +void dbox_storage_get_list_settings(const struct mail_namespace *ns, + struct mailbox_list_settings *set); +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); +void dbox_notify_changes(struct mailbox *box); +int dbox_mailbox_open(struct mailbox *box); +int dbox_mailbox_create(struct mailbox *box, + const struct mailbox_update *update, bool directory); +int dbox_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx, + const char *dir, const char *fname, + const char *mailbox_name, + enum mailbox_list_file_type type, + enum mailbox_info_flags *flags); +int dbox_list_rename_mailbox_pre(struct mailbox_list *oldlist, + const char *oldname, + struct mailbox_list *newlist, + const char *newname); +int dbox_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname, + struct mailbox_list *newlist, const char *newname, + bool rename_children); + +int dbox_list_delete_mailbox1(struct mailbox_list *list, const char *name, + const char **trash_dest_r); +int dbox_list_delete_mailbox2(struct mailbox_list *list, const char *name, + int ret, const char *trash_dest); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-sync-rebuild.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,132 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "index-storage.h" +#include "dbox-storage.h" +#include "dbox-sync-rebuild.h" + +static void +dbox_sync_index_copy_cache(struct dbox_sync_rebuild_context *ctx, + struct mail_index_view *view, + uint32_t old_seq, uint32_t new_seq) +{ + struct mail_index_map *map; + const void *data; + uint32_t reset_id; + bool expunged; + + if (ctx->cache_ext_id == (uint32_t)-1) + return; + + mail_index_lookup_ext_full(view, old_seq, ctx->cache_ext_id, + &map, &data, &expunged); + if (expunged) + return; + + if (!mail_index_ext_get_reset_id(view, map, ctx->cache_ext_id, + &reset_id) || reset_id == 0) + return; + + if (!ctx->cache_used) { + /* set reset id */ + ctx->cache_used = TRUE; + ctx->cache_reset_id = reset_id; + mail_index_ext_reset(ctx->trans, ctx->cache_ext_id, + ctx->cache_reset_id, TRUE); + } + if (ctx->cache_reset_id == reset_id) { + mail_index_update_ext(ctx->trans, new_seq, + ctx->cache_ext_id, data, NULL); + } +} + +static void +dbox_sync_index_copy_from_old(struct dbox_sync_rebuild_context *ctx, + struct mail_index_view *view, + uint32_t old_seq, uint32_t new_seq) +{ + struct mail_index *index = mail_index_view_get_index(view); + const struct mail_index_record *rec; + ARRAY_TYPE(keyword_indexes) old_keywords; + struct mail_keywords *kw; + + /* copy flags */ + rec = mail_index_lookup(view, old_seq); + mail_index_update_flags(ctx->trans, new_seq, + MODIFY_REPLACE, rec->flags); + + /* copy keywords */ + t_array_init(&old_keywords, 32); + mail_index_lookup_keywords(view, old_seq, &old_keywords); + kw = mail_index_keywords_create_from_indexes(index, &old_keywords); + mail_index_update_keywords(ctx->trans, new_seq, MODIFY_REPLACE, kw); + mail_index_keywords_unref(&kw); + + dbox_sync_index_copy_cache(ctx, view, old_seq, new_seq); +} + +void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx, + uint32_t new_seq, uint32_t uid) +{ + uint32_t old_seq; + + if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) { + /* the message exists in the old index. + copy the metadata from it. */ + dbox_sync_index_copy_from_old(ctx, ctx->view, old_seq, new_seq); + } else if (ctx->backup_view != NULL && + mail_index_lookup_seq(ctx->backup_view, uid, &old_seq)) { + /* copy the metadata from backup index. */ + dbox_sync_index_copy_from_old(ctx, ctx->backup_view, + old_seq, new_seq); + } +} + +struct dbox_sync_rebuild_context * +dbox_sync_index_rebuild_init(struct index_mailbox *ibox, + struct mail_index_view *view, + struct mail_index_transaction *trans) +{ + struct mailbox *box = &ibox->box; + struct dbox_sync_rebuild_context *ctx; + const char *index_dir; + enum mail_index_open_flags open_flags = MAIL_INDEX_OPEN_FLAG_READONLY; + + ctx = i_new(struct dbox_sync_rebuild_context, 1); + ctx->ibox = ibox; + ctx->view = view; + ctx->trans = trans; + mail_index_reset(ctx->trans); + index_mailbox_reset_uidvalidity(ibox); + mail_index_ext_lookup(ibox->index, "cache", &ctx->cache_ext_id); + + /* if backup index file exists, try to use it */ + index_dir = mailbox_list_get_path(box->list, box->name, + MAILBOX_LIST_PATH_TYPE_INDEX); + ctx->backup_index = + mail_index_alloc(index_dir, DBOX_INDEX_PREFIX".backup"); + +#ifndef MMAP_CONFLICTS_WRITE + if (box->storage->set->mmap_disable) +#endif + open_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE; + if (mail_index_open(ctx->backup_index, open_flags, + box->storage->set->parsed_lock_method) <= 0) + mail_index_free(&ctx->backup_index); + else + ctx->backup_view = mail_index_view_open(ctx->backup_index); + return ctx; +} + +void dbox_sync_index_rebuild_deinit(struct dbox_sync_rebuild_context **_ctx) +{ + struct dbox_sync_rebuild_context *ctx = *_ctx; + + *_ctx = NULL; + if (ctx->backup_index != NULL) { + mail_index_view_close(&ctx->backup_view); + mail_index_free(&ctx->backup_index); + } + i_free(ctx); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-sync-rebuild.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,27 @@ +#ifndef DBOX_SYNC_REBUILD_H +#define DBOX_SYNC_REBUILD_H + +struct dbox_sync_rebuild_context { + struct index_mailbox *ibox; + + struct mail_index_view *view; + struct mail_index_transaction *trans; + uint32_t cache_ext_id; + uint32_t cache_reset_id; + + struct mail_index *backup_index; + struct mail_index_view *backup_view; + + unsigned int cache_used:1; +}; + +struct dbox_sync_rebuild_context * +dbox_sync_index_rebuild_init(struct index_mailbox *mbox, + struct mail_index_view *view, + struct mail_index_transaction *trans); +void dbox_sync_index_rebuild_deinit(struct dbox_sync_rebuild_context **ctx); + +void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx, + uint32_t new_seq, uint32_t uid); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/Makefile.am Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,38 @@ +noinst_LTLIBRARIES = libstorage_dbox_multi.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_multi_la_SOURCES = \ + mdbox-file.c \ + mdbox-file-purge.c \ + mdbox-mail.c \ + mdbox-map.c \ + mdbox-save.c \ + mdbox-settings.c \ + mdbox-sync.c \ + mdbox-storage.c \ + mdbox-storage-rebuild.c + +headers = \ + mdbox-file.h \ + mdbox-map.h \ + mdbox-map-private.h \ + mdbox-settings.h \ + mdbox-storage.h \ + mdbox-storage-rebuild.h \ + mdbox-sync.h + +if INSTALL_HEADERS + pkginc_libdir=$(pkgincludedir) + pkginc_lib_HEADERS = $(headers) +else + noinst_HEADERS = $(headers) +endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-file-purge.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,227 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "hex-binary.h" +#include "mdbox-storage.h" +#include "mdbox-file.h" +#include "mdbox-map.h" +#include "mdbox-sync.h" + +#include <stdlib.h> + +struct dbox_mail_move { + struct dbox_file *file; + uint32_t offset; +}; +ARRAY_DEFINE_TYPE(dbox_mail_move, struct dbox_mail_move); + +static int mdbox_map_file_msg_offset_cmp(const void *p1, const void *p2) +{ + const struct dbox_map_file_msg *m1 = p1, *m2 = p2; + + if (m1->offset < m2->offset) + return -1; + else if (m1->offset > m2->offset) + return 1; + else + return 0; +} + +static int +mdbox_file_copy_metadata(struct dbox_file *file, struct ostream *output) +{ + struct dbox_metadata_header meta_hdr; + const char *line; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_data(file->input, &data, &size, + sizeof(meta_hdr)); + if (ret <= 0) { + i_assert(ret == -1); + if (file->input->stream_errno == 0) { + dbox_file_set_corrupted(file, "missing metadata"); + return 0; + } + mail_storage_set_critical(&file->storage->storage, + "read(%s) failed: %m", file->cur_path); + return -1; + } + + memcpy(&meta_hdr, data, sizeof(meta_hdr)); + if (memcmp(meta_hdr.magic_post, DBOX_MAGIC_POST, + sizeof(meta_hdr.magic_post)) != 0) { + dbox_file_set_corrupted(file, "invalid metadata magic"); + return 0; + } + i_stream_skip(file->input, sizeof(meta_hdr)); + if (output != NULL) + o_stream_send(output, &meta_hdr, sizeof(meta_hdr)); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + break; + } + if (output != NULL) { + o_stream_send_str(output, line); + o_stream_send(output, "\n", 1); + } + } + if (line == NULL) { + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return 0; + } + if (output != NULL) + o_stream_send(output, "\n", 1); + return 1; +} + +int mdbox_file_purge(struct dbox_file *file) +{ + struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage; + struct dbox_file_append_context *out_file_append; + struct stat st; + struct istream *input; + struct ostream *output = NULL; + struct dbox_map_append_context *append_ctx; + ARRAY_TYPE(dbox_map_file_msg) msgs_arr; + const struct dbox_map_file_msg *msgs; + ARRAY_TYPE(seq_range) expunged_map_uids; + ARRAY_TYPE(uint32_t) copied_map_uids; + unsigned int i, count; + uoff_t offset, physical_size, msg_size; + int ret; + + if ((ret = dbox_file_try_lock(file)) <= 0) + return ret; + + /* make sure the file still exists. another process may have already + deleted it. */ + if (stat(file->cur_path, &st) < 0) { + dbox_file_unlock(file); + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(&file->storage->storage, + "stat(%s) failed: %m", file->cur_path); + return -1; + } + + i_array_init(&msgs_arr, 128); + if (dbox_map_get_file_msgs(dstorage->map, + ((struct mdbox_file *)file)->file_id, + &msgs_arr) < 0) { + array_free(&msgs_arr); + dbox_file_unlock(file); + return -1; + } + /* sort messages by their offset */ + array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp); + + msgs = array_get(&msgs_arr, &count); + append_ctx = dbox_map_append_begin(dstorage->map); + i_array_init(&copied_map_uids, I_MIN(count, 1)); + i_array_init(&expunged_map_uids, I_MIN(count, 1)); + offset = file->file_header_size; + for (i = 0; i < count; i++) { + if ((ret = dbox_file_get_mail_stream(file, offset, + &physical_size, + NULL)) <= 0) + break; + msg_size = file->msg_header_size + physical_size; + + if (msgs[i].offset != offset) { + /* map doesn't match file's actual contents */ + dbox_file_set_corrupted(file, + "purging found mismatched offsets " + "(%"PRIuUOFF_T" vs %u, %u/%u)", + offset, msgs[i].offset, i, count); + ret = 0; + break; + } + + if (msgs[i].refcount == 0) { + seq_range_array_add(&expunged_map_uids, 0, + msgs[i].map_uid); + output = NULL; + } else { + /* non-expunged message. write it to output file. */ + if (dbox_map_append_next(append_ctx, physical_size, + &out_file_append, &output) < 0) { + ret = -1; + break; + } + i_assert(file != out_file_append->file); + + i_stream_seek(file->input, offset); + input = i_stream_create_limit(file->input, msg_size); + ret = o_stream_send_istream(output, input); + if (input->stream_errno != 0) { + errno = input->stream_errno; + mail_storage_set_critical(&file->storage->storage, + "read(%s) failed: %m", + file->cur_path); + i_stream_unref(&input); + break; + } + i_stream_unref(&input); + if (output->stream_errno != 0) { + errno = output->stream_errno; + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %m", + out_file_append->file->cur_path); + break; + } + i_assert(ret == (off_t)msg_size); + } + + /* copy/skip metadata */ + i_stream_seek(file->input, offset + msg_size); + if ((ret = mdbox_file_copy_metadata(file, output)) <= 0) + break; + + if (output != NULL) { + dbox_map_append_finish(append_ctx); + array_append(&copied_map_uids, &msgs[i].map_uid, 1); + } + offset = file->input->v_offset; + } + if (offset != (uoff_t)st.st_size && ret > 0) { + /* file has more messages than what map tells us */ + dbox_file_set_corrupted(file, + "more messages available than in map " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", offset, st.st_size); + ret = 0; + } + array_free(&msgs_arr); msgs = NULL; + + if (ret <= 0) { + dbox_map_append_free(&append_ctx); + dbox_file_unlock(file); + ret = -1; + } else if (array_count(&copied_map_uids) == 0) { + /* everything expunged in this file, unlink it */ + ret = dbox_file_unlink(file); + dbox_map_append_free(&append_ctx); + } else { + /* assign new file_id + offset to moved messages */ + if (dbox_map_append_move(append_ctx, &copied_map_uids, + &expunged_map_uids) < 0 || + dbox_map_append_commit(append_ctx) < 0) { + dbox_file_unlock(file); + ret = -1; + } else { + ret = 1; + (void)dbox_file_unlink(file); + } + dbox_map_append_free(&append_ctx); + } + array_free(&copied_map_uids); + array_free(&expunged_map_uids); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-file.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,263 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "hex-dec.h" +#include "hex-binary.h" +#include "hostpid.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "mkdir-parents.h" +#include "fdatasync-path.h" +#include "eacces-error.h" +#include "str.h" +#include "mdbox-storage.h" +#include "mdbox-map-private.h" +#include "mdbox-file.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <ctype.h> +#include <fcntl.h> + +static struct mdbox_file * +mdbox_find_and_move_open_file(struct mdbox_storage *storage, uint32_t file_id) +{ + struct mdbox_file *const *files, *file; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) { + if (files[i]->file_id == file_id) { + /* move to last in the array */ + file = files[i]; + array_delete(&storage->open_files, i, 1); + array_append(&storage->open_files, &file, 1); + return file; + } + } + return NULL; +} + +void mdbox_files_free(struct mdbox_storage *storage) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) + dbox_file_free(&files[i]->file); + array_clear(&storage->open_files); +} + +void mdbox_files_sync_input(struct mdbox_storage *storage) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) { + if (files[i]->file.input != NULL) + i_stream_sync(files[i]->file.input); + } +} + +static void +mdbox_close_open_files(struct mdbox_storage *storage, unsigned int close_count) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count;) { + if (files[i]->file.refcount == 0) { + dbox_file_free(&files[i]->file); + array_delete(&storage->open_files, i, 1); + + if (--close_count == 0) + break; + + files = array_get(&storage->open_files, &count); + } else { + i++; + } + } +} + +static void mdbox_file_init_paths(struct mdbox_file *file, const char *fname) +{ + i_free(file->file.primary_path); + i_free(file->file.alt_path); + file->file.primary_path = + i_strdup_printf("%s/%s", file->storage->storage_dir, fname); + if (file->storage->alt_storage_dir != NULL) { + file->file.alt_path = + i_strdup_printf("%s/%s", file->storage->alt_storage_dir, + fname); + } + file->file.cur_path = file->file.primary_path; +} + +static int mdbox_file_create(struct dbox_file *file) +{ + int ret; + + file->fd = file->storage->v. + file_create_fd(file, file->primary_path, FALSE); + + /* even though we don't need it locked while writing to it, by the + time we rename() it it needs to be locked. so we might as well do + it here. */ + if ((ret = dbox_file_try_lock(file)) <= 0) { + if (ret < 0) + return -1; + mail_storage_set_critical(&file->storage->storage, + "dbox: Couldn't lock created file: %s", + file->cur_path); + return -1; + } + return 0; +} + +struct dbox_file * +mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id) +{ + struct mdbox_file *file; + const char *fname; + unsigned int count; + + file = file_id == 0 ? NULL : + mdbox_find_and_move_open_file(storage, file_id); + if (file != NULL) { + file->file.refcount++; + return &file->file; + } + + count = array_count(&storage->open_files); + if (count > storage->set->mdbox_max_open_files) { + mdbox_close_open_files(storage, count - + storage->set->mdbox_max_open_files); + } + + file = i_new(struct mdbox_file, 1); + file->storage = storage; + file->file.storage = &storage->storage; + file->file_id = file_id; + fname = file_id == 0 ? dbox_generate_tmp_filename() : + t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id); + mdbox_file_init_paths(file, fname); + dbox_file_init(&file->file); + + if (file_id != 0) + array_append(&storage->open_files, &file, 1); + else + (void)mdbox_file_create(&file->file); + return &file->file; +} + +int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id) +{ + const char *old_path; + const char *new_fname, *new_path; + + i_assert(file->file_id == 0); + i_assert(file_id != 0); + + old_path = file->file.cur_path; + new_fname = t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id); + new_path = t_strdup_printf("%s/%s", file->storage->storage_dir, + new_fname); + if (rename(old_path, new_path) < 0) { + mail_storage_set_critical(&file->storage->storage.storage, + "rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + mdbox_file_init_paths(file, new_fname); + file->file_id = file_id; + array_append(&file->storage->open_files, &file, 1); + return 0; +} + +void mdbox_file_unrefed(struct dbox_file *file) +{ + struct mdbox_file *mfile = (struct mdbox_file *)file; + struct mdbox_file *const *files, *oldest_file; + unsigned int i, count; + + /* don't cache metadata seeks while file isn't being referenced */ + file->metadata_read_offset = (uoff_t)-1; + + if (mfile->file_id != 0) { + files = array_get(&mfile->storage->open_files, &count); + if (!file->deleted && + count <= mfile->storage->set->mdbox_max_open_files) { + /* we can leave this file open for now */ + return; + } + + /* close the oldest file with refcount=0 */ + for (i = 0; i < count; i++) { + if (files[i]->file.refcount == 0) + break; + } + oldest_file = files[i]; + array_delete(&mfile->storage->open_files, i, 1); + if (oldest_file != mfile) { + dbox_file_free(&oldest_file->file); + return; + } + /* have to close ourself */ + } + dbox_file_free(file); +} + +int mdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents) +{ + struct mdbox_file *mfile = (struct mdbox_file *)file; + struct dbox_map *map = mfile->storage->map; + mode_t old_mask; + const char *p, *dir; + int fd; + + old_mask = umask(0666 & ~map->create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + if (fd == -1 && errno == ENOENT && parents && + (p = strrchr(path, '/')) != NULL) { + dir = t_strdup_until(path, p); + if (mkdir_parents_chgrp(dir, map->create_dir_mode, + map->create_gid, + map->create_gid_origin) < 0) { + mail_storage_set_critical(&file->storage->storage, + "mkdir_parents(%s) failed: %m", dir); + return -1; + } + /* try again */ + old_mask = umask(0666 & ~map->create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + } + if (fd == -1) { + mail_storage_set_critical(&file->storage->storage, + "open(%s, O_CREAT) failed: %m", path); + } else if (map->create_gid == (gid_t)-1) { + /* no group change */ + } else if (fchown(fd, (uid_t)-1, map->create_gid) < 0) { + if (errno == EPERM) { + mail_storage_set_critical(&file->storage->storage, "%s", + eperm_error_get_chgrp("fchown", path, + map->create_gid, + map->create_gid_origin)); + } else { + mail_storage_set_critical(&file->storage->storage, + "fchown(%s, -1, %ld) failed: %m", + path, (long)map->create_gid); + } + /* continue anyway */ + } + return fd; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-file.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,27 @@ +#ifndef MDBOX_FILE_H +#define MDBOX_FILE_H + +#include "dbox-file.h" + +struct mdbox_file { + struct dbox_file file; + struct mdbox_storage *storage; + + uint32_t file_id; +}; + +struct dbox_file * +mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id); + +/* Assign file ID for a newly created file. */ +int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id); + +void mdbox_file_unrefed(struct dbox_file *file); +int mdbox_file_create_fd(struct dbox_file *file, const char *path, + bool parents); + +void mdbox_files_free(struct mdbox_storage *storage); +void mdbox_files_sync_input(struct mdbox_storage *storage); +int mdbox_file_purge(struct dbox_file *file); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,205 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "str.h" +#include "index-mail.h" +#include "dbox-mail.h" +#include "mdbox-storage.h" +#include "mdbox-map.h" +#include "mdbox-file.h" + +#include <stdlib.h> +#include <sys/stat.h> + +int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, uint32_t *map_uid_r) +{ + const struct mdbox_mail_index_record *dbox_rec; + struct mdbox_index_header hdr; + const void *data; + uint32_t uid, cur_map_uid_validity; + bool expunged; + + mail_index_lookup_ext(view, seq, mbox->ext_id, &data, &expunged); + dbox_rec = data; + if (dbox_rec == NULL || dbox_rec->map_uid == 0) { + mail_index_lookup_uid(view, seq, &uid); + mail_storage_set_critical(&mbox->storage->storage.storage, + "dbox %s: map uid lost for uid %u", + mbox->ibox.box.path, uid); + mbox->storage->storage.files_corrupted = TRUE; + return -1; + } + + if (mbox->map_uid_validity == 0) { + if (mdbox_read_header(mbox, &hdr) < 0) { + mbox->storage->storage.files_corrupted = TRUE; + return -1; + } + mbox->map_uid_validity = hdr.map_uid_validity; + } + if (dbox_map_open(mbox->storage->map, TRUE) < 0) + return -1; + + cur_map_uid_validity = dbox_map_get_uid_validity(mbox->storage->map); + if (cur_map_uid_validity != mbox->map_uid_validity) { + mail_storage_set_critical(&mbox->storage->storage.storage, + "dbox %s: map uidvalidity mismatch (%u vs %u)", + mbox->ibox.box.path, mbox->map_uid_validity, + cur_map_uid_validity); + mbox->storage->storage.files_corrupted = TRUE; + return -1; + } + *map_uid_r = dbox_rec->map_uid; + return 0; +} + +static void dbox_mail_set_expunged(struct dbox_mail *mail, uint32_t map_uid) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)mail->imail.ibox; + struct mail *_mail = &mail->imail.mail.mail; + + (void)mail_index_refresh(mbox->ibox.index); + if (mail_index_is_expunged(mbox->ibox.view, _mail->seq)) { + mail_set_expunged(_mail); + return; + } + + dbox_map_set_corrupted(mbox->storage->map, + "Unexpectedly lost uid=%u map_uid=%u", + _mail->uid, map_uid); + mbox->storage->storage.files_corrupted = TRUE; +} + +static int dbox_mail_open_init(struct dbox_mail *mail, uint32_t map_uid) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)mail->imail.ibox; + uint32_t file_id; + int ret; + + if ((ret = dbox_map_lookup(mbox->storage->map, map_uid, + &file_id, &mail->offset)) <= 0) { + if (ret < 0) + return -1; + + /* map_uid doesn't exist anymore. either it + got just expunged or the map index is + corrupted. */ + dbox_mail_set_expunged(mail, map_uid); + return -1; + } else { + mail->open_file = mdbox_file_init(mbox->storage, file_id); + } + return 0; +} + +int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)mail->imail.ibox; + struct mail *_mail = &mail->imail.mail.mail; + uint32_t prev_file_id = 0, map_uid = 0; + bool deleted; + + if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) { + mail_set_aborted(_mail); + return -1; + } + + do { + if (mail->open_file != NULL) { + /* already open */ + } else if (_mail->uid != 0) { + if (mdbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq, + &map_uid) < 0) + return -1; + if (dbox_mail_open_init(mail, map_uid) < 0) + return -1; + } else { + /* mail is being saved in this transaction */ + mail->open_file = + mdbox_save_file_get_file(_mail->transaction, + _mail->seq, + &mail->offset); + mail->open_file->refcount++; + break; + } + + if (!dbox_file_is_open(mail->open_file)) + mail->imail.mail.stats_open_lookup_count++; + if (dbox_file_open(mail->open_file, &deleted) <= 0) + return -1; + if (deleted) { + /* either it's expunged now or moved to another file. */ + struct mdbox_file *mfile = + (struct mdbox_file *)mail->open_file; + + if (mfile->file_id == prev_file_id) { + dbox_mail_set_expunged(mail, map_uid); + return -1; + } + prev_file_id = mfile->file_id; + if (dbox_map_refresh(mbox->storage->map) < 0) + return -1; + dbox_file_unref(&mail->open_file); + } + } while (mail->open_file == NULL); + + *file_r = mail->open_file; + *offset_r = mail->offset; + return 0; +} + +static int mdbox_mail_get_save_date(struct mail *mail, time_t *date_r) +{ + struct mdbox_mailbox *mbox = + (struct mdbox_mailbox *)mail->transaction->box; + const struct mdbox_mail_index_record *dbox_rec; + const void *data; + bool expunged; + + mail_index_lookup_ext(mbox->ibox.view, mail->seq, + mbox->ext_id, &data, &expunged); + dbox_rec = data; + if (dbox_rec == NULL || dbox_rec->map_uid == 0) { + /* lost for some reason, use fallback */ + return dbox_mail_get_save_date(mail, date_r); + } + + *date_r = dbox_rec->save_date; + return TRUE; +} + +struct mail_vfuncs mdbox_mail_vfuncs = { + dbox_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_parts, + index_mail_get_date, + dbox_mail_get_received_date, + mdbox_mail_get_save_date, + dbox_mail_get_virtual_size, + dbox_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + dbox_mail_get_stream, + dbox_mail_get_special, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_uid, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_get_index_mail +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-map-private.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,57 @@ +#ifndef MDBOX_MAP_PRIVATE_H +#define MDBOX_MAP_PRIVATE_H + +#include "mdbox-map.h" + +struct dbox_mail_lookup_rec { + uint32_t map_uid; + uint16_t refcount; + struct dbox_map_mail_index_record rec; +}; + +struct dbox_map { + struct mdbox_storage *storage; + const struct mdbox_settings *set; + char *path; + + struct mail_index *index; + struct mail_index_view *view; + uint32_t created_uid_validity; + + uint32_t map_ext_id, ref_ext_id; + ARRAY_TYPE(seq_range) ref0_file_ids; + + mode_t create_mode, create_dir_mode; + gid_t create_gid; + const char *create_gid_origin; +}; + +struct dbox_map_append { + struct dbox_file_append_context *file_append; + uoff_t offset, size; +}; + +struct dbox_map_append_context { + struct dbox_map *map; + + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *sync_trans, *trans; + + ARRAY_DEFINE(file_appends, struct dbox_file_append_context *); + ARRAY_DEFINE(files, struct dbox_file *); + ARRAY_DEFINE(appends, struct dbox_map_append); + + uint32_t first_new_file_id; + uint32_t orig_next_uid; + + unsigned int files_nonappendable_count; + + unsigned int failed:1; + unsigned int committed:1; +}; + +int dbox_map_view_lookup_rec(struct dbox_map *map, struct mail_index_view *view, + uint32_t seq, struct dbox_mail_lookup_rec *rec_r); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,1143 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "ostream.h" +#include "mkdir-parents.h" +#include "mdbox-storage.h" +#include "mdbox-file.h" +#include "mdbox-map-private.h" + +#define MAX_BACKWARDS_LOOKUPS 10 + +#define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10) +#define DBOX_FORCE_PURGE_MIN_RATIO 0.5 + +#define MAP_STORAGE(map) (&(map)->storage->storage.storage) + +struct dbox_map_transaction_context { + struct dbox_map *map; + struct mail_index_transaction *trans; + struct mail_index_sync_ctx *sync_ctx; + + unsigned int changed:1; + unsigned int success:1; +}; + +void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...) +{ + va_list args; + + map->storage->storage.files_corrupted = TRUE; + + va_start(args, format); + mail_storage_set_critical(MAP_STORAGE(map), + "dbox map %s corrupted: %s", + map->index->filepath, + t_strdup_vprintf(format, args)); + va_end(args); +} + +struct dbox_map * +dbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list, + const char *path) +{ + struct dbox_map *map; + gid_t tmp_gid; + const char *tmp_origin; + + map = i_new(struct dbox_map, 1); + map->storage = storage; + map->set = storage->set; + map->path = i_strdup(path); + map->index = mail_index_alloc(path, MDBOX_GLOBAL_INDEX_PREFIX); + map->map_ext_id = mail_index_ext_register(map->index, "map", + sizeof(struct dbox_map_mail_index_header), + sizeof(struct dbox_map_mail_index_record), + sizeof(uint32_t)); + map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0, + sizeof(uint16_t), sizeof(uint16_t)); + map->created_uid_validity = ioloop_time; + + mailbox_list_get_permissions(root_list, NULL, &map->create_mode, + &map->create_gid, &map->create_gid_origin); + mailbox_list_get_dir_permissions(root_list, NULL, &map->create_dir_mode, + &tmp_gid, &tmp_origin); + mail_index_set_permissions(map->index, map->create_mode, + map->create_gid, map->create_gid_origin); + return map; +} + +void dbox_map_deinit(struct dbox_map **_map) +{ + struct dbox_map *map = *_map; + + *_map = NULL; + + if (array_is_created(&map->ref0_file_ids)) + array_free(&map->ref0_file_ids); + if (map->view != NULL) + mail_index_view_close(&map->view); + mail_index_free(&map->index); + i_free(map->path); + i_free(map); +} + +static int dbox_map_mkdir_storage(struct dbox_map *map) +{ + if (mkdir_parents_chgrp(map->path, map->create_dir_mode, + map->create_gid, map->create_gid_origin) < 0 && + errno != EEXIST) { + mail_storage_set_critical(MAP_STORAGE(map), + "mkdir(%s) failed: %m", map->path); + return -1; + } + return 0; +} + +int dbox_map_open(struct dbox_map *map, bool create_missing) +{ + enum mail_index_open_flags open_flags; + int ret; + + if (map->view != NULL) { + /* already opened */ + return 0; + } + + open_flags = MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY | + mail_storage_settings_to_index_flags(MAP_STORAGE(map)->set); + if (create_missing) { + open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; + if (dbox_map_mkdir_storage(map) < 0) + return -1; + } + ret = mail_index_open(map->index, open_flags, + MAP_STORAGE(map)->set->parsed_lock_method); + if (ret < 0) { + mail_storage_set_internal_error(MAP_STORAGE(map)); + mail_index_reset_error(map->index); + return -1; + } + if (ret == 0) { + /* index not found - for now just return failure */ + return -1; + } + + map->view = mail_index_view_open(map->index); + return 0; +} + +int dbox_map_refresh(struct dbox_map *map) +{ + struct mail_index_view_sync_ctx *ctx; + bool delayed_expunges; + + /* some open files may have read partially written mails. now that + map syncing makes the new mails visible, we need to make sure the + partial data is flushed out of memory */ + mdbox_files_sync_input(map->storage); + + if (mail_index_refresh(map->view->index) < 0) { + mail_storage_set_internal_error(MAP_STORAGE(map)); + mail_index_reset_error(map->index); + return -1; + } + ctx = mail_index_view_sync_begin(map->view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { + mail_storage_set_internal_error(MAP_STORAGE(map)); + mail_index_reset_error(map->index); + return -1; + } + return 0; +} + +static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq, + uint32_t *file_id_r, uoff_t *offset_r, + uoff_t *size_r) +{ + const struct dbox_map_mail_index_record *rec; + const void *data; + uint32_t uid; + bool expunged; + + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + rec = data; + + if (rec == NULL || rec->file_id == 0) { + mail_index_lookup_uid(map->view, seq, &uid); + dbox_map_set_corrupted(map, "file_id=0 for map_uid=%u", uid); + return -1; + } + + *file_id_r = rec->file_id; + *offset_r = rec->offset; + *size_r = rec->size; + return 0; +} + +static int +dbox_map_get_seq(struct dbox_map *map, uint32_t map_uid, uint32_t *seq_r) +{ + if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) { + /* not found - try again after a refresh */ + if (dbox_map_refresh(map) < 0) + return -1; + if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) + return 0; + } + return 1; +} + +int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r) +{ + uint32_t seq; + uoff_t size; + int ret; + + if (dbox_map_open(map, TRUE) < 0) + return -1; + + if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0) + return ret; + + if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r, &size) < 0) + return -1; + return 1; +} + +int dbox_map_view_lookup_rec(struct dbox_map *map, struct mail_index_view *view, + uint32_t seq, struct dbox_mail_lookup_rec *rec_r) +{ + const uint16_t *ref16_p; + const void *data; + bool expunged; + + memset(rec_r, 0, sizeof(*rec_r)); + mail_index_lookup_uid(view, seq, &rec_r->map_uid); + + mail_index_lookup_ext(view, seq, map->map_ext_id, &data, &expunged); + if (data == NULL) { + dbox_map_set_corrupted(map, "missing map extension"); + return -1; + } + memcpy(&rec_r->rec, data, sizeof(rec_r->rec)); + + mail_index_lookup_ext(view, seq, map->ref_ext_id, &data, &expunged); + if (data == NULL) { + dbox_map_set_corrupted(map, "missing ref extension"); + return -1; + } + ref16_p = data; + rec_r->refcount = *ref16_p; + return 0; +} + +int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id, + ARRAY_TYPE(dbox_map_file_msg) *recs) +{ + const struct mail_index_header *hdr; + struct dbox_mail_lookup_rec rec; + struct dbox_map_file_msg msg; + uint32_t seq; + + if (dbox_map_refresh(map) < 0) + return -1; + hdr = mail_index_get_header(map->view); + + memset(&msg, 0, sizeof(msg)); + for (seq = 1; seq <= hdr->messages_count; seq++) { + if (dbox_map_view_lookup_rec(map, map->view, seq, &rec) < 0) + return -1; + + if (rec.rec.file_id == file_id) { + msg.map_uid = rec.map_uid; + msg.offset = rec.rec.offset; + msg.refcount = rec.refcount; + array_append(recs, &msg, 1); + } + } + return 0; +} + +struct dbox_file_size { + uoff_t file_size; + uoff_t ref0_size; +}; + +static void dbox_map_filter_zero_refs(struct dbox_map *map) +{ + ARRAY_TYPE(seq_range) new_ref0_file_ids; + struct hash_table *hash; + struct dbox_file_size *size; + struct seq_range_iter iter; + const struct mail_index_header *hdr; + const struct dbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + uint32_t seq, file_id; + unsigned int i; + bool expunged; + pool_t pool; + + pool = pool_alloconly_create("dbox zero ref count", 8*1024); + hash = hash_table_create(default_pool, pool, 0, NULL, NULL); + + /* count file sizes */ + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data == NULL || expunged) + continue; + rec = data; + + if (!seq_range_exists(&map->ref0_file_ids, rec->file_id)) + continue; + + /* this file has at least some zero references. count how many + bytes it has in total and how much of it has refcount=0. */ + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + if (data == NULL || expunged) + continue; + ref16_p = data; + + size = hash_table_lookup(hash, POINTER_CAST(rec->file_id)); + if (size == NULL) { + size = p_new(pool, struct dbox_file_size, 1); + hash_table_insert(hash, POINTER_CAST(rec->file_id), + size); + } + if (*ref16_p == 0) + size->ref0_size += rec->size; + if (size->file_size < rec->offset + rec->size) + size->file_size = rec->offset + rec->size; + } + + /* now drop the files that don't have enough deleted space */ + seq_range_array_iter_init(&iter, &map->ref0_file_ids); i = 0; + p_array_init(&new_ref0_file_ids, pool, + array_count(&map->ref0_file_ids)); + while (seq_range_array_iter_nth(&iter, i++, &file_id)) { + size = hash_table_lookup(hash, POINTER_CAST(file_id)); + if (size->ref0_size*100 / size->file_size >= + map->set->mdbox_purge_min_percentage) + seq_range_array_add(&new_ref0_file_ids, 0, file_id); + } + seq_range_array_intersect(&map->ref0_file_ids, &new_ref0_file_ids); + + hash_table_destroy(&hash); + pool_unref(&pool); +} + +bool dbox_map_want_purge(struct dbox_map *map) +{ + const struct mail_index_header *hdr; + const struct dbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + uoff_t ref0_size, total_size; + bool expunged; + uint32_t seq; + + if (map->set->mdbox_purge_min_percentage >= 100) { + /* we never purge anything */ + return FALSE; + } + + ref0_size = total_size = 0; + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data == NULL || expunged) + continue; + rec = data; + + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + if (data == NULL || expunged) + continue; + ref16_p = data; + + if (*ref16_p == 0) + ref0_size += rec->size; + total_size += rec->size; + } + + if (ref0_size < DBOX_FORCE_PURGE_MIN_BYTES) + return FALSE; + if ((float)ref0_size / (float)total_size < DBOX_FORCE_PURGE_MIN_RATIO) + return FALSE; + return TRUE; +} + +const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map) +{ + const struct mail_index_header *hdr; + const struct dbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + uint32_t seq; + bool expunged; + + if (array_is_created(&map->ref0_file_ids)) + array_clear(&map->ref0_file_ids); + else + i_array_init(&map->ref0_file_ids, 64); + + if (map->set->mdbox_purge_min_percentage >= 100) { + /* we're never purging anything */ + return &map->ref0_file_ids; + } + + if (dbox_map_open(map, FALSE) < 0) { + /* some internal error */ + return &map->ref0_file_ids; + } + (void)dbox_map_refresh(map); + + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + if (data != NULL && !expunged) { + ref16_p = data; + if (*ref16_p != 0) + continue; + } + + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data != NULL && !expunged) { + rec = data; + seq_range_array_add(&map->ref0_file_ids, 0, + rec->file_id); + } + } + if (map->set->mdbox_purge_min_percentage > 0 && + array_count(&map->ref0_file_ids) > 0) + dbox_map_filter_zero_refs(map); + return &map->ref0_file_ids; +} + +struct dbox_map_transaction_context * +dbox_map_transaction_begin(struct dbox_map *map, bool external) +{ + struct dbox_map_transaction_context *ctx; + enum mail_index_transaction_flags flags = + MAIL_INDEX_TRANSACTION_FLAG_FSYNC; + + if (external) + flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + + ctx = i_new(struct dbox_map_transaction_context, 1); + ctx->map = map; + if (dbox_map_open(map, FALSE) == 0 && + dbox_map_refresh(map) == 0) + ctx->trans = mail_index_transaction_begin(map->view, flags); + return ctx; +} + +static void +dbox_map_sync_handle(struct dbox_map *map, struct mail_index_sync_ctx *sync_ctx) +{ + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + uoff_t offset1, offset2; + + mail_index_sync_get_offsets(sync_ctx, &seq1, &offset1, &seq2, &offset2); + if (offset1 != offset2 || seq1 != seq2) { + /* something had crashed. need a full resync. */ + i_warning("dbox %s: Inconsistency in map index " + "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")", + map->path, seq1, offset1, seq2, offset2); + map->storage->storage.files_corrupted = TRUE; + } else { + while (mail_index_sync_next(sync_ctx, &sync_rec)) ; + } +} + +int dbox_map_transaction_commit(struct dbox_map_transaction_context *ctx) +{ + struct dbox_map *map = ctx->map; + struct mail_index_view *view; + struct mail_index_transaction *sync_trans; + int ret; + + if (!ctx->changed) + return 0; + + /* use syncing to lock the transaction log, so that we always see + log's head_offset = tail_offset */ + ret = mail_index_sync_begin(map->index, &ctx->sync_ctx, + &view, &sync_trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_internal_error(MAP_STORAGE(map)); + mail_index_reset_error(map->index); + mail_index_transaction_rollback(&ctx->trans); + return -1; + } + dbox_map_sync_handle(map, ctx->sync_ctx); + + if (mail_index_transaction_commit(&ctx->trans) < 0) { + mail_storage_set_internal_error(MAP_STORAGE(map)); + mail_index_reset_error(map->index); + return -1; + } + ctx->success = TRUE; + return 0; +} + +void dbox_map_transaction_free(struct dbox_map_transaction_context **_ctx) +{ + struct dbox_map_transaction_context *ctx = *_ctx; + struct dbox_map *map = ctx->map; + + *_ctx = NULL; + if (ctx->success) { + if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { + mail_storage_set_internal_error(MAP_STORAGE(map)); + mail_index_reset_error(map->index); + } + } else if (ctx->sync_ctx != NULL) { + mail_index_sync_rollback(&ctx->sync_ctx); + } + if (ctx->trans != NULL) + mail_index_transaction_rollback(&ctx->trans); + i_free(ctx); +} + +int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, int diff) +{ + struct dbox_map *map = ctx->map; + const uint32_t *uidp; + unsigned int i, count; + const void *data; + uint32_t seq; + bool expunged; + int cur_diff; + + if (ctx->trans == NULL) + return -1; + + count = array_count(map_uids); + for (i = 0; i < count; i++) { + uidp = array_idx(map_uids, i); + if (!mail_index_lookup_seq(map->view, *uidp, &seq)) { + /* we can't refresh map here since view has a + transaction open. */ + dbox_map_set_corrupted(map, + "refcount update lost map_uid=%u", *uidp); + return -1; + } + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + cur_diff = data == NULL ? 0 : *((const uint16_t *)data); + ctx->changed = TRUE; + cur_diff += mail_index_atomic_inc_ext(ctx->trans, seq, + map->ref_ext_id, diff); + i_assert(cur_diff >= 0); + if (cur_diff >= 32768) { + /* we're getting close to the 64k limit. fail early + to make it less likely that two processes increase + the refcount enough times to cross the limit */ + mail_storage_set_error(MAP_STORAGE(map), + MAIL_ERROR_NOTPOSSIBLE, + "Message has been copied too many times"); + return -1; + } + } + return 0; +} + +int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id) +{ + struct dbox_map_transaction_context *map_trans; + const struct mail_index_header *hdr; + const struct dbox_map_mail_index_record *rec; + const void *data; + bool expunged; + uint32_t seq; + int ret = 0; + + /* make sure the map is refreshed, otherwise we might be expunging + messages that have already been moved to other files. */ + + /* we need a per-file transaction, otherwise we can't refresh the map */ + map_trans = dbox_map_transaction_begin(map, TRUE); + + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data == NULL) { + dbox_map_set_corrupted(map, "missing map extension"); + ret = -1; + break; + } + + rec = data; + if (rec->file_id == file_id) { + map_trans->changed = TRUE; + mail_index_expunge(map_trans->trans, seq); + } + } + if (ret == 0) + (void)dbox_map_transaction_commit(map_trans); + dbox_map_transaction_free(&map_trans); + return ret; +} + +struct dbox_map_append_context * +dbox_map_append_begin(struct dbox_map *map) +{ + struct dbox_map_append_context *ctx; + + ctx = i_new(struct dbox_map_append_context, 1); + ctx->map = map; + ctx->first_new_file_id = (uint32_t)-1; + i_array_init(&ctx->file_appends, 64); + i_array_init(&ctx->files, 64); + i_array_init(&ctx->appends, 128); + + if (dbox_map_open(ctx->map, TRUE) < 0) + ctx->failed = TRUE; + else { + /* refresh the map so we can try appending to the + latest files */ + (void)dbox_map_refresh(ctx->map); + } + return ctx; +} + +static time_t day_begin_stamp(unsigned int days) +{ + struct tm tm; + time_t stamp; + + if (days == 0) + return 0; + + /* get beginning of today */ + tm = *localtime(&ioloop_time); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + stamp = mktime(&tm); + if (stamp == (time_t)-1) + i_panic("mktime(today) failed"); + + return stamp - (3600*24 * (days-1)); +} + +static bool +dbox_map_file_try_append(struct dbox_map_append_context *ctx, + uint32_t file_id, time_t stamp, uoff_t mail_size, + struct dbox_file_append_context **file_append_r, + struct ostream **output_r, bool *retry_later_r) +{ + struct dbox_map *map = ctx->map; + struct mdbox_storage *storage = map->storage; + struct dbox_file *file; + struct dbox_file_append_context *file_append; + struct stat st; + bool deleted, file_too_old = FALSE; + int ret; + + *file_append_r = NULL; + *output_r = NULL; + *retry_later_r = FALSE; + + file = mdbox_file_init(storage, file_id); + if (dbox_file_open(file, &deleted) <= 0 || deleted) { + dbox_file_unref(&file); + return TRUE; + } + if (file->lock != NULL) { + /* already locked, we're possibly in the middle of purging it + in which case we really don't want to write there. */ + dbox_file_unref(&file); + return TRUE; + } + + if (file->create_time < stamp) + file_too_old = TRUE; + else if ((ret = dbox_file_try_lock(file)) <= 0) { + /* locking failed */ + *retry_later_r = ret == 0; + } else if (stat(file->cur_path, &st) < 0) { + if (errno != ENOENT) + i_error("stat(%s) failed: %m", file->cur_path); + /* the file was unlinked between opening and locking it. */ + } else { + file_append = dbox_file_append_init(file); + if (dbox_file_get_append_stream(file_append, output_r) <= 0) { + /* couldn't append to this file */ + } else if ((*output_r)->offset + mail_size > map->set->mdbox_rotate_size) { + /* file was too large after all */ + } else { + /* success */ + *file_append_r = file_append; + return TRUE; + } + dbox_file_append_rollback(&file_append); + } + + /* failure */ + dbox_file_unlock(file); + dbox_file_unref(&file); + return !file_too_old; +} + +static bool +dbox_map_is_appending(struct dbox_map_append_context *ctx, uint32_t file_id) +{ + struct dbox_file_append_context *const *file_appends; + unsigned int i, count; + + /* there shouldn't be many files open, don't bother with anything + faster. */ + file_appends = array_get(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)file_appends[i]->file; + + if (mfile->file_id == file_id) + return TRUE; + } + return FALSE; +} + +static int +dbox_map_find_appendable_file(struct dbox_map_append_context *ctx, + uoff_t mail_size, + struct dbox_file_append_context **file_append_r, + struct ostream **output_r, bool *existing_r) +{ + struct dbox_map *map = ctx->map; + ARRAY_TYPE(seq_range) checked_file_ids; + struct dbox_file_append_context *const *file_appends; + const struct mail_index_header *hdr; + unsigned int i, count, backwards_lookup_count; + uint32_t seq, seq1, uid, file_id; + uoff_t offset, append_offset, size; + time_t stamp; + bool retry_later; + + *existing_r = FALSE; + + if (mail_size >= map->set->mdbox_rotate_size) + return 0; + + /* first try to use files already used in this append */ + file_appends = array_get(&ctx->file_appends, &count); + for (i = count; i > ctx->files_nonappendable_count; i--) { + append_offset = file_appends[i-1]->output->offset; + if (append_offset + mail_size <= map->set->mdbox_rotate_size && + dbox_file_get_append_stream(file_appends[i-1], output_r) > 0) { + *file_append_r = file_appends[i-1]; + *existing_r = TRUE; + return 1; + } + /* can't append to this file anymore. we could close it + otherwise, except that would also lose our lock too early. */ + } + ctx->files_nonappendable_count = count; + + /* try to find an existing appendable file */ + stamp = day_begin_stamp(map->set->mdbox_rotate_days); + hdr = mail_index_get_header(map->view); + + ctx->orig_next_uid = hdr->next_uid; + backwards_lookup_count = 0; + t_array_init(&checked_file_ids, 16); + for (seq = hdr->messages_count; seq > 0; seq--) { + if (dbox_map_lookup_seq(map, seq, &file_id, &offset, &size) < 0) + return -1; + + if (seq_range_exists(&checked_file_ids, file_id)) + continue; + seq_range_array_add(&checked_file_ids, 0, file_id); + + if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) { + /* we've wasted enough time here */ + break; + } + + /* first lookup: this should be enough usually, but we can't + be sure until after locking. also if messages were recently + moved, this message might not be the last one in the file. */ + if (offset + size + mail_size >= map->set->mdbox_rotate_size) + continue; + + if (dbox_map_is_appending(ctx, file_id)) { + /* already checked this */ + continue; + } + + mail_index_lookup_uid(map->view, seq, &uid); + if (!dbox_map_file_try_append(ctx, file_id, stamp, mail_size, + file_append_r, output_r, + &retry_later)) { + /* file is too old. the rest of the files are too. */ + break; + } + /* NOTE: we've now refreshed map view. there are no guarantees + about sequences anymore. */ + if (*file_append_r != NULL) + return 1; + /* FIXME: use retry_later somehow */ + if (uid == 1 || + !mail_index_lookup_seq_range(map->view, 1, uid-1, + &seq1, &seq)) + break; + seq++; + } + return 0; +} + +int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, + struct dbox_file_append_context **file_append_ctx_r, + struct ostream **output_r) +{ + struct dbox_file *file; + struct dbox_map_append *append; + struct dbox_file_append_context *file_append; + bool existing; + int ret; + + if (ctx->failed) + return -1; + + ret = dbox_map_find_appendable_file(ctx, mail_size, &file_append, + output_r, &existing); + if (ret > 0) + file = file_append->file; + else if (ret < 0) + return -1; + else { + /* create a new file */ + file = mdbox_file_init(ctx->map->storage, 0); + file_append = dbox_file_append_init(file); + + ret = dbox_file_get_append_stream(file_append, output_r); + if (ret <= 0) { + i_assert(ret < 0); + dbox_file_append_rollback(&file_append); + dbox_file_unref(&file); + return -1; + } + } + + append = array_append_space(&ctx->appends); + append->file_append = file_append; + append->offset = (*output_r)->offset; + append->size = (uint32_t)-1; + if (!existing) { + i_assert(file_append->first_append_offset == 0); + file_append->first_append_offset = file_append->output->offset; + array_append(&ctx->file_appends, &file_append, 1); + array_append(&ctx->files, &file, 1); + } + *file_append_ctx_r = file_append; + return 0; +} + +void dbox_map_append_finish(struct dbox_map_append_context *ctx) +{ + struct dbox_map_append *appends; + unsigned int count; + uoff_t cur_offset; + + appends = array_get_modifiable(&ctx->appends, &count); + i_assert(count > 0 && appends[count-1].size == (uint32_t)-1); + cur_offset = appends[count-1].file_append->output->offset; + i_assert(cur_offset >= appends[count-1].offset); + appends[count-1].size = cur_offset - appends[count-1].offset; +} + +static int +dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view, + uint32_t *file_id_r) +{ + const struct dbox_map_mail_index_header *hdr; + const void *data; + size_t data_size; + + mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size); + if (data_size != sizeof(*hdr)) { + if (data_size != 0) { + dbox_map_set_corrupted(map, "hdr size=%"PRIuSIZE_T, + data_size); + return -1; + } + /* first file */ + *file_id_r = 1; + } else { + hdr = data; + *file_id_r = hdr->highest_file_id + 1; + } + return 0; +} + +static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx, + bool separate_transaction) +{ + struct dbox_file_append_context *const *file_appends; + unsigned int i, count; + uint32_t first_file_id, file_id; + int ret; + + /* start the syncing. we'll need it even if there are no file ids to + be assigned. */ + ret = mail_index_sync_begin(ctx->map->index, &ctx->sync_ctx, + &ctx->sync_view, &ctx->sync_trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_internal_error(MAP_STORAGE(ctx->map)); + mail_index_reset_error(ctx->map->index); + return -1; + } + dbox_map_sync_handle(ctx->map, ctx->sync_ctx); + + if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) { + mail_index_sync_rollback(&ctx->sync_ctx); + return -1; + } + + /* assign file_ids for newly created files */ + first_file_id = file_id; + file_appends = array_get(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)file_appends[i]->file; + + if (dbox_file_append_flush(file_appends[i]) < 0) { + ret = -1; + break; + } + + if (mfile->file_id == 0) { + if (mdbox_file_assign_file_id(mfile, file_id++) < 0) { + ret = -1; + break; + } + } + } + + if (ret < 0) { + mail_index_sync_rollback(&ctx->sync_ctx); + return -1; + } + + ctx->trans = !separate_transaction ? NULL : + mail_index_transaction_begin(ctx->map->view, + MAIL_INDEX_TRANSACTION_FLAG_FSYNC); + + /* update the highest used file_id */ + if (first_file_id != file_id) { + file_id--; + mail_index_update_header_ext(ctx->trans != NULL ? ctx->trans : + ctx->sync_trans, + ctx->map->map_ext_id, + 0, &file_id, sizeof(file_id)); + } + return 0; +} + +int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r) +{ + const struct dbox_map_append *appends; + const struct mail_index_header *hdr; + struct dbox_map_mail_index_record rec; + unsigned int i, count; + ARRAY_TYPE(seq_range) uids; + const struct seq_range *range; + uint32_t seq; + uint16_t ref16; + int ret = 0; + + if (array_count(&ctx->appends) == 0) { + *first_map_uid_r = 0; + *last_map_uid_r = 0; + return 0; + } + + if (dbox_map_assign_file_ids(ctx, TRUE) < 0) + return -1; + + /* append map records to index */ + memset(&rec, 0, sizeof(rec)); + ref16 = 1; + appends = array_get(&ctx->appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)appends[i].file_append->file; + + i_assert(appends[i].offset <= (uint32_t)-1); + i_assert(appends[i].size <= (uint32_t)-1); + + rec.file_id = mfile->file_id; + rec.offset = appends[i].offset; + rec.size = appends[i].size; + + mail_index_append(ctx->trans, 0, &seq); + mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id, + &rec, NULL); + mail_index_update_ext(ctx->trans, seq, ctx->map->ref_ext_id, + &ref16, NULL); + } + + /* assign map UIDs for appended records */ + hdr = mail_index_get_header(ctx->sync_view); + t_array_init(&uids, 1); + mail_index_append_finish_uids(ctx->trans, hdr->next_uid, &uids); + range = array_idx(&uids, 0); + i_assert(range[0].seq2 - range[0].seq1 + 1 == count); + + if (hdr->uid_validity == 0) { + /* we don't really care about uidvalidity, but it can't be 0 */ + uint32_t uid_validity = ioloop_time; + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + if (mail_index_transaction_commit(&ctx->trans) < 0) { + mail_storage_set_internal_error(MAP_STORAGE(ctx->map)); + mail_index_reset_error(ctx->map->index); + return -1; + } + + *first_map_uid_r = range[0].seq1; + *last_map_uid_r = range[0].seq2; + return ret; +} + +int dbox_map_append_move(struct dbox_map_append_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, + const ARRAY_TYPE(seq_range) *expunge_map_uids) +{ + const struct dbox_map_append *appends; + struct dbox_map_mail_index_record rec; + struct seq_range_iter iter; + const uint32_t *uids; + unsigned int i, j, map_uids_count, appends_count; + uint32_t uid, seq; + + if (dbox_map_assign_file_ids(ctx, FALSE) < 0) + return -1; + + memset(&rec, 0, sizeof(rec)); + appends = array_get(&ctx->appends, &appends_count); + + uids = array_get(map_uids, &map_uids_count); + for (i = j = 0; i < map_uids_count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)appends[j].file_append->file; + + i_assert(j < appends_count); + rec.file_id = mfile->file_id; + rec.offset = appends[j].offset; + rec.size = appends[j].size; + j++; + + if (!mail_index_lookup_seq(ctx->sync_view, uids[i], &seq)) + i_unreached(); + mail_index_update_ext(ctx->sync_trans, seq, + ctx->map->map_ext_id, &rec, NULL); + } + + seq_range_array_iter_init(&iter, expunge_map_uids); i = 0; + while (seq_range_array_iter_nth(&iter, i++, &uid)) { + if (!mail_index_lookup_seq(ctx->sync_view, uid, &seq)) + i_unreached(); + mail_index_expunge(ctx->sync_trans, seq); + } + return 0; +} + +int dbox_map_append_commit(struct dbox_map_append_context *ctx) +{ + struct dbox_map *map = ctx->map; + struct dbox_file_append_context **file_appends; + unsigned int i, count; + + i_assert(ctx->trans == NULL); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (dbox_file_append_commit(&file_appends[i]) < 0) + return -1; + } + + if (ctx->sync_ctx != NULL) { + if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { + mail_storage_set_internal_error(MAP_STORAGE(map)); + mail_index_reset_error(map->index); + return -1; + } + } + + ctx->committed = TRUE; + return 0; +} + +void dbox_map_append_free(struct dbox_map_append_context **_ctx) +{ + struct dbox_map_append_context *ctx = *_ctx; + struct dbox_file_append_context **file_appends; + struct dbox_file **files; + unsigned int i, count; + + *_ctx = NULL; + + if (ctx->trans != NULL) + mail_index_transaction_rollback(&ctx->trans); + if (ctx->sync_ctx != NULL) + mail_index_sync_rollback(&ctx->sync_ctx); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (file_appends[i] != NULL) + dbox_file_append_rollback(&file_appends[i]); + } + + files = array_get_modifiable(&ctx->files, &count); + for (i = 0; i < count; i++) { + dbox_file_unlock(files[i]); + dbox_file_unref(&files[i]); + } + + array_free(&ctx->appends); + array_free(&ctx->file_appends); + array_free(&ctx->files); + i_free(ctx); +} + +uint32_t dbox_map_get_uid_validity(struct dbox_map *map) +{ + const struct mail_index_header *hdr; + + i_assert(map->view != NULL); + + hdr = mail_index_get_header(map->view); + if (hdr->uid_validity != 0) + return hdr->uid_validity; + + /* refresh index in case it was just changed */ + (void)dbox_map_refresh(map); + hdr = mail_index_get_header(map->view); + return hdr->uid_validity != 0 ? hdr->uid_validity : + map->created_uid_validity; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,93 @@ +#ifndef MDBOX_MAP_H +#define MDBOX_MAP_H + +#include "seq-range-array.h" + +struct dbox_map_append_context; +struct dbox_file_append_context; +struct mdbox_storage; + +struct dbox_map_mail_index_header { + uint32_t highest_file_id; +}; + +struct dbox_map_mail_index_record { + uint32_t file_id; + uint32_t offset; + uint32_t size; /* including pre/post metadata */ +}; + +struct dbox_map_file_msg { + uint32_t map_uid; + uint32_t offset; + uint32_t refcount; +}; +ARRAY_DEFINE_TYPE(dbox_map_file_msg, struct dbox_map_file_msg); + +struct dbox_map * +dbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list, + const char *path); +void dbox_map_deinit(struct dbox_map **map); + +/* Open the map. This is done automatically for most operations. + Returns 0 if ok, -1 if error. */ +int dbox_map_open(struct dbox_map *map, bool create_missing); +/* Refresh the map. Returns 0 if ok, -1 if error. */ +int dbox_map_refresh(struct dbox_map *map); + +/* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID + is already expunged, -1 if error. */ +int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r); + +/* Get all messages from file */ +int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id, + ARRAY_TYPE(dbox_map_file_msg) *recs); + +struct dbox_map_transaction_context * +dbox_map_transaction_begin(struct dbox_map *map, bool external); +/* Write transaction to map and leave it locked. Call _free() to update tail + offset and unlock. */ +int dbox_map_transaction_commit(struct dbox_map_transaction_context *ctx); +void dbox_map_transaction_free(struct dbox_map_transaction_context **ctx); + +int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, int diff); +int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id); + +/* Returns TRUE if there's enough pressure to purge immediately. */ +bool dbox_map_want_purge(struct dbox_map *map); +/* Return all files containing messages with zero refcount. */ +const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map); + +struct dbox_map_append_context * +dbox_map_append_begin(struct dbox_map *map); +/* Request file for saving a new message with given size (if available). If an + existing file can be used, the record is locked and updated in index. + Returns 0 if ok, -1 if error. */ +int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, + struct dbox_file_append_context **file_append_ctx_r, + struct ostream **output_r); +/* Finished saving the last mail. Saves the message size. */ +void dbox_map_append_finish(struct dbox_map_append_context *ctx); +/* Assign map UIDs to all appended msgs to multi-files. */ +int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r); +/* The appends are existing messages that were simply moved to a new file. + map_uids contains the moved messages' map UIDs. */ +int dbox_map_append_move(struct dbox_map_append_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, + const ARRAY_TYPE(seq_range) *expunge_map_uids); +/* Returns 0 if ok, -1 if error. */ +int dbox_map_append_commit(struct dbox_map_append_context *ctx); +void dbox_map_append_free(struct dbox_map_append_context **ctx); + +/* Get either existing uidvalidity or create a new one if map was + just created. */ +uint32_t dbox_map_get_uid_validity(struct dbox_map *map); + +void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...) + ATTR_FORMAT(2, 3); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,353 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "fdatasync-path.h" +#include "hex-binary.h" +#include "hex-dec.h" +#include "str.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "write-full.h" +#include "index-mail.h" +#include "mail-copy.h" +#include "dbox-save.h" +#include "mdbox-storage.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" + +#include <stdlib.h> + +struct dbox_save_mail { + struct dbox_file_append_context *file_append; + uint32_t seq; + uint32_t append_offset; +}; + +struct mdbox_save_context { + struct dbox_save_context ctx; + + struct mdbox_mailbox *mbox; + struct mdbox_sync_context *sync_ctx; + + struct dbox_file_append_context *cur_file_append; + struct dbox_map_append_context *append_ctx; + + ARRAY_TYPE(uint32_t) copy_map_uids; + struct dbox_map_transaction_context *map_trans; + + ARRAY_DEFINE(mails, struct dbox_save_mail); +}; + +struct dbox_file * +mdbox_save_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r) +{ + struct mdbox_save_context *ctx = + (struct mdbox_save_context *)t->save_ctx; + const struct dbox_save_mail *mails, *mail; + unsigned int count; + + mails = array_get(&ctx->mails, &count); + i_assert(count > 0); + i_assert(seq >= mails[0].seq); + + mail = &mails[mails[0].seq - seq]; + i_assert(mail->seq == seq); + + if (dbox_file_append_flush(mail->file_append) < 0) + ctx->ctx.failed = TRUE; + + *offset_r = mail->append_offset; + return mail->file_append->file; +} + +struct mail_save_context * +mdbox_save_alloc(struct mailbox_transaction_context *t) +{ + struct index_transaction_context *it = + (struct index_transaction_context *)t; + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)t->box; + struct mdbox_save_context *ctx = + (struct mdbox_save_context *)t->save_ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (ctx != NULL) { + /* use the existing allocated structure */ + ctx->ctx.finished = FALSE; + return &ctx->ctx.ctx; + } + + ctx = i_new(struct mdbox_save_context, 1); + ctx->ctx.ctx.transaction = t; + ctx->ctx.trans = it->trans; + ctx->mbox = mbox; + ctx->append_ctx = dbox_map_append_begin(mbox->storage->map); + i_array_init(&ctx->mails, 32); + t->save_ctx = &ctx->ctx.ctx; + return t->save_ctx; +} + +int mdbox_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + struct dbox_save_mail *save_mail; + uoff_t mail_size, append_offset; + + /* get the size of the mail to be saved, if possible */ + if (i_stream_get_size(input, TRUE, &mail_size) <= 0) + mail_size = 0; + if (dbox_map_append_next(ctx->append_ctx, mail_size, + &ctx->cur_file_append, + &ctx->ctx.cur_output) < 0) { + ctx->ctx.failed = TRUE; + return -1; + } + i_assert(ctx->ctx.cur_output->offset <= (uint32_t)-1); + append_offset = ctx->ctx.cur_output->offset; + + ctx->ctx.cur_file = ctx->cur_file_append->file; + dbox_save_begin(&ctx->ctx, input); + + save_mail = array_append_space(&ctx->mails); + save_mail->file_append = ctx->cur_file_append; + save_mail->seq = ctx->ctx.seq; + save_mail->append_offset = append_offset; + return ctx->ctx.failed ? -1 : 0; +} + +static int mdbox_save_mail_write_metadata(struct mdbox_save_context *ctx, + struct dbox_save_mail *mail) +{ + struct dbox_file *file = mail->file_append->file; + struct dbox_message_header dbox_msg_hdr; + uoff_t message_size; + uint8_t guid_128[MAIL_GUID_128_SIZE]; + + i_assert(file->msg_header_size == sizeof(dbox_msg_hdr)); + + message_size = ctx->ctx.cur_output->offset - + mail->append_offset - mail->file_append->file->msg_header_size; + + dbox_save_write_metadata(&ctx->ctx.ctx, ctx->ctx.cur_output, + ctx->mbox->ibox.box.name, guid_128); + /* save the 128bit GUID to index so if the map index gets corrupted + we can still find the message */ + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_128, NULL); + + dbox_msg_header_fill(&dbox_msg_hdr, message_size); + if (o_stream_pwrite(ctx->ctx.cur_output, &dbox_msg_hdr, + sizeof(dbox_msg_hdr), mail->append_offset) < 0) { + dbox_file_set_syscall_error(file, "pwrite()"); + return -1; + } + return 0; +} + +static int mdbox_save_finish_write(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + struct dbox_save_mail *mails; + + ctx->ctx.finished = TRUE; + if (ctx->ctx.cur_output == NULL) + return -1; + + index_mail_cache_parse_deinit(_ctx->dest_mail, + _ctx->received_date, !ctx->ctx.failed); + + mails = array_idx_modifiable(&ctx->mails, array_count(&ctx->mails) - 1); + if (!ctx->ctx.failed) T_BEGIN { + if (mdbox_save_mail_write_metadata(ctx, mails) < 0) + ctx->ctx.failed = TRUE; + else + dbox_map_append_finish(ctx->append_ctx); + } T_END; + + i_stream_unref(&ctx->ctx.input); + + if (ctx->ctx.failed) { + array_delete(&ctx->mails, array_count(&ctx->mails) - 1, 1); + return -1; + } + return 0; +} + +int mdbox_save_finish(struct mail_save_context *ctx) +{ + int ret; + + ret = mdbox_save_finish_write(ctx); + index_save_context_free(ctx); + return ret; +} + +void mdbox_save_cancel(struct mail_save_context *_ctx) +{ + struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; + + ctx->failed = TRUE; + (void)mdbox_save_finish(_ctx); +} + +int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + struct mailbox_transaction_context *_t = _ctx->transaction; + struct mdbox_mailbox *mbox = ctx->mbox; + const struct mail_index_header *hdr; + uint32_t first_map_uid, last_map_uid; + + i_assert(ctx->ctx.finished); + + /* lock the mailbox before map to avoid deadlocks */ + if (mdbox_sync_begin(mbox, MDBOX_SYNC_FLAG_NO_PURGE | + MDBOX_SYNC_FLAG_FORCE | + MDBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* get map UIDs for messages saved to multi-files. they're written + to transaction log immediately within this function, but the map + is left locked. */ + if (dbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* assign UIDs for new messages */ + hdr = mail_index_get_header(ctx->sync_ctx->sync_view); + mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, + &_t->changes->saved_uids); + + /* add map_uids for all messages saved to multi-files */ + if (first_map_uid != 0) { + struct mdbox_mail_index_record rec; + const struct dbox_save_mail *mails; + unsigned int i, count; + uint32_t next_map_uid = first_map_uid; + + mdbox_update_header(mbox, ctx->ctx.trans, NULL); + + memset(&rec, 0, sizeof(rec)); + rec.save_date = ioloop_time; + mails = array_get(&ctx->mails, &count); + for (i = 0; i < count; i++) { + rec.map_uid = next_map_uid++; + i_assert(i == 0 || + mails[i-1].append_offset != mails[i].append_offset); + mail_index_update_ext(ctx->ctx.trans, mails[i].seq, + mbox->ext_id, &rec, NULL); + } + i_assert(next_map_uid == last_map_uid + 1); + } + + /* increase map's refcount for copied mails */ + if (array_is_created(&ctx->copy_map_uids)) { + ctx->map_trans = + dbox_map_transaction_begin(mbox->storage->map, FALSE); + if (dbox_map_update_refcounts(ctx->map_trans, + &ctx->copy_map_uids, 1) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + } + + if (ctx->ctx.mail != NULL) + mail_free(&ctx->ctx.mail); + + _t->changes->uid_validity = hdr->uid_validity; + return 0; +} + +void mdbox_transaction_save_commit_post(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + + _ctx->transaction = NULL; /* transaction is already freed */ + + /* finish writing the mailbox APPENDs */ + if (mdbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) { + if (ctx->map_trans != NULL) + (void)dbox_map_transaction_commit(ctx->map_trans); + /* commit only updates the sync tail offset, everything else + was already written at this point. */ + (void)dbox_map_append_commit(ctx->append_ctx); + } + dbox_map_append_free(&ctx->append_ctx); + + if (!ctx->mbox->storage->storage.storage.set->fsync_disable) { + if (fdatasync_path(ctx->mbox->ibox.box.path) < 0) { + i_error("fdatasync_path(%s) failed: %m", + ctx->mbox->ibox.box.path); + } + } + mdbox_transaction_save_rollback(_ctx); +} + +void mdbox_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + + if (!ctx->ctx.finished) + mdbox_save_cancel(&ctx->ctx.ctx); + if (ctx->append_ctx != NULL) + dbox_map_append_free(&ctx->append_ctx); + if (ctx->map_trans != NULL) + dbox_map_transaction_free(&ctx->map_trans); + if (array_is_created(&ctx->copy_map_uids)) + array_free(&ctx->copy_map_uids); + + if (ctx->sync_ctx != NULL) + (void)mdbox_sync_finish(&ctx->sync_ctx, FALSE); + + if (ctx->ctx.mail != NULL) + mail_free(&ctx->ctx.mail); + array_free(&ctx->mails); + i_free(ctx); +} + +int mdbox_copy(struct mail_save_context *_ctx, struct mail *mail) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + struct mdbox_mailbox *src_mbox; + struct mdbox_mail_index_record rec; + const void *data; + bool expunged; + + ctx->ctx.finished = TRUE; + + if (mail->box->storage != _ctx->transaction->box->storage) + return mail_storage_copy(_ctx, mail); + src_mbox = (struct mdbox_mailbox *)mail->box; + + memset(&rec, 0, sizeof(rec)); + rec.save_date = ioloop_time; + if (mdbox_mail_lookup(src_mbox, src_mbox->ibox.view, mail->seq, + &rec.map_uid) < 0) + return -1; + + /* remember the map_uid so we can later increase its refcount */ + if (!array_is_created(&ctx->copy_map_uids)) + i_array_init(&ctx->copy_map_uids, 32); + array_append(&ctx->copy_map_uids, &rec.map_uid, 1); + + /* add message to mailbox index */ + dbox_save_add_to_index(&ctx->ctx); + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->ext_id, &rec, NULL); + + mail_index_lookup_ext(src_mbox->ibox.view, mail->seq, + src_mbox->guid_ext_id, &data, &expunged); + if (data != NULL) { + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, data, NULL); + } + return 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,65 @@ +/* Copyright (c) 2006-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "mdbox-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + { type, #name, offsetof(struct mdbox_settings, name), NULL } + +static bool mdbox_settings_verify(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r); + +static struct setting_define mdbox_setting_defines[] = { + DEF(SET_UINT, mdbox_rotate_size), + DEF(SET_UINT, mdbox_rotate_min_size), + DEF(SET_UINT, mdbox_rotate_days), + DEF(SET_UINT, mdbox_max_open_files), + DEF(SET_UINT, mdbox_purge_min_percentage), + + SETTING_DEFINE_LIST_END +}; + +static struct mdbox_settings mdbox_default_settings = { + MEMBER(mdbox_rotate_size) 2048*1024, + MEMBER(mdbox_rotate_min_size) 16*1024, + MEMBER(mdbox_rotate_days) 0, + MEMBER(mdbox_max_open_files) 64, + MEMBER(mdbox_purge_min_percentage) 0 +}; + +static struct setting_parser_info mdbox_setting_parser_info = { + MEMBER(defines) mdbox_setting_defines, + MEMBER(defaults) &mdbox_default_settings, + + MEMBER(parent) &mail_user_setting_parser_info, + MEMBER(dynamic_parsers) NULL, + + MEMBER(parent_offset) (size_t)-1, + MEMBER(type_offset) (size_t)-1, + MEMBER(struct_size) sizeof(struct mdbox_settings), + MEMBER(check_func) mdbox_settings_verify +}; + +/* <settings checks> */ +static bool mdbox_settings_verify(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) +{ + const struct mdbox_settings *set = _set; + + if (set->mdbox_max_open_files < 2) { + *error_r = "mdbox_max_open_files must be at least 2"; + return FALSE; + } + return TRUE; +} +/* </settings checks> */ + +const struct setting_parser_info *mdbox_get_setting_parser_info(void) +{ + return &mdbox_setting_parser_info; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,14 @@ +#ifndef MDBOX_SETTINGS_H +#define MDBOX_SETTINGS_H + +struct mdbox_settings { + unsigned int mdbox_rotate_size; + unsigned int mdbox_rotate_min_size; + unsigned int mdbox_rotate_days; + unsigned int mdbox_max_open_files; + unsigned int mdbox_purge_min_percentage; +}; + +const struct setting_parser_info *mdbox_get_setting_parser_info(void); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,777 @@ +/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "hash.h" +#include "str.h" +#include "dbox-sync-rebuild.h" +#include "mdbox-storage.h" +#include "mdbox-file.h" +#include "mdbox-map-private.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" + +#include <stdlib.h> +#include <dirent.h> +#include <unistd.h> + +struct mdbox_rebuild_msg { + uint8_t guid_128[MAIL_GUID_128_SIZE]; + uint32_t file_id; + uint32_t offset; + uint32_t size; + uint32_t map_uid; + + uint16_t refcount; + unsigned int seen_zero_ref_in_map:1; +}; + +struct rebuild_msg_mailbox { + struct mailbox *box; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + uint32_t next_uid; +}; + +struct mdbox_storage_rebuild_context { + struct mdbox_storage *storage; + pool_t pool; + + struct hash_table *guid_hash; + ARRAY_DEFINE(msgs, struct mdbox_rebuild_msg *); + + uint32_t prev_file_id; + uint32_t highest_seen_map_uid; + + struct mailbox_list *default_list; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + + struct rebuild_msg_mailbox prev_msg; + + unsigned int msgs_unsorted:1; +}; + +static unsigned int guid_hash(const void *p) +{ + const uint8_t *s = p; + unsigned int i, g, h = 0; + + for (i = 0; i < MAIL_GUID_128_SIZE; i++) { + h = (h << 4) + s[i]; + if ((g = h & 0xf0000000UL)) { + h = h ^ (g >> 24); + h = h ^ g; + } + } + return h; +} + +static int guid_cmp(const void *p1, const void *p2) +{ + return memcmp(p1, p2, MAIL_GUID_128_SIZE); +} + +static struct mdbox_storage_rebuild_context * +mdbox_storage_rebuild_init(struct mdbox_storage *storage) +{ + struct mdbox_storage_rebuild_context *ctx; + + ctx = i_new(struct mdbox_storage_rebuild_context, 1); + ctx->storage = storage; + ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256); + ctx->guid_hash = hash_table_create(default_pool, ctx->pool, 0, + guid_hash, guid_cmp); + i_array_init(&ctx->msgs, 512); + return ctx; +} + +static void +mdbox_storage_rebuild_deinit(struct mdbox_storage_rebuild_context *ctx) +{ + if (ctx->sync_ctx != NULL) + mail_index_sync_rollback(&ctx->sync_ctx); + + hash_table_destroy(&ctx->guid_hash); + pool_unref(&ctx->pool); + array_free(&ctx->msgs); + i_free(ctx); +} + +static int mdbox_rebuild_msg_offset_cmp(const void *p1, const void *p2) +{ + const struct mdbox_rebuild_msg *const *m1 = p1, *const *m2 = p2; + + if ((*m1)->file_id < (*m2)->file_id) + return -1; + if ((*m1)->file_id > (*m2)->file_id) + return 1; + + if ((*m1)->offset < (*m2)->offset) + return -1; + if ((*m1)->offset > (*m2)->offset) + return 1; + return 0; +} + +static int mdbox_rebuild_msg_uid_cmp(struct mdbox_rebuild_msg *const *m1, + struct mdbox_rebuild_msg *const *m2) +{ + if ((*m1)->map_uid < (*m2)->map_uid) + return -1; + if ((*m1)->map_uid > (*m2)->map_uid) + return 1; + return 0; +} + +static int rebuild_add_file(struct mdbox_storage_rebuild_context *ctx, + const char *path) +{ + struct dbox_file *file; + const char *fname, *guid; + struct mdbox_rebuild_msg *rec; + uint32_t file_id; + uoff_t offset, prev_offset, size; + bool last, first, fixed = FALSE; + unsigned int len; + int ret = 0; + + fname = strrchr(path, '/'); + i_assert(fname != NULL); + fname += strlen(MDBOX_MAIL_FILE_PREFIX) + 1; + + file_id = strtoul(fname, NULL, 10); + if (!is_numeric(fname, '\0') || file_id == 0) { + len = strlen(fname); + if (len > 7 && strcmp(fname + len - 7, ".broken") != 0) { + i_warning("dbox rebuild: File name is missing ID: %s", + path); + } + return 0; + } + + /* small optimization: typically files are returned sorted. in that + case we don't need to sort them ourself. */ + if (file_id < ctx->prev_file_id) + ctx->msgs_unsorted = TRUE; + ctx->prev_file_id = file_id; + + file = mdbox_file_init(ctx->storage, file_id); + prev_offset = 0; + dbox_file_seek_rewind(file); + while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) { + if (ret > 0) { + if ((ret = dbox_file_metadata_read(file)) < 0) + break; + } + + if (ret == 0) { + /* file is corrupted. fix it and retry. */ + if (fixed || last) + break; + first = prev_offset == 0; + if (prev_offset == 0) { + /* use existing file header if it was ok */ + prev_offset = offset; + } + if (dbox_file_fix(file, prev_offset) < 0) { + ret = -1; + break; + } + fixed = TRUE; + if (!first) { + /* seek to the offset where we last left off */ + ret = dbox_file_get_mail_stream(file, + prev_offset, &size, NULL); + if (ret <= 0) + break; + } + continue; + } + prev_offset = offset; + + guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID); + if (guid == NULL) { + dbox_file_set_corrupted(file, + "Message is missing GUID"); + ret = 0; + break; + } + + rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1); + rec->file_id = file_id; + rec->offset = offset; + rec->size = file->input->v_offset - offset; + mail_generate_guid_128_hash(guid, rec->guid_128); + array_append(&ctx->msgs, &rec, 1); + + if (hash_table_lookup(ctx->guid_hash, rec->guid_128) != NULL) { + /* duplicate. save this as a refcount=0 to map, + so it will eventually be deleted. */ + rec->seen_zero_ref_in_map = TRUE; + } else { + hash_table_insert(ctx->guid_hash, rec->guid_128, rec); + } + } + if (ret == 0 && !last) + i_error("dbox rebuild: Failed to fix file %s", path); + dbox_file_unref(&file); + return ret < 0 ? -1 : 0; +} + +static void +rebuild_add_missing_map_uids(struct mdbox_storage_rebuild_context *ctx, + uint32_t next_uid) +{ + struct mdbox_rebuild_msg **msgs; + struct dbox_map_mail_index_record rec; + unsigned int i, count; + uint32_t seq; + + memset(&rec, 0, sizeof(rec)); + msgs = array_get_modifiable(&ctx->msgs, &count); + for (i = 0; i < count; i++) { + if (msgs[i]->map_uid != 0) + continue; + + rec.file_id = msgs[i]->file_id; + rec.offset = msgs[i]->offset; + rec.size = msgs[i]->size; + + msgs[i]->map_uid = next_uid++; + mail_index_append(ctx->trans, msgs[i]->map_uid, &seq); + mail_index_update_ext(ctx->trans, seq, + ctx->storage->map->map_ext_id, + &rec, NULL); + } +} + +static int rebuild_apply_map(struct mdbox_storage_rebuild_context *ctx) +{ + struct dbox_map *map = ctx->storage->map; + const struct mail_index_header *hdr; + struct mdbox_rebuild_msg *const *msgs, **pos; + struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg; + struct dbox_mail_lookup_rec rec; + uint32_t seq; + unsigned int count; + + if (ctx->msgs_unsorted) + array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp); + + msgs = array_get_modifiable(&ctx->msgs, &count); + hdr = mail_index_get_header(ctx->sync_view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + if (dbox_map_view_lookup_rec(map, ctx->sync_view, + seq, &rec) < 0) + return -1; + + /* look up the rebuild msg record for this message */ + search_msg.file_id = rec.rec.file_id; + search_msg.offset = rec.rec.offset; + pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs), + mdbox_rebuild_msg_offset_cmp); + if (pos == NULL) { + /* map record points to non-existing message. */ + mail_index_expunge(ctx->trans, seq); + } else { + (*pos)->map_uid = rec.map_uid; + if (rec.refcount == 0) + (*pos)->seen_zero_ref_in_map = TRUE; + } + } + rebuild_add_missing_map_uids(ctx, hdr->next_uid); + + /* afterwards we're interested in looking up map_uids. + re-sort the messages to make it easier. */ + array_sort(&ctx->msgs, mdbox_rebuild_msg_uid_cmp); + return 0; +} + +static struct mdbox_rebuild_msg * +rebuild_lookup_map_uid(struct mdbox_storage_rebuild_context *ctx, + uint32_t map_uid) +{ + struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg; + struct mdbox_rebuild_msg **pos; + + search_msg.map_uid = map_uid; + pos = array_bsearch(&ctx->msgs, &search_msgp, + mdbox_rebuild_msg_uid_cmp); + return pos == NULL ? NULL : *pos; +} + +static void +rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx, + struct dbox_sync_rebuild_context *rebuild_ctx, + struct mdbox_mailbox *mbox, + struct mail_index_view *view, + struct mail_index_transaction *trans) +{ + const struct mdbox_mail_index_record *dbox_rec; + struct mdbox_mail_index_record new_dbox_rec; + const struct mail_index_header *hdr; + struct mdbox_rebuild_msg *rec; + const void *data; + bool expunged; + uint32_t seq, uid, new_seq, map_uid; + + memset(&new_dbox_rec, 0, sizeof(new_dbox_rec)); + hdr = mail_index_get_header(view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(view, seq, mbox->ext_id, + &data, &expunged); + dbox_rec = data; + map_uid = dbox_rec == NULL ? 0 : dbox_rec->map_uid; + + mail_index_lookup_ext(view, seq, mbox->guid_ext_id, + &data, &expunged); + + /* see if we can find this message based on + 1) GUID, 2) map_uid */ + rec = data == NULL ? NULL : + hash_table_lookup(ctx->guid_hash, data); + if (rec == NULL) { + if (map_uid == 0) { + /* not a multi-dbox message, ignore. */ + continue; + } + /* multi-dbox message that wasn't found with GUID. + either it's lost or GUID has been corrupted. we can + still try to look it up using map_uid. */ + rec = rebuild_lookup_map_uid(ctx, map_uid); + if (rec != NULL) { + mail_index_update_ext(trans, seq, + mbox->guid_ext_id, + rec->guid_128, NULL); + } + } else if (map_uid != rec->map_uid) { + /* map_uid is wrong, update it */ + i_assert(rec->map_uid != 0); + new_dbox_rec.map_uid = rec->map_uid; + mail_index_update_ext(trans, seq, mbox->ext_id, + &new_dbox_rec, NULL); + } else { + /* everything was ok */ + } + + if (rec != NULL) T_BEGIN { + /* keep this message */ + rec->refcount++; + + mail_index_lookup_uid(view, seq, &uid); + mail_index_append(trans, uid, &new_seq); + dbox_sync_rebuild_index_metadata(rebuild_ctx, + new_seq, uid); + + new_dbox_rec.map_uid = rec->map_uid; + mail_index_update_ext(trans, new_seq, mbox->ext_id, + &new_dbox_rec, NULL); + } T_END; + } +} + +static int +rebuild_mailbox(struct mdbox_storage_rebuild_context *ctx, + struct mail_namespace *ns, const char *name) +{ + struct mailbox *box; + struct mdbox_mailbox *mbox; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + struct dbox_sync_rebuild_context *rebuild_ctx; + enum mail_error error; + int ret; + + box = mdbox_mailbox_alloc(&ctx->storage->storage.storage, + ns->list, name, NULL, + MAILBOX_FLAG_READONLY | + MAILBOX_FLAG_KEEP_RECENT | + MAILBOX_FLAG_IGNORE_ACLS); + if (dbox_mailbox_open(box) < 0) { + (void)mail_storage_get_last_error(box->storage, &error); + mailbox_close(&box); + if (error == MAIL_ERROR_TEMP) + return -1; + /* non-temporary error, ignore */ + return 0; + } + mbox = (struct mdbox_mailbox *)box; + + ret = mail_index_sync_begin(mbox->ibox.index, &sync_ctx, &view, &trans, + MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_index_error(&mbox->ibox); + mailbox_close(&box); + return -1; + } + + rebuild_ctx = dbox_sync_index_rebuild_init(&mbox->ibox, view, trans); + rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans); + dbox_sync_index_rebuild_deinit(&rebuild_ctx); + + if (mail_index_sync_commit(&sync_ctx) < 0) { + mail_storage_set_index_error(&mbox->ibox); + ret = -1; + } + + mailbox_close(&box); + return ret < 0 ? -1 : 0; +} + +static int +rebuild_namespace_mailboxes(struct mdbox_storage_rebuild_context *ctx, + struct mail_namespace *ns) +{ + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + int ret = 0; + + if (ctx->default_list == NULL || + (ns->flags & NAMESPACE_FLAG_INBOX) != 0) + ctx->default_list = ns->list; + + iter = mailbox_list_iter_init(ns->list, "*", + MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & (MAILBOX_NONEXISTENT | + MAILBOX_NOSELECT)) == 0) { + T_BEGIN { + ret = rebuild_mailbox(ctx, ns, info->name); + } T_END; + if (ret < 0) { + ret = -1; + break; + } + } + } + if (mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int rebuild_mailboxes(struct mdbox_storage_rebuild_context *ctx) +{ + struct mail_user *user = ctx->storage->storage.storage.user; + struct mail_namespace *ns; + + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + if (ns->storage == &ctx->storage->storage.storage && + ns->alias_for == NULL) { + if (rebuild_namespace_mailboxes(ctx, ns) < 0) + return -1; + } + } + return 0; +} + +static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg) +{ + if (mail_index_sync_commit(&msg->sync_ctx) < 0) + return -1; + mailbox_close(&msg->box); + memset(msg, 0, sizeof(*msg)); + return 0; +} + +static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx, + struct mdbox_rebuild_msg *msg) +{ + struct mail_storage *storage = &ctx->storage->storage.storage; + struct dbox_file *file; + const struct mail_index_header *hdr; + struct mdbox_mail_index_record dbox_rec; + const char *mailbox = NULL; + struct mailbox *box; + struct mdbox_mailbox *mbox; + enum mail_error error; + bool deleted, created; + uoff_t size; + int ret; + uint32_t seq; + + /* first see if message contains the mailbox it was originally + saved to */ + file = mdbox_file_init(ctx->storage, msg->file_id); + ret = dbox_file_open(file, &deleted); + if (ret > 0) + ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL); + if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) { + mailbox = dbox_file_metadata_get(file, + DBOX_METADATA_ORIG_MAILBOX); + } + dbox_file_unref(&file); + if (ret <= 0 || deleted) { + if (ret < 0) + return -1; + /* we shouldn't get here, so apparently we couldn't fix + something. just ignore the mail.. */ + return 0; + } + + if (mailbox == NULL) + mailbox = "INBOX"; + + /* we have the destination mailbox. now open it and add the message + there. */ + created = FALSE; + box = ctx->prev_msg.box != NULL && + strcmp(mailbox, ctx->prev_msg.box->name) == 0 ? + ctx->prev_msg.box : NULL; + while (box == NULL) { + box = mdbox_mailbox_alloc(storage, ctx->default_list, + mailbox, NULL, + MAILBOX_FLAG_READONLY | + MAILBOX_FLAG_KEEP_RECENT | + MAILBOX_FLAG_IGNORE_ACLS); + if (dbox_mailbox_open(box) == 0) + break; + + (void)mail_storage_get_last_error(box->storage, &error); + if (error == MAIL_ERROR_NOTFOUND && !created) { + /* mailbox doesn't exist currently? see if creating + it helps. */ + created = TRUE; + (void)mailbox_create(box, NULL, FALSE); + mailbox_close(&box); + continue; + } + + mailbox_close(&box); + if (error == MAIL_ERROR_TEMP) + return -1; + + if (strcmp(mailbox, "INBOX") != 0) { + /* see if we can save to INBOX instead. */ + mailbox = "INBOX"; + } else { + /* this shouldn't happen */ + return -1; + } + } + mbox = (struct mdbox_mailbox *)box; + + /* switch the mailbox cache if necessary */ + if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) { + if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) + return -1; + } + if (ctx->prev_msg.box == NULL) { + ret = mail_index_sync_begin(mbox->ibox.index, + &ctx->prev_msg.sync_ctx, + &ctx->prev_msg.view, + &ctx->prev_msg.trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_index_error(&mbox->ibox); + mailbox_close(&box); + return -1; + } + ctx->prev_msg.box = box; + hdr = mail_index_get_header(ctx->prev_msg.view); + ctx->prev_msg.next_uid = hdr->next_uid; + } + + /* add the new message */ + memset(&dbox_rec, 0, sizeof(dbox_rec)); + dbox_rec.map_uid = msg->map_uid; + dbox_rec.save_date = ioloop_time; + mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq); + mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->ext_id, + &dbox_rec, NULL); + mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id, + msg->guid_128, NULL); + + msg->refcount++; + return 0; +} + +static int rebuild_handle_zero_refs(struct mdbox_storage_rebuild_context *ctx) +{ + struct mdbox_rebuild_msg **msgs; + unsigned int i, count; + + /* if we have messages at this point which have refcount=0, they're + either already expunged or they were somehow lost for some reason. + we'll need to figure out what to do about them. */ + msgs = array_get_modifiable(&ctx->msgs, &count); + for (i = 0; i < count; i++) { + if (msgs[i]->refcount != 0) + continue; + + if (msgs[i]->seen_zero_ref_in_map) { + /* we've seen the map record, trust it. */ + continue; + } + /* either map record was lost for this message or the message + was lost from its mailbox. safest way to handle this is to + restore the message. */ + if (rebuild_restore_msg(ctx, msgs[i]) < 0) + return -1; + } + if (ctx->prev_msg.box != NULL) { + if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) + return -1; + } + return 0; +} + +static void rebuild_update_refcounts(struct mdbox_storage_rebuild_context *ctx) +{ + const struct mail_index_header *hdr; + const void *data; + struct mdbox_rebuild_msg **msgs; + const uint16_t *ref16_p; + bool expunged; + uint32_t seq, map_uid; + unsigned int i, count; + + /* update refcounts for existing map records */ + msgs = array_get_modifiable(&ctx->msgs, &count); + hdr = mail_index_get_header(ctx->sync_view); + for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) { + mail_index_lookup_uid(ctx->sync_view, seq, &map_uid); + if (map_uid != msgs[i]->map_uid) { + /* we've already expunged this map record */ + i_assert(map_uid < msgs[i]->map_uid); + continue; + } + + mail_index_lookup_ext(ctx->sync_view, seq, + ctx->storage->map->ref_ext_id, + &data, &expunged); + ref16_p = data; + if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) { + mail_index_update_ext(ctx->trans, seq, + ctx->storage->map->ref_ext_id, + &msgs[i]->refcount, NULL); + } + i++; + } + + /* update refcounts for newly created map records */ + for (; i < count; i++, seq++) { + mail_index_update_ext(ctx->trans, seq, + ctx->storage->map->ref_ext_id, + &msgs[i]->refcount, NULL); + } +} + +static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx) +{ + if (rebuild_handle_zero_refs(ctx) < 0) + return -1; + rebuild_update_refcounts(ctx); + return 0; +} + +static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx) +{ + const struct mail_index_header *hdr; + DIR *dir; + struct dirent *d; + string_t *path; + unsigned int dir_len; + uint32_t uid_validity; + int ret = 0; + + if (dbox_map_open(ctx->storage->map, TRUE) < 0) + return -1; + + /* begin by locking the map, so that other processes can't try to + rebuild at the same time. */ + ret = mail_index_sync_begin(ctx->storage->map->index, &ctx->sync_ctx, + &ctx->sync_view, &ctx->trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_internal_error(&ctx->storage->storage.storage); + mail_index_reset_error(ctx->storage->map->index); + return -1; + } + + uid_validity = dbox_map_get_uid_validity(ctx->storage->map); + hdr = mail_index_get_header(ctx->sync_view); + if (hdr->uid_validity != uid_validity) { + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + dir = opendir(ctx->storage->storage_dir); + if (dir == NULL) { + mail_storage_set_critical(&ctx->storage->storage.storage, + "opendir(%s) failed: %m", ctx->storage->storage_dir); + return -1; + } + path = t_str_new(256); + str_append(path, ctx->storage->storage_dir); + str_append_c(path, '/'); + dir_len = str_len(path); + + for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) { + if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, + strlen(MDBOX_MAIL_FILE_PREFIX)) == 0) { + str_truncate(path, dir_len); + str_append(path, d->d_name); + T_BEGIN { + ret = rebuild_add_file(ctx, str_c(path)); + } T_END; + if (ret < 0) { + ret = -1; + break; + } + } + } + if (ret == 0 && errno != 0) { + mail_storage_set_critical(&ctx->storage->storage.storage, + "readdir(%s) failed: %m", ctx->storage->storage_dir); + ret = -1; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(&ctx->storage->storage.storage, + "closedir(%s) failed: %m", ctx->storage->storage_dir); + ret = -1; + } + + if (ret < 0 || + rebuild_apply_map(ctx) < 0 || + rebuild_mailboxes(ctx) < 0 || + rebuild_finish(ctx) < 0 || + mail_index_sync_commit(&ctx->sync_ctx) < 0) + return -1; + return 0; +} + +int mdbox_storage_rebuild(struct mdbox_storage *storage) +{ + struct mdbox_storage_rebuild_context *ctx; + struct stat st; + int ret; + + if (stat(storage->storage_dir, &st) < 0) { + if (errno == ENOENT) { + /* no multi-dbox files */ + return 0; + } + + mail_storage_set_critical(&storage->storage.storage, + "stat(%s) failed: %m", storage->storage_dir); + return -1; + } + + i_warning("dbox %s: rebuilding indexes", storage->storage_dir); + + ctx = mdbox_storage_rebuild_init(storage); + ret = mdbox_storage_rebuild_scan(ctx); + mdbox_storage_rebuild_deinit(ctx); + + if (ret == 0) + storage->storage.files_corrupted = FALSE; + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,6 @@ +#ifndef MDBOX_STORAGE_REBUILD_H +#define MDBOX_STORAGE_REBUILD_H + +int mdbox_storage_rebuild(struct mdbox_storage *storage); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,505 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "str.h" +#include "hex-binary.h" +#include "randgen.h" +#include "mkdir-parents.h" +#include "unlink-directory.h" +#include "unlink-old-files.h" +#include "index-mail.h" +#include "mail-copy.h" +#include "mail-index-modseq.h" +#include "mailbox-uidvalidity.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-storage.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <dirent.h> +#include <sys/stat.h> + +#define MDBOX_LIST_CONTEXT(obj) \ + MODULE_CONTEXT(obj, mdbox_mailbox_list_module) + +struct mdbox_mailbox_list { + union mailbox_list_module_context module_ctx; +}; + +extern struct mail_storage mdbox_storage; +extern struct mailbox mdbox_mailbox; +extern struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs; + +static MODULE_CONTEXT_DEFINE_INIT(mdbox_mailbox_list_module, + &mailbox_list_module_register); + +static struct mail_storage *mdbox_storage_alloc(void) +{ + struct mdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("dbox storage", 512+256); + storage = p_new(pool, struct mdbox_storage, 1); + storage->storage.v = mdbox_dbox_storage_vfuncs; + storage->storage.storage = mdbox_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +static int +mdbox_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, + const char **error_r) +{ + struct mdbox_storage *storage = (struct mdbox_storage *)_storage; + const char *dir; + + storage->set = mail_storage_get_driver_settings(_storage); + i_assert(storage->set->mdbox_max_open_files >= 2); + + if (*ns->list->set.mailbox_dir_name == '\0') { + *error_r = "dbox: MAILBOXDIR must not be empty"; + return -1; + } + + _storage->unique_root_dir = + p_strdup(_storage->pool, ns->list->set.root_dir); + + dir = mailbox_list_get_path(ns->list, NULL, MAILBOX_LIST_PATH_TYPE_DIR); + storage->storage_dir = p_strconcat(_storage->pool, dir, + "/"MDBOX_GLOBAL_DIR_NAME, NULL); + storage->alt_storage_dir = p_strconcat(_storage->pool, + ns->list->set.alt_dir, + "/"MDBOX_GLOBAL_DIR_NAME, NULL); + i_array_init(&storage->open_files, + I_MIN(storage->set->mdbox_max_open_files, 128)); + + storage->map = dbox_map_init(storage, ns->list, storage->storage_dir); + return 0; +} + +static void mdbox_storage_destroy(struct mail_storage *_storage) +{ + struct mdbox_storage *storage = (struct mdbox_storage *)_storage; + + if (storage->storage.files_corrupted) { + if (mdbox_storage_rebuild(storage) < 0) + return; + } + + mdbox_files_free(storage); + dbox_map_deinit(&storage->map); + array_free(&storage->open_files); + index_storage_destroy(_storage); +} + +struct mailbox * +mdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *name, struct istream *input, + enum mailbox_flags flags) +{ + struct mdbox_mailbox *mbox; + pool_t pool; + + /* dbox can't work without index files */ + flags &= ~MAILBOX_FLAG_NO_INDEX_FILES; + + pool = pool_alloconly_create("mdbox mailbox", 1024+512); + mbox = p_new(pool, struct mdbox_mailbox, 1); + mbox->ibox.box = mdbox_mailbox; + mbox->ibox.box.pool = pool; + mbox->ibox.box.storage = storage; + mbox->ibox.box.list = list; + mbox->ibox.mail_vfuncs = &mdbox_mail_vfuncs; + + mbox->ibox.save_commit_pre = mdbox_transaction_save_commit_pre; + mbox->ibox.save_commit_post = mdbox_transaction_save_commit_post; + mbox->ibox.save_rollback = mdbox_transaction_save_rollback; + + index_storage_mailbox_alloc(&mbox->ibox, name, input, flags, + DBOX_INDEX_PREFIX); + mail_index_set_fsync_types(mbox->ibox.index, + MAIL_INDEX_SYNC_TYPE_APPEND | + MAIL_INDEX_SYNC_TYPE_EXPUNGE); + + mbox->ibox.index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY; + + mbox->storage = (struct mdbox_storage *)storage; + mbox->ext_id = + mail_index_ext_register(mbox->ibox.index, "mdbox", 0, + sizeof(struct mdbox_mail_index_record), + sizeof(uint32_t)); + mbox->hdr_ext_id = + mail_index_ext_register(mbox->ibox.index, "mdbox-hdr", + sizeof(struct mdbox_index_header), 0, 0); + mbox->guid_ext_id = + mail_index_ext_register(mbox->ibox.index, "guid", + 0, MAIL_GUID_128_SIZE, 1); + return &mbox->ibox.box; +} + +int mdbox_read_header(struct mdbox_mailbox *mbox, + struct mdbox_index_header *hdr) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(mbox->ibox.view, mbox->hdr_ext_id, + &data, &data_size); + if (data_size < MDBOX_INDEX_HEADER_MIN_SIZE && + (!mbox->creating || data_size != 0)) { + mail_storage_set_critical(&mbox->storage->storage.storage, + "dbox %s: Invalid dbox header size", + mbox->ibox.box.path); + return -1; + } + memset(hdr, 0, sizeof(*hdr)); + memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr))); + return 0; +} + +void mdbox_update_header(struct mdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update) +{ + struct mdbox_index_header hdr, new_hdr; + + if (mdbox_read_header(mbox, &hdr) < 0) + memset(&hdr, 0, sizeof(hdr)); + + new_hdr = hdr; + + if (update != NULL && !mail_guid_128_is_empty(update->mailbox_guid)) { + memcpy(new_hdr.mailbox_guid, update->mailbox_guid, + sizeof(new_hdr.mailbox_guid)); + } else if (mail_guid_128_is_empty(new_hdr.mailbox_guid)) { + mail_generate_guid_128(new_hdr.mailbox_guid); + } + + new_hdr.map_uid_validity = + dbox_map_get_uid_validity(mbox->storage->map); + if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) { + mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0, + &new_hdr, sizeof(new_hdr)); + } +} + +static int mdbox_write_index_header(struct mailbox *box, + const struct mailbox_update *update) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + struct mail_index_transaction *trans; + const struct mail_index_header *hdr; + uint32_t uid_validity, uid_next; + + if (dbox_map_open(mbox->storage->map, TRUE) < 0) + return -1; + + hdr = mail_index_get_header(mbox->ibox.view); + trans = mail_index_transaction_begin(mbox->ibox.view, 0); + mdbox_update_header(mbox, trans, update); + + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + else if (hdr->uid_validity == 0) { + /* set uidvalidity */ + uid_validity = dbox_get_uidvalidity_next(box->list); + } + + if (hdr->uid_validity != uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + if (update != NULL && hdr->next_uid < update->min_next_uid) { + uid_next = update->min_next_uid; + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + } + if (update != NULL && update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(mbox->ibox.view) < + update->min_highest_modseq) { + mail_index_update_highest_modseq(trans, + update->min_highest_modseq); + } + + if (mail_index_transaction_commit(&trans) < 0) { + mail_storage_set_internal_error(box->storage); + mail_index_reset_error(mbox->ibox.index); + return -1; + } + return 0; +} + +static int mdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + const char *origin; + mode_t mode; + gid_t gid; + int ret; + + mailbox_list_get_dir_permissions(box->list, NULL, &mode, &gid, &origin); + if (mkdir_parents_chgrp(box->path, mode, gid, origin) == 0) { + /* create indexes immediately with the dbox header */ + if (index_storage_mailbox_open(box) < 0) + return -1; + mbox->creating = TRUE; + ret = mdbox_write_index_header(box, update); + mbox->creating = FALSE; + if (ret < 0) + return -1; + } else if (errno != EEXIST) { + if (!mail_storage_set_error_from_errno(box->storage)) { + mail_storage_set_critical(box->storage, + "mkdir(%s) failed: %m", box->path); + } + return -1; + } + return 0; +} + +static void mdbox_storage_get_status_guid(struct mailbox *box, + struct mailbox_status *status_r) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + struct mdbox_index_header hdr; + + if (mdbox_read_header(mbox, &hdr) < 0) + memset(&hdr, 0, sizeof(hdr)); + + if (mail_guid_128_is_empty(hdr.mailbox_guid)) { + /* regenerate it */ + if (mdbox_write_index_header(box, NULL) < 0 || + mdbox_read_header(mbox, &hdr) < 0) + return; + } + memcpy(status_r->mailbox_guid, hdr.mailbox_guid, + sizeof(status_r->mailbox_guid)); +} + +static void +mdbox_storage_get_status(struct mailbox *box, enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + index_storage_get_status(box, items, status_r); + + if ((items & STATUS_GUID) != 0) + mdbox_storage_get_status_guid(box, status_r); +} + +static int +mdbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + if (!box->opened) { + if (index_storage_mailbox_open(box) < 0) + return -1; + } + return mdbox_write_index_header(box, update); +} + +static int +mdbox_mailbox_unref_mails(struct mailbox_list *list, const char *path) +{ + struct mdbox_storage *storage = + (struct mdbox_storage *)list->ns->storage; + const struct mail_storage_settings *old_set; + struct mail_storage_settings tmp_set; + struct mailbox *box; + struct mdbox_mailbox *mbox; + const struct mail_index_header *hdr; + const struct mdbox_mail_index_record *dbox_rec; + struct dbox_map_transaction_context *map_trans; + ARRAY_TYPE(uint32_t) map_uids; + const void *data; + bool expunged; + uint32_t seq; + int ret; + + old_set = list->mail_set; + tmp_set = *list->mail_set; + tmp_set.mail_full_filesystem_access = TRUE; + list->mail_set = &tmp_set; + box = mdbox_mailbox_alloc(&storage->storage.storage, list, path, NULL, + MAILBOX_FLAG_IGNORE_ACLS | + MAILBOX_FLAG_KEEP_RECENT); + ret = mailbox_open(box); + list->mail_set = old_set; + if (ret < 0) { + mailbox_close(&box); + return -1; + } + mbox = (struct mdbox_mailbox *)box; + + /* get a list of all map_uids in this mailbox */ + i_array_init(&map_uids, 128); + hdr = mail_index_get_header(mbox->ibox.view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(mbox->ibox.view, seq, mbox->ext_id, + &data, &expunged); + dbox_rec = data; + if (dbox_rec == NULL) { + /* no multi-mails */ + break; + } + if (dbox_rec->map_uid != 0) + array_append(&map_uids, &dbox_rec->map_uid, 1); + } + + /* unreference the map_uids */ + map_trans = dbox_map_transaction_begin(storage->map, FALSE); + ret = dbox_map_update_refcounts(map_trans, &map_uids, -1); + if (ret == 0) + ret = dbox_map_transaction_commit(map_trans); + dbox_map_transaction_free(&map_trans); + array_free(&map_uids); + mailbox_close(&box); + return ret; +} + +static int +mdbox_list_delete_mailbox(struct mailbox_list *list, const char *name) +{ + struct mdbox_mailbox_list *mlist = MDBOX_LIST_CONTEXT(list); + const char *trash_dest; + int ret; + + /* Make sure the indexes are closed before trying to delete the + directory that contains them. It can still fail with some NFS + implementations if indexes are opened by another session, but + that can't really be helped. */ + index_storage_destroy_unrefed(); + + /* delete the index and control directories */ + if (mlist->module_ctx.super.delete_mailbox(list, name) < 0) + return -1; + + if ((ret = dbox_list_delete_mailbox1(list, name, &trash_dest)) < 0) + return -1; + if (ret > 0) { + if (mdbox_mailbox_unref_mails(list, trash_dest) < 0) { + /* we've already renamed it. there's no going back. */ + mailbox_list_set_internal_error(list); + ret = -1; + } + } + return dbox_list_delete_mailbox2(list, name, ret, trash_dest); +} + +static int +mdbox_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname, + struct mailbox_list *newlist, const char *newname, + bool rename_children) +{ + struct mdbox_mailbox_list *oldmlist = MDBOX_LIST_CONTEXT(oldlist); + + if (oldmlist->module_ctx.super. + rename_mailbox(oldlist, oldname, newlist, newname, + rename_children) < 0) + return -1; + return dbox_list_rename_mailbox(oldlist, oldname, newlist, newname, + rename_children); +} + +static void dbox_storage_add_list(struct mail_storage *storage ATTR_UNUSED, + struct mailbox_list *list) +{ + struct mdbox_mailbox_list *mlist; + + mlist = p_new(list->pool, struct mdbox_mailbox_list, 1); + mlist->module_ctx.super = list->v; + + list->v.iter_is_mailbox = dbox_list_iter_is_mailbox; + list->v.delete_mailbox = mdbox_list_delete_mailbox; + list->v.rename_mailbox = mdbox_list_rename_mailbox; + list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre; + + MODULE_CONTEXT_SET(list, mdbox_mailbox_list_module, mlist); +} + +struct mail_storage mdbox_storage = { + MEMBER(name) MDBOX_STORAGE_NAME, + MEMBER(class_flags) MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT, + + { + mdbox_get_setting_parser_info, + mdbox_storage_alloc, + mdbox_storage_create, + mdbox_storage_destroy, + dbox_storage_add_list, + dbox_storage_get_list_settings, + NULL, + mdbox_mailbox_alloc, + mdbox_sync_purge + } +}; + +struct mailbox mdbox_mailbox = { + MEMBER(name) NULL, + MEMBER(storage) NULL, + MEMBER(list) NULL, + + { + index_storage_is_readonly, + index_storage_allow_new_keywords, + index_storage_mailbox_enable, + dbox_mailbox_open, + index_storage_mailbox_close, + dbox_mailbox_create, + mdbox_mailbox_update, + mdbox_storage_get_status, + NULL, + NULL, + mdbox_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + index_transaction_set_max_modseq, + index_keywords_create, + index_keywords_create_from_indexes, + index_keywords_ref, + index_keywords_unref, + index_keyword_is_valid, + index_storage_get_seq_range, + index_storage_get_uid_range, + index_storage_get_expunges, + NULL, + NULL, + NULL, + dbox_mail_alloc, + index_header_lookup_init, + index_header_lookup_deinit, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + mdbox_save_alloc, + mdbox_save_begin, + dbox_save_continue, + mdbox_save_finish, + mdbox_save_cancel, + mdbox_copy, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs = { + mdbox_file_unrefed, + mdbox_file_create_fd, + mdbox_mail_open, + mdbox_mailbox_create_indexes +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,86 @@ +#ifndef MDBOX_STORAGE_H +#define MDBOX_STORAGE_H + +#include "index-storage.h" +#include "mailbox-list-private.h" +#include "dbox-storage.h" +#include "mdbox-settings.h" + +#define MDBOX_STORAGE_NAME "mdbox" +#define MDBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index" +#define MDBOX_GLOBAL_DIR_NAME "storage" +#define MDBOX_MAIL_FILE_PREFIX "m." +#define MDBOX_MAIL_FILE_FORMAT MDBOX_MAIL_FILE_PREFIX"%u" + +#define MDBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t)) +struct mdbox_index_header { + uint32_t map_uid_validity; + uint8_t mailbox_guid[MAIL_GUID_128_SIZE]; +}; + +struct mdbox_storage { + struct dbox_storage storage; + union mailbox_list_module_context list_module_ctx; + const struct mdbox_settings *set; + + /* paths for storage directories */ + const char *storage_dir, *alt_storage_dir; + struct dbox_map *map; + + ARRAY_DEFINE(open_files, struct mdbox_file *); +}; + +struct mdbox_mail_index_record { + uint32_t map_uid; + /* UNIX timestamp of when the message was saved/copied to this + mailbox */ + uint32_t save_date; +}; + +struct mdbox_mailbox { + struct index_mailbox ibox; + struct mdbox_storage *storage; + + uint32_t map_uid_validity; + uint32_t ext_id, hdr_ext_id, guid_ext_id; + + unsigned int creating:1; +}; + +extern struct mail_vfuncs mdbox_mail_vfuncs; + +struct mailbox * +mdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *name, struct istream *input, + enum mailbox_flags flags); + +int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r); + +/* Get map_uid for wanted message. */ +int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, uint32_t *map_uid_r); +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); +int mdbox_read_header(struct mdbox_mailbox *mbox, + struct mdbox_index_header *hdr); +void mdbox_update_header(struct mdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update); + +struct mail_save_context * +mdbox_save_alloc(struct mailbox_transaction_context *_t); +int mdbox_save_begin(struct mail_save_context *ctx, struct istream *input); +int mdbox_save_finish(struct mail_save_context *ctx); +void mdbox_save_cancel(struct mail_save_context *ctx); + +struct dbox_file * +mdbox_save_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r); + +int mdbox_transaction_save_commit_pre(struct mail_save_context *ctx); +void mdbox_transaction_save_commit_post(struct mail_save_context *ctx); +void mdbox_transaction_save_rollback(struct mail_save_context *ctx); + +int mdbox_copy(struct mail_save_context *ctx, struct mail *mail); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,349 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "hex-binary.h" +#include "str.h" +#include "mdbox-storage.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" + +#define DBOX_REBUILD_COUNT 3 + +static int +dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq, + const uint8_t guid_128[MAIL_GUID_128_SIZE]) +{ + const void *data; + uint32_t uid; + + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mail_index_lookup_ext(ctx->sync_view, seq, + ctx->mbox->guid_ext_id, &data, NULL); + if (mail_guid_128_is_empty(guid_128) || + memcmp(data, guid_128, MAIL_GUID_128_SIZE) == 0) + return 0; + + mail_storage_set_critical(&ctx->mbox->storage->storage.storage, + "Mailbox %s: Expunged GUID mismatch for UID %u: %s vs %s", + ctx->mbox->ibox.box.vname, uid, + binary_to_hex(data, MAIL_GUID_128_SIZE), + binary_to_hex(guid_128, MAIL_GUID_128_SIZE)); + ctx->mbox->storage->storage.files_corrupted = TRUE; + return -1; +} + +static int mdbox_sync_expunge(struct mdbox_sync_context *ctx, uint32_t seq, + const uint8_t guid_128[MAIL_GUID_128_SIZE]) +{ + uint32_t map_uid; + + if (seq_range_exists(&ctx->expunged_seqs, seq)) { + /* already marked as expunged in this sync */ + return 0; + } + + if (dbox_sync_verify_expunge_guid(ctx, seq, guid_128) < 0) + return -1; + + if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0) + return -1; + + seq_range_array_add(&ctx->expunged_seqs, 0, seq); + array_append(&ctx->expunged_map_uids, &map_uid, 1); + return 0; +} + +static int mdbox_sync_add(struct mdbox_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec) +{ + uint32_t seq, seq1, seq2; + + if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + /* not interested */ + return 0; + } + + if (!mail_index_lookup_seq_range(ctx->sync_view, + sync_rec->uid1, sync_rec->uid2, + &seq1, &seq2)) { + /* already expunged everything. nothing to do. */ + return 0; + } + + for (seq = seq1; seq <= seq2; seq++) { + if (mdbox_sync_expunge(ctx, seq, sync_rec->guid_128) < 0) + return -1; + } + return 0; +} + +static void dbox_sync_mark_expunges(struct mdbox_sync_context *ctx) +{ + struct mailbox *box = &ctx->mbox->ibox.box; + struct seq_range_iter iter; + unsigned int n; + const void *data; + uint32_t seq, uid; + + seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &seq)) { + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mail_index_lookup_ext(ctx->sync_view, seq, + ctx->mbox->guid_ext_id, &data, NULL); + mail_index_expunge_guid(ctx->trans, seq, data); + + if (box->v.sync_notify != NULL) + box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE); + } +} + +static int mdbox_sync_index_finish_expunges(struct mdbox_sync_context *ctx) +{ + struct dbox_map_transaction_context *map_trans; + int ret; + + /* prevent a user from saving + expunging messages all the time and + using lots of disk space. but avoid doing this in situations where + a user simply expunges a lot of mail for the first time. that's why + we do this calculation before sync, not after: the purging is + triggered only after the second expunge. */ + if ((ctx->flags & MDBOX_SYNC_FLAG_NO_PURGE) == 0 && + dbox_map_want_purge(ctx->mbox->storage->map)) + ctx->purge = TRUE; + + map_trans = dbox_map_transaction_begin(ctx->mbox->storage->map, FALSE); + ret = dbox_map_update_refcounts(map_trans, &ctx->expunged_map_uids, -1); + if (ret == 0) { + ret = dbox_map_transaction_commit(map_trans); + if (ret == 0) + dbox_sync_mark_expunges(ctx); + } + + dbox_map_transaction_free(&map_trans); + return ret; +} + +static int mdbox_sync_index(struct mdbox_sync_context *ctx) +{ + struct mailbox *box = &ctx->mbox->ibox.box; + const struct mail_index_header *hdr; + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + int ret = 0; + + hdr = mail_index_get_header(ctx->sync_view); + if (hdr->uid_validity == 0) { + /* newly created index file */ + return 0; + } + + /* mark the newly seen messages as recent */ + if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, + hdr->next_uid, &seq1, &seq2)) { + index_mailbox_set_recent_seq(&ctx->mbox->ibox, ctx->sync_view, + seq1, seq2); + } + + /* read all changes and group changes to same file_id together */ + i_array_init(&ctx->expunged_seqs, 64); + i_array_init(&ctx->expunged_map_uids, 64); + while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) { + if ((ret = mdbox_sync_add(ctx, &sync_rec)) < 0) + break; + } + if (ret == 0 && array_count(&ctx->expunged_seqs) > 0) + ret = mdbox_sync_index_finish_expunges(ctx); + + if (box->v.sync_notify != NULL) + box->v.sync_notify(box, 0, 0); + + array_free(&ctx->expunged_seqs); + array_free(&ctx->expunged_map_uids); + return ret == 0 ? 1 : + (ctx->mbox->storage->storage.files_corrupted ? 0 : -1); +} + +static int mdbox_refresh_header(struct mdbox_mailbox *mbox, bool retry) +{ + struct mail_index_view *view; + struct mdbox_index_header hdr; + int ret; + + view = mail_index_view_open(mbox->ibox.index); + ret = mdbox_read_header(mbox, &hdr); + mail_index_view_close(&view); + + if (ret == 0) { + ret = mbox->storage->storage.files_corrupted ? -1 : 0; + } else if (retry) { + (void)mail_index_refresh(mbox->ibox.index); + return mdbox_refresh_header(mbox, FALSE); + } + return ret; +} + +int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, + struct mdbox_sync_context **ctx_r) +{ + struct mail_storage *storage = mbox->ibox.box.storage; + struct mdbox_sync_context *ctx; + enum mail_index_sync_flags sync_flags = 0; + unsigned int i; + int ret; + bool rebuild, storage_rebuilt = FALSE; + + rebuild = mdbox_refresh_header(mbox, TRUE) < 0 || + (flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0; + if (rebuild) { + if (mdbox_storage_rebuild(mbox->storage) < 0) + return -1; + index_mailbox_reset_uidvalidity(&mbox->ibox); + storage_rebuilt = TRUE; + } + + ctx = i_new(struct mdbox_sync_context, 1); + ctx->mbox = mbox; + ctx->flags = flags; + + if ((mbox->ibox.box.flags & MAILBOX_FLAG_KEEP_RECENT) == 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; + if (!rebuild && (flags & MDBOX_SYNC_FLAG_FORCE) == 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + if ((flags & MDBOX_SYNC_FLAG_FSYNC) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC; + /* don't write unnecessary dirty flag updates */ + sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; + + for (i = 0;; i++) { + ret = mail_index_sync_begin(mbox->ibox.index, + &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, + sync_flags); + if (ret <= 0) { + if (ret < 0) + mail_storage_set_index_error(&mbox->ibox); + i_free(ctx); + *ctx_r = NULL; + return ret; + } + + /* now that we're locked, check again if we want to rebuild */ + if (mdbox_refresh_header(mbox, FALSE) < 0) + ret = 0; + else { + if ((ret = mdbox_sync_index(ctx)) > 0) + break; + } + + /* failure. keep the index locked while we're doing a + rebuild. */ + if (ret == 0) { + if (!storage_rebuilt) { + /* we'll need to rebuild storage too. + try again from the beginning. */ + mbox->storage->storage.files_corrupted = TRUE; + mail_index_sync_rollback(&ctx->index_sync_ctx); + i_free(ctx); + return mdbox_sync_begin(mbox, flags, ctx_r); + } + mail_storage_set_critical(storage, + "dbox %s: Storage keeps breaking", + ctx->mbox->ibox.box.path); + ret = -1; + } + mail_index_sync_rollback(&ctx->index_sync_ctx); + if (ret < 0) { + i_free(ctx); + return -1; + } + } + + *ctx_r = ctx; + return 0; +} + +int mdbox_sync_finish(struct mdbox_sync_context **_ctx, bool success) +{ + struct mdbox_sync_context *ctx = *_ctx; + int ret = success ? 0 : -1; + + *_ctx = NULL; + + if (success) { + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { + mail_storage_set_index_error(&ctx->mbox->ibox); + ret = -1; + } + } else { + mail_index_sync_rollback(&ctx->index_sync_ctx); + } + + if (ctx->purge) + (void)mdbox_sync_purge(&ctx->mbox->storage->storage.storage); + i_free(ctx); + return ret; +} + +int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags) +{ + struct mdbox_sync_context *sync_ctx; + + if (mdbox_sync_begin(mbox, flags, &sync_ctx) < 0) + return -1; + + if (sync_ctx == NULL) + return 0; + return mdbox_sync_finish(&sync_ctx, TRUE); +} + +struct mailbox_sync_context * +mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + enum mdbox_sync_flags mdbox_sync_flags = 0; + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + ret = -1; + } + + if (ret == 0 && (index_mailbox_want_full_sync(&mbox->ibox, flags) || + mbox->storage->storage.files_corrupted)) { + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) + mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD; + ret = mdbox_sync(mbox, mdbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +} + +int mdbox_sync_purge(struct mail_storage *_storage) +{ + struct mdbox_storage *storage = (struct mdbox_storage *)_storage; + const ARRAY_TYPE(seq_range) *ref0_file_ids; + struct dbox_file *file; + struct seq_range_iter iter; + unsigned int i = 0; + uint32_t file_id; + bool deleted; + int ret = 0; + + ref0_file_ids = dbox_map_get_zero_ref_files(storage->map); + seq_range_array_iter_init(&iter, ref0_file_ids); i = 0; + while (seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN { + file = mdbox_file_init(storage, file_id); + if (dbox_file_open(file, &deleted) > 0 && !deleted) { + if (mdbox_file_purge(file) < 0) + ret = -1; + } else { + dbox_map_remove_file_id(storage->map, file_id); + } + dbox_file_unref(&file); + } T_END; + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,39 @@ +#ifndef MDBOX_SYNC_H +#define MDBOX_SYNC_H + +struct mailbox; +struct mdbox_mailbox; + +enum mdbox_sync_flags { + MDBOX_SYNC_FLAG_FORCE = 0x01, + MDBOX_SYNC_FLAG_FSYNC = 0x02, + MDBOX_SYNC_FLAG_FORCE_REBUILD = 0x04, + MDBOX_SYNC_FLAG_NO_PURGE = 0x08 +}; + +struct mdbox_sync_context { + struct mdbox_mailbox *mbox; + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + enum mdbox_sync_flags flags; + + ARRAY_TYPE(seq_range) expunged_seqs; + /* list of expunged map_uids. the same map_uid may be listed more than + once in case message has been copied multiple times to mailbox. */ + ARRAY_TYPE(uint32_t) expunged_map_uids; + + unsigned int purge:1; +}; + +int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, + struct mdbox_sync_context **ctx_r); +int mdbox_sync_finish(struct mdbox_sync_context **ctx, bool success); +int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags); + +int mdbox_sync_purge(struct mail_storage *storage); + +struct mailbox_sync_context * +mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/Makefile.am Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,32 @@ +noinst_LTLIBRARIES = libstorage_dbox_single.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_single_la_SOURCES = \ + sdbox-file.c \ + sdbox-mail.c \ + sdbox-save.c \ + sdbox-sync.c \ + sdbox-sync-file.c \ + sdbox-sync-rebuild.c \ + sdbox-storage.c + +headers = \ + sdbox-file.h \ + sdbox-storage.h \ + sdbox-sync.h + +if INSTALL_HEADERS + pkginc_libdir=$(pkgincludedir) + pkginc_lib_HEADERS = $(headers) +else + noinst_HEADERS = $(headers) +endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-file.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,120 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "eacces-error.h" +#include "mkdir-parents.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" + +#include <stdio.h> + +static void sdbox_file_init_paths(struct sdbox_file *file, const char *fname) +{ + i_free(file->file.primary_path); + i_free(file->file.alt_path); + file->file.primary_path = + i_strdup_printf("%s/%s", file->mbox->ibox.box.path, fname); + if (file->mbox->alt_path != NULL) { + file->file.alt_path = + i_strdup_printf("%s/%s", file->mbox->alt_path, fname); + } +} + +struct dbox_file *sdbox_file_init(struct sdbox_mailbox *mbox, uint32_t uid) +{ + struct sdbox_file *file; + const char *fname; + + file = i_new(struct sdbox_file, 1); + file->file.storage = &mbox->storage->storage; + file->mbox = mbox; + T_BEGIN { + if (uid != 0) { + fname = t_strdup_printf(SDBOX_MAIL_FILE_FORMAT, uid); + sdbox_file_init_paths(file, fname); + file->uid = uid; + } else { + file->file.primary_path = + i_strdup_printf("%s/%s", + file->mbox->ibox.box.path, + dbox_generate_tmp_filename()); + } + } T_END; + dbox_file_init(&file->file); + + if (uid == 0) { + file->file.fd = file->file.storage->v. + file_create_fd(&file->file, file->file.primary_path, + FALSE); + } + return &file->file; +} + +int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid) +{ + const char *old_path, *new_fname, *new_path; + + i_assert(file->uid == 0); + i_assert(uid != 0); + + old_path = file->file.cur_path; + new_fname = t_strdup_printf(SDBOX_MAIL_FILE_FORMAT, uid); + new_path = t_strdup_printf("%s/%s", file->mbox->ibox.box.path, + new_fname); + if (rename(old_path, new_path) < 0) { + mail_storage_set_critical(&file->file.storage->storage, + "rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + sdbox_file_init_paths(file, new_fname); + file->uid = uid; + return 0; +} + +int sdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents) +{ + struct sdbox_file *sfile = (struct sdbox_file *)file; + struct mailbox *box = &sfile->mbox->ibox.box; + const char *p, *dir; + mode_t old_mask; + int fd; + + old_mask = umask(0666 & ~box->file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + if (fd == -1 && errno == ENOENT && parents && + (p = strrchr(path, '/')) != NULL) { + dir = t_strdup_until(path, p); + if (mkdir_parents_chgrp(dir, box->dir_create_mode, + box->file_create_gid, + box->file_create_gid_origin) < 0) { + mail_storage_set_critical(box->storage, + "mkdir_parents(%s) failed: %m", dir); + return -1; + } + /* try again */ + old_mask = umask(0666 & ~box->file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + } + if (fd == -1) { + mail_storage_set_critical(box->storage, + "open(%s, O_CREAT) failed: %m", path); + } else if (box->file_create_gid == (gid_t)-1) { + /* no group change */ + } else if (fchown(fd, (uid_t)-1, box->file_create_gid) < 0) { + if (errno == EPERM) { + mail_storage_set_critical(box->storage, "%s", + eperm_error_get_chgrp("fchown", path, + box->file_create_gid, + box->file_create_gid_origin)); + } else { + mail_storage_set_critical(box->storage, + "fchown(%s, -1, %ld) failed: %m", + path, (long)box->file_create_gid); + } + /* continue anyway */ + } + return fd; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-file.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,22 @@ +#ifndef SDBOX_FILE_H +#define SDBOX_FILE_H + +#include "dbox-file.h" + +struct sdbox_file { + struct dbox_file file; + struct sdbox_mailbox *mbox; + + /* 0 while file is being created */ + uint32_t uid; +}; + +struct dbox_file *sdbox_file_init(struct sdbox_mailbox *mbox, uint32_t uid); + +/* Assign UID for a newly created file (by renaming it) */ +int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid); + +int sdbox_file_create_fd(struct dbox_file *file, const char *path, + bool parents); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-mail.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,109 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "str.h" +#include "index-mail.h" +#include "dbox-mail.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" + +#include <stdlib.h> +#include <sys/stat.h> + +static void sdbox_mail_set_expunged(struct dbox_mail *mail) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)mail->imail.ibox; + struct mail *_mail = &mail->imail.mail.mail; + + (void)mail_index_refresh(mbox->ibox.index); + if (mail_index_is_expunged(mbox->ibox.view, _mail->seq)) { + mail_set_expunged(_mail); + return; + } + + mail_storage_set_critical(_mail->box->storage, + "Unexpectedly lost uid=%u", _mail->uid); + mbox->sync_rebuild = TRUE; +} + +static bool sdbox_mail_file_set(struct dbox_mail *mail) +{ + struct mail *_mail = &mail->imail.mail.mail; + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)_mail->box; + + if (mail->open_file != NULL) { + /* already set */ + return FALSE; + } else if (_mail->uid != 0) { + mail->open_file = sdbox_file_init(mbox, _mail->uid); + return FALSE; + } else { + /* mail is being saved in this transaction */ + mail->open_file = + sdbox_save_file_get_file(_mail->transaction, + _mail->seq); + mail->open_file->refcount++; + return TRUE; + } +} + +int sdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r) +{ + struct mail *_mail = &mail->imail.mail.mail; + bool deleted; + + if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) { + mail_set_aborted(_mail); + return -1; + } + + if (!sdbox_mail_file_set(mail)) { + if (!dbox_file_is_open(mail->open_file)) + mail->imail.mail.stats_open_lookup_count++; + if (dbox_file_open(mail->open_file, &deleted) <= 0) + return -1; + if (deleted) { + sdbox_mail_set_expunged(mail); + return -1; + } + } + + *file_r = mail->open_file; + *offset_r = 0; + return 0; +} + +struct mail_vfuncs sdbox_mail_vfuncs = { + dbox_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_parts, + index_mail_get_date, + dbox_mail_get_received_date, + dbox_mail_get_save_date, + dbox_mail_get_virtual_size, + dbox_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + dbox_mail_get_stream, + dbox_mail_get_special, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_uid, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_get_index_mail +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,296 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "fdatasync-path.h" +#include "hex-binary.h" +#include "hex-dec.h" +#include "str.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "write-full.h" +#include "index-mail.h" +#include "mail-copy.h" +#include "dbox-save.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" + +#include <stdlib.h> + +struct sdbox_save_context { + struct dbox_save_context ctx; + + struct sdbox_mailbox *mbox; + struct sdbox_sync_context *sync_ctx; + struct dbox_file_append_context *append_ctx; + + uint32_t first_saved_seq; + ARRAY_DEFINE(files, struct dbox_file *); +}; + +struct dbox_file * +sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq) +{ + struct sdbox_save_context *ctx = + (struct sdbox_save_context *)t->save_ctx; + struct dbox_file *const *files; + unsigned int count; + + i_assert(seq >= ctx->first_saved_seq); + + files = array_get(&ctx->files, &count); + i_assert(count > 0); + + return files[ctx->first_saved_seq - seq]; +} + +struct mail_save_context * +sdbox_save_alloc(struct mailbox_transaction_context *t) +{ + struct index_transaction_context *it = + (struct index_transaction_context *)t; + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)t->box; + struct sdbox_save_context *ctx = + (struct sdbox_save_context *)t->save_ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (ctx != NULL) { + /* use the existing allocated structure */ + ctx->ctx.finished = FALSE; + return &ctx->ctx.ctx; + } + + ctx = i_new(struct sdbox_save_context, 1); + ctx->ctx.ctx.transaction = t; + ctx->ctx.trans = it->trans; + ctx->mbox = mbox; + i_array_init(&ctx->files, 32); + t->save_ctx = &ctx->ctx.ctx; + return t->save_ctx; +} + +int sdbox_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; + struct dbox_file *file; + int ret; + + file = sdbox_file_init(ctx->mbox, 0); + ctx->append_ctx = dbox_file_append_init(file); + ret = dbox_file_get_append_stream(ctx->append_ctx, + &ctx->ctx.cur_output); + if (ret <= 0) { + i_assert(ret != 0); + dbox_file_append_rollback(&ctx->append_ctx); + dbox_file_unref(&file); + ctx->ctx.failed = TRUE; + return -1; + } + ctx->ctx.cur_file = file; + dbox_save_begin(&ctx->ctx, input); + + if (ctx->first_saved_seq == 0) + ctx->first_saved_seq = ctx->ctx.seq; + + array_append(&ctx->files, &file, 1); + return ctx->ctx.failed ? -1 : 0; +} + +static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx, + struct dbox_file *file) +{ + struct dbox_message_header dbox_msg_hdr; + uoff_t message_size; + uint8_t guid_128[MAIL_GUID_128_SIZE]; + + i_assert(file->msg_header_size == sizeof(dbox_msg_hdr)); + + message_size = ctx->cur_output->offset - + file->msg_header_size - file->file_header_size; + + dbox_save_write_metadata(&ctx->ctx, ctx->cur_output, NULL, guid_128); + dbox_msg_header_fill(&dbox_msg_hdr, message_size); + if (o_stream_pwrite(ctx->cur_output, &dbox_msg_hdr, + sizeof(dbox_msg_hdr), + file->file_header_size) < 0) { + dbox_file_set_syscall_error(file, "pwrite()"); + return -1; + } + return 0; +} + +static int dbox_save_finish_write(struct mail_save_context *_ctx) +{ + struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; + struct dbox_file *const *files; + + ctx->ctx.finished = TRUE; + if (ctx->ctx.cur_output == NULL) + return -1; + + index_mail_cache_parse_deinit(_ctx->dest_mail, + _ctx->received_date, !ctx->ctx.failed); + + files = array_idx_modifiable(&ctx->files, array_count(&ctx->files) - 1); + + if (!ctx->ctx.failed) T_BEGIN { + if (dbox_save_mail_write_metadata(&ctx->ctx, *files) < 0) + ctx->ctx.failed = TRUE; + } T_END; + + if (ctx->ctx.failed) + dbox_file_append_rollback(&ctx->append_ctx); + else if (dbox_file_append_commit(&ctx->append_ctx) < 0) + ctx->ctx.failed = TRUE; + + i_stream_unref(&ctx->ctx.input); + dbox_file_close(*files); + ctx->ctx.cur_output = NULL; + + return ctx->ctx.failed ? -1 : 0; +} + +int sdbox_save_finish(struct mail_save_context *ctx) +{ + int ret; + + ret = dbox_save_finish_write(ctx); + index_save_context_free(ctx); + return ret; +} + +void sdbox_save_cancel(struct mail_save_context *_ctx) +{ + struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; + + ctx->failed = TRUE; + (void)sdbox_save_finish(_ctx); +} + +static int dbox_save_assign_uids(struct sdbox_save_context *ctx, + const ARRAY_TYPE(seq_range) *uids) +{ + struct dbox_file *const *files; + struct seq_range_iter iter; + unsigned int i, count, n = 0; + uint32_t uid; + bool ret; + + seq_range_array_iter_init(&iter, uids); + files = array_get(&ctx->files, &count); + for (i = 0; i < count; i++) { + struct sdbox_file *sfile = (struct sdbox_file *)files[i]; + + ret = seq_range_array_iter_nth(&iter, n++, &uid); + i_assert(ret); + if (sdbox_file_assign_uid(sfile, uid) < 0) + return -1; + } + i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); + return 0; +} + +static void dbox_save_unref_files(struct sdbox_save_context *ctx) +{ + struct mail_storage *storage = &ctx->mbox->storage->storage.storage; + struct dbox_file **files; + unsigned int i, count; + + files = array_get_modifiable(&ctx->files, &count); + for (i = 0; i < count; i++) { + if (ctx->ctx.failed) { + if (unlink(files[i]->cur_path) < 0) { + mail_storage_set_critical(storage, + "unlink(%s) failed: %m", + files[i]->cur_path); + } + } + dbox_file_unref(&files[i]); + } + array_free(&ctx->files); +} + +int sdbox_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; + struct mailbox_transaction_context *_t = _ctx->transaction; + const struct mail_index_header *hdr; + + i_assert(ctx->ctx.finished); + + if (array_count(&ctx->files) == 0) + return 0; + + if (sdbox_sync_begin(ctx->mbox, SDBOX_SYNC_FLAG_FORCE | + SDBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) { + sdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* assign UIDs for new messages */ + hdr = mail_index_get_header(ctx->sync_ctx->sync_view); + mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, + &_t->changes->saved_uids); + if (dbox_save_assign_uids(ctx, &_t->changes->saved_uids) < 0) { + sdbox_transaction_save_rollback(_ctx); + return -1; + } + + if (ctx->ctx.mail != NULL) + mail_free(&ctx->ctx.mail); + + _t->changes->uid_validity = hdr->uid_validity; + return 0; +} + +void sdbox_transaction_save_commit_post(struct mail_save_context *_ctx) +{ + struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; + + _ctx->transaction = NULL; /* transaction is already freed */ + + if (array_count(&ctx->files) == 0) { + sdbox_transaction_save_rollback(_ctx); + return; + } + + if (sdbox_sync_finish(&ctx->sync_ctx, TRUE) < 0) + ctx->ctx.failed = TRUE; + + if (!ctx->mbox->storage->storage.storage.set->fsync_disable) { + if (fdatasync_path(ctx->mbox->ibox.box.path) < 0) { + i_error("fdatasync_path(%s) failed: %m", + ctx->mbox->ibox.box.path); + } + } + sdbox_transaction_save_rollback(_ctx); +} + +void sdbox_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; + + if (!ctx->ctx.finished) + sdbox_save_cancel(_ctx); + dbox_save_unref_files(ctx); + + if (ctx->sync_ctx != NULL) + (void)sdbox_sync_finish(&ctx->sync_ctx, FALSE); + + if (ctx->ctx.mail != NULL) + mail_free(&ctx->ctx.mail); + i_free(ctx); +} + +int sdbox_copy(struct mail_save_context *_ctx, struct mail *mail) +{ + struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; + + /* FIXME: use hard linking */ + + ctx->finished = TRUE; + return mail_storage_copy(_ctx, mail); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,378 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "hex-binary.h" +#include "randgen.h" +#include "mkdir-parents.h" +#include "unlink-directory.h" +#include "unlink-old-files.h" +#include "index-mail.h" +#include "mail-index-modseq.h" +#include "mailbox-uidvalidity.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" +#include "sdbox-storage.h" + +#include <stdio.h> +#include <unistd.h> +#include <sys/stat.h> + +#define SDBOX_LIST_CONTEXT(obj) \ + MODULE_CONTEXT(obj, sdbox_mailbox_list_module) + +struct sdbox_mailbox_list { + union mailbox_list_module_context module_ctx; +}; + +extern struct mail_storage dbox_storage; +extern struct mailbox sdbox_mailbox; +extern struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs; + +static MODULE_CONTEXT_DEFINE_INIT(sdbox_mailbox_list_module, + &mailbox_list_module_register); + +static struct mail_storage *sdbox_storage_alloc(void) +{ + struct sdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("dbox storage", 512+256); + storage = p_new(pool, struct sdbox_storage, 1); + storage->storage.v = sdbox_dbox_storage_vfuncs; + storage->storage.storage = dbox_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +struct mailbox * +sdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *name, struct istream *input, + enum mailbox_flags flags) +{ + struct sdbox_mailbox *mbox; + pool_t pool; + + /* dbox can't work without index files */ + flags &= ~MAILBOX_FLAG_NO_INDEX_FILES; + + pool = pool_alloconly_create("dbox mailbox", 1024+512); + mbox = p_new(pool, struct sdbox_mailbox, 1); + mbox->ibox.box = sdbox_mailbox; + mbox->ibox.box.pool = pool; + mbox->ibox.box.storage = storage; + mbox->ibox.box.list = list; + mbox->ibox.mail_vfuncs = &sdbox_mail_vfuncs; + + mbox->ibox.save_commit_pre = sdbox_transaction_save_commit_pre; + mbox->ibox.save_commit_post = sdbox_transaction_save_commit_post; + mbox->ibox.save_rollback = sdbox_transaction_save_rollback; + + index_storage_mailbox_alloc(&mbox->ibox, name, input, flags, + DBOX_INDEX_PREFIX); + mail_index_set_fsync_types(mbox->ibox.index, + MAIL_INDEX_SYNC_TYPE_APPEND | + MAIL_INDEX_SYNC_TYPE_EXPUNGE); + + mbox->ibox.index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY; + + mbox->storage = (struct sdbox_storage *)storage; + mbox->alt_path = + p_strconcat(pool, list->set.alt_dir, "/", + list->set.maildir_name, NULL); + mbox->hdr_ext_id = + mail_index_ext_register(mbox->ibox.index, "dbox-hdr", + sizeof(struct sdbox_index_header), 0, 0); + return &mbox->ibox.box; +} + +int sdbox_read_header(struct sdbox_mailbox *mbox, + struct sdbox_index_header *hdr) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(mbox->ibox.view, mbox->hdr_ext_id, + &data, &data_size); + if (data_size < SDBOX_INDEX_HEADER_MIN_SIZE && + (!mbox->creating || data_size != 0)) { + mail_storage_set_critical(&mbox->storage->storage.storage, + "dbox %s: Invalid dbox header size", + mbox->ibox.box.path); + return -1; + } + memset(hdr, 0, sizeof(*hdr)); + memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr))); + return 0; +} + +void sdbox_update_header(struct sdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update) +{ + struct sdbox_index_header hdr, new_hdr; + + if (sdbox_read_header(mbox, &hdr) < 0) + memset(&hdr, 0, sizeof(hdr)); + + new_hdr = hdr; + + if (update != NULL && !mail_guid_128_is_empty(update->mailbox_guid)) { + memcpy(new_hdr.mailbox_guid, update->mailbox_guid, + sizeof(new_hdr.mailbox_guid)); + } else if (mail_guid_128_is_empty(new_hdr.mailbox_guid)) { + mail_generate_guid_128(new_hdr.mailbox_guid); + } + + if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) { + mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0, + &new_hdr, sizeof(new_hdr)); + } +} + +static int sdbox_write_index_header(struct mailbox *box, + const struct mailbox_update *update) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; + struct mail_index_transaction *trans; + const struct mail_index_header *hdr; + uint32_t uid_validity, uid_next; + + hdr = mail_index_get_header(mbox->ibox.view); + trans = mail_index_transaction_begin(mbox->ibox.view, 0); + sdbox_update_header(mbox, trans, update); + + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + else if (hdr->uid_validity == 0) { + /* set uidvalidity */ + uid_validity = dbox_get_uidvalidity_next(box->list); + } + + if (hdr->uid_validity != uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + if (update != NULL && hdr->next_uid < update->min_next_uid) { + uid_next = update->min_next_uid; + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + } + if (update != NULL && update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(mbox->ibox.view) < + update->min_highest_modseq) { + mail_index_update_highest_modseq(trans, + update->min_highest_modseq); + } + + if (mail_index_transaction_commit(&trans) < 0) { + mail_storage_set_internal_error(box->storage); + mail_index_reset_error(mbox->ibox.index); + return -1; + } + return 0; +} + +static int sdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; + const char *origin; + mode_t mode; + gid_t gid; + int ret; + + mailbox_list_get_dir_permissions(box->list, NULL, &mode, &gid, &origin); + if (mkdir_parents_chgrp(box->path, mode, gid, origin) == 0) { + /* create indexes immediately with the dbox header */ + if (index_storage_mailbox_open(box) < 0) + return -1; + mbox->creating = TRUE; + ret = sdbox_write_index_header(box, update); + mbox->creating = FALSE; + if (ret < 0) + return -1; + } else if (errno != EEXIST) { + if (!mail_storage_set_error_from_errno(box->storage)) { + mail_storage_set_critical(box->storage, + "mkdir(%s) failed: %m", box->path); + } + return -1; + } + return 0; +} + +static void sdbox_storage_get_status_guid(struct mailbox *box, + struct mailbox_status *status_r) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; + struct sdbox_index_header hdr; + + if (sdbox_read_header(mbox, &hdr) < 0) + memset(&hdr, 0, sizeof(hdr)); + + if (mail_guid_128_is_empty(hdr.mailbox_guid)) { + /* regenerate it */ + if (sdbox_write_index_header(box, NULL) < 0 || + sdbox_read_header(mbox, &hdr) < 0) + return; + } + memcpy(status_r->mailbox_guid, hdr.mailbox_guid, + sizeof(status_r->mailbox_guid)); +} + +static void +dbox_storage_get_status(struct mailbox *box, enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + index_storage_get_status(box, items, status_r); + + if ((items & STATUS_GUID) != 0) + sdbox_storage_get_status_guid(box, status_r); +} + +static int +dbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + if (!box->opened) { + if (index_storage_mailbox_open(box) < 0) + return -1; + } + return sdbox_write_index_header(box, update); +} + +static int +sdbox_list_delete_mailbox(struct mailbox_list *list, const char *name) +{ + struct sdbox_mailbox_list *mlist = SDBOX_LIST_CONTEXT(list); + const char *trash_dest; + int ret; + + /* Make sure the indexes are closed before trying to delete the + directory that contains them. It can still fail with some NFS + implementations if indexes are opened by another session, but + that can't really be helped. */ + index_storage_destroy_unrefed(); + + /* delete the index and control directories */ + if (mlist->module_ctx.super.delete_mailbox(list, name) < 0) + return -1; + + if ((ret = dbox_list_delete_mailbox1(list, name, &trash_dest)) < 0) + return -1; + return dbox_list_delete_mailbox2(list, name, ret, trash_dest); +} + +static int +sdbox_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname, + struct mailbox_list *newlist, const char *newname, + bool rename_children) +{ + struct sdbox_mailbox_list *oldmlist = SDBOX_LIST_CONTEXT(oldlist); + + if (oldmlist->module_ctx.super. + rename_mailbox(oldlist, oldname, newlist, newname, + rename_children) < 0) + return -1; + return dbox_list_rename_mailbox(oldlist, oldname, newlist, newname, + rename_children); +} + +static void sdbox_storage_add_list(struct mail_storage *storage ATTR_UNUSED, + struct mailbox_list *list) +{ + struct sdbox_mailbox_list *mlist; + + mlist = p_new(list->pool, struct sdbox_mailbox_list, 1); + mlist->module_ctx.super = list->v; + + list->v.iter_is_mailbox = dbox_list_iter_is_mailbox; + list->v.delete_mailbox = sdbox_list_delete_mailbox; + list->v.rename_mailbox = sdbox_list_rename_mailbox; + list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre; + + MODULE_CONTEXT_SET(list, sdbox_mailbox_list_module, mlist); +} + +struct mail_storage dbox_storage = { + MEMBER(name) SDBOX_STORAGE_NAME, + MEMBER(class_flags) 0, + + { + NULL, + sdbox_storage_alloc, + NULL, + index_storage_destroy, + sdbox_storage_add_list, + dbox_storage_get_list_settings, + NULL, + sdbox_mailbox_alloc, + NULL + } +}; + +struct mailbox sdbox_mailbox = { + MEMBER(name) NULL, + MEMBER(storage) NULL, + MEMBER(list) NULL, + + { + index_storage_is_readonly, + index_storage_allow_new_keywords, + index_storage_mailbox_enable, + dbox_mailbox_open, + index_storage_mailbox_close, + dbox_mailbox_create, + dbox_mailbox_update, + dbox_storage_get_status, + NULL, + NULL, + sdbox_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + index_transaction_set_max_modseq, + index_keywords_create, + index_keywords_create_from_indexes, + index_keywords_ref, + index_keywords_unref, + index_keyword_is_valid, + index_storage_get_seq_range, + index_storage_get_uid_range, + index_storage_get_expunges, + NULL, + NULL, + NULL, + dbox_mail_alloc, + index_header_lookup_init, + index_header_lookup_deinit, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + sdbox_save_alloc, + sdbox_save_begin, + dbox_save_continue, + sdbox_save_finish, + sdbox_save_cancel, + sdbox_copy, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs = { + dbox_file_free, + sdbox_file_create_fd, + sdbox_mail_open, + sdbox_mailbox_create_indexes +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,69 @@ +#ifndef SDBOX_STORAGE_H +#define SDBOX_STORAGE_H + +#include "index-storage.h" +#include "dbox-storage.h" +#include "mailbox-list-private.h" + +#define SDBOX_STORAGE_NAME "dbox" +#define SDBOX_MAIL_FILE_PREFIX "u." +#define SDBOX_MAIL_FILE_FORMAT SDBOX_MAIL_FILE_PREFIX"%u" + +/* Flag specifies if the message should be in primary or alternative storage */ +#define SDBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND + +#define SDBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t)) +struct sdbox_index_header { + uint32_t oldv1_highest_maildir_uid; + uint8_t mailbox_guid[MAIL_GUID_128_SIZE]; +}; + +struct sdbox_storage { + struct dbox_storage storage; + union mailbox_list_module_context list_module_ctx; +}; + +struct sdbox_mailbox { + struct index_mailbox ibox; + struct sdbox_storage *storage; + + uint32_t hdr_ext_id; + const char *alt_path; + + unsigned int creating:1; + unsigned int sync_rebuild:1; +}; + +extern struct mail_vfuncs sdbox_mail_vfuncs; + +struct mailbox * +sdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *name, struct istream *input, + enum mailbox_flags flags); + +int sdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r); + +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); +int sdbox_read_header(struct sdbox_mailbox *mbox, + struct sdbox_index_header *hdr); +void sdbox_update_header(struct sdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update); + +struct mail_save_context * +sdbox_save_alloc(struct mailbox_transaction_context *_t); +int sdbox_save_begin(struct mail_save_context *ctx, struct istream *input); +int sdbox_save_finish(struct mail_save_context *ctx); +void sdbox_save_cancel(struct mail_save_context *ctx); + +struct dbox_file * +sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq); + +int sdbox_transaction_save_commit_pre(struct mail_save_context *ctx); +void sdbox_transaction_save_commit_post(struct mail_save_context *ctx); +void sdbox_transaction_save_rollback(struct mail_save_context *ctx); + +int sdbox_copy(struct mail_save_context *ctx, struct mail *mail); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-sync-file.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,67 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "hex-binary.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" + +#include <stdlib.h> + +static void +dbox_sync_file_move_if_needed(struct dbox_file *file, + enum sdbox_sync_entry_type type) +{ + bool move_to_alt = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT; + + if (move_to_alt != dbox_file_is_in_alt(file)) { + /* move the file. if it fails, nothing broke so + don't worry about it. */ + if (dbox_file_try_lock(file) > 0) { + (void)dbox_file_move(file, move_to_alt); + dbox_file_unlock(file); + } + } +} + +static void +dbox_sync_mark_single_file_expunged(struct sdbox_sync_context *ctx, + const struct sdbox_sync_file_entry *entry) +{ + struct mailbox *box = &ctx->mbox->ibox.box; + uint32_t seq; + + mail_index_lookup_seq(ctx->sync_view, entry->uid, &seq); + mail_index_expunge(ctx->trans, seq); + + if (box->v.sync_notify != NULL) + box->v.sync_notify(box, entry->uid, MAILBOX_SYNC_TYPE_EXPUNGE); +} + +int sdbox_sync_file(struct sdbox_sync_context *ctx, + const struct sdbox_sync_file_entry *entry) +{ + struct sdbox_mailbox *mbox = ctx->mbox; + struct dbox_file *file; + int ret = 1; + + file = sdbox_file_init(mbox, entry->uid); + switch (entry->type) { + case SDBOX_SYNC_ENTRY_TYPE_EXPUNGE: + if (dbox_file_unlink(file) >= 0) { + dbox_sync_mark_single_file_expunged(ctx, entry); + ret = 1; + } + break; + case SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT: + case SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT: + dbox_sync_file_move_if_needed(file, entry->type); + break; + } + dbox_file_unref(&file); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,182 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dbox-sync-rebuild.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" + +#include <stdlib.h> +#include <dirent.h> + +static void sdbox_sync_set_uidvalidity(struct dbox_sync_rebuild_context *ctx) +{ + struct mailbox *box = &ctx->ibox->box; + uint32_t uid_validity; + + /* if uidvalidity is set in the old index, use it */ + uid_validity = mail_index_get_header(ctx->view)->uid_validity; + if (uid_validity == 0) + uid_validity = dbox_get_uidvalidity_next(box->list); + + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); +} + +static int sdbox_sync_add_file_index(struct dbox_sync_rebuild_context *ctx, + struct dbox_file *file, uint32_t uid) +{ + uint32_t seq; + uoff_t size; + bool deleted; + int ret; + + if ((ret = dbox_file_open(file, &deleted)) > 0) { + if (deleted) + return 0; + ret = dbox_file_get_mail_stream(file, 0, &size, NULL); + } + + if (ret <= 0) { + if (ret < 0) + return -1; + + i_warning("dbox: Ignoring broken file: %s", file->cur_path); + return 0; + } + + mail_index_append(ctx->trans, uid, &seq); + T_BEGIN { + dbox_sync_rebuild_index_metadata(ctx, seq, uid); + } T_END; + return 0; +} + +static int +sdbox_sync_add_file(struct dbox_sync_rebuild_context *ctx, + const char *fname, bool primary) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)ctx->ibox; + struct dbox_file *file; + unsigned long uid; + char *p; + int ret; + + if (strncmp(fname, SDBOX_MAIL_FILE_PREFIX, + strlen(SDBOX_MAIL_FILE_PREFIX)) != 0) + return 0; + fname += strlen(SDBOX_MAIL_FILE_PREFIX); + + uid = strtoul(fname, &p, 10); + if (*p != '\0' || uid == 0 || uid >= (uint32_t)-1) { + i_warning("dbox %s: Ignoring invalid filename %s", + ctx->ibox->box.path, fname); + return 0; + } + + file = sdbox_file_init(mbox, uid); + if (!primary) + file->cur_path = file->alt_path; + ret = sdbox_sync_add_file_index(ctx, file, uid); + dbox_file_unref(&file); + return ret; +} + +static int sdbox_sync_index_rebuild_dir(struct dbox_sync_rebuild_context *ctx, + const char *path, bool primary) +{ + struct mail_storage *storage = ctx->ibox->box.storage; + DIR *dir; + struct dirent *d; + int ret = 0; + + dir = opendir(path); + if (dir == NULL) { + if (errno == ENOENT) { + if (!primary) { + /* alt directory doesn't exist, ignore */ + return 0; + } + mailbox_set_deleted(&ctx->ibox->box); + return -1; + } + mail_storage_set_critical(storage, + "opendir(%s) failed: %m", path); + return -1; + } + do { + errno = 0; + if ((d = readdir(dir)) == NULL) + break; + + ret = sdbox_sync_add_file(ctx, d->d_name, primary); + } while (ret >= 0); + if (errno != 0) { + mail_storage_set_critical(storage, + "readdir(%s) failed: %m", path); + ret = -1; + } + + if (closedir(dir) < 0) { + mail_storage_set_critical(storage, + "closedir(%s) failed: %m", path); + ret = -1; + } + return ret; +} + +static void sdbox_sync_update_header(struct dbox_sync_rebuild_context *ctx) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)ctx->ibox; + struct sdbox_index_header hdr; + + if (sdbox_read_header(mbox, &hdr) < 0) + memset(&hdr, 0, sizeof(hdr)); + if (!mail_guid_128_is_empty(hdr.mailbox_guid)) + mail_generate_guid_128(hdr.mailbox_guid); + mail_index_update_header_ext(ctx->trans, mbox->hdr_ext_id, 0, + &hdr, sizeof(hdr)); +} + +static int +sdbox_sync_index_rebuild_singles(struct dbox_sync_rebuild_context *ctx) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)ctx->ibox; + int ret = 0; + + sdbox_sync_set_uidvalidity(ctx); + if (sdbox_sync_index_rebuild_dir(ctx, ctx->ibox->box.path, TRUE) < 0) + ret = -1; + else if (mbox->alt_path != NULL) + ret = sdbox_sync_index_rebuild_dir(ctx, mbox->alt_path, FALSE); + sdbox_sync_update_header(ctx); + return ret; +} + +int sdbox_sync_index_rebuild(struct sdbox_mailbox *mbox) +{ + struct dbox_sync_rebuild_context *ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + int ret; + + view = mail_index_view_open(mbox->ibox.index); + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + + ctx = dbox_sync_index_rebuild_init(&mbox->ibox, view, trans); + ret = sdbox_sync_index_rebuild_singles(ctx); + dbox_sync_index_rebuild_deinit(&ctx); + + if (ret < 0) + mail_index_transaction_rollback(&trans); + else + ret = mail_index_transaction_commit(&trans); + mail_index_view_close(&view); + + if (ret == 0) + mbox->sync_rebuild = FALSE; + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,291 @@ +/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "str.h" +#include "hash.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" + +#define SDBOX_REBUILD_COUNT 3 + +static unsigned int sdbox_sync_file_entry_hash(const void *p) +{ + const struct sdbox_sync_file_entry *entry = p; + + return entry->uid; +} + +static int sdbox_sync_file_entry_cmp(const void *p1, const void *p2) +{ + const struct sdbox_sync_file_entry *entry1 = p1, *entry2 = p2; + + /* this is only for hashing, don't bother ever returning 1. */ + if (entry1->uid != entry2->uid) + return -1; + return 0; +} + +static int sdbox_sync_add_seq(struct sdbox_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec, + uint32_t seq) +{ + struct sdbox_sync_file_entry *entry, lookup_entry; + + i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE || + sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); + + memset(&lookup_entry, 0, sizeof(lookup_entry)); + mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid); + + entry = hash_table_lookup(ctx->syncs, &lookup_entry); + if (entry == NULL) { + entry = p_new(ctx->pool, struct sdbox_sync_file_entry, 1); + *entry = lookup_entry; + hash_table_insert(ctx->syncs, entry, entry); + } + + if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) + entry->type = SDBOX_SYNC_ENTRY_TYPE_EXPUNGE; + else if ((sync_rec->add_flags & SDBOX_INDEX_FLAG_ALT) != 0) + entry->type = SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT; + else + entry->type = SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT; + return 1; +} + +static int sdbox_sync_add(struct sdbox_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec) +{ + uint32_t seq, seq1, seq2; + int ret; + + if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + /* we're interested */ + } else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) { + /* we care only about alt flag changes */ + if ((sync_rec->add_flags & SDBOX_INDEX_FLAG_ALT) == 0 && + (sync_rec->remove_flags & SDBOX_INDEX_FLAG_ALT) == 0) + return 1; + } else { + /* not interested */ + return 1; + } + + if (!mail_index_lookup_seq_range(ctx->sync_view, + sync_rec->uid1, sync_rec->uid2, + &seq1, &seq2)) { + /* already expunged everything. nothing to do. */ + return 1; + } + + for (seq = seq1; seq <= seq2; seq++) { + if ((ret = sdbox_sync_add_seq(ctx, sync_rec, seq)) <= 0) + return ret; + } + return 1; +} + +static int sdbox_sync_index(struct sdbox_sync_context *ctx) +{ + struct mailbox *box = &ctx->mbox->ibox.box; + const struct mail_index_header *hdr; + struct mail_index_sync_rec sync_rec; + struct hash_iterate_context *iter; + void *key, *value; + uint32_t seq1, seq2; + int ret = 1; + + hdr = mail_index_get_header(ctx->sync_view); + if (hdr->uid_validity == 0) { + /* newly created index file */ + return 0; + } + + /* mark the newly seen messages as recent */ + if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, + hdr->next_uid, &seq1, &seq2)) { + index_mailbox_set_recent_seq(&ctx->mbox->ibox, ctx->sync_view, + seq1, seq2); + } + + /* read all changes and group changes to same file_id together */ + ctx->pool = pool_alloconly_create("dbox sync pool", 1024*32); + ctx->syncs = hash_table_create(default_pool, ctx->pool, 0, + sdbox_sync_file_entry_hash, + sdbox_sync_file_entry_cmp); + + while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) { + if ((ret = sdbox_sync_add(ctx, &sync_rec)) <= 0) + break; + } + + if (ret > 0) { + /* now sync each file separately */ + iter = hash_table_iterate_init(ctx->syncs); + while (hash_table_iterate(iter, &key, &value)) { + const struct sdbox_sync_file_entry *entry = value; + + if ((ret = sdbox_sync_file(ctx, entry)) <= 0) + break; + } + hash_table_iterate_deinit(&iter); + } + + if (box->v.sync_notify != NULL) + box->v.sync_notify(box, 0, 0); + + hash_table_destroy(&ctx->syncs); + pool_unref(&ctx->pool); + return ret; +} + +static int sdbox_refresh_header(struct sdbox_mailbox *mbox, bool retry) +{ + struct mail_index_view *view; + struct sdbox_index_header hdr; + int ret; + + view = mail_index_view_open(mbox->ibox.index); + ret = sdbox_read_header(mbox, &hdr); + mail_index_view_close(&view); + + if (ret == 0) { + ret = mbox->sync_rebuild ? -1 : 0; + } else if (retry) { + (void)mail_index_refresh(mbox->ibox.index); + return sdbox_refresh_header(mbox, FALSE); + } + return ret; +} + +int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags, + struct sdbox_sync_context **ctx_r) +{ + struct mail_storage *storage = mbox->ibox.box.storage; + struct sdbox_sync_context *ctx; + enum mail_index_sync_flags sync_flags = 0; + unsigned int i; + int ret; + bool rebuild; + + rebuild = sdbox_refresh_header(mbox, TRUE) < 0 || + (flags & SDBOX_SYNC_FLAG_FORCE_REBUILD) != 0; + + ctx = i_new(struct sdbox_sync_context, 1); + ctx->mbox = mbox; + ctx->flags = flags; + + if ((mbox->ibox.box.flags & MAILBOX_FLAG_KEEP_RECENT) == 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; + if (!rebuild && (flags & SDBOX_SYNC_FLAG_FORCE) == 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + if ((flags & SDBOX_SYNC_FLAG_FSYNC) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC; + /* don't write unnecessary dirty flag updates */ + sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; + + for (i = 0;; i++) { + ret = mail_index_sync_begin(mbox->ibox.index, + &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, + sync_flags); + if (ret <= 0) { + if (ret < 0) + mail_storage_set_index_error(&mbox->ibox); + i_free(ctx); + *ctx_r = NULL; + return ret; + } + + /* now that we're locked, check again if we want to rebuild */ + if (sdbox_refresh_header(mbox, FALSE) < 0) + ret = 0; + else { + if ((ret = sdbox_sync_index(ctx)) > 0) + break; + } + + /* failure. keep the index locked while we're doing a + rebuild. */ + if (ret == 0) { + if (i >= SDBOX_REBUILD_COUNT) { + mail_storage_set_critical(storage, + "dbox %s: Index keeps breaking", + ctx->mbox->ibox.box.path); + ret = -1; + } else { + /* do a full resync and try again. */ + i_warning("dbox %s: Rebuilding index", + ctx->mbox->ibox.box.path); + ret = sdbox_sync_index_rebuild(mbox); + } + } + mail_index_sync_rollback(&ctx->index_sync_ctx); + if (ret < 0) { + i_free(ctx); + return -1; + } + } + + *ctx_r = ctx; + return 0; +} + +int sdbox_sync_finish(struct sdbox_sync_context **_ctx, bool success) +{ + struct sdbox_sync_context *ctx = *_ctx; + int ret = success ? 0 : -1; + + *_ctx = NULL; + + if (success) { + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { + mail_storage_set_index_error(&ctx->mbox->ibox); + ret = -1; + } + } else { + mail_index_sync_rollback(&ctx->index_sync_ctx); + } + if (ctx->path != NULL) + str_free(&ctx->path); + + i_free(ctx); + return ret; +} + +int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags) +{ + struct sdbox_sync_context *sync_ctx; + + if (sdbox_sync_begin(mbox, flags, &sync_ctx) < 0) + return -1; + + if (sync_ctx == NULL) + return 0; + return sdbox_sync_finish(&sync_ctx, TRUE); +} + +struct mailbox_sync_context * +sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; + enum sdbox_sync_flags sdbox_sync_flags = 0; + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + ret = -1; + } + + if (ret == 0 && (index_mailbox_want_full_sync(&mbox->ibox, flags) || + mbox->sync_rebuild)) { + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) + sdbox_sync_flags |= SDBOX_SYNC_FLAG_FORCE_REBUILD; + ret = sdbox_sync(mbox, sdbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.h Tue Oct 06 19:22:42 2009 -0400 @@ -0,0 +1,51 @@ +#ifndef SDBOX_SYNC_H +#define SDBOX_SYNC_H + +struct mailbox; +struct sdbox_mailbox; + +enum sdbox_sync_flags { + SDBOX_SYNC_FLAG_FORCE = 0x01, + SDBOX_SYNC_FLAG_FSYNC = 0x02, + SDBOX_SYNC_FLAG_FORCE_REBUILD = 0x04 +}; + +enum sdbox_sync_entry_type { + SDBOX_SYNC_ENTRY_TYPE_EXPUNGE, + SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT, + SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT +}; + +struct sdbox_sync_file_entry { + uint32_t uid; + enum sdbox_sync_entry_type type; +}; + +struct sdbox_sync_context { + struct sdbox_mailbox *mbox; + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + enum sdbox_sync_flags flags; + + string_t *path; + unsigned int path_dir_prefix_len; + + pool_t pool; + struct hash_table *syncs; /* struct sdbox_sync_file_entry */ +}; + +int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags, + struct sdbox_sync_context **ctx_r); +int sdbox_sync_finish(struct sdbox_sync_context **ctx, bool success); +int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags); + +int sdbox_sync_file(struct sdbox_sync_context *ctx, + const struct sdbox_sync_file_entry *entry); + +int sdbox_sync_index_rebuild(struct sdbox_mailbox *mbox); + +struct mailbox_sync_context * +sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +#endif
--- a/src/lib-storage/index/dbox/Makefile.am Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,41 +0,0 @@ -noinst_LTLIBRARIES = libstorage_dbox.la - -AM_CPPFLAGS = \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-settings \ - -I$(top_srcdir)/src/lib-mail \ - -I$(top_srcdir)/src/lib-imap \ - -I$(top_srcdir)/src/lib-index \ - -I$(top_srcdir)/src/lib-storage \ - -I$(top_srcdir)/src/lib-storage/index - -libstorage_dbox_la_SOURCES = \ - dbox-file.c \ - dbox-file-fix.c \ - dbox-file-maildir.c \ - dbox-mail.c \ - dbox-map.c \ - dbox-save.c \ - dbox-settings.c \ - dbox-sync.c \ - dbox-sync-file.c \ - dbox-sync-rebuild.c \ - dbox-storage.c \ - dbox-storage-rebuild.c - -headers = \ - dbox-file.h \ - dbox-file-maildir.h \ - dbox-map.h \ - dbox-map-private.h \ - dbox-settings.h \ - dbox-storage.h \ - dbox-storage-rebuild.h \ - dbox-sync.h - -if INSTALL_HEADERS - pkginc_libdir=$(pkgincludedir) - pkginc_lib_HEADERS = $(headers) -else - noinst_HEADERS = $(headers) -endif
--- a/src/lib-storage/index/dbox/dbox-file-fix.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,352 +0,0 @@ -/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "istream.h" -#include "ostream.h" -#include "str-find.h" -#include "hex-binary.h" -#include "message-size.h" -#include "dbox-storage.h" -#include "dbox-file.h" - -#include <stdio.h> - -static int -dbox_file_find_next_magic(struct dbox_file *file, uoff_t *offset_r, bool *pre_r) -{ - struct istream *input = file->input; - struct str_find_context *pre_ctx, *post_ctx; - uoff_t orig_offset, pre_offset, post_offset; - const unsigned char *data; - size_t size; - int ret; - - *pre_r = FALSE; - - pre_ctx = str_find_init(default_pool, "\n"DBOX_MAGIC_PRE); - post_ctx = str_find_init(default_pool, DBOX_MAGIC_POST); - - /* \n isn't part of the DBOX_MAGIC_PRE, but it always preceds it. - assume that at this point we've already just read the \n. when - scanning for it later we'll need to find the \n though. */ - str_find_more(pre_ctx, (const unsigned char *)"\n", 1); - - orig_offset = input->v_offset; - while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { - pre_offset = (uoff_t)-1; - post_offset = (uoff_t)-1; - if (str_find_more(pre_ctx, data, size)) { - pre_offset = input->v_offset + - str_find_get_match_end_pos(pre_ctx) - - (strlen(DBOX_MAGIC_PRE) + 1); - *pre_r = TRUE; - } - if (str_find_more(post_ctx, data, size)) { - post_offset = input->v_offset + - str_find_get_match_end_pos(post_ctx) - - strlen(DBOX_MAGIC_POST); - if (pre_offset == (uoff_t)-1 || - post_offset < pre_offset) { - pre_offset = post_offset; - *pre_r = FALSE; - } - } - - if (pre_offset != (uoff_t)-1) { - if (*pre_r) { - /* LF isn't part of the magic */ - pre_offset++; - } - *offset_r = pre_offset; - break; - } - i_stream_skip(input, size); - } - if (ret <= 0) { - i_assert(ret == -1); - if (input->stream_errno != 0) - dbox_file_set_syscall_error(file, "read()"); - else { - ret = 0; - *offset_r = input->v_offset; - } - } - i_stream_seek(input, orig_offset); - str_find_deinit(&pre_ctx); - str_find_deinit(&post_ctx); - return ret; -} - -static int -stream_copy(struct dbox_file *file, struct ostream *output, - const char *path, uoff_t count) -{ - struct istream *input; - off_t bytes; - - input = i_stream_create_limit(file->input, count); - bytes = o_stream_send_istream(output, input); - i_stream_unref(&input); - - if (bytes < 0) { - mail_storage_set_critical(&file->storage->storage, - "o_stream_send_istream(%s, %s) failed: %m", - file->current_path, path); - return -1; - } - if ((uoff_t)bytes != count) { - mail_storage_set_critical(&file->storage->storage, - "o_stream_send_istream(%s) copied only %" - PRIuUOFF_T" of %"PRIuUOFF_T" bytes", - path, bytes, count); - return -1; - } - return 0; -} - -static void dbox_file_skip_broken_header(struct dbox_file *file) -{ - const unsigned int magic_len = strlen(DBOX_MAGIC_PRE); - const unsigned char *data; - size_t i, size; - - /* if there's LF close to our position, assume that the header ends - there. */ - data = i_stream_get_data(file->input, &size); - if (size > file->msg_header_size + 16) - size = file->msg_header_size + 16; - for (i = 0; i < size; i++) { - if (data[i] == '\n') { - i_stream_skip(file->input, i); - return; - } - } - - /* skip at least the magic bytes if possible */ - if (size > magic_len && memcmp(data, DBOX_MAGIC_PRE, magic_len) == 0) - i_stream_skip(file->input, magic_len); -} - -static void -dbox_file_copy_metadata(struct dbox_file *file, struct ostream *output, - bool *have_guid_r) -{ - const char *line; - uoff_t prev_offset = file->input->v_offset; - - *have_guid_r = FALSE; - while ((line = i_stream_read_next_line(file->input)) != NULL) { - if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { - /* end of metadata */ - return; - } - if (*line < 32) { - /* broken - possibly a new pre-magic block */ - i_stream_seek(file->input, prev_offset); - return; - } - if (*line == DBOX_METADATA_VIRTUAL_SIZE) { - /* it may be wrong - recreate it */ - continue; - } - if (*line == DBOX_METADATA_GUID) - *have_guid_r = TRUE; - o_stream_send_str(output, line); - o_stream_send_str(output, "\n"); - } -} - -static int -dbox_file_fix_write_stream(struct dbox_file *file, uoff_t start_offset, - const char *temp_path, struct ostream *output) -{ - struct dbox_message_header msg_hdr; - uoff_t offset, msg_size, hdr_offset, body_offset; - bool pre, write_header, have_guid; - struct message_size body; - struct istream *body_input; - uint8_t guid_128[MAIL_GUID_128_SIZE]; - int ret; - - i_stream_seek(file->input, 0); - if (start_offset > 0) { - /* copy the valid data */ - if (stream_copy(file, output, temp_path, start_offset) < 0) - return -1; - } else { - /* the file header is broken. recreate it */ - if (dbox_file_header_write(file, output) < 0) { - dbox_file_set_syscall_error(file, "write()"); - return -1; - } - } - - while ((ret = dbox_file_find_next_magic(file, &offset, &pre)) > 0) { - msg_size = offset - file->input->v_offset; - if (msg_size < 256 && pre) { - /* probably some garbage or some broken headers. - we most likely don't miss anything by skipping - over this data. */ - i_stream_skip(file->input, msg_size); - hdr_offset = file->input->v_offset; - ret = dbox_file_read_mail_header(file, &msg_size); - if (ret <= 0) { - if (ret < 0) - return -1; - dbox_file_skip_broken_header(file); - body_offset = file->input->v_offset; - msg_size = (uoff_t)-1; - } else { - i_stream_skip(file->input, - file->msg_header_size); - body_offset = file->input->v_offset; - i_stream_skip(file->input, msg_size); - } - - ret = dbox_file_find_next_magic(file, &offset, &pre); - if (ret <= 0) - break; - - if (!pre && msg_size == offset - body_offset) { - /* msg header ok, copy it */ - i_stream_seek(file->input, hdr_offset); - if (stream_copy(file, output, temp_path, - file->msg_header_size) < 0) - return -1; - write_header = FALSE; - } else { - /* msg header is broken. write our own. */ - i_stream_seek(file->input, body_offset); - if (msg_size != (uoff_t)-1) { - /* previous magic find might have - skipped too much. seek back and - make sure */ - ret = dbox_file_find_next_magic(file, &offset, &pre); - if (ret <= 0) - break; - } - - write_header = TRUE; - msg_size = offset - body_offset; - } - } else { - /* treat this data as a separate message. */ - write_header = TRUE; - body_offset = file->input->v_offset; - } - /* write msg header */ - if (write_header) { - dbox_msg_header_fill(&msg_hdr, msg_size); - (void)o_stream_send(output, &msg_hdr, sizeof(msg_hdr)); - } - /* write msg body */ - i_assert(file->input->v_offset == body_offset); - if (stream_copy(file, output, temp_path, msg_size) < 0) - return -1; - i_assert(file->input->v_offset == offset); - - /* get message body size */ - i_stream_seek(file->input, body_offset); - body_input = i_stream_create_limit(file->input, msg_size); - ret = message_get_body_size(body_input, &body, NULL); - i_stream_unref(&body_input); - if (ret < 0) { - errno = output->stream_errno; - mail_storage_set_critical(&file->storage->storage, - "read(%s) failed: %m", file->current_path); - return -1; - } - - /* write msg metadata. */ - i_assert(file->input->v_offset == offset); - ret = dbox_file_metadata_skip_header(file); - if (ret < 0) - return -1; - o_stream_send_str(output, DBOX_MAGIC_POST); - if (ret == 0) - have_guid = FALSE; - else - dbox_file_copy_metadata(file, output, &have_guid); - if (!have_guid) { - mail_generate_guid_128(guid_128); - o_stream_send_str(output, - t_strdup_printf("%c%s\n", DBOX_METADATA_GUID, - binary_to_hex(guid_128, sizeof(guid_128)))); - } - o_stream_send_str(output, - t_strdup_printf("%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, - (unsigned long long)body.virtual_size)); - o_stream_send_str(output, "\n"); - if (output->stream_errno != 0) { - errno = output->stream_errno; - mail_storage_set_critical(&file->storage->storage, - "write(%s) failed: %m", temp_path); - return -1; - } - } - return 0; -} - -int dbox_file_fix(struct dbox_file *file, uoff_t start_offset) -{ - struct ostream *output; - const char *temp_path, *broken_path; - char *temp_fname; - bool deleted; - int fd, ret; - - i_assert(file->input != NULL); - - temp_fname = dbox_generate_tmp_filename(); - temp_path = t_strdup_printf("%s/%s", file->storage->storage_dir, - temp_fname); - i_free(temp_fname); - - fd = dbox_create_fd(file->storage, temp_path); - if (fd == -1) - return -1; - - output = o_stream_create_fd_file(fd, 0, FALSE); - ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output); - o_stream_unref(&output); - if (close(fd) < 0) { - mail_storage_set_critical(&file->storage->storage, - "close(%s) failed: %m", temp_path); - ret = -1; - } - if (ret < 0) { - if (unlink(temp_path) < 0) { - mail_storage_set_critical(&file->storage->storage, - "unlink(%s) failed: %m", temp_path); - } - return -1; - } - /* keep a copy of the original file in case someone wants to look - at it */ - broken_path = t_strconcat(file->current_path, - DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX, NULL); - if (link(file->current_path, broken_path) < 0) { - mail_storage_set_critical(&file->storage->storage, - "link(%s, %s) failed: %m", - file->current_path, broken_path); - } else { - i_warning("dbox: Copy of the broken file saved to %s", - broken_path); - } - if (rename(temp_path, file->current_path) < 0) { - mail_storage_set_critical(&file->storage->storage, - "rename(%s, %s) failed: %m", - temp_path, file->current_path); - return -1; - } - - /* file was successfully recreated - reopen it */ - dbox_file_close(file); - if (dbox_file_open(file, &deleted) <= 0) { - mail_storage_set_critical(&file->storage->storage, - "dbox_file_fix(%s): reopening file failed", - file->current_path); - return -1; - } - return 0; -}
--- a/src/lib-storage/index/dbox/dbox-file-maildir.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,91 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "maildir/maildir-storage.h" -#include "maildir/maildir-uidlist.h" -#include "maildir/maildir-filename.h" -#include "dbox-storage.h" -#include "dbox-file.h" -#include "dbox-file-maildir.h" - -#include <stdlib.h> - -const char *dbox_file_maildir_metadata_get(struct dbox_file *file, - enum dbox_metadata_key key) -{ - struct stat st; - uoff_t size; - const char *p, *value = NULL; - - switch (key) { - case DBOX_METADATA_GUID: - p = strchr(file->fname, MAILDIR_INFO_SEP); - value = p == NULL ? file->fname : - t_strdup_until(file->fname, p); - break; - case DBOX_METADATA_RECEIVED_TIME: - case DBOX_METADATA_SAVE_TIME: - if (file->fd != -1) { - if (fstat(file->fd, &st) < 0) { - dbox_file_set_syscall_error(file, "fstat()"); - return NULL; - } - } else { - if (stat(file->current_path, &st) < 0) { - if (errno == ENOENT) - return NULL; - dbox_file_set_syscall_error(file, "stat()"); - return NULL; - } - } - value = t_strdup_printf("%lx", (unsigned long) - (key == DBOX_METADATA_RECEIVED_TIME ? - st.st_mtime : st.st_ctime)); - break; - case DBOX_METADATA_VIRTUAL_SIZE: - if (!maildir_filename_get_size(file->fname, - MAILDIR_EXTRA_VIRTUAL_SIZE, - &size)) { - value = maildir_uidlist_lookup_ext( - file->single_mbox->maildir_uidlist, - file->uid, MAILDIR_UIDLIST_REC_EXT_VSIZE); - if (value == NULL) - break; - size = strtoull(value, NULL, 10); - } - value = t_strdup_printf("%llx", (unsigned long long)size); - break; - case DBOX_METADATA_POP3_UIDL: - value = maildir_uidlist_lookup_ext( - file->single_mbox->maildir_uidlist, - file->uid, MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); - if (value != NULL && *value == '\0') { - /* special case: use base filename */ - p = strchr(file->fname, MAILDIR_INFO_SEP); - if (p == NULL) - value = file->fname; - else - value = t_strdup_until(file->fname, p); - } - break; - case DBOX_METADATA_ORIG_MAILBOX: - case DBOX_METADATA_OLDV1_EXPUNGED: - case DBOX_METADATA_OLDV1_FLAGS: - case DBOX_METADATA_OLDV1_KEYWORDS: - case DBOX_METADATA_OLDV1_SPACE: - case DBOX_METADATA_EXT_REF: - break; - } - return value; -} - -bool dbox_maildir_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid, - const char **fname_r) -{ - enum maildir_uidlist_rec_flag flags; - - if (maildir_uidlist_lookup(mbox->maildir_uidlist, uid, &flags, - fname_r) <= 0) - return FALSE; - return TRUE; -}
--- a/src/lib-storage/index/dbox/dbox-file-maildir.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,9 +0,0 @@ -#ifndef DBOX_FILE_MAILDIR_H -#define DBOX_FILE_MAILDIR_H - -const char *dbox_file_maildir_metadata_get(struct dbox_file *file, - enum dbox_metadata_key key); -bool dbox_maildir_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid, - const char **fname_r); - -#endif
--- a/src/lib-storage/index/dbox/dbox-file.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1033 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "ioloop.h" -#include "array.h" -#include "hex-dec.h" -#include "hex-binary.h" -#include "hostpid.h" -#include "istream.h" -#include "ostream.h" -#include "file-lock.h" -#include "mkdir-parents.h" -#include "fdatasync-path.h" -#include "eacces-error.h" -#include "str.h" -#include "dbox-storage.h" -#include "dbox-file.h" -#include "dbox-file-maildir.h" - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <ctype.h> -#include <fcntl.h> - -#define DBOX_READ_BLOCK_SIZE 4096 - -char *dbox_generate_tmp_filename(void) -{ - static unsigned int create_count = 0; - - return i_strdup_printf("temp.%lu.P%sQ%uM%u.%s", - (unsigned long)ioloop_timeval.tv_sec, my_pid, - create_count++, - (unsigned int)ioloop_timeval.tv_usec, - my_hostname); -} - -void dbox_file_set_syscall_error(struct dbox_file *file, const char *function) -{ - mail_storage_set_critical(&file->storage->storage, - "%s failed for file %s: %m", - function, file->current_path); -} - -void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) -{ - va_list args; - - if (file->single_mbox == NULL) - file->storage->sync_rebuild = TRUE; - - va_start(args, reason); - mail_storage_set_critical(&file->storage->storage, - "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s", - file->current_path, - file->input == NULL ? 0 : file->input->v_offset, - t_strdup_vprintf(reason, args)); - va_end(args); -} - -static struct dbox_file * -dbox_find_and_move_open_file(struct dbox_storage *storage, uint32_t file_id) -{ - struct dbox_file *const *files, *file; - unsigned int i, count; - - files = array_get(&storage->open_files, &count); - for (i = 0; i < count; i++) { - if (files[i]->file_id == file_id) { - /* move to last in the array */ - file = files[i]; - array_delete(&storage->open_files, i, 1); - array_append(&storage->open_files, &file, 1); - return file; - } - } - return NULL; -} - -static void dbox_file_free(struct dbox_file *file) -{ - i_assert(file->refcount == 0); - - if (file->metadata_pool != NULL) - pool_unref(&file->metadata_pool); - dbox_file_close(file); - i_free(file->current_path); - i_free(file->fname); - i_free(file); -} - -void dbox_files_free(struct dbox_storage *storage) -{ - struct dbox_file *const *files; - unsigned int i, count; - - files = array_get(&storage->open_files, &count); - for (i = 0; i < count; i++) - dbox_file_free(files[i]); - array_clear(&storage->open_files); -} - -void dbox_files_sync_input(struct dbox_storage *storage) -{ - struct dbox_file *const *files; - unsigned int i, count; - - files = array_get(&storage->open_files, &count); - for (i = 0; i < count; i++) { - if (files[i]->input != NULL) - i_stream_sync(files[i]->input); - } -} - -static void -dbox_close_open_files(struct dbox_storage *storage, unsigned int close_count) -{ - struct dbox_file *const *files; - unsigned int i, count; - - files = array_get(&storage->open_files, &count); - for (i = 0; i < count;) { - if (files[i]->refcount == 0) { - dbox_file_free(files[i]); - array_delete(&storage->open_files, i, 1); - - if (--close_count == 0) - break; - - files = array_get(&storage->open_files, &count); - } else { - i++; - } - } -} - -static char * -dbox_file_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid, - bool *maildir_file_r) -{ - const char *fname; - - if (uid <= mbox->highest_maildir_uid && - dbox_maildir_uid_get_fname(mbox, uid, &fname)) { - *maildir_file_r = TRUE; - return i_strdup(fname); - } else { - *maildir_file_r = FALSE; - return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, uid); - } -} - -const char *dbox_file_get_primary_path(struct dbox_file *file) -{ - const char *dir; - - dir = file->single_mbox != NULL ? file->single_mbox->ibox.box.path : - file->storage->storage_dir; - return t_strdup_printf("%s/%s", dir, file->fname); -} - -const char *dbox_file_get_alt_path(struct dbox_file *file) -{ - const char *dir; - - dir = file->single_mbox != NULL ? file->single_mbox->alt_path : - file->storage->alt_storage_dir; - return t_strdup_printf("%s/%s", dir, file->fname); -} - -struct dbox_file * -dbox_file_init_single(struct dbox_mailbox *mbox, uint32_t uid) -{ - struct dbox_file *file; - bool maildir; - - file = i_new(struct dbox_file, 1); - file->refcount = 1; - file->storage = mbox->storage; - file->single_mbox = mbox; - file->fd = -1; - file->cur_offset = (uoff_t)-1; - if (uid != 0) { - file->uid = uid; - file->fname = dbox_file_uid_get_fname(mbox, uid, &maildir); - file->maildir_file = maildir; - } else { - file->fname = dbox_generate_tmp_filename(); - } - file->current_path = i_strdup_printf("%s/%s", mbox->ibox.box.path, - file->fname); - return file; -} - -struct dbox_file * -dbox_file_init_multi(struct dbox_storage *storage, uint32_t file_id) -{ - struct dbox_file *file; - unsigned int count; - - file = file_id == 0 ? NULL : - dbox_find_and_move_open_file(storage, file_id); - if (file != NULL) { - file->refcount++; - return file; - } - - count = array_count(&storage->open_files); - if (count > storage->set->dbox_max_open_files) { - dbox_close_open_files(storage, count - - storage->set->dbox_max_open_files); - } - - file = i_new(struct dbox_file, 1); - file->refcount = 1; - file->storage = storage; - file->file_id = file_id; - file->fd = -1; - file->cur_offset = (uoff_t)-1; - file->fname = file_id == 0 ? dbox_generate_tmp_filename() : - i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id); - file->current_path = - i_strdup_printf("%s/%s", storage->storage_dir, file->fname); - - if (file_id != 0) - array_append(&storage->open_files, &file, 1); - return file; -} - -int dbox_file_assign_id(struct dbox_file *file, uint32_t id) -{ - const char *old_path; - char *new_fname, *new_path; - bool maildir; - - i_assert(!file->maildir_file); - i_assert(file->uid == 0 && file->file_id == 0); - i_assert(id != 0); - - old_path = file->current_path; - if (file->single_mbox != NULL) { - new_fname = dbox_file_uid_get_fname(file->single_mbox, - id, &maildir); - new_path = i_strdup_printf("%s/%s", - file->single_mbox->ibox.box.path, - new_fname); - } else { - new_fname = i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, id); - new_path = i_strdup_printf("%s/%s", file->storage->storage_dir, - new_fname); - } - if (rename(old_path, new_path) < 0) { - mail_storage_set_critical(&file->storage->storage, - "rename(%s, %s) failed: %m", - old_path, new_path); - i_free(new_fname); - i_free(new_path); - return -1; - } - i_free(file->fname); - i_free(file->current_path); - file->fname = new_fname; - file->current_path = new_path; - - if (file->single_mbox != NULL) - file->uid = id; - else { - file->file_id = id; - array_append(&file->storage->open_files, &file, 1); - } - return 0; -} - -void dbox_file_unref(struct dbox_file **_file) -{ - struct dbox_file *file = *_file; - struct dbox_file *const *files, *oldest_file; - unsigned int i, count; - - *_file = NULL; - - i_assert(file->refcount > 0); - if (--file->refcount > 0) - return; - - /* don't cache metadata seeks while file isn't being referenced */ - file->metadata_read_offset = (uoff_t)-1; - - if (file->file_id != 0) { - files = array_get(&file->storage->open_files, &count); - if (!file->deleted && - count <= file->storage->set->dbox_max_open_files) { - /* we can leave this file open for now */ - return; - } - - /* close the oldest file with refcount=0 */ - for (i = 0; i < count; i++) { - if (files[i]->refcount == 0) - break; - } - oldest_file = files[i]; - array_delete(&file->storage->open_files, i, 1); - if (oldest_file != file) { - dbox_file_free(oldest_file); - return; - } - /* have to close ourself */ - } - - dbox_file_free(file); -} - -static int dbox_file_parse_header(struct dbox_file *file, const char *line) -{ - const char *const *tmp, *value; - unsigned int pos; - enum dbox_header_key key; - - file->file_version = *line - '0'; - if (!i_isdigit(line[0]) || line[1] != ' ' || - (file->file_version != 1 && file->file_version != DBOX_VERSION)) { - dbox_file_set_corrupted(file, "Invalid dbox version"); - return -1; - } - line += 2; - pos = 2; - - file->msg_header_size = 0; - - for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) { - key = **tmp; - value = *tmp + 1; - - switch (key) { - case DBOX_HEADER_OLDV1_APPEND_OFFSET: - break; - case DBOX_HEADER_MSG_HEADER_SIZE: - file->msg_header_size = strtoul(value, NULL, 16); - break; - case DBOX_HEADER_CREATE_STAMP: - file->create_time = strtoul(value, NULL, 16); - break; - } - pos += strlen(value) + 2; - } - - if (file->msg_header_size == 0) { - dbox_file_set_corrupted(file, "Missing message header size"); - return -1; - } - return 0; -} - -static int dbox_file_read_header(struct dbox_file *file) -{ - const char *line; - unsigned int hdr_size; - int ret; - - i_stream_seek(file->input, 0); - line = i_stream_read_next_line(file->input); - if (line == NULL) { - if (file->input->stream_errno == 0) { - dbox_file_set_corrupted(file, - "EOF while reading file header"); - return 0; - } - - dbox_file_set_syscall_error(file, "read()"); - return -1; - } - hdr_size = file->input->v_offset; - T_BEGIN { - ret = dbox_file_parse_header(file, line) < 0 ? 0 : 1; - } T_END; - if (ret > 0) - file->file_header_size = hdr_size; - return ret; -} - -static int dbox_file_open_fd(struct dbox_file *file) -{ - const char *path; - bool alt = FALSE; - - /* try the primary path first */ - path = dbox_file_get_primary_path(file); - while ((file->fd = open(path, O_RDWR)) == -1) { - if (errno != ENOENT) { - mail_storage_set_critical(&file->storage->storage, - "open(%s) failed: %m", path); - return -1; - } - - if (file->storage->alt_storage_dir == NULL || alt) { - /* not found */ - return 0; - } - - /* try the alternative path */ - path = dbox_file_get_alt_path(file); - alt = TRUE; - } - i_free(file->current_path); - file->current_path = i_strdup(path); - file->alt_path = alt; - return 1; -} - -int dbox_file_open(struct dbox_file *file, bool *deleted_r) -{ - int ret; - - *deleted_r = FALSE; - if (file->input != NULL) - return 1; - - if (file->fd == -1) { - T_BEGIN { - ret = dbox_file_open_fd(file); - } T_END; - if (ret <= 0) { - if (ret < 0) - return -1; - *deleted_r = TRUE; - return 1; - } - } - - file->input = i_stream_create_fd(file->fd, 0, FALSE); - i_stream_set_init_buffer_size(file->input, DBOX_READ_BLOCK_SIZE); - return file->maildir_file ? 1 : - dbox_file_read_header(file); -} - -int dbox_create_fd(struct dbox_storage *storage, const char *path) -{ - mode_t old_mask; - int fd; - - old_mask = umask(0666 & ~storage->dir_create_mode); - fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); - umask(old_mask); - if (fd == -1) { - mail_storage_set_critical(&storage->storage, - "open(%s, O_CREAT) failed: %m", path); - } else if (storage->create_gid == (gid_t)-1) { - /* no group change */ - } else if (fchown(fd, (uid_t)-1, storage->create_gid) < 0) { - if (errno == EPERM) { - mail_storage_set_critical(&storage->storage, "%s", - eperm_error_get_chgrp("fchown", path, - storage->create_gid, - storage->create_gid_origin)); - } else { - mail_storage_set_critical(&storage->storage, - "fchown(%s, -1, %ld) failed: %m", - path, (long)storage->create_gid); - } - /* continue anyway */ - } - return fd; -} - -int dbox_file_header_write(struct dbox_file *file, struct ostream *output) -{ - string_t *hdr; - - hdr = t_str_new(128); - str_printfa(hdr, "%u %c%x %c%x\n", DBOX_VERSION, - DBOX_HEADER_MSG_HEADER_SIZE, - (unsigned int)sizeof(struct dbox_message_header), - DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time); - - file->file_header_size = str_len(hdr); - file->msg_header_size = sizeof(struct dbox_message_header); - return o_stream_send(output, str_data(hdr), str_len(hdr)); -} - -static int dbox_file_create(struct dbox_file *file) -{ - i_assert(file->fd == -1); - - file->fd = dbox_create_fd(file->storage, file->current_path); - if (file->fd == -1) - return -1; - file->output = o_stream_create_fd_file(file->fd, 0, FALSE); - if (dbox_file_header_write(file, file->output) < 0) { - dbox_file_set_syscall_error(file, "write()"); - return -1; - } - return 0; -} - -int dbox_file_open_or_create(struct dbox_file *file, bool *deleted_r) -{ - int ret; - - *deleted_r = FALSE; - - if (file->file_id == 0 && file->uid == 0) { - T_BEGIN { - ret = dbox_file_create(file) < 0 ? -1 : 1; - } T_END; - return ret; - } else if (file->input != NULL) - return 1; - else - return dbox_file_open(file, deleted_r); -} - -void dbox_file_close(struct dbox_file *file) -{ - dbox_file_unlock(file); - if (file->input != NULL) - i_stream_unref(&file->input); - if (file->output != NULL) - o_stream_unref(&file->output); - if (file->fd != -1) { - if (close(file->fd) < 0) - dbox_file_set_syscall_error(file, "close()"); - file->fd = -1; - } - file->cur_offset = (uoff_t)-1; -} - -int dbox_file_try_lock(struct dbox_file *file) -{ - int ret; - - i_assert(file->fd != -1); - - ret = file_try_lock(file->fd, file->current_path, F_WRLCK, - FILE_LOCK_METHOD_FCNTL, &file->lock); - if (ret < 0) { - mail_storage_set_critical(&file->storage->storage, - "file_try_lock(%s) failed: %m", file->current_path); - } - return ret; -} - -void dbox_file_unlock(struct dbox_file *file) -{ - struct stat st; - - if (file->lock != NULL) { - if (file->output != NULL) { - i_assert(o_stream_get_buffer_used_size(file->output) == 0); - if (fstat(file->fd, &st) == 0 && - (uoff_t)st.st_size != file->output->offset) - i_fatal("dbox file modified while locked"); - o_stream_unref(&file->output); - } - file_unlock(&file->lock); - } - if (file->input != NULL) - i_stream_sync(file->input); -} - -int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) -{ - struct dbox_message_header hdr; - struct stat st; - const unsigned char *data; - size_t size; - int ret; - - if (file->maildir_file) { - if (fstat(file->fd, &st) < 0) { - dbox_file_set_syscall_error(file, "fstat()"); - return -1; - } - *physical_size_r = st.st_size; - return 1; - } - - ret = i_stream_read_data(file->input, &data, &size, - file->msg_header_size - 1); - if (ret <= 0) { - if (file->input->stream_errno == 0) { - /* EOF, broken offset or file truncated */ - dbox_file_set_corrupted(file, "EOF reading msg header " - "(got %"PRIuSIZE_T"/%u bytes)", - size, file->msg_header_size); - return 0; - } - dbox_file_set_syscall_error(file, "read()"); - return -1; - } - memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size)); - if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) { - /* probably broken offset */ - dbox_file_set_corrupted(file, "msg header has bad magic value"); - return 0; - } - - if (data[file->msg_header_size-1] != '\n') { - dbox_file_set_corrupted(file, "msg header doesn't end with LF"); - return 0; - } - - *physical_size_r = hex2dec(hdr.message_size_hex, - sizeof(hdr.message_size_hex)); - return 1; -} - -int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uoff_t *physical_size_r, - struct istream **stream_r, bool *expunged_r) -{ - uoff_t size; - int ret; - - *expunged_r = FALSE; - - if (file->input == NULL) { - if ((ret = dbox_file_open(file, expunged_r)) <= 0 || - *expunged_r) - return ret; - } - - if (offset == 0) - offset = file->file_header_size; - - if (offset != file->cur_offset) { - i_stream_seek(file->input, offset); - ret = dbox_file_read_mail_header(file, &size); - if (ret <= 0) - return ret; - file->cur_offset = offset; - file->cur_physical_size = size; - } - i_stream_seek(file->input, offset + file->msg_header_size); - if (stream_r != NULL) { - *stream_r = i_stream_create_limit(file->input, - file->cur_physical_size); - } - *physical_size_r = file->cur_physical_size; - return 1; -} - -static int -dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset) -{ - const char *line; - int ret; - - i_stream_seek(file->input, *offset); - if ((ret = dbox_file_metadata_skip_header(file)) <= 0) - return ret; - - /* skip over the actual metadata */ - while ((line = i_stream_read_next_line(file->input)) != NULL) { - if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { - /* end of metadata */ - break; - } - } - *offset = file->input->v_offset; - return 1; -} - -void dbox_file_seek_rewind(struct dbox_file *file) -{ - file->cur_offset = (uoff_t)-1; -} - -int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r) -{ - uoff_t offset, size; - bool expunged; - int ret; - - if (file->cur_offset == (uoff_t)-1) { - /* first mail. we may not have read the file at all yet, - so set the offset afterwards. */ - offset = 0; - } else { - offset = file->cur_offset + file->msg_header_size + - file->cur_physical_size; - if ((ret = dbox_file_seek_next_at_metadata(file, &offset)) <= 0) { - *offset_r = file->cur_offset; - return ret; - } - } - *offset_r = offset; - - if (file->input != NULL && i_stream_is_eof(file->input)) { - *last_r = TRUE; - return 0; - } - *last_r = FALSE; - - ret = dbox_file_get_mail_stream(file, offset, &size, NULL, &expunged); - if (*offset_r == 0) - *offset_r = file->file_header_size; - return ret; -} - -static int -dbox_file_seek_append_pos(struct dbox_file *file, uoff_t *append_offset_r) -{ - struct stat st; - - if (file->file_version != DBOX_VERSION || - file->msg_header_size != sizeof(struct dbox_message_header)) { - /* created by an incompatible version, can't append */ - return 0; - } - - if (fstat(file->fd, &st) < 0) { - dbox_file_set_syscall_error(file, "fstat()"); - return -1; - } - *append_offset_r = st.st_size; - - file->output = o_stream_create_fd_file(file->fd, 0, FALSE); - o_stream_seek(file->output, st.st_size); - return 1; -} - -int dbox_file_get_append_stream(struct dbox_file *file, uoff_t *append_offset_r, - struct ostream **stream_r) -{ - int ret; - - if (file->fd == -1) { - /* creating a new file */ - i_assert(file->output == NULL); - i_assert(file->file_id == 0 && file->uid == 0); - - if (dbox_file_create(file) < 0) - return -1; - - if (file->single_mbox == NULL) { - /* creating a new multi-file. even though we don't - need it locked while writing to it, by the time - we rename() it it needs to be locked. so we might - as well do it here. */ - if ((ret = dbox_file_try_lock(file)) <= 0) { - if (ret < 0) - return -1; - mail_storage_set_critical( - &file->storage->storage, - "dbox: Couldn't lock created file: %s", - file->current_path); - return -1; - } - } - i_assert(file->output != NULL); - } else if (file->output == NULL) { - i_assert(file->lock != NULL || file->single_mbox != NULL); - - ret = dbox_file_seek_append_pos(file, append_offset_r); - if (ret <= 0) - return ret; - } - - o_stream_ref(file->output); - *stream_r = file->output; - return 1; -} - -uoff_t dbox_file_get_next_append_offset(struct dbox_file *file) -{ - i_assert(file->output != NULL); - - return file->output->offset; -} - -void dbox_file_cancel_append(struct dbox_file *file, uoff_t append_offset) -{ - (void)o_stream_flush(file->output); - - if (file->output->offset != append_offset) { - if (ftruncate(file->fd, append_offset) < 0) - dbox_file_set_syscall_error(file, "ftruncate()"); - o_stream_seek(file->output, append_offset); - } -} - -int dbox_file_flush_append(struct dbox_file *file) -{ - i_assert(file->output != NULL); - - if (o_stream_flush(file->output) < 0) { - dbox_file_set_syscall_error(file, "write()"); - return -1; - } - - if (!file->storage->storage.set->fsync_disable) { - if (fdatasync(file->fd) < 0) { - dbox_file_set_syscall_error(file, "fdatasync()"); - return -1; - } - } - return 0; -} - -int dbox_file_metadata_skip_header(struct dbox_file *file) -{ - struct dbox_metadata_header metadata_hdr; - const unsigned char *data; - size_t size; - int ret; - - ret = i_stream_read_data(file->input, &data, &size, - sizeof(metadata_hdr) - 1); - if (ret <= 0) { - if (file->input->stream_errno == 0) { - /* EOF, broken offset */ - dbox_file_set_corrupted(file, - "Unexpected EOF while reading metadata header"); - return 0; - } - dbox_file_set_syscall_error(file, "read()"); - return -1; - } - memcpy(&metadata_hdr, data, sizeof(metadata_hdr)); - if (memcmp(metadata_hdr.magic_post, DBOX_MAGIC_POST, - sizeof(metadata_hdr.magic_post)) != 0) { - /* probably broken offset */ - dbox_file_set_corrupted(file, - "metadata header has bad magic value"); - return 0; - } - i_stream_skip(file->input, sizeof(metadata_hdr)); - return 1; -} - -static int -dbox_file_metadata_read_at(struct dbox_file *file, uoff_t metadata_offset) -{ - const char *line; - int ret; - - if (file->metadata_pool != NULL) - p_clear(file->metadata_pool); - else { - file->metadata_pool = - pool_alloconly_create("dbox metadata", 1024); - } - p_array_init(&file->metadata, file->metadata_pool, 16); - - i_stream_seek(file->input, metadata_offset); - if ((ret = dbox_file_metadata_skip_header(file)) <= 0) - return ret; - - ret = 0; - while ((line = i_stream_read_next_line(file->input)) != NULL) { - if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { - /* end of metadata */ - ret = 1; - break; - } - line = p_strdup(file->metadata_pool, line); - array_append(&file->metadata, &line, 1); - } - if (ret == 0) - dbox_file_set_corrupted(file, "missing end-of-metadata line"); - return ret; -} - -int dbox_file_metadata_read(struct dbox_file *file) -{ - uoff_t metadata_offset; - int ret; - - i_assert(file->cur_offset != (uoff_t)-1); - - if (file->metadata_read_offset == file->cur_offset || - file->maildir_file) - return 1; - - metadata_offset = file->cur_offset + file->msg_header_size + - file->cur_physical_size; - ret = dbox_file_metadata_read_at(file, metadata_offset); - if (ret <= 0) - return ret; - - file->metadata_read_offset = file->cur_offset; - return 1; -} - -const char *dbox_file_metadata_get(struct dbox_file *file, - enum dbox_metadata_key key) -{ - const char *const *metadata; - unsigned int i, count; - - if (file->maildir_file) - return dbox_file_maildir_metadata_get(file, key); - - metadata = array_get(&file->metadata, &count); - for (i = 0; i < count; i++) { - if (*metadata[i] == (char)key) - return metadata[i] + 1; - } - return NULL; -} - -int dbox_file_move(struct dbox_file *file, bool alt_path) -{ - struct ostream *output; - const char *dest_dir, *temp_path, *dest_path; - struct stat st; - bool deleted; - int out_fd, ret = 0; - - i_assert(file->input != NULL); - i_assert(file->lock != NULL); - - if (file->alt_path == alt_path) - return 0; - - if (stat(file->current_path, &st) < 0 && errno == ENOENT) { - /* already expunged/moved by another session */ - dbox_file_unlock(file); - return 0; - } - - dest_dir = !alt_path ? dbox_file_get_primary_path(file) : - dbox_file_get_alt_path(file); - temp_path = t_strdup_printf("%s/%s", dest_dir, - dbox_generate_tmp_filename()); - - /* first copy the file. make sure to catch every possible error - since we really don't want to break the file. */ - out_fd = open(temp_path, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (out_fd == -1 && errno == ENOENT) { - if (mkdir_parents(dest_dir, 0700) < 0 && errno != EEXIST) { - mail_storage_set_critical(&file->storage->storage, - "mkdir_parents(%s) failed: %m", dest_dir); - return -1; - } - out_fd = open(temp_path, O_WRONLY | O_CREAT | O_TRUNC, 0600); - } - if (out_fd == -1) { - mail_storage_set_critical(&file->storage->storage, - "open(%s, O_CREAT) failed: %m", temp_path); - return -1; - } - output = o_stream_create_fd_file(out_fd, 0, FALSE); - i_stream_seek(file->input, 0); - while ((ret = o_stream_send_istream(output, file->input)) > 0) ; - if (ret == 0) - ret = o_stream_flush(output); - if (output->stream_errno != 0) { - errno = output->stream_errno; - mail_storage_set_critical(&file->storage->storage, - "write(%s) failed: %m", temp_path); - ret = -1; - } else if (file->input->stream_errno != 0) { - errno = file->input->stream_errno; - dbox_file_set_syscall_error(file, "ftruncate()"); - ret = -1; - } else if (ret < 0) { - mail_storage_set_critical(&file->storage->storage, - "o_stream_send_istream(%s, %s) " - "failed with unknown error", - temp_path, file->current_path); - } - o_stream_unref(&output); - - if (!file->storage->storage.set->fsync_disable && ret == 0) { - if (fsync(out_fd) < 0) { - mail_storage_set_critical(&file->storage->storage, - "fsync(%s) failed: %m", temp_path); - ret = -1; - } - } - if (close(out_fd) < 0) { - mail_storage_set_critical(&file->storage->storage, - "close(%s) failed: %m", temp_path); - ret = -1; - } - if (ret < 0) { - (void)unlink(temp_path); - return -1; - } - - /* the temp file was successfully written. rename it now to the - destination file. the destination shouldn't exist, but if it does - its contents should be the same (except for maybe older metadata) */ - dest_path = t_strdup_printf("%s/%s", dest_dir, file->fname); - if (rename(temp_path, dest_path) < 0) { - mail_storage_set_critical(&file->storage->storage, - "rename(%s, %s) failed: %m", temp_path, dest_path); - (void)unlink(temp_path); - return -1; - } - if (!file->storage->storage.set->fsync_disable) { - if (fdatasync_path(dest_dir) < 0) { - mail_storage_set_critical(&file->storage->storage, - "fdatasync(%s) failed: %m", dest_dir); - (void)unlink(dest_path); - return -1; - } - } - if (unlink(file->current_path) < 0) { - dbox_file_set_syscall_error(file, "unlink()"); - if (errno == EACCES) { - /* configuration problem? revert the write */ - (void)unlink(dest_path); - } - /* who knows what happened to the file. keep both just to be - sure both won't get deleted. */ - return -1; - } - - /* file was successfully moved - reopen it */ - dbox_file_close(file); - if (dbox_file_open(file, &deleted) <= 0) { - mail_storage_set_critical(&file->storage->storage, - "dbox_file_move(%s): reopening file failed", dest_path); - return -1; - } - return 0; -} - -void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, - uoff_t message_size) -{ - memset(dbox_msg_hdr, ' ', sizeof(*dbox_msg_hdr)); - memcpy(dbox_msg_hdr->magic_pre, DBOX_MAGIC_PRE, - sizeof(dbox_msg_hdr->magic_pre)); - dbox_msg_hdr->type = DBOX_MESSAGE_TYPE_NORMAL; - dec2hex(dbox_msg_hdr->message_size_hex, message_size, - sizeof(dbox_msg_hdr->message_size_hex)); - dbox_msg_hdr->save_lf = '\n'; -}
--- a/src/lib-storage/index/dbox/dbox-file.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,210 +0,0 @@ -#ifndef DBOX_FILE_H -#define DBOX_FILE_H - -/* The file begins with a header followed by zero or more messages: - - <dbox message header> - <LF> - <message body> - <metadata> - - Metadata block begins with DBOX_MAGIC_POST, followed by zero or more lines - in format <key character><value><LF>. The block ends with an empty line. - Unknown metadata should be ignored, but preserved when copying. - - There should be no duplicates for the current metadata, but future - extensions may need them so they should be preserved. -*/ -#define DBOX_VERSION 2 -#define DBOX_MAGIC_PRE "\001\002" -#define DBOX_MAGIC_POST "\n\001\003\n" - -enum dbox_header_key { - /* Offset for appending next message. In %08x format so it can be - updated without moving data in header. If messages have been - expunged and file must not be appended anymore, the value is filled - with 'X'. */ - DBOX_HEADER_OLDV1_APPEND_OFFSET = 'A', - /* Must be sizeof(struct dbox_message_header) when appending (hex) */ - DBOX_HEADER_MSG_HEADER_SIZE = 'M', - /* Creation UNIX timestamp (hex) */ - DBOX_HEADER_CREATE_STAMP = 'C' -}; - -enum dbox_metadata_key { - /* Globally unique identifier for the message. Preserved when - copying. */ - DBOX_METADATA_GUID = 'G', - /* POP3 UIDL overriding the default format */ - DBOX_METADATA_POP3_UIDL = 'P', - /* Received UNIX timestamp in hex */ - DBOX_METADATA_RECEIVED_TIME = 'R', - /* Saved UNIX timestamp in hex */ - DBOX_METADATA_SAVE_TIME = 'S', - /* Virtual message size in hex (line feeds counted as CRLF) */ - DBOX_METADATA_VIRTUAL_SIZE = 'V', - /* Pointer to external message data. Format is: - 1*(<start offset> <byte count> <ref>) */ - DBOX_METADATA_EXT_REF = 'X', - /* Mailbox name where this message was originally saved to. - When rebuild finds a message whose mailbox is unknown, it's - placed to this mailbox. */ - DBOX_METADATA_ORIG_MAILBOX = 'B', - - /* metadata used by old Dovecot versions */ - DBOX_METADATA_OLDV1_EXPUNGED = 'E', - DBOX_METADATA_OLDV1_FLAGS = 'F', - DBOX_METADATA_OLDV1_KEYWORDS = 'K', - DBOX_METADATA_OLDV1_SPACE = ' ' -}; - -enum dbox_message_type { - /* Normal message */ - DBOX_MESSAGE_TYPE_NORMAL = 'N', - /* Parts of the message exists outside the following data. - See the metadata for how to find them. */ - DBOX_MESSAGE_TYPE_EXT_REFS = 'E' -}; - -struct dbox_message_header { - unsigned char magic_pre[2]; - unsigned char type; - unsigned char space1; - unsigned char oldv1_uid_hex[8]; - unsigned char space2; - unsigned char message_size_hex[16]; - /* <space reserved for future extensions, LF is always last> */ - unsigned char save_lf; -}; - -struct dbox_metadata_header { - unsigned char magic_post[sizeof(DBOX_MAGIC_POST)-1]; -}; - -struct dbox_file { - struct dbox_storage *storage; - /* set only for single-msg-per-file */ - struct dbox_mailbox *single_mbox; - - int refcount; - /* uid is for single-msg-per-file, file_id for multi-msgs-per-file */ - uint32_t uid, file_id; - - time_t create_time; - unsigned int file_version; - unsigned int file_header_size; - unsigned int msg_header_size; - - uoff_t cur_offset; - uoff_t cur_physical_size; - /* first appended message's offset (while appending) */ - uoff_t first_append_offset; - - char *fname; - char *current_path; - - int fd; - struct istream *input; - struct ostream *output; - struct file_lock *lock; - - /* Metadata for the currently seeked metadata block. */ - pool_t metadata_pool; - ARRAY_DEFINE(metadata, const char *); - uoff_t metadata_read_offset; - - unsigned int alt_path:1; - unsigned int maildir_file:1; - unsigned int deleted:1; - unsigned int corrupted:1; -}; - -#define dbox_file_is_open(file) ((file)->input != NULL) - -struct dbox_file * -dbox_file_init_single(struct dbox_mailbox *mbox, uint32_t uid); -struct dbox_file * -dbox_file_init_multi(struct dbox_storage *storage, uint32_t file_id); -void dbox_file_unref(struct dbox_file **file); - -/* Free all currently opened files. */ -void dbox_files_free(struct dbox_storage *storage); -/* Flush all cached input data from opened files. */ -void dbox_files_sync_input(struct dbox_storage *storage); - -/* Assign a newly created file a new id. For single files assign UID, - for multi files assign map UID. */ -int dbox_file_assign_id(struct dbox_file *file, uint32_t id); - -/* Open the file. Returns 1 if ok, 0 if file header is corrupted, -1 if error. - If file is deleted, deleted_r=TRUE and 1 is returned. */ -int dbox_file_open(struct dbox_file *file, bool *deleted_r); -/* Open the file if uid or file_id is not 0, otherwise create it. */ -int dbox_file_open_or_create(struct dbox_file *file, bool *deleted_r); -/* Close the file handle from the file, but don't free it. */ -void dbox_file_close(struct dbox_file *file); - -/* Try to lock the dbox file. Returns 1 if ok, 0 if already locked by someone - else, -1 if error. */ -int dbox_file_try_lock(struct dbox_file *file); -void dbox_file_unlock(struct dbox_file *file); - -/* Seek to given offset in file and return the message's input stream - and physical size. Returns 1 if ok/expunged, 0 if file/offset is corrupted, - -1 if I/O error. */ -int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uoff_t *physical_size_r, - struct istream **stream_r, bool *expunged_r); -/* Start seeking at the beginning of the file. */ -void dbox_file_seek_rewind(struct dbox_file *file); -/* Seek to next message after current one. If there are no more messages, - returns 0 and last_r is set to TRUE. Returns 1 if ok, 0 if file is - corrupted, -1 if I/O error. */ -int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r); - -/* Returns TRUE if mail_size bytes can be appended to the file. */ -bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size); -/* Get output stream for appending a new message. Returns 1 if ok, 0 if file - can't be appended to (old file version or corruption) or -1 if error. */ -int dbox_file_get_append_stream(struct dbox_file *file, uoff_t *append_offset_r, - struct ostream **stream_r); -/* Returns the next offset for append a message. dbox_file_get_append_stream() - must have been called for this file already at least once. */ -uoff_t dbox_file_get_next_append_offset(struct dbox_file *file); -/* Truncate file to append_offset */ -void dbox_file_cancel_append(struct dbox_file *file, uoff_t append_offset); -/* Flush writes to dbox file. */ -int dbox_file_flush_append(struct dbox_file *file); - -/* Read current message's metadata. Returns 1 if ok, 0 if metadata is - corrupted, -1 if I/O error. */ -int dbox_file_metadata_read(struct dbox_file *file); -/* Return wanted metadata value, or NULL if not found. */ -const char *dbox_file_metadata_get(struct dbox_file *file, - enum dbox_metadata_key key); - -/* Move the file to alt path or back. */ -int dbox_file_move(struct dbox_file *file, bool alt_path); -/* Fix a broken dbox file by rename()ing over it with a fixed file. Everything - before start_offset is assumed to be valid and is simply copied. The file - is reopened afterwards. Returns 0 if ok, -1 if I/O error. */ -int dbox_file_fix(struct dbox_file *file, uoff_t start_offset); - -/* Fill dbox_message_header with given size. */ -void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, - uoff_t message_size); - -const char *dbox_file_get_primary_path(struct dbox_file *file); -const char *dbox_file_get_alt_path(struct dbox_file *file); -void dbox_file_set_syscall_error(struct dbox_file *file, const char *function); -void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) - ATTR_FORMAT(2, 3); - -/* private: */ -char *dbox_generate_tmp_filename(void); -int dbox_create_fd(struct dbox_storage *storage, const char *path); -int dbox_file_header_write(struct dbox_file *file, struct ostream *output); -int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r); -int dbox_file_metadata_skip_header(struct dbox_file *file); - -#endif
--- a/src/lib-storage/index/dbox/dbox-mail.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,433 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "ioloop.h" -#include "istream.h" -#include "str.h" -#include "index-mail.h" -#include "dbox-storage.h" -#include "dbox-map.h" -#include "dbox-file.h" - -#include <stdlib.h> -#include <sys/stat.h> - -struct dbox_mail { - struct index_mail imail; - - struct dbox_file *open_file; - uoff_t offset; - uint32_t map_uid; -}; - -struct mail * -dbox_mail_alloc(struct mailbox_transaction_context *t, - enum mail_fetch_field wanted_fields, - struct mailbox_header_lookup_ctx *wanted_headers) -{ - struct dbox_mail *mail; - pool_t pool; - - pool = pool_alloconly_create("mail", 1024); - mail = p_new(pool, struct dbox_mail, 1); - mail->imail.mail.pool = pool; - - index_mail_init(&mail->imail, t, wanted_fields, wanted_headers); - return &mail->imail.mail.mail; -} - -static void dbox_mail_close(struct mail *_mail) -{ - struct dbox_mail *mail = (struct dbox_mail *)_mail; - - if (mail->open_file != NULL) - dbox_file_unref(&mail->open_file); - index_mail_close(_mail); -} - -int dbox_mail_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view, - uint32_t seq, uint32_t *map_uid_r) -{ - const struct dbox_mail_index_record *dbox_rec; - struct dbox_index_header hdr; - const void *data; - uint32_t cur_map_uid_validity; - bool expunged; - - mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged); - dbox_rec = data; - if (dbox_rec == NULL || dbox_rec->map_uid == 0) { - *map_uid_r = 0; - return 0; - } - - if (mbox->map_uid_validity == 0) { - if (dbox_read_header(mbox, &hdr) < 0) { - mbox->storage->sync_rebuild = TRUE; - return -1; - } - mbox->map_uid_validity = hdr.map_uid_validity; - } - if (dbox_map_open(mbox->storage->map, TRUE) < 0) - return -1; - - cur_map_uid_validity = dbox_map_get_uid_validity(mbox->storage->map); - if (cur_map_uid_validity != mbox->map_uid_validity) { - mail_storage_set_critical(&mbox->storage->storage, - "dbox %s: map uidvalidity mismatch (%u vs %u)", - mbox->ibox.box.path, mbox->map_uid_validity, - cur_map_uid_validity); - mbox->storage->sync_rebuild = TRUE; - return -1; - } - *map_uid_r = dbox_rec->map_uid; - return 0; -} - -static void dbox_mail_set_expunged(struct dbox_mail *mail) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox; - struct mail *_mail = &mail->imail.mail.mail; - - (void)mail_index_refresh(mbox->ibox.index); - if (mail_index_is_expunged(mbox->ibox.view, _mail->seq)) { - mail_set_expunged(_mail); - return; - } - - if (mail->map_uid != 0) { - dbox_map_set_corrupted(mbox->storage->map, - "Unexpectedly lost uid=%u map_uid=%u", - _mail->uid, mail->map_uid); - } else { - mail_storage_set_critical(&mbox->storage->storage, - "Unexpectedly lost uid=%u", _mail->uid); - } - mbox->storage->sync_rebuild = TRUE; -} - -static int dbox_mail_open_init(struct dbox_mail *mail) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox; - struct mail *_mail = &mail->imail.mail.mail; - uint32_t file_id; - int ret; - - if (mail->map_uid == 0) - mail->open_file = dbox_file_init_single(mbox, _mail->uid); - else if ((ret = dbox_map_lookup(mbox->storage->map, mail->map_uid, - &file_id, &mail->offset)) <= 0) { - if (ret < 0) - return -1; - - /* map_uid doesn't exist anymore. either it - got just expunged or the map index is - corrupted. */ - dbox_mail_set_expunged(mail); - return -1; - } else { - mail->open_file = dbox_file_init_multi(mbox->storage, file_id); - } - return 0; -} - -static int dbox_mail_open(struct dbox_mail *mail, - uoff_t *offset_r, struct dbox_file **file_r) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox; - struct mail *_mail = &mail->imail.mail.mail; - uint32_t prev_file_id = 0; - bool deleted; - - if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) - return mail_set_aborted(_mail); - - do { - if (mail->open_file != NULL) { - /* already open */ - } else if (_mail->uid != 0) { - if (dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq, - &mail->map_uid) < 0) - return -1; - if (dbox_mail_open_init(mail) < 0) - return -1; - } else { - /* mail is being saved in this transaction */ - mail->open_file = - dbox_save_file_get_file(_mail->transaction, - _mail->seq, - &mail->offset); - mail->open_file->refcount++; - break; - } - - if (!dbox_file_is_open(mail->open_file)) - mail->imail.mail.stats_open_lookup_count++; - if (dbox_file_open(mail->open_file, &deleted) <= 0) - return -1; - if (deleted) { - /* either it's expunged now or moved to another file. */ - if (mail->open_file->file_id == prev_file_id) { - dbox_mail_set_expunged(mail); - return -1; - } - prev_file_id = mail->open_file->file_id; - if (dbox_map_refresh(mbox->storage->map) < 0) - return -1; - dbox_file_unref(&mail->open_file); - } - } while (mail->open_file == NULL); - - *file_r = mail->open_file; - *offset_r = mail->offset; - return 0; -} - -static int -dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r) -{ - uoff_t offset, size; - bool expunged; - - if (dbox_mail_open(mail, &offset, file_r) < 0) - return -1; - - if (dbox_file_get_mail_stream(*file_r, offset, &size, - NULL, &expunged) <= 0) - return -1; - i_assert(!expunged); - if (dbox_file_metadata_read(*file_r) <= 0) - return -1; - return 0; -} - -static int dbox_mail_get_received_date(struct mail *_mail, time_t *date_r) -{ - struct dbox_mail *mail = (struct dbox_mail *)_mail; - struct index_mail_data *data = &mail->imail.data; - struct dbox_file *file; - const char *value; - - if (index_mail_get_received_date(_mail, date_r) == 0) - return 0; - - if (dbox_mail_metadata_read(mail, &file) < 0) - return -1; - - value = dbox_file_metadata_get(file, DBOX_METADATA_RECEIVED_TIME); - data->received_date = value == NULL ? 0 : strtoul(value, NULL, 16); - *date_r = data->received_date; - return 0; -} - -static bool multi_dbox_get_save_date(struct mail *mail, time_t *date_r) -{ - struct dbox_mailbox *mbox = - (struct dbox_mailbox *)mail->transaction->box; - const struct dbox_mail_index_record *dbox_rec; - const void *data; - bool expunged; - - mail_index_lookup_ext(mbox->ibox.view, mail->seq, - mbox->dbox_ext_id, &data, &expunged); - dbox_rec = data; - if (dbox_rec == NULL || dbox_rec->map_uid == 0) - return FALSE; - - *date_r = dbox_rec->save_date; - return TRUE; -} - -static int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r) -{ - struct dbox_mail *mail = (struct dbox_mail *)_mail; - struct index_mail_data *data = &mail->imail.data; - struct dbox_file *file; - struct stat st; - const char *value; - - if (multi_dbox_get_save_date(_mail, date_r)) - return 0; - - if (index_mail_get_save_date(_mail, date_r) == 0) - return 0; - - if (dbox_mail_metadata_read(mail, &file) < 0) - return -1; - - value = dbox_file_metadata_get(file, DBOX_METADATA_SAVE_TIME); - data->save_date = value == NULL ? 0 : strtoul(value, NULL, 16); - - if (data->save_date == 0) { - /* missing / corrupted save time - use the file's ctime */ - i_assert(file->fd != -1); - mail->imail.mail.stats_fstat_lookup_count++; - if (fstat(file->fd, &st) < 0) { - mail_storage_set_critical(_mail->box->storage, - "fstat(%s) failed: %m", file->current_path); - return -1; - } - data->save_date = st.st_ctime; - } - *date_r = data->save_date; - return 0; -} - -static int dbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) -{ - struct dbox_mail *mail = (struct dbox_mail *)_mail; - struct index_mail_data *data = &mail->imail.data; - struct dbox_file *file; - const char *value; - - if (index_mail_get_cached_virtual_size(&mail->imail, size_r)) - return 0; - - if (dbox_mail_metadata_read(mail, &file) < 0) - return -1; - - value = dbox_file_metadata_get(file, DBOX_METADATA_VIRTUAL_SIZE); - if (value == NULL) - return index_mail_get_virtual_size(_mail, size_r); - - data->virtual_size = strtoul(value, NULL, 16); - *size_r = data->virtual_size; - return 0; -} - -static int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) -{ - struct index_mail *mail = (struct index_mail *)_mail; - struct index_mail_data *data = &mail->data; - struct istream *input; - - if (index_mail_get_physical_size(_mail, size_r) == 0) - return 0; - - if (mail_get_stream(_mail, NULL, NULL, &input) < 0) - return -1; - - i_assert(data->physical_size != (uoff_t)-1); - *size_r = data->physical_size; - return 0; -} - -static int -dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key, - enum index_cache_field cache_field, - const char **value_r) -{ - struct index_mail *imail = &mail->imail; - const struct mail_cache_field *cache_fields = imail->ibox->cache_fields; - struct dbox_file *file; - const char *value; - string_t *str; - - str = str_new(imail->data_pool, 64); - if (mail_cache_lookup_field(imail->trans->cache_view, str, - imail->mail.mail.seq, - cache_fields[cache_field].idx) > 0) { - *value_r = str_c(str); - return 0; - } - - if (dbox_mail_metadata_read(mail, &file) < 0) - return -1; - - value = dbox_file_metadata_get(file, key); - if (value == NULL) - value = ""; - index_mail_cache_add_idx(imail, cache_fields[cache_field].idx, - value, strlen(value)+1); - *value_r = value; - return 0; -} - -static int -dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, - const char **value_r) -{ - struct dbox_mail *mail = (struct dbox_mail *)_mail; - - /* keep the UIDL in cache file, otherwise POP3 would open all - mail files and read the metadata. same for GUIDs if they're - used. */ - switch (field) { - case MAIL_FETCH_UIDL_BACKEND: - return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, - MAIL_CACHE_POP3_UIDL, value_r); - case MAIL_FETCH_GUID: - return dbox_get_cached_metadata(mail, DBOX_METADATA_GUID, - MAIL_CACHE_GUID, value_r); - default: - break; - } - - return index_mail_get_special(_mail, field, value_r); -} - -static int -dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, - struct message_size *body_size, struct istream **stream_r) -{ - struct dbox_mail *mail = (struct dbox_mail *)_mail; - struct index_mail_data *data = &mail->imail.data; - struct istream *input; - uoff_t offset, size; - bool expunged; - int ret; - - if (data->stream == NULL) { - if (dbox_mail_open(mail, &offset, &mail->open_file) < 0) - return -1; - - ret = dbox_file_get_mail_stream(mail->open_file, offset, - &size, &input, &expunged); - if (ret <= 0) { - if (ret < 0) - return -1; - dbox_file_set_corrupted(mail->open_file, - "uid=%u points to broken data at offset=" - "%"PRIuUOFF_T, _mail->uid, offset); - return -1; - } - i_assert(!expunged); - data->physical_size = size; - data->stream = input; - } - - return index_mail_init_stream(&mail->imail, hdr_size, body_size, - stream_r); -} - -struct mail_vfuncs dbox_mail_vfuncs = { - dbox_mail_close, - index_mail_free, - index_mail_set_seq, - index_mail_set_uid, - index_mail_set_uid_cache_updates, - - index_mail_get_flags, - index_mail_get_keywords, - index_mail_get_keyword_indexes, - index_mail_get_modseq, - index_mail_get_parts, - index_mail_get_date, - dbox_mail_get_received_date, - dbox_mail_get_save_date, - dbox_mail_get_virtual_size, - dbox_mail_get_physical_size, - index_mail_get_first_header, - index_mail_get_headers, - index_mail_get_header_stream, - dbox_mail_get_stream, - dbox_mail_get_special, - index_mail_update_flags, - index_mail_update_keywords, - index_mail_update_modseq, - index_mail_update_uid, - NULL, - index_mail_expunge, - index_mail_set_cache_corrupted, - index_mail_get_index_mail -};
--- a/src/lib-storage/index/dbox/dbox-map-private.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,50 +0,0 @@ -#ifndef DBOX_MAP_PRIVATE_H -#define DBOX_MAP_PRIVATE_H - -#include "dbox-map.h" - -struct dbox_mail_lookup_rec { - uint32_t map_uid; - uint16_t refcount; - struct dbox_mail_index_map_record rec; -}; - -struct dbox_map { - struct dbox_storage *storage; - struct mail_index *index; - struct mail_index_view *view; - uint32_t created_uid_validity; - - uint32_t map_ext_id, ref_ext_id; - ARRAY_TYPE(seq_range) ref0_file_ids; -}; - -struct dbox_map_append { - struct dbox_file *file; - uoff_t offset, size; -}; - -struct dbox_map_append_context { - struct dbox_mailbox *mbox; - struct dbox_map *map; - - struct mail_index_sync_ctx *sync_ctx; - struct mail_index_view *sync_view; - struct mail_index_transaction *sync_trans, *trans; - - ARRAY_DEFINE(files, struct dbox_file *); - ARRAY_DEFINE(appends, struct dbox_map_append); - - uint32_t first_new_file_id; - uint32_t orig_next_uid; - - unsigned int files_nonappendable_count; - - unsigned int failed:1; - unsigned int committed:1; -}; - -int dbox_map_view_lookup_rec(struct dbox_map *map, struct mail_index_view *view, - uint32_t seq, struct dbox_mail_lookup_rec *rec_r); - -#endif
--- a/src/lib-storage/index/dbox/dbox-map.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1175 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "hash.h" -#include "ostream.h" -#include "mkdir-parents.h" -#include "dbox-storage.h" -#include "dbox-file.h" -#include "dbox-map-private.h" - -#define MAX_BACKWARDS_LOOKUPS 10 - -#define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10) -#define DBOX_FORCE_PURGE_MIN_RATIO 0.5 - -struct dbox_map_transaction_context { - struct dbox_map *map; - struct mail_index_transaction *trans; - struct mail_index_sync_ctx *sync_ctx; - - unsigned int changed:1; - unsigned int success:1; -}; - -void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...) -{ - va_list args; - - va_start(args, format); - mail_storage_set_critical(&map->storage->storage, - "dbox map %s corrupted: %s", - map->index->filepath, - t_strdup_vprintf(format, args)); - va_end(args); -} - -struct dbox_map *dbox_map_init(struct dbox_storage *storage) -{ - struct dbox_map *map; - - map = i_new(struct dbox_map, 1); - map->storage = storage; - map->index = mail_index_alloc(storage->storage_dir, - DBOX_GLOBAL_INDEX_PREFIX); - map->map_ext_id = mail_index_ext_register(map->index, "map", - sizeof(struct dbox_mail_index_map_header), - sizeof(struct dbox_mail_index_map_record), - sizeof(uint32_t)); - map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0, - sizeof(uint16_t), sizeof(uint16_t)); - map->created_uid_validity = ioloop_time; - return map; -} - -void dbox_map_deinit(struct dbox_map **_map) -{ - struct dbox_map *map = *_map; - - *_map = NULL; - - if (array_is_created(&map->ref0_file_ids)) - array_free(&map->ref0_file_ids); - if (map->view != NULL) - mail_index_view_close(&map->view); - mail_index_free(&map->index); - i_free(map); -} - -static int dbox_map_mkdir_storage(struct dbox_storage *storage) -{ - if (mkdir_parents_chgrp(storage->storage_dir, storage->dir_create_mode, - storage->create_gid, - storage->create_gid_origin) < 0 && - errno != EEXIST) { - mail_storage_set_critical(&storage->storage, - "mkdir(%s) failed: %m", storage->storage_dir); - return -1; - } - return 0; -} - -int dbox_map_open(struct dbox_map *map, bool create_missing) -{ - struct mail_storage *storage = &map->storage->storage; - enum mail_index_open_flags open_flags; - int ret; - - if (map->view != NULL) { - /* already opened */ - return 0; - } - - open_flags = MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY | - mail_storage_settings_to_index_flags(storage->set); - if (create_missing) { - open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; - if (dbox_map_mkdir_storage(map->storage) < 0) - return -1; - } - ret = mail_index_open(map->index, open_flags, - storage->set->parsed_lock_method); - if (ret < 0) { - mail_storage_set_internal_error(storage); - mail_index_reset_error(map->index); - return -1; - } - if (ret == 0) { - /* index not found - for now just return failure */ - return -1; - } - - map->view = mail_index_view_open(map->index); - return 0; -} - -int dbox_map_refresh(struct dbox_map *map) -{ - struct mail_index_view_sync_ctx *ctx; - bool delayed_expunges; - - /* some open files may have read partially written mails. now that - map syncing makes the new mails visible, we need to make sure the - partial data is flushed out of memory */ - dbox_files_sync_input(map->storage); - - if (mail_index_refresh(map->view->index) < 0) { - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - return -1; - } - ctx = mail_index_view_sync_begin(map->view, - MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); - if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - return -1; - } - return 0; -} - -static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq, - uint32_t *file_id_r, uoff_t *offset_r, - uoff_t *size_r) -{ - const struct dbox_mail_index_map_record *rec; - const void *data; - uint32_t uid; - bool expunged; - - mail_index_lookup_ext(map->view, seq, map->map_ext_id, - &data, &expunged); - rec = data; - - if (rec == NULL || rec->file_id == 0) { - mail_index_lookup_uid(map->view, seq, &uid); - dbox_map_set_corrupted(map, "file_id=0 for map_uid=%u", uid); - return -1; - } - - *file_id_r = rec->file_id; - *offset_r = rec->offset; - *size_r = rec->size; - return 0; -} - -static int -dbox_map_get_seq(struct dbox_map *map, uint32_t map_uid, uint32_t *seq_r) -{ - if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) { - /* not found - try again after a refresh */ - if (dbox_map_refresh(map) < 0) - return -1; - if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) - return 0; - } - return 1; -} - -int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, - uint32_t *file_id_r, uoff_t *offset_r) -{ - uint32_t seq; - uoff_t size; - int ret; - - if (dbox_map_open(map, TRUE) < 0) - return -1; - - if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0) - return ret; - - if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r, &size) < 0) - return -1; - return 1; -} - -int dbox_map_view_lookup_rec(struct dbox_map *map, struct mail_index_view *view, - uint32_t seq, struct dbox_mail_lookup_rec *rec_r) -{ - const uint16_t *ref16_p; - const void *data; - bool expunged; - - memset(rec_r, 0, sizeof(*rec_r)); - mail_index_lookup_uid(view, seq, &rec_r->map_uid); - - mail_index_lookup_ext(view, seq, map->map_ext_id, &data, &expunged); - if (data == NULL) { - dbox_map_set_corrupted(map, "missing map extension"); - return -1; - } - memcpy(&rec_r->rec, data, sizeof(rec_r->rec)); - - mail_index_lookup_ext(view, seq, map->ref_ext_id, &data, &expunged); - if (data == NULL) { - dbox_map_set_corrupted(map, "missing ref extension"); - return -1; - } - ref16_p = data; - rec_r->refcount = *ref16_p; - return 0; -} - -int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id, - ARRAY_TYPE(dbox_map_file_msg) *recs) -{ - const struct mail_index_header *hdr; - struct dbox_mail_lookup_rec rec; - struct dbox_map_file_msg msg; - uint32_t seq; - - if (dbox_map_refresh(map) < 0) - return -1; - hdr = mail_index_get_header(map->view); - - memset(&msg, 0, sizeof(msg)); - for (seq = 1; seq <= hdr->messages_count; seq++) { - if (dbox_map_view_lookup_rec(map, map->view, seq, &rec) < 0) - return -1; - - if (rec.rec.file_id == file_id) { - msg.map_uid = rec.map_uid; - msg.offset = rec.rec.offset; - msg.refcount = rec.refcount; - array_append(recs, &msg, 1); - } - } - return 0; -} - -struct dbox_file_size { - uoff_t file_size; - uoff_t ref0_size; -}; - -static void dbox_map_filter_zero_refs(struct dbox_map *map) -{ - ARRAY_TYPE(seq_range) new_ref0_file_ids; - struct hash_table *hash; - struct dbox_file_size *size; - struct seq_range_iter iter; - const struct mail_index_header *hdr; - const struct dbox_mail_index_map_record *rec; - const uint16_t *ref16_p; - const void *data; - uint32_t seq, file_id; - unsigned int i; - bool expunged; - pool_t pool; - - pool = pool_alloconly_create("dbox zero ref count", 8*1024); - hash = hash_table_create(default_pool, pool, 0, NULL, NULL); - - /* count file sizes */ - hdr = mail_index_get_header(map->view); - for (seq = 1; seq <= hdr->messages_count; seq++) { - mail_index_lookup_ext(map->view, seq, map->map_ext_id, - &data, &expunged); - if (data == NULL || expunged) - continue; - rec = data; - - if (!seq_range_exists(&map->ref0_file_ids, rec->file_id)) - continue; - - /* this file has at least some zero references. count how many - bytes it has in total and how much of it has refcount=0. */ - mail_index_lookup_ext(map->view, seq, map->ref_ext_id, - &data, &expunged); - if (data == NULL || expunged) - continue; - ref16_p = data; - - size = hash_table_lookup(hash, POINTER_CAST(rec->file_id)); - if (size == NULL) { - size = p_new(pool, struct dbox_file_size, 1); - hash_table_insert(hash, POINTER_CAST(rec->file_id), - size); - } - if (*ref16_p == 0) - size->ref0_size += rec->size; - if (size->file_size < rec->offset + rec->size) - size->file_size = rec->offset + rec->size; - } - - /* now drop the files that don't have enough deleted space */ - seq_range_array_iter_init(&iter, &map->ref0_file_ids); i = 0; - p_array_init(&new_ref0_file_ids, pool, - array_count(&map->ref0_file_ids)); - while (seq_range_array_iter_nth(&iter, i++, &file_id)) { - size = hash_table_lookup(hash, POINTER_CAST(file_id)); - if (size->ref0_size*100 / size->file_size >= - map->storage->set->dbox_purge_min_percentage) - seq_range_array_add(&new_ref0_file_ids, 0, file_id); - } - seq_range_array_intersect(&map->ref0_file_ids, &new_ref0_file_ids); - - hash_table_destroy(&hash); - pool_unref(&pool); -} - -bool dbox_map_want_purge(struct dbox_map *map) -{ - const struct mail_index_header *hdr; - const struct dbox_mail_index_map_record *rec; - const uint16_t *ref16_p; - const void *data; - uoff_t ref0_size, total_size; - bool expunged; - uint32_t seq; - - if (map->storage->set->dbox_purge_min_percentage >= 100) { - /* we never purge anything */ - return FALSE; - } - - ref0_size = total_size = 0; - hdr = mail_index_get_header(map->view); - for (seq = 1; seq <= hdr->messages_count; seq++) { - mail_index_lookup_ext(map->view, seq, map->map_ext_id, - &data, &expunged); - if (data == NULL || expunged) - continue; - rec = data; - - mail_index_lookup_ext(map->view, seq, map->ref_ext_id, - &data, &expunged); - if (data == NULL || expunged) - continue; - ref16_p = data; - - if (*ref16_p == 0) - ref0_size += rec->size; - total_size += rec->size; - } - - if (ref0_size < DBOX_FORCE_PURGE_MIN_BYTES) - return FALSE; - if ((float)ref0_size / (float)total_size < DBOX_FORCE_PURGE_MIN_RATIO) - return FALSE; - return TRUE; -} - -const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map) -{ - const struct mail_index_header *hdr; - const struct dbox_mail_index_map_record *rec; - const uint16_t *ref16_p; - const void *data; - uint32_t seq; - bool expunged; - - if (array_is_created(&map->ref0_file_ids)) - array_clear(&map->ref0_file_ids); - else - i_array_init(&map->ref0_file_ids, 64); - - if (map->storage->set->dbox_purge_min_percentage >= 100) { - /* we're never purging anything */ - return &map->ref0_file_ids; - } - - if (dbox_map_open(map, FALSE) < 0) { - /* some internal error */ - return &map->ref0_file_ids; - } - (void)dbox_map_refresh(map); - - hdr = mail_index_get_header(map->view); - for (seq = 1; seq <= hdr->messages_count; seq++) { - mail_index_lookup_ext(map->view, seq, map->ref_ext_id, - &data, &expunged); - if (data != NULL && !expunged) { - ref16_p = data; - if (*ref16_p != 0) - continue; - } - - mail_index_lookup_ext(map->view, seq, map->map_ext_id, - &data, &expunged); - if (data != NULL && !expunged) { - rec = data; - seq_range_array_add(&map->ref0_file_ids, 0, - rec->file_id); - } - } - if (map->storage->set->dbox_purge_min_percentage > 0 && - array_count(&map->ref0_file_ids) > 0) - dbox_map_filter_zero_refs(map); - return &map->ref0_file_ids; -} - -struct dbox_map_transaction_context * -dbox_map_transaction_begin(struct dbox_map *map, bool external) -{ - struct dbox_map_transaction_context *ctx; - enum mail_index_transaction_flags flags = - MAIL_INDEX_TRANSACTION_FLAG_FSYNC; - - if (external) - flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; - - ctx = i_new(struct dbox_map_transaction_context, 1); - ctx->map = map; - if (dbox_map_open(map, FALSE) == 0 && - dbox_map_refresh(map) == 0) - ctx->trans = mail_index_transaction_begin(map->view, flags); - return ctx; -} - -static void -dbox_map_sync_handle(struct dbox_map *map, struct mail_index_sync_ctx *sync_ctx) -{ - struct mail_index_sync_rec sync_rec; - uint32_t seq1, seq2; - uoff_t offset1, offset2; - - mail_index_sync_get_offsets(sync_ctx, &seq1, &offset1, &seq2, &offset2); - if (offset1 != offset2 || seq1 != seq2) { - /* something had crashed. need a full resync. */ - i_warning("dbox %s: Inconsistency in map index " - "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")", - map->storage->storage_dir, - seq1, offset1, seq2, offset2); - map->storage->sync_rebuild = TRUE; - } else { - while (mail_index_sync_next(sync_ctx, &sync_rec)) ; - } -} - -int dbox_map_transaction_commit(struct dbox_map_transaction_context *ctx) -{ - struct dbox_map *map = ctx->map; - struct mail_index_view *view; - struct mail_index_transaction *sync_trans; - int ret; - - if (!ctx->changed) - return 0; - - /* use syncing to lock the transaction log, so that we always see - log's head_offset = tail_offset */ - ret = mail_index_sync_begin(map->index, &ctx->sync_ctx, - &view, &sync_trans, 0); - if (ret <= 0) { - i_assert(ret != 0); - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - mail_index_transaction_rollback(&ctx->trans); - return -1; - } - dbox_map_sync_handle(map, ctx->sync_ctx); - - if (mail_index_transaction_commit(&ctx->trans) < 0) { - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - return -1; - } - ctx->success = TRUE; - return 0; -} - -void dbox_map_transaction_free(struct dbox_map_transaction_context **_ctx) -{ - struct dbox_map_transaction_context *ctx = *_ctx; - struct dbox_map *map = ctx->map; - - *_ctx = NULL; - if (ctx->success) { - if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - } - } else if (ctx->sync_ctx != NULL) { - mail_index_sync_rollback(&ctx->sync_ctx); - } - if (ctx->trans != NULL) - mail_index_transaction_rollback(&ctx->trans); - i_free(ctx); -} - -int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, - const ARRAY_TYPE(uint32_t) *map_uids, int diff) -{ - struct dbox_map *map = ctx->map; - const uint32_t *uidp; - unsigned int i, count; - const void *data; - uint32_t seq; - bool expunged; - int cur_diff; - - if (ctx->trans == NULL) - return -1; - - count = array_count(map_uids); - for (i = 0; i < count; i++) { - uidp = array_idx(map_uids, i); - if (!mail_index_lookup_seq(map->view, *uidp, &seq)) { - /* we can't refresh map here since view has a - transaction open. */ - dbox_map_set_corrupted(map, - "refcount update lost map_uid=%u", *uidp); - return -1; - } - mail_index_lookup_ext(map->view, seq, map->ref_ext_id, - &data, &expunged); - cur_diff = data == NULL ? 0 : *((const uint16_t *)data); - ctx->changed = TRUE; - cur_diff += mail_index_atomic_inc_ext(ctx->trans, seq, - map->ref_ext_id, diff); - if (cur_diff >= 32768) { - /* we're getting close to the 64k limit. fail early - to make it less likely that two processes increase - the refcount enough times to cross the limit */ - mail_storage_set_error(&map->storage->storage, - MAIL_ERROR_NOTPOSSIBLE, - "Message has been copied too many times"); - return -1; - } - } - return 0; -} - -int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id) -{ - struct dbox_map_transaction_context *map_trans; - const struct mail_index_header *hdr; - const struct dbox_mail_index_map_record *rec; - const void *data; - bool expunged; - uint32_t seq; - int ret = 0; - - /* make sure the map is refreshed, otherwise we might be expunging - messages that have already been moved to other files. */ - - /* we need a per-file transaction, otherwise we can't refresh the map */ - map_trans = dbox_map_transaction_begin(map, TRUE); - - hdr = mail_index_get_header(map->view); - for (seq = 1; seq <= hdr->messages_count; seq++) { - mail_index_lookup_ext(map->view, seq, map->map_ext_id, - &data, &expunged); - if (data == NULL) { - dbox_map_set_corrupted(map, "missing map extension"); - ret = -1; - break; - } - - rec = data; - if (rec->file_id == file_id) { - map_trans->changed = TRUE; - mail_index_expunge(map_trans->trans, seq); - } - } - if (ret == 0) - (void)dbox_map_transaction_commit(map_trans); - dbox_map_transaction_free(&map_trans); - return ret; -} - -struct dbox_map_append_context * -dbox_map_append_begin_storage(struct dbox_storage *storage) -{ - struct dbox_map_append_context *ctx; - - ctx = i_new(struct dbox_map_append_context, 1); - ctx->map = storage->map; - ctx->first_new_file_id = (uint32_t)-1; - i_array_init(&ctx->files, 64); - i_array_init(&ctx->appends, 128); - - if (dbox_map_open(ctx->map, TRUE) < 0) - ctx->failed = TRUE; - else { - /* refresh the map so we can try appending to the - latest files */ - (void)dbox_map_refresh(ctx->map); - } - return ctx; -} - -struct dbox_map_append_context * -dbox_map_append_begin(struct dbox_mailbox *mbox) -{ - struct dbox_map_append_context *ctx; - - ctx = dbox_map_append_begin_storage(mbox->storage); - ctx->mbox = mbox; - return ctx; -} - -static time_t day_begin_stamp(unsigned int days) -{ - struct tm tm; - time_t stamp; - - if (days == 0) - return 0; - - /* get beginning of today */ - tm = *localtime(&ioloop_time); - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; - stamp = mktime(&tm); - if (stamp == (time_t)-1) - i_panic("mktime(today) failed"); - - return stamp - (3600*24 * (days-1)); -} - -static bool -dbox_map_file_try_append(struct dbox_map_append_context *ctx, - uint32_t file_id, time_t stamp, uoff_t mail_size, - struct dbox_file **file_r, struct ostream **output_r, - bool *retry_later_r) -{ - struct dbox_map *map = ctx->map; - struct dbox_storage *storage = map->storage; - struct dbox_file *file; - struct stat st; - uoff_t append_offset; - bool deleted, file_too_old = FALSE; - int ret; - - *file_r = NULL; - *retry_later_r = FALSE; - - file = dbox_file_init_multi(storage, file_id); - if (dbox_file_open_or_create(file, &deleted) <= 0 || deleted) { - dbox_file_unref(&file); - return TRUE; - } - if (file->lock != NULL) { - /* already locked, we're possibly in the middle of purging it - in which case we really don't want to write there. */ - dbox_file_unref(&file); - return TRUE; - } - - if (file->create_time < stamp) - file_too_old = TRUE; - else if ((ret = dbox_file_try_lock(file)) <= 0) { - /* locking failed */ - *retry_later_r = ret == 0; - } else if (stat(file->current_path, &st) < 0) { - if (errno != ENOENT) - i_error("stat(%s) failed: %m", file->current_path); - /* the file was unlinked between opening and locking it. */ - } else if (dbox_file_get_append_stream(file, &append_offset, - output_r) <= 0) { - /* couldn't append to this file */ - } else if (append_offset + mail_size > storage->set->dbox_rotate_size) { - /* file was too large after all */ - dbox_file_cancel_append(file, append_offset); - } else { - /* success */ - *file_r = file; - return TRUE; - } - - /* failure */ - dbox_file_unlock(file); - dbox_file_unref(&file); - return !file_too_old; -} - -static bool -dbox_map_is_appending(struct dbox_map_append_context *ctx, uint32_t file_id) -{ - struct dbox_file *const *files; - unsigned int i, count; - - /* there shouldn't be many files open, don't bother with anything - faster. */ - files = array_get(&ctx->files, &count); - for (i = 0; i < count; i++) { - if (files[i]->file_id == file_id) - return TRUE; - } - return FALSE; -} - -static int -dbox_map_find_appendable_file(struct dbox_map_append_context *ctx, - uoff_t mail_size, struct dbox_file **file_r, - struct ostream **output_r, bool *existing_r) -{ - struct dbox_map *map = ctx->map; - const struct dbox_settings *set = map->storage->set; - ARRAY_TYPE(seq_range) checked_file_ids; - struct dbox_file *const *files; - const struct mail_index_header *hdr; - unsigned int i, count, backwards_lookup_count; - uint32_t seq, seq1, uid, file_id; - uoff_t offset, append_offset, size; - time_t stamp; - bool retry_later; - - *existing_r = FALSE; - - if (mail_size >= set->dbox_rotate_size) - return 0; - - /* first try to use files already used in this append */ - files = array_get(&ctx->files, &count); - for (i = count; i > ctx->files_nonappendable_count; i--) { - if (files[i-1]->output == NULL) { - /* we already decided we can't append to this */ - continue; - } - - append_offset = dbox_file_get_next_append_offset(files[i-1]); - if (append_offset + mail_size <= set->dbox_rotate_size && - dbox_file_get_append_stream(files[i-1], &append_offset, - output_r) > 0) { - *file_r = files[i-1]; - *existing_r = TRUE; - return 1; - } - /* can't append to this file anymore */ -#if 0 /* FIXME: we can't close files, otherwise we lose the lock too early */ - if (files[i-1]->fd != -1) { - /* avoid wasting fds by closing the file, but not if - we're also reading from it. */ - if (dbox_file_flush_append(files[i-1]) < 0) - return -1; - dbox_file_unlock(files[i-1]); - if (files[i-1]->refcount == 1) - dbox_file_close(files[i-1]); - } -#endif - } - ctx->files_nonappendable_count = count; - - /* try to find an existing appendable file */ - stamp = day_begin_stamp(set->dbox_rotate_days); - hdr = mail_index_get_header(map->view); - - ctx->orig_next_uid = hdr->next_uid; - backwards_lookup_count = 0; - t_array_init(&checked_file_ids, 16); - for (seq = hdr->messages_count; seq > 0; seq--) { - if (dbox_map_lookup_seq(map, seq, &file_id, &offset, &size) < 0) - return -1; - - if (seq_range_exists(&checked_file_ids, file_id)) - continue; - seq_range_array_add(&checked_file_ids, 0, file_id); - - if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) { - /* we've wasted enough time here */ - break; - } - - /* first lookup: this should be enough usually, but we can't - be sure until after locking. also if messages were recently - moved, this message might not be the last one in the file. */ - if (offset + size + mail_size >= set->dbox_rotate_size) - continue; - - if (dbox_map_is_appending(ctx, file_id)) { - /* already checked this */ - continue; - } - - mail_index_lookup_uid(map->view, seq, &uid); - if (!dbox_map_file_try_append(ctx, file_id, stamp, mail_size, - file_r, output_r, &retry_later)) { - /* file is too old. the rest of the files are too. */ - break; - } - /* NOTE: we've now refreshed map view. there are no guarantees - about sequences anymore. */ - if (*file_r != NULL) - return 1; - /* FIXME: use retry_later somehow */ - if (uid == 1 || - !mail_index_lookup_seq_range(map->view, 1, uid-1, - &seq1, &seq)) - break; - seq++; - } - return 0; -} - -int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, - struct dbox_file **file_r, struct ostream **output_r) -{ - struct dbox_file *file = NULL; - struct dbox_map_append *append; - uoff_t append_offset; - bool existing; - int ret; - - if (ctx->failed) - return -1; - - ret = dbox_map_find_appendable_file(ctx, mail_size, &file, - output_r, &existing); - if (ret < 0) - return -1; - - if (ret == 0) { - /* create a new file */ - file = ctx->map->storage->set->dbox_rotate_size == 0 ? - dbox_file_init_single(ctx->mbox, 0) : - dbox_file_init_multi(ctx->map->storage, 0); - ret = dbox_file_get_append_stream(file, &append_offset, - output_r); - if (ret <= 0) { - i_assert(ret < 0); - (void)unlink(file->current_path); - dbox_file_unref(&file); - return -1; - } - } - - if (file->single_mbox == NULL) { - append = array_append_space(&ctx->appends); - append->file = file; - append->offset = (*output_r)->offset; - append->size = (uint32_t)-1; - } - if (!existing) { - i_assert(file->first_append_offset == 0); - i_assert(file->output != NULL); - file->first_append_offset = file->output->offset; - array_append(&ctx->files, &file, 1); - } - *file_r = file; - return 0; -} - -void dbox_map_append_finish_multi_mail(struct dbox_map_append_context *ctx) -{ - struct dbox_map_append *appends; - unsigned int count; - - appends = array_get_modifiable(&ctx->appends, &count); - i_assert(count > 0 && appends[count-1].size == (uint32_t)-1); - appends[count-1].size = appends[count-1].file->output->offset - - appends[count-1].offset; -} - -static int -dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view, - uint32_t *file_id_r) -{ - const struct dbox_mail_index_map_header *hdr; - const void *data; - size_t data_size; - - mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size); - if (data_size != sizeof(*hdr)) { - if (data_size != 0) { - dbox_map_set_corrupted(map, "hdr size=%"PRIuSIZE_T, - data_size); - return -1; - } - /* first file */ - *file_id_r = 1; - } else { - hdr = data; - *file_id_r = hdr->highest_file_id + 1; - } - return 0; -} - -static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx, - bool separate_transaction) -{ - struct dbox_file *const *files; - unsigned int i, count; - uint32_t first_file_id, file_id; - int ret; - - /* start the syncing. we'll need it even if there are no file ids to - be assigned. */ - ret = mail_index_sync_begin(ctx->map->index, &ctx->sync_ctx, - &ctx->sync_view, &ctx->sync_trans, 0); - if (ret <= 0) { - i_assert(ret != 0); - mail_storage_set_internal_error(&ctx->map->storage->storage); - mail_index_reset_error(ctx->map->index); - return -1; - } - dbox_map_sync_handle(ctx->map, ctx->sync_ctx); - - if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) { - mail_index_sync_rollback(&ctx->sync_ctx); - return -1; - } - - /* assign file_ids for newly created multi-files */ - first_file_id = file_id; - files = array_get(&ctx->files, &count); - for (i = 0; i < count; i++) { - if (files[i]->single_mbox != NULL) - continue; - - if (files[i]->output != NULL) { - if (dbox_file_flush_append(files[i]) < 0) { - ret = -1; - break; - } - } - - if (files[i]->file_id == 0) { - if (dbox_file_assign_id(files[i], file_id++) < 0) { - ret = -1; - break; - } - } - } - - if (ret < 0) { - mail_index_sync_rollback(&ctx->sync_ctx); - return -1; - } - - ctx->trans = !separate_transaction ? NULL : - mail_index_transaction_begin(ctx->map->view, - MAIL_INDEX_TRANSACTION_FLAG_FSYNC); - - /* update the highest used file_id */ - if (first_file_id != file_id) { - file_id--; - mail_index_update_header_ext(ctx->trans != NULL ? ctx->trans : - ctx->sync_trans, - ctx->map->map_ext_id, - 0, &file_id, sizeof(file_id)); - } - return 0; -} - -int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, - uint32_t *first_map_uid_r, - uint32_t *last_map_uid_r) -{ - const struct dbox_map_append *appends; - const struct mail_index_header *hdr; - struct dbox_mail_index_map_record rec; - unsigned int i, count; - ARRAY_TYPE(seq_range) uids; - const struct seq_range *range; - uint32_t seq; - uint16_t ref16; - int ret = 0; - - if (array_count(&ctx->appends) == 0) { - *first_map_uid_r = 0; - *last_map_uid_r = 0; - return 0; - } - - if (dbox_map_assign_file_ids(ctx, TRUE) < 0) - return -1; - - /* append map records to index */ - memset(&rec, 0, sizeof(rec)); - ref16 = 1; - appends = array_get(&ctx->appends, &count); - for (i = 0; i < count; i++) { - i_assert(appends[i].offset <= (uint32_t)-1); - i_assert(appends[i].size <= (uint32_t)-1); - - rec.file_id = appends[i].file->file_id; - rec.offset = appends[i].offset; - rec.size = appends[i].size; - - mail_index_append(ctx->trans, 0, &seq); - mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id, - &rec, NULL); - mail_index_update_ext(ctx->trans, seq, ctx->map->ref_ext_id, - &ref16, NULL); - } - - /* assign map UIDs for appended records */ - hdr = mail_index_get_header(ctx->sync_view); - t_array_init(&uids, 1); - mail_index_append_finish_uids(ctx->trans, hdr->next_uid, &uids); - range = array_idx(&uids, 0); - i_assert(range[0].seq2 - range[0].seq1 + 1 == count); - - if (hdr->uid_validity == 0) { - /* we don't really care about uidvalidity, but it can't be 0 */ - uint32_t uid_validity = ioloop_time; - mail_index_update_header(ctx->trans, - offsetof(struct mail_index_header, uid_validity), - &uid_validity, sizeof(uid_validity), TRUE); - } - - if (mail_index_transaction_commit(&ctx->trans) < 0) { - mail_storage_set_internal_error(&ctx->map->storage->storage); - mail_index_reset_error(ctx->map->index); - return -1; - } - - *first_map_uid_r = range[0].seq1; - *last_map_uid_r = range[0].seq2; - return ret; -} - -int dbox_map_append_move(struct dbox_map_append_context *ctx, - const ARRAY_TYPE(uint32_t) *map_uids, - const ARRAY_TYPE(seq_range) *expunge_map_uids) -{ - const struct dbox_map_append *appends; - struct dbox_mail_index_map_record rec; - struct seq_range_iter iter; - const uint32_t *uids; - unsigned int i, j, map_uids_count, appends_count; - uint32_t uid, seq; - - if (dbox_map_assign_file_ids(ctx, FALSE) < 0) - return -1; - - memset(&rec, 0, sizeof(rec)); - appends = array_get(&ctx->appends, &appends_count); - - uids = array_get(map_uids, &map_uids_count); - for (i = j = 0; i < map_uids_count; i++) { - i_assert(j < appends_count); - rec.file_id = appends[j].file->file_id; - rec.offset = appends[j].offset; - rec.size = appends[j].size; - j++; - - if (!mail_index_lookup_seq(ctx->sync_view, uids[i], &seq)) - i_unreached(); - mail_index_update_ext(ctx->sync_trans, seq, - ctx->map->map_ext_id, &rec, NULL); - } - - seq_range_array_iter_init(&iter, expunge_map_uids); i = 0; - while (seq_range_array_iter_nth(&iter, i++, &uid)) { - if (!mail_index_lookup_seq(ctx->sync_view, uid, &seq)) - i_unreached(); - mail_index_expunge(ctx->sync_trans, seq); - } - return 0; -} - -int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx, - const ARRAY_TYPE(seq_range) *uids) -{ - struct dbox_file *const *files; - struct seq_range_iter iter; - unsigned int i, count, n = 0; - uint32_t uid; - bool ret; - - seq_range_array_iter_init(&iter, uids); - files = array_get(&ctx->files, &count); - for (i = 0; i < count; i++) { - if (files[i]->single_mbox == NULL) - continue; - - ret = seq_range_array_iter_nth(&iter, n++, &uid); - i_assert(ret); - if (dbox_file_assign_id(files[i], uid) < 0) - return -1; - } - i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); - return 0; -} - -int dbox_map_append_commit(struct dbox_map_append_context *ctx) -{ - struct dbox_map *map = ctx->map; - - i_assert(ctx->trans == NULL); - - if (ctx->sync_ctx != NULL) { - if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { - mail_storage_set_internal_error(&map->storage->storage); - mail_index_reset_error(map->index); - return -1; - } - } - ctx->committed = TRUE; - return 0; -} - -static void dbox_map_append_file_rollback(struct dbox_file *file) -{ - struct mail_storage *storage = &file->storage->storage; - - if (file->output != NULL) { - /* flush before truncating */ - (void)o_stream_flush(file->output); - } - - if (file->file_id != 0 && - file->first_append_offset > file->file_header_size) { - if (ftruncate(file->fd, file->first_append_offset) < 0) { - mail_storage_set_critical(storage, - "ftruncate(%s, %"PRIuUOFF_T") failed: %m", - file->current_path, file->first_append_offset); - } - } else { - if (unlink(file->current_path) < 0) { - mail_storage_set_critical(storage, - "unlink(%s) failed: %m", file->current_path); - } - } -} - -void dbox_map_append_free(struct dbox_map_append_context **_ctx) -{ - struct dbox_map_append_context *ctx = *_ctx; - struct dbox_file **files; - unsigned int i, count; - - *_ctx = NULL; - - if (ctx->trans != NULL) - mail_index_transaction_rollback(&ctx->trans); - if (ctx->sync_ctx != NULL) - mail_index_sync_rollback(&ctx->sync_ctx); - - files = array_get_modifiable(&ctx->files, &count); - for (i = 0; i < count; i++) { - if (!ctx->committed) - dbox_map_append_file_rollback(files[i]); - - files[i]->first_append_offset = 0; - dbox_file_unlock(files[i]); - dbox_file_unref(&files[i]); - } - array_free(&ctx->appends); - array_free(&ctx->files); - i_free(ctx); -} - -uint32_t dbox_map_get_uid_validity(struct dbox_map *map) -{ - const struct mail_index_header *hdr; - - i_assert(map->view != NULL); - - hdr = mail_index_get_header(map->view); - if (hdr->uid_validity != 0) - return hdr->uid_validity; - - /* refresh index in case it was just changed */ - (void)dbox_map_refresh(map); - hdr = mail_index_get_header(map->view); - return hdr->uid_validity != 0 ? hdr->uid_validity : - map->created_uid_validity; -}
--- a/src/lib-storage/index/dbox/dbox-map.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,97 +0,0 @@ -#ifndef DBOX_MAP_H -#define DBOX_MAP_H - -#include "seq-range-array.h" - -struct dbox_storage; -struct dbox_mailbox; -struct dbox_file; -struct dbox_map_append_context; -struct dbox_mail_lookup_rec; - -struct dbox_mail_index_map_header { - uint32_t highest_file_id; -}; - -struct dbox_mail_index_map_record { - uint32_t file_id; - uint32_t offset; - uint32_t size; /* including pre/post metadata */ -}; - -struct dbox_map_file_msg { - uint32_t map_uid; - uint32_t offset; - uint32_t refcount; -}; -ARRAY_DEFINE_TYPE(dbox_map_file_msg, struct dbox_map_file_msg); - -struct dbox_map *dbox_map_init(struct dbox_storage *storage); -void dbox_map_deinit(struct dbox_map **map); - -/* Open the map. This is done automatically for most operations. - Returns 0 if ok, -1 if error. */ -int dbox_map_open(struct dbox_map *map, bool create_missing); -/* Refresh the map. Returns 0 if ok, -1 if error. */ -int dbox_map_refresh(struct dbox_map *map); - -/* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID - is already expunged, -1 if error. */ -int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, - uint32_t *file_id_r, uoff_t *offset_r); - -/* Get all messages from file */ -int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id, - ARRAY_TYPE(dbox_map_file_msg) *recs); - -struct dbox_map_transaction_context * -dbox_map_transaction_begin(struct dbox_map *map, bool external); -/* Write transaction to map and leave it locked. Call _free() to update tail - offset and unlock. */ -int dbox_map_transaction_commit(struct dbox_map_transaction_context *ctx); -void dbox_map_transaction_free(struct dbox_map_transaction_context **ctx); - -int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, - const ARRAY_TYPE(uint32_t) *map_uids, int diff); -int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id); - -/* Returns TRUE if there's enough pressure to purge immediately. */ -bool dbox_map_want_purge(struct dbox_map *map); -/* Return all files containing messages with zero refcount. */ -const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map); - -struct dbox_map_append_context * -dbox_map_append_begin(struct dbox_mailbox *mbox); -struct dbox_map_append_context * -dbox_map_append_begin_storage(struct dbox_storage *storage); -/* Request file for saving a new message with given size (if available). If an - existing file can be used, the record is locked and updated in index. - Returns 0 if ok, -1 if error. */ -int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, - struct dbox_file **file_r, struct ostream **output_r); -/* Finished saving the last mail. Saves the message size. */ -void dbox_map_append_finish_multi_mail(struct dbox_map_append_context *ctx); -/* Assign map UIDs to all appended msgs to multi-files. */ -int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, - uint32_t *first_map_uid_r, - uint32_t *last_map_uid_r); -/* Assign UIDs to all created single-files. */ -int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx, - const ARRAY_TYPE(seq_range) *uids); -/* The appends are existing messages that were simply moved to a new file. - map_uids contains the moved messages' map UIDs. */ -int dbox_map_append_move(struct dbox_map_append_context *ctx, - const ARRAY_TYPE(uint32_t) *map_uids, - const ARRAY_TYPE(seq_range) *expunge_map_uids); -/* Returns 0 if ok, -1 if error. */ -int dbox_map_append_commit(struct dbox_map_append_context *ctx); -void dbox_map_append_free(struct dbox_map_append_context **ctx); - -/* Get either existing uidvalidity or create a new one if map was - just created. */ -uint32_t dbox_map_get_uid_validity(struct dbox_map *map); - -void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...) - ATTR_FORMAT(2, 3); - -#endif
--- a/src/lib-storage/index/dbox/dbox-save.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,515 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "fdatasync-path.h" -#include "hex-binary.h" -#include "hex-dec.h" -#include "str.h" -#include "istream.h" -#include "istream-crlf.h" -#include "ostream.h" -#include "write-full.h" -#include "index-mail.h" -#include "mail-copy.h" -#include "dbox-storage.h" -#include "dbox-map.h" -#include "dbox-file.h" -#include "dbox-sync.h" - -#include <stdlib.h> - -struct dbox_save_mail { - struct dbox_file *file; - uint32_t seq; - uint32_t append_offset; - uoff_t message_size; -}; - -struct dbox_save_context { - struct mail_save_context ctx; - - struct dbox_mailbox *mbox; - struct mail_index_transaction *trans; - - struct dbox_map_append_context *append_ctx; - struct dbox_sync_context *sync_ctx; - - ARRAY_TYPE(uint32_t) copy_map_uids; - struct dbox_map_transaction_context *map_trans; - - /* updated for each appended mail: */ - uint32_t seq; - struct istream *input; - struct mail *mail; - - struct dbox_file *cur_file; - struct ostream *cur_output; - - ARRAY_DEFINE(mails, struct dbox_save_mail); - unsigned int single_count; - - unsigned int failed:1; - unsigned int finished:1; -}; - -struct dbox_file * -dbox_save_file_get_file(struct mailbox_transaction_context *t, - uint32_t seq, uoff_t *offset_r) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)t->save_ctx; - const struct dbox_save_mail *mails, *mail; - unsigned int count; - - mails = array_get(&ctx->mails, &count); - i_assert(count > 0); - i_assert(seq >= mails[0].seq); - - mail = &mails[mails[0].seq - seq]; - i_assert(mail->seq == seq); - - if (o_stream_flush(mail->file->output) < 0) { - dbox_file_set_syscall_error(mail->file, "write()"); - ctx->failed = TRUE; - } - - *offset_r = mail->append_offset; - return mail->file; -} - -struct mail_save_context * -dbox_save_alloc(struct mailbox_transaction_context *t) -{ - struct index_transaction_context *it = - (struct index_transaction_context *)t; - struct dbox_mailbox *mbox = (struct dbox_mailbox *)t->box; - struct dbox_save_context *ctx = (struct dbox_save_context *)t->save_ctx; - - i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); - - if (ctx != NULL) { - /* use the existing allocated structure */ - ctx->finished = FALSE; - return &ctx->ctx; - } - - ctx = i_new(struct dbox_save_context, 1); - ctx->ctx.transaction = t; - ctx->mbox = mbox; - ctx->trans = it->trans; - ctx->append_ctx = dbox_map_append_begin(mbox); - i_array_init(&ctx->mails, 32); - t->save_ctx = &ctx->ctx; - return t->save_ctx; -} - -static void dbox_save_add_to_index(struct dbox_save_context *ctx) -{ - enum mail_flags save_flags; - - save_flags = ctx->ctx.flags & ~MAIL_RECENT; - mail_index_append(ctx->trans, ctx->ctx.uid, &ctx->seq); - mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE, - save_flags); - if (ctx->ctx.keywords != NULL) { - mail_index_update_keywords(ctx->trans, ctx->seq, - MODIFY_REPLACE, ctx->ctx.keywords); - } - if (ctx->ctx.min_modseq != 0) { - mail_index_update_modseq(ctx->trans, ctx->seq, - ctx->ctx.min_modseq); - } -} - -int dbox_save_begin(struct mail_save_context *_ctx, struct istream *input) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - struct dbox_message_header dbox_msg_hdr; - struct dbox_save_mail *save_mail; - struct istream *crlf_input; - uoff_t mail_size; - - /* get the size of the mail to be saved, if possible */ - if (i_stream_get_size(input, TRUE, &mail_size) <= 0) - mail_size = 0; - if (dbox_map_append_next(ctx->append_ctx, mail_size, - &ctx->cur_file, &ctx->cur_output) < 0) { - ctx->failed = TRUE; - return -1; - } - - dbox_save_add_to_index(ctx); - - if (_ctx->dest_mail == NULL) { - if (ctx->mail == NULL) - ctx->mail = mail_alloc(_ctx->transaction, 0, NULL); - _ctx->dest_mail = ctx->mail; - } - mail_set_seq(_ctx->dest_mail, ctx->seq); - - crlf_input = i_stream_create_lf(input); - ctx->input = index_mail_cache_parse_init(_ctx->dest_mail, crlf_input); - i_stream_unref(&crlf_input); - - save_mail = array_append_space(&ctx->mails); - save_mail->file = ctx->cur_file; - save_mail->seq = ctx->seq; - i_assert(ctx->cur_output->offset <= (uint32_t)-1); - save_mail->append_offset = ctx->cur_output->offset; - - /* write a dummy header. it'll get rewritten when we're finished */ - memset(&dbox_msg_hdr, 0, sizeof(dbox_msg_hdr)); - o_stream_cork(ctx->cur_output); - if (o_stream_send(ctx->cur_output, &dbox_msg_hdr, - sizeof(dbox_msg_hdr)) < 0) { - mail_storage_set_critical(_ctx->transaction->box->storage, - "o_stream_send(%s) failed: %m", - ctx->cur_file->current_path); - ctx->failed = TRUE; - } - - if (_ctx->received_date == (time_t)-1) - _ctx->received_date = ioloop_time; - return ctx->failed ? -1 : 0; -} - -int dbox_save_continue(struct mail_save_context *_ctx) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - struct mail_storage *storage = &ctx->mbox->storage->storage; - - if (ctx->failed) - return -1; - - do { - if (o_stream_send_istream(ctx->cur_output, ctx->input) < 0) { - if (!mail_storage_set_error_from_errno(storage)) { - mail_storage_set_critical(storage, - "o_stream_send_istream(%s) failed: %m", - ctx->cur_file->current_path); - } - ctx->failed = TRUE; - return -1; - } - index_mail_cache_parse_continue(_ctx->dest_mail); - - /* both tee input readers may consume data from our primary - input stream. we'll have to make sure we don't return with - one of the streams still having data in them. */ - } while (i_stream_read(ctx->input) > 0); - return 0; -} - -static void dbox_save_write_metadata(struct dbox_save_context *ctx) -{ - struct dbox_metadata_header metadata_hdr; - uint8_t guid_128[MAIL_GUID_128_SIZE]; - const char *guid; - string_t *str; - uoff_t vsize; - - memset(&metadata_hdr, 0, sizeof(metadata_hdr)); - memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST, - sizeof(metadata_hdr.magic_post)); - o_stream_send(ctx->cur_output, &metadata_hdr, sizeof(metadata_hdr)); - - str = t_str_new(256); - str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME, - (unsigned long)ctx->ctx.received_date); - str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME, - (unsigned long)ioloop_time); - if (mail_get_virtual_size(ctx->ctx.dest_mail, &vsize) < 0) - i_unreached(); - str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, - (unsigned long long)vsize); - - guid = ctx->ctx.guid; - if (ctx->ctx.guid != NULL) - mail_generate_guid_128_hash(guid, guid_128); - else { - mail_generate_guid_128(guid_128); - guid = binary_to_hex(guid_128, sizeof(guid_128)); - } - if (ctx->cur_file->single_mbox == NULL) { - /* multi-file: save the 128bit GUID to index so if the map - index gets corrupted we can still find the message */ - mail_index_update_ext(ctx->trans, ctx->seq, - ctx->mbox->guid_ext_id, - guid_128, NULL); - } - str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid); - if (ctx->cur_file->single_mbox == NULL && - strchr(ctx->mbox->ibox.box.name, '\r') == NULL && - strchr(ctx->mbox->ibox.box.name, '\n') == NULL) { - /* multi-file: save the original mailbox name so if mailbox - indexes get corrupted we can place at least some - (hopefully most) of the messages to correct mailboxes. */ - str_printfa(str, "%c%s\n", DBOX_METADATA_ORIG_MAILBOX, - ctx->mbox->ibox.box.name); - } - - str_append_c(str, '\n'); - o_stream_send(ctx->cur_output, str_data(str), str_len(str)); -} - -static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx, - struct dbox_save_mail *mail) -{ - struct dbox_message_header dbox_msg_hdr; - - i_assert(mail->file->msg_header_size == sizeof(dbox_msg_hdr)); - - dbox_save_write_metadata(ctx); - dbox_msg_header_fill(&dbox_msg_hdr, mail->message_size); - if (o_stream_pwrite(ctx->cur_output, &dbox_msg_hdr, - sizeof(dbox_msg_hdr), mail->append_offset) < 0) { - dbox_file_set_syscall_error(mail->file, "pwrite()"); - return -1; - } - if (mail->file->single_mbox != NULL) { - /* we're done writing to single-files now */ - if (dbox_file_flush_append(mail->file) < 0) - return -1; - } - return 0; -} - -static int dbox_save_finish_write(struct mail_save_context *_ctx) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - struct dbox_save_mail *save_mail; - unsigned int count; - - ctx->finished = TRUE; - if (ctx->cur_output == NULL) - return -1; - - index_mail_cache_parse_deinit(_ctx->dest_mail, - _ctx->received_date, !ctx->failed); - - count = array_count(&ctx->mails); - save_mail = array_idx_modifiable(&ctx->mails, count - 1); - - if (!ctx->failed) T_BEGIN { - save_mail->message_size = ctx->cur_output->offset - - save_mail->append_offset - - save_mail->file->msg_header_size; - - if (dbox_save_mail_write_metadata(ctx, save_mail) < 0) - ctx->failed = TRUE; - } T_END; - - if (o_stream_flush(ctx->cur_output) < 0) { - dbox_file_set_syscall_error(save_mail->file, "write()"); - ctx->failed = TRUE; - } - - o_stream_unref(&ctx->cur_output); - i_stream_unref(&ctx->input); - - if (ctx->failed) { - dbox_file_cancel_append(save_mail->file, - save_mail->append_offset); - array_delete(&ctx->mails, count - 1, 1); - return -1; - } - - if (save_mail->file->single_mbox != NULL) { - dbox_file_close(save_mail->file); - ctx->single_count++; - } else { - dbox_map_append_finish_multi_mail(ctx->append_ctx); - } - return 0; -} - -int dbox_save_finish(struct mail_save_context *ctx) -{ - int ret; - - ret = dbox_save_finish_write(ctx); - index_save_context_free(ctx); - return ret; -} - -void dbox_save_cancel(struct mail_save_context *_ctx) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - - ctx->failed = TRUE; - (void)dbox_save_finish(_ctx); -} - -int dbox_transaction_save_commit_pre(struct mail_save_context *_ctx) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - struct mailbox_transaction_context *_t = _ctx->transaction; - const struct mail_index_header *hdr; - uint32_t first_map_uid, last_map_uid; - - i_assert(ctx->finished); - - /* lock the mailbox before map to avoid deadlocks */ - if (dbox_sync_begin(ctx->mbox, DBOX_SYNC_FLAG_NO_PURGE | - DBOX_SYNC_FLAG_FORCE | - DBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) { - dbox_transaction_save_rollback(_ctx); - return -1; - } - - /* get map UIDs for messages saved to multi-files. they're written - to transaction log immediately within this function, but the map - is left locked. */ - if (dbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, - &last_map_uid) < 0) { - dbox_transaction_save_rollback(_ctx); - return -1; - } - - /* assign UIDs for new messages */ - hdr = mail_index_get_header(ctx->sync_ctx->sync_view); - mail_index_append_finish_uids(ctx->trans, hdr->next_uid, - &_t->changes->saved_uids); - - /* if we saved any single-files, rename the files to contain UIDs */ - if (ctx->single_count > 0) { - if (dbox_map_append_assign_uids(ctx->append_ctx, - &_t->changes->saved_uids) < 0) { - dbox_transaction_save_rollback(_ctx); - return -1; - } - } - - /* add map_uids for all messages saved to multi-files */ - if (first_map_uid != 0) { - struct dbox_mail_index_record rec; - const struct dbox_save_mail *mails; - unsigned int i, count; - uint32_t next_map_uid = first_map_uid; - - dbox_update_header(ctx->mbox, ctx->trans, NULL); - - memset(&rec, 0, sizeof(rec)); - rec.save_date = ioloop_time; - mails = array_get(&ctx->mails, &count); - for (i = 0; i < count; i++) { - if (mails[i].file->single_mbox != NULL) - continue; - - rec.map_uid = next_map_uid++; - mail_index_update_ext(ctx->trans, mails[i].seq, - ctx->mbox->dbox_ext_id, - &rec, NULL); - } - i_assert(next_map_uid == last_map_uid + 1); - } - - /* increase map's refcount for copied mails */ - if (array_is_created(&ctx->copy_map_uids)) { - ctx->map_trans = - dbox_map_transaction_begin(ctx->mbox->storage->map, - FALSE); - if (dbox_map_update_refcounts(ctx->map_trans, - &ctx->copy_map_uids, 1) < 0) { - dbox_transaction_save_rollback(_ctx); - return -1; - } - } - - if (ctx->mail != NULL) - mail_free(&ctx->mail); - - _t->changes->uid_validity = hdr->uid_validity; - return 0; -} - -void dbox_transaction_save_commit_post(struct mail_save_context *_ctx) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - - _ctx->transaction = NULL; /* transaction is already freed */ - - /* finish writing the mailbox APPENDs */ - if (dbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) { - if (ctx->map_trans != NULL) - (void)dbox_map_transaction_commit(ctx->map_trans); - /* commit only updates the sync tail offset, everything else - was already written at this point. */ - (void)dbox_map_append_commit(ctx->append_ctx); - } - dbox_map_append_free(&ctx->append_ctx); - - if (!ctx->mbox->storage->storage.set->fsync_disable) { - if (fdatasync_path(ctx->mbox->ibox.box.path) < 0) { - i_error("fdatasync_path(%s) failed: %m", - ctx->mbox->ibox.box.path); - } - } - dbox_transaction_save_rollback(_ctx); -} - -void dbox_transaction_save_rollback(struct mail_save_context *_ctx) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - - if (!ctx->finished) - dbox_save_cancel(&ctx->ctx); - if (ctx->append_ctx != NULL) - dbox_map_append_free(&ctx->append_ctx); - if (ctx->map_trans != NULL) - dbox_map_transaction_free(&ctx->map_trans); - if (array_is_created(&ctx->copy_map_uids)) - array_free(&ctx->copy_map_uids); - - if (ctx->sync_ctx != NULL) - (void)dbox_sync_finish(&ctx->sync_ctx, FALSE); - - if (ctx->mail != NULL) - mail_free(&ctx->mail); - array_free(&ctx->mails); - i_free(ctx); -} - -int dbox_copy(struct mail_save_context *_ctx, struct mail *mail) -{ - struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; - struct dbox_mailbox *src_mbox; - struct dbox_mail_index_record rec; - const void *data; - bool expunged; - - ctx->finished = TRUE; - - if (mail->box->storage != _ctx->transaction->box->storage) - return mail_storage_copy(_ctx, mail); - src_mbox = (struct dbox_mailbox *)mail->box; - - memset(&rec, 0, sizeof(rec)); - rec.save_date = ioloop_time; - if (dbox_mail_lookup(src_mbox, src_mbox->ibox.view, mail->seq, - &rec.map_uid) < 0) - return -1; - - if (rec.map_uid == 0) { - /* FIXME: we could hard link */ - return mail_storage_copy(_ctx, mail); - } - - /* remember the map_uid so we can later increase its refcount */ - if (!array_is_created(&ctx->copy_map_uids)) - i_array_init(&ctx->copy_map_uids, 32); - array_append(&ctx->copy_map_uids, &rec.map_uid, 1); - - /* add message to mailbox index */ - dbox_save_add_to_index(ctx); - mail_index_update_ext(ctx->trans, ctx->seq, ctx->mbox->dbox_ext_id, - &rec, NULL); - - mail_index_lookup_ext(src_mbox->ibox.view, mail->seq, - src_mbox->guid_ext_id, &data, &expunged); - if (data != NULL) { - mail_index_update_ext(ctx->trans, ctx->seq, - ctx->mbox->guid_ext_id, data, NULL); - } - return 0; -}
--- a/src/lib-storage/index/dbox/dbox-settings.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,65 +0,0 @@ -/* Copyright (c) 2006-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "settings-parser.h" -#include "mail-storage-settings.h" -#include "dbox-settings.h" - -#include <stddef.h> - -#undef DEF -#define DEF(type, name) \ - { type, #name, offsetof(struct dbox_settings, name), NULL } - -static bool dbox_settings_verify(void *_set, pool_t pool ATTR_UNUSED, - const char **error_r); - -static struct setting_define dbox_setting_defines[] = { - DEF(SET_UINT, dbox_rotate_size), - DEF(SET_UINT, dbox_rotate_min_size), - DEF(SET_UINT, dbox_rotate_days), - DEF(SET_UINT, dbox_max_open_files), - DEF(SET_UINT, dbox_purge_min_percentage), - - SETTING_DEFINE_LIST_END -}; - -static struct dbox_settings dbox_default_settings = { - MEMBER(dbox_rotate_size) 2048*1024, - MEMBER(dbox_rotate_min_size) 16*1024, - MEMBER(dbox_rotate_days) 0, - MEMBER(dbox_max_open_files) 64, - MEMBER(dbox_purge_min_percentage) 0 -}; - -static struct setting_parser_info dbox_setting_parser_info = { - MEMBER(defines) dbox_setting_defines, - MEMBER(defaults) &dbox_default_settings, - - MEMBER(parent) &mail_user_setting_parser_info, - MEMBER(dynamic_parsers) NULL, - - MEMBER(parent_offset) (size_t)-1, - MEMBER(type_offset) (size_t)-1, - MEMBER(struct_size) sizeof(struct dbox_settings), - MEMBER(check_func) dbox_settings_verify -}; - -/* <settings checks> */ -static bool dbox_settings_verify(void *_set, pool_t pool ATTR_UNUSED, - const char **error_r) -{ - const struct dbox_settings *set = _set; - - if (set->dbox_max_open_files < 2) { - *error_r = "dbox_max_open_files must be at least 2"; - return FALSE; - } - return TRUE; -} -/* </settings checks> */ - -const struct setting_parser_info *dbox_get_setting_parser_info(void) -{ - return &dbox_setting_parser_info; -}
--- a/src/lib-storage/index/dbox/dbox-settings.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,14 +0,0 @@ -#ifndef DBOX_SETTINGS_H -#define DBOX_SETTINGS_H - -struct dbox_settings { - unsigned int dbox_rotate_size; - unsigned int dbox_rotate_min_size; - unsigned int dbox_rotate_days; - unsigned int dbox_max_open_files; - unsigned int dbox_purge_min_percentage; -}; - -const struct setting_parser_info *dbox_get_setting_parser_info(void); - -#endif
--- a/src/lib-storage/index/dbox/dbox-storage-rebuild.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,770 +0,0 @@ -/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "ioloop.h" -#include "istream.h" -#include "hash.h" -#include "str.h" -#include "dbox-storage.h" -#include "dbox-file.h" -#include "dbox-map-private.h" -#include "dbox-sync.h" -#include "dbox-storage-rebuild.h" - -#include <stdlib.h> -#include <dirent.h> -#include <unistd.h> - -struct dbox_rebuild_msg { - uint8_t guid_128[MAIL_GUID_128_SIZE]; - uint32_t file_id; - uint32_t offset; - uint32_t size; - uint32_t map_uid; - - uint16_t refcount; - unsigned int seen_zero_ref_in_map:1; -}; - -struct rebuild_msg_mailbox { - struct mailbox *box; - struct mail_index_sync_ctx *sync_ctx; - struct mail_index_view *view; - struct mail_index_transaction *trans; - uint32_t next_uid; -}; - -struct dbox_storage_rebuild_context { - struct dbox_storage *storage; - pool_t pool; - - struct hash_table *guid_hash; - ARRAY_DEFINE(msgs, struct dbox_rebuild_msg *); - - uint32_t prev_file_id; - uint32_t highest_seen_map_uid; - - struct mailbox_list *default_list; - struct mail_index_sync_ctx *sync_ctx; - struct mail_index_view *sync_view; - struct mail_index_transaction *trans; - - struct rebuild_msg_mailbox prev_msg; - - unsigned int msgs_unsorted:1; -}; - -static unsigned int guid_hash(const void *p) -{ - const uint8_t *s = p; - unsigned int i, g, h = 0; - - for (i = 0; i < MAIL_GUID_128_SIZE; i++) { - h = (h << 4) + s[i]; - if ((g = h & 0xf0000000UL)) { - h = h ^ (g >> 24); - h = h ^ g; - } - } - return h; -} - -static int guid_cmp(const void *p1, const void *p2) -{ - return memcmp(p1, p2, MAIL_GUID_128_SIZE); -} - -static struct dbox_storage_rebuild_context * -dbox_storage_rebuild_init(struct dbox_storage *storage) -{ - struct dbox_storage_rebuild_context *ctx; - - ctx = i_new(struct dbox_storage_rebuild_context, 1); - ctx->storage = storage; - ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256); - ctx->guid_hash = hash_table_create(default_pool, ctx->pool, 0, - guid_hash, guid_cmp); - i_array_init(&ctx->msgs, 512); - return ctx; -} - -static void dbox_storage_rebuild_deinit(struct dbox_storage_rebuild_context *ctx) -{ - if (ctx->sync_ctx != NULL) - mail_index_sync_rollback(&ctx->sync_ctx); - - hash_table_destroy(&ctx->guid_hash); - pool_unref(&ctx->pool); - array_free(&ctx->msgs); - i_free(ctx); -} - -static int dbox_rebuild_msg_offset_cmp(const void *p1, const void *p2) -{ - const struct dbox_rebuild_msg *const *m1 = p1, *const *m2 = p2; - - if ((*m1)->file_id < (*m2)->file_id) - return -1; - if ((*m1)->file_id > (*m2)->file_id) - return 1; - - if ((*m1)->offset < (*m2)->offset) - return -1; - if ((*m1)->offset > (*m2)->offset) - return 1; - return 0; -} - -static int dbox_rebuild_msg_uid_cmp(struct dbox_rebuild_msg *const *m1, - struct dbox_rebuild_msg *const *m2) -{ - if ((*m1)->map_uid < (*m2)->map_uid) - return -1; - if ((*m1)->map_uid > (*m2)->map_uid) - return 1; - return 0; -} - -static int rebuild_add_file(struct dbox_storage_rebuild_context *ctx, - const char *path) -{ - struct dbox_file *file; - const char *fname, *guid; - struct dbox_rebuild_msg *rec; - uint32_t file_id; - uoff_t offset, prev_offset, size; - bool last, expunged, first, fixed = FALSE; - int ret = 0; - - fname = strrchr(path, '/'); - i_assert(fname != NULL); - fname += strlen(DBOX_MAIL_FILE_MULTI_PREFIX) + 1; - - file_id = strtoul(fname, NULL, 10); - if (!is_numeric(fname, '\0') || file_id == 0) { - i_warning("dbox rebuild: File name is missing ID: %s", path); - return 0; - } - - /* small optimization: typically files are returned sorted. in that - case we don't need to sort them ourself. */ - if (file_id < ctx->prev_file_id) - ctx->msgs_unsorted = TRUE; - ctx->prev_file_id = file_id; - - file = dbox_file_init_multi(ctx->storage, file_id); - prev_offset = 0; - dbox_file_seek_rewind(file); - while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) { - if (ret > 0) { - if ((ret = dbox_file_metadata_read(file)) < 0) - break; - } - - if (ret == 0) { - /* file is corrupted. fix it and retry. */ - if (fixed || last) - break; - first = prev_offset == 0; - if (prev_offset == 0) { - /* use existing file header if it was ok */ - prev_offset = offset; - } - if (dbox_file_fix(file, prev_offset) < 0) { - ret = -1; - break; - } - fixed = TRUE; - if (!first) { - /* seek to the offset where we last left off */ - ret = dbox_file_get_mail_stream(file, - prev_offset, &size, NULL, &expunged); - if (ret <= 0) - break; - } - continue; - } - prev_offset = offset; - - guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID); - if (guid == NULL) { - dbox_file_set_corrupted(file, - "Message is missing GUID"); - ret = 0; - break; - } - - rec = p_new(ctx->pool, struct dbox_rebuild_msg, 1); - rec->file_id = file_id; - rec->offset = offset; - rec->size = file->input->v_offset - offset; - mail_generate_guid_128_hash(guid, rec->guid_128); - array_append(&ctx->msgs, &rec, 1); - - if (hash_table_lookup(ctx->guid_hash, rec->guid_128) != NULL) { - /* duplicate. save this as a refcount=0 to map, - so it will eventually be deleted. */ - rec->seen_zero_ref_in_map = TRUE; - } else { - hash_table_insert(ctx->guid_hash, rec->guid_128, rec); - } - } - if (ret == 0 && !last) - i_error("dbox rebuild: Failed to fix file %s", path); - dbox_file_unref(&file); - return ret < 0 ? -1 : 0; -} - -static void -rebuild_add_missing_map_uids(struct dbox_storage_rebuild_context *ctx, - uint32_t next_uid) -{ - struct dbox_rebuild_msg **msgs; - struct dbox_mail_index_map_record rec; - unsigned int i, count; - uint32_t seq; - - memset(&rec, 0, sizeof(rec)); - msgs = array_get_modifiable(&ctx->msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i]->map_uid != 0) - continue; - - rec.file_id = msgs[i]->file_id; - rec.offset = msgs[i]->offset; - rec.size = msgs[i]->size; - - msgs[i]->map_uid = next_uid++; - mail_index_append(ctx->trans, msgs[i]->map_uid, &seq); - mail_index_update_ext(ctx->trans, seq, - ctx->storage->map->map_ext_id, - &rec, NULL); - } -} - -static int rebuild_apply_map(struct dbox_storage_rebuild_context *ctx) -{ - struct dbox_map *map = ctx->storage->map; - const struct mail_index_header *hdr; - struct dbox_rebuild_msg *const *msgs, **pos; - struct dbox_rebuild_msg search_msg, *search_msgp = &search_msg; - struct dbox_mail_lookup_rec rec; - uint32_t seq; - unsigned int count; - - if (ctx->msgs_unsorted) - array_sort(&ctx->msgs, dbox_rebuild_msg_offset_cmp); - - msgs = array_get_modifiable(&ctx->msgs, &count); - hdr = mail_index_get_header(ctx->sync_view); - for (seq = 1; seq <= hdr->messages_count; seq++) { - if (dbox_map_view_lookup_rec(map, ctx->sync_view, - seq, &rec) < 0) - return -1; - - /* look up the rebuild msg record for this message */ - search_msg.file_id = rec.rec.file_id; - search_msg.offset = rec.rec.offset; - pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs), - dbox_rebuild_msg_offset_cmp); - if (pos == NULL) { - /* map record points to non-existing message. */ - mail_index_expunge(ctx->trans, seq); - } else { - (*pos)->map_uid = rec.map_uid; - if (rec.refcount == 0) - (*pos)->seen_zero_ref_in_map = TRUE; - } - } - rebuild_add_missing_map_uids(ctx, hdr->next_uid); - - /* afterwards we're interested in looking up map_uids. - re-sort the messages to make it easier. */ - array_sort(&ctx->msgs, dbox_rebuild_msg_uid_cmp); - return 0; -} - -static struct dbox_rebuild_msg * -rebuild_lookup_map_uid(struct dbox_storage_rebuild_context *ctx, - uint32_t map_uid) -{ - struct dbox_rebuild_msg search_msg, *search_msgp = &search_msg; - struct dbox_rebuild_msg **pos; - - search_msg.map_uid = map_uid; - pos = array_bsearch(&ctx->msgs, &search_msgp, dbox_rebuild_msg_uid_cmp); - return pos == NULL ? NULL : *pos; -} - -static void -rebuild_mailbox_multi(struct dbox_storage_rebuild_context *ctx, - struct dbox_sync_rebuild_context *rebuild_ctx, - struct dbox_mailbox *mbox, - struct mail_index_view *view, - struct mail_index_transaction *trans) -{ - const struct dbox_mail_index_record *dbox_rec; - struct dbox_mail_index_record new_dbox_rec; - const struct mail_index_header *hdr; - struct dbox_rebuild_msg *rec; - const void *data; - bool expunged; - uint32_t seq, uid, new_seq, map_uid; - - memset(&new_dbox_rec, 0, sizeof(new_dbox_rec)); - hdr = mail_index_get_header(view); - for (seq = 1; seq <= hdr->messages_count; seq++) { - mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, - &data, &expunged); - dbox_rec = data; - map_uid = dbox_rec == NULL ? 0 : dbox_rec->map_uid; - - mail_index_lookup_ext(view, seq, mbox->guid_ext_id, - &data, &expunged); - - /* see if we can find this message based on - 1) GUID, 2) map_uid */ - rec = data == NULL ? NULL : - hash_table_lookup(ctx->guid_hash, data); - if (rec == NULL) { - if (map_uid == 0) { - /* not a multi-dbox message, ignore. */ - continue; - } - /* multi-dbox message that wasn't found with GUID. - either it's lost or GUID has been corrupted. we can - still try to look it up using map_uid. */ - rec = rebuild_lookup_map_uid(ctx, map_uid); - if (rec != NULL) { - mail_index_update_ext(trans, seq, - mbox->guid_ext_id, - rec->guid_128, NULL); - } - } else if (map_uid != rec->map_uid) { - /* map_uid is wrong, update it */ - i_assert(rec->map_uid != 0); - new_dbox_rec.map_uid = rec->map_uid; - mail_index_update_ext(trans, seq, mbox->dbox_ext_id, - &new_dbox_rec, NULL); - } else { - /* everything was ok */ - } - - if (rec != NULL) T_BEGIN { - /* keep this message */ - rec->refcount++; - - mail_index_lookup_uid(view, seq, &uid); - mail_index_append(trans, uid, &new_seq); - dbox_sync_rebuild_index_metadata(rebuild_ctx, - NULL, new_seq, uid); - - new_dbox_rec.map_uid = rec->map_uid; - mail_index_update_ext(trans, new_seq, - mbox->dbox_ext_id, - &new_dbox_rec, NULL); - } T_END; - } -} - -static int -rebuild_mailbox(struct dbox_storage_rebuild_context *ctx, - struct mail_namespace *ns, const char *name) -{ - struct mailbox *box; - struct dbox_mailbox *mbox; - struct mail_index_sync_ctx *sync_ctx; - struct mail_index_view *view; - struct mail_index_transaction *trans; - struct dbox_sync_rebuild_context *rebuild_ctx; - enum mail_error error; - int ret; - - box = dbox_mailbox_alloc(&ctx->storage->storage, ns->list, name, NULL, - MAILBOX_FLAG_READONLY | - MAILBOX_FLAG_KEEP_RECENT | - MAILBOX_FLAG_IGNORE_ACLS); - if (dbox_mailbox_open(box) < 0) { - (void)mail_storage_get_last_error(box->storage, &error); - mailbox_close(&box); - if (error == MAIL_ERROR_TEMP) - return -1; - /* non-temporary error, ignore */ - return 0; - } - mbox = (struct dbox_mailbox *)box; - - ret = mail_index_sync_begin(mbox->ibox.index, &sync_ctx, &view, &trans, - MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES); - if (ret <= 0) { - i_assert(ret != 0); - mail_storage_set_index_error(&mbox->ibox); - mailbox_close(&box); - return -1; - } - - rebuild_ctx = dbox_sync_index_rebuild_init(mbox, view, trans, TRUE); - ret = dbox_sync_index_rebuild_singles(rebuild_ctx); - if (ret == 0) - rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans); - dbox_sync_index_rebuild_deinit(&rebuild_ctx); - - if (mail_index_sync_commit(&sync_ctx) < 0) { - mail_storage_set_index_error(&mbox->ibox); - ret = -1; - } - - mailbox_close(&box); - return ret < 0 ? -1 : 0; -} - -static int rebuild_namespace_mailboxes(struct dbox_storage_rebuild_context *ctx, - struct mail_namespace *ns) -{ - struct mailbox_list_iterate_context *iter; - const struct mailbox_info *info; - int ret = 0; - - if (ctx->default_list == NULL || - (ns->flags & NAMESPACE_FLAG_INBOX) != 0) - ctx->default_list = ns->list; - - iter = mailbox_list_iter_init(ns->list, "*", - MAILBOX_LIST_ITER_RETURN_NO_FLAGS); - while ((info = mailbox_list_iter_next(iter)) != NULL) { - if ((info->flags & (MAILBOX_NONEXISTENT | - MAILBOX_NOSELECT)) == 0) { - T_BEGIN { - ret = rebuild_mailbox(ctx, ns, info->name); - } T_END; - if (ret < 0) { - ret = -1; - break; - } - } - } - if (mailbox_list_iter_deinit(&iter) < 0) - ret = -1; - return ret; -} - -static int rebuild_mailboxes(struct dbox_storage_rebuild_context *ctx) -{ - struct mail_user *user = ctx->storage->storage.user; - struct mail_namespace *ns; - - for (ns = user->namespaces; ns != NULL; ns = ns->next) { - if (ns->storage == &ctx->storage->storage && - ns->alias_for == NULL) { - if (rebuild_namespace_mailboxes(ctx, ns) < 0) - return -1; - } - } - return 0; -} - -static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg) -{ - if (mail_index_sync_commit(&msg->sync_ctx) < 0) - return -1; - mailbox_close(&msg->box); - memset(msg, 0, sizeof(*msg)); - return 0; -} - -static int rebuild_restore_msg(struct dbox_storage_rebuild_context *ctx, - struct dbox_rebuild_msg *msg) -{ - struct mail_storage *storage = &ctx->storage->storage; - struct dbox_file *file; - const struct mail_index_header *hdr; - struct dbox_mail_index_record dbox_rec; - const char *mailbox = NULL; - struct mailbox *box; - struct dbox_mailbox *mbox; - enum mail_error error; - bool expunged, created; - uoff_t size; - int ret; - uint32_t seq; - - /* first see if message contains the mailbox it was originally - saved to */ - file = dbox_file_init_multi(ctx->storage, msg->file_id); - ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL, - &expunged); - if (ret > 0 && !expunged && dbox_file_metadata_read(file) > 0) { - mailbox = dbox_file_metadata_get(file, - DBOX_METADATA_ORIG_MAILBOX); - } - dbox_file_unref(&file); - if (ret <= 0 || expunged) { - if (ret < 0) - return -1; - /* we shouldn't get here, so apparently we couldn't fix - something. just ignore the mail.. */ - return 0; - } - - if (mailbox == NULL) - mailbox = "INBOX"; - - /* we have the destination mailbox. now open it and add the message - there. */ - created = FALSE; - box = ctx->prev_msg.box != NULL && - strcmp(mailbox, ctx->prev_msg.box->name) == 0 ? - ctx->prev_msg.box : NULL; - while (box == NULL) { - box = dbox_mailbox_alloc(storage, ctx->default_list, - mailbox, NULL, - MAILBOX_FLAG_READONLY | - MAILBOX_FLAG_KEEP_RECENT | - MAILBOX_FLAG_IGNORE_ACLS); - if (dbox_mailbox_open(box) == 0) - break; - - (void)mail_storage_get_last_error(box->storage, &error); - if (error == MAIL_ERROR_NOTFOUND && !created) { - /* mailbox doesn't exist currently? see if creating - it helps. */ - created = TRUE; - (void)mailbox_create(box, NULL, FALSE); - mailbox_close(&box); - continue; - } - - mailbox_close(&box); - if (error == MAIL_ERROR_TEMP) - return -1; - - if (strcmp(mailbox, "INBOX") != 0) { - /* see if we can save to INBOX instead. */ - mailbox = "INBOX"; - } else { - /* this shouldn't happen */ - return -1; - } - } - mbox = (struct dbox_mailbox *)box; - - /* switch the mailbox cache if necessary */ - if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) { - if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) - return -1; - } - if (ctx->prev_msg.box == NULL) { - ret = mail_index_sync_begin(mbox->ibox.index, - &ctx->prev_msg.sync_ctx, - &ctx->prev_msg.view, - &ctx->prev_msg.trans, 0); - if (ret <= 0) { - i_assert(ret != 0); - mail_storage_set_index_error(&mbox->ibox); - mailbox_close(&box); - return -1; - } - ctx->prev_msg.box = box; - hdr = mail_index_get_header(ctx->prev_msg.view); - ctx->prev_msg.next_uid = hdr->next_uid; - } - - /* add the new message */ - memset(&dbox_rec, 0, sizeof(dbox_rec)); - dbox_rec.map_uid = msg->map_uid; - dbox_rec.save_date = ioloop_time; - mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq); - mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->dbox_ext_id, - &dbox_rec, NULL); - mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id, - msg->guid_128, NULL); - - msg->refcount++; - return 0; -} - -static int rebuild_handle_zero_refs(struct dbox_storage_rebuild_context *ctx) -{ - struct dbox_rebuild_msg **msgs; - unsigned int i, count; - - /* if we have messages at this point which have refcount=0, they're - either already expunged or they were somehow lost for some reason. - we'll need to figure out what to do about them. */ - msgs = array_get_modifiable(&ctx->msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i]->refcount != 0) - continue; - - if (msgs[i]->seen_zero_ref_in_map) { - /* we've seen the map record, trust it. */ - continue; - } - /* either map record was lost for this message or the message - was lost from its mailbox. safest way to handle this is to - restore the message. */ - if (rebuild_restore_msg(ctx, msgs[i]) < 0) - return -1; - } - if (ctx->prev_msg.box != NULL) { - if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) - return -1; - } - return 0; -} - -static void rebuild_update_refcounts(struct dbox_storage_rebuild_context *ctx) -{ - const struct mail_index_header *hdr; - const void *data; - struct dbox_rebuild_msg **msgs; - const uint16_t *ref16_p; - bool expunged; - uint32_t seq, map_uid; - unsigned int i, count; - - /* update refcounts for existing map records */ - msgs = array_get_modifiable(&ctx->msgs, &count); - hdr = mail_index_get_header(ctx->sync_view); - for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) { - mail_index_lookup_uid(ctx->sync_view, seq, &map_uid); - if (map_uid != msgs[i]->map_uid) { - /* we've already expunged this map record */ - i_assert(map_uid < msgs[i]->map_uid); - continue; - } - - mail_index_lookup_ext(ctx->sync_view, seq, - ctx->storage->map->ref_ext_id, - &data, &expunged); - ref16_p = data; - if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) { - mail_index_update_ext(ctx->trans, seq, - ctx->storage->map->ref_ext_id, - &msgs[i]->refcount, NULL); - } - i++; - } - - /* update refcounts for newly created map records */ - for (; i < count; i++, seq++) { - mail_index_update_ext(ctx->trans, seq, - ctx->storage->map->ref_ext_id, - &msgs[i]->refcount, NULL); - } -} - -static int rebuild_finish(struct dbox_storage_rebuild_context *ctx) -{ - if (rebuild_handle_zero_refs(ctx) < 0) - return -1; - rebuild_update_refcounts(ctx); - return 0; -} - -static int dbox_storage_rebuild_scan(struct dbox_storage_rebuild_context *ctx) -{ - const struct mail_index_header *hdr; - DIR *dir; - struct dirent *d; - string_t *path; - unsigned int dir_len; - uint32_t uid_validity; - int ret = 0; - - if (dbox_map_open(ctx->storage->map, TRUE) < 0) - return -1; - - /* begin by locking the map, so that other processes can't try to - rebuild at the same time. */ - ret = mail_index_sync_begin(ctx->storage->map->index, &ctx->sync_ctx, - &ctx->sync_view, &ctx->trans, 0); - if (ret <= 0) { - i_assert(ret != 0); - mail_storage_set_internal_error(&ctx->storage->storage); - mail_index_reset_error(ctx->storage->map->index); - return -1; - } - - uid_validity = dbox_map_get_uid_validity(ctx->storage->map); - hdr = mail_index_get_header(ctx->sync_view); - if (hdr->uid_validity != uid_validity) { - mail_index_update_header(ctx->trans, - offsetof(struct mail_index_header, uid_validity), - &uid_validity, sizeof(uid_validity), TRUE); - } - - dir = opendir(ctx->storage->storage_dir); - if (dir == NULL) { - mail_storage_set_critical(&ctx->storage->storage, - "opendir(%s) failed: %m", ctx->storage->storage_dir); - return -1; - } - path = t_str_new(256); - str_append(path, ctx->storage->storage_dir); - str_append_c(path, '/'); - dir_len = str_len(path); - - for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) { - if (strncmp(d->d_name, DBOX_MAIL_FILE_MULTI_PREFIX, - strlen(DBOX_MAIL_FILE_MULTI_PREFIX)) == 0) { - str_truncate(path, dir_len); - str_append(path, d->d_name); - T_BEGIN { - ret = rebuild_add_file(ctx, str_c(path)); - } T_END; - if (ret < 0) { - ret = -1; - break; - } - } - } - if (ret == 0 && errno != 0) { - mail_storage_set_critical(&ctx->storage->storage, - "readdir(%s) failed: %m", ctx->storage->storage_dir); - ret = -1; - } - if (closedir(dir) < 0) { - mail_storage_set_critical(&ctx->storage->storage, - "closedir(%s) failed: %m", ctx->storage->storage_dir); - ret = -1; - } - - if (ret < 0 || - rebuild_apply_map(ctx) < 0 || - rebuild_mailboxes(ctx) < 0 || - rebuild_finish(ctx) < 0 || - mail_index_sync_commit(&ctx->sync_ctx) < 0) - return -1; - return 0; -} - -int dbox_storage_rebuild(struct dbox_storage *storage) -{ - struct dbox_storage_rebuild_context *ctx; - struct stat st; - int ret; - - if (stat(storage->storage_dir, &st) < 0) { - if (errno == ENOENT) { - /* no multi-dbox files */ - return 0; - } - - mail_storage_set_critical(&storage->storage, - "stat(%s) failed: %m", storage->storage_dir); - return -1; - } - storage->have_multi_msgs = TRUE; - - i_warning("dbox %s: rebuilding indexes", storage->storage_dir); - - ctx = dbox_storage_rebuild_init(storage); - ret = dbox_storage_rebuild_scan(ctx); - dbox_storage_rebuild_deinit(ctx); - - if (ret == 0) - storage->sync_rebuild = FALSE; - return ret; -}
--- a/src/lib-storage/index/dbox/dbox-storage-rebuild.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ -#ifndef DBOX_STORAGE_REBUILD_H -#define DBOX_STORAGE_REBUILD_H - -int dbox_storage_rebuild(struct dbox_storage *storage); - -#endif
--- a/src/lib-storage/index/dbox/dbox-storage.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,925 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "ioloop.h" -#include "str.h" -#include "hex-binary.h" -#include "randgen.h" -#include "mkdir-parents.h" -#include "unlink-directory.h" -#include "unlink-old-files.h" -#include "index-mail.h" -#include "mail-copy.h" -#include "mail-index-modseq.h" -#include "mailbox-uidvalidity.h" -#include "maildir/maildir-uidlist.h" -#include "dbox-map.h" -#include "dbox-file.h" -#include "dbox-sync.h" -#include "dbox-storage-rebuild.h" -#include "dbox-storage.h" - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <dirent.h> -#include <sys/stat.h> - -#define DBOX_LIST_CONTEXT(obj) \ - MODULE_CONTEXT(obj, dbox_mailbox_list_module) - -struct dbox_mailbox_list { - union mailbox_list_module_context module_ctx; - const struct dbox_settings *set; -}; - -extern struct mail_storage dbox_storage; -extern struct mailbox dbox_mailbox; - -static MODULE_CONTEXT_DEFINE_INIT(dbox_mailbox_list_module, - &mailbox_list_module_register); - -static struct mail_storage *dbox_storage_alloc(void) -{ - struct dbox_storage *storage; - pool_t pool; - - pool = pool_alloconly_create("dbox storage", 512+256); - storage = p_new(pool, struct dbox_storage, 1); - storage->storage = dbox_storage; - storage->storage.pool = pool; - return &storage->storage; -} - -static int -dbox_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, - const char **error_r) -{ - struct dbox_storage *storage = (struct dbox_storage *)_storage; - const char *dir, *origin; - - storage->set = mail_storage_get_driver_settings(_storage); - i_assert(storage->set->dbox_max_open_files >= 2); - - if (*ns->list->set.mailbox_dir_name == '\0') { - *error_r = "dbox: MAILBOXDIR must not be empty"; - return -1; - } - - _storage->unique_root_dir = - p_strdup(_storage->pool, ns->list->set.root_dir); - - dir = mailbox_list_get_path(ns->list, NULL, MAILBOX_LIST_PATH_TYPE_DIR); - storage->storage_dir = p_strconcat(_storage->pool, dir, - "/"DBOX_GLOBAL_DIR_NAME, NULL); - storage->alt_storage_dir = p_strconcat(_storage->pool, - ns->list->set.alt_dir, - "/"DBOX_GLOBAL_DIR_NAME, NULL); - i_array_init(&storage->open_files, - I_MIN(storage->set->dbox_max_open_files, 128)); - - storage->map = dbox_map_init(storage); - mailbox_list_get_dir_permissions(ns->list, NULL, - &storage->dir_create_mode, - &storage->create_gid, &origin); - storage->create_gid_origin = p_strdup(_storage->pool, origin); - return 0; -} - -static void dbox_storage_destroy(struct mail_storage *_storage) -{ - struct dbox_storage *storage = (struct dbox_storage *)_storage; - - if (storage->sync_rebuild) { - if (dbox_storage_rebuild(storage) < 0) - return; - } - - dbox_files_free(storage); - dbox_map_deinit(&storage->map); - array_free(&storage->open_files); - index_storage_destroy(_storage); -} - -static void -dbox_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, - struct mailbox_list_settings *set) -{ - if (set->layout == NULL) - set->layout = MAILBOX_LIST_NAME_FS; - if (set->subscription_fname == NULL) - set->subscription_fname = DBOX_SUBSCRIPTION_FILE_NAME; - if (set->dir_guid_fname == NULL) - set->dir_guid_fname = DBOX_DIR_GUID_FILE_NAME; - if (set->maildir_name == NULL) - set->maildir_name = DBOX_MAILDIR_NAME; - if (set->mailbox_dir_name == NULL) - set->mailbox_dir_name = DBOX_MAILBOX_DIR_NAME; -} - -static const char * -dbox_get_alt_path(struct mailbox_list *list, const char *path) -{ -#if 0 // FIXME - const char *root; - unsigned int len; - - if (storage->alt_dir == NULL) - return NULL; - - root = mailbox_list_get_path(storage->storage.list, NULL, - MAILBOX_LIST_PATH_TYPE_DIR); - - len = strlen(root); - if (strncmp(path, root, len) != 0 && path[len] == '/') { - /* can't determine the alt path - shouldn't happen */ - return NULL; - } - return t_strconcat(storage->alt_dir, path + len, NULL); -#endif - return NULL; -} - -struct mailbox * -dbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, - const char *name, struct istream *input, - enum mailbox_flags flags) -{ - struct dbox_mailbox *mbox; - pool_t pool; - - /* dbox can't work without index files */ - flags &= ~MAILBOX_FLAG_NO_INDEX_FILES; - - pool = pool_alloconly_create("dbox mailbox", 1024+512); - mbox = p_new(pool, struct dbox_mailbox, 1); - mbox->ibox.box = dbox_mailbox; - mbox->ibox.box.pool = pool; - mbox->ibox.box.storage = storage; - mbox->ibox.box.list = list; - mbox->ibox.mail_vfuncs = &dbox_mail_vfuncs; - - mbox->ibox.save_commit_pre = dbox_transaction_save_commit_pre; - mbox->ibox.save_commit_post = dbox_transaction_save_commit_post; - mbox->ibox.save_rollback = dbox_transaction_save_rollback; - - index_storage_mailbox_alloc(&mbox->ibox, name, input, flags, - DBOX_INDEX_PREFIX); - mail_index_set_fsync_types(mbox->ibox.index, - MAIL_INDEX_SYNC_TYPE_APPEND | - MAIL_INDEX_SYNC_TYPE_EXPUNGE); - - mbox->ibox.index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | - MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY; - - mbox->storage = (struct dbox_storage *)storage; - mbox->alt_path = - p_strdup(pool, dbox_get_alt_path(list, mbox->ibox.box.path)); - mbox->dbox_ext_id = - mail_index_ext_register(mbox->ibox.index, "dbox", 0, - sizeof(struct dbox_mail_index_record), - sizeof(uint32_t)); - mbox->dbox_hdr_ext_id = - mail_index_ext_register(mbox->ibox.index, "dbox-hdr", - sizeof(struct dbox_index_header), 0, 0); - mbox->guid_ext_id = - mail_index_ext_register(mbox->ibox.index, "guid", - 0, MAIL_GUID_128_SIZE, 1); - - mbox->maildir_uidlist = maildir_uidlist_init_readonly(&mbox->ibox); - return &mbox->ibox.box; -} - -uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list) -{ - const char *path; - - path = mailbox_list_get_path(list, NULL, - MAILBOX_LIST_PATH_TYPE_CONTROL); - path = t_strconcat(path, "/"DBOX_UIDVALIDITY_FILE_NAME, NULL); - return mailbox_uidvalidity_next(path); -} - -int dbox_read_header(struct dbox_mailbox *mbox, struct dbox_index_header *hdr) -{ - const void *data; - size_t data_size; - - mail_index_get_header_ext(mbox->ibox.view, mbox->dbox_hdr_ext_id, - &data, &data_size); - if (data_size < DBOX_INDEX_HEADER_MIN_SIZE && - (!mbox->creating || data_size != 0)) { - mail_storage_set_critical(&mbox->storage->storage, - "dbox %s: Invalid dbox header size", - mbox->ibox.box.path); - return -1; - } - memset(hdr, 0, sizeof(*hdr)); - memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr))); - return 0; -} - -void dbox_update_header(struct dbox_mailbox *mbox, - struct mail_index_transaction *trans, - const struct mailbox_update *update) -{ - struct dbox_index_header hdr, new_hdr; - - if (dbox_read_header(mbox, &hdr) < 0) - memset(&hdr, 0, sizeof(hdr)); - - new_hdr = hdr; - - if (update != NULL && !mail_guid_128_is_empty(update->mailbox_guid)) { - memcpy(new_hdr.mailbox_guid, update->mailbox_guid, - sizeof(new_hdr.mailbox_guid)); - } else if (mail_guid_128_is_empty(new_hdr.mailbox_guid)) { - mail_generate_guid_128(new_hdr.mailbox_guid); - } - - new_hdr.map_uid_validity = - dbox_map_get_uid_validity(mbox->storage->map); - if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) { - mail_index_update_header_ext(trans, mbox->dbox_hdr_ext_id, 0, - &new_hdr, sizeof(new_hdr)); - } -} - -static int dbox_write_index_header(struct mailbox *box, - const struct mailbox_update *update) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; - struct mail_index_transaction *trans; - const struct mail_index_header *hdr; - uint32_t uid_validity, uid_next; - - if (dbox_map_open(mbox->storage->map, TRUE) < 0) - return -1; - - hdr = mail_index_get_header(mbox->ibox.view); - trans = mail_index_transaction_begin(mbox->ibox.view, 0); - dbox_update_header(mbox, trans, update); - - if (update != NULL && update->uid_validity != 0) - uid_validity = update->uid_validity; - else if (hdr->uid_validity == 0) { - /* set uidvalidity */ - uid_validity = dbox_get_uidvalidity_next(box->list); - } - - if (hdr->uid_validity != uid_validity) { - mail_index_update_header(trans, - offsetof(struct mail_index_header, uid_validity), - &uid_validity, sizeof(uid_validity), TRUE); - } - if (update != NULL && hdr->next_uid < update->min_next_uid) { - uid_next = update->min_next_uid; - mail_index_update_header(trans, - offsetof(struct mail_index_header, next_uid), - &uid_next, sizeof(uid_next), TRUE); - } - if (update != NULL && update->min_highest_modseq != 0 && - mail_index_modseq_get_highest(mbox->ibox.view) < - update->min_highest_modseq) { - mail_index_update_highest_modseq(trans, - update->min_highest_modseq); - } - - if (mail_index_transaction_commit(&trans) < 0) { - mail_storage_set_internal_error(box->storage); - mail_index_reset_error(mbox->ibox.index); - return -1; - } - return 0; -} - -static int dbox_mailbox_create_indexes(struct mailbox *box, - const struct mailbox_update *update) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; - const char *origin; - mode_t mode; - gid_t gid; - int ret; - - mailbox_list_get_dir_permissions(box->list, NULL, &mode, &gid, &origin); - if (mkdir_parents_chgrp(box->path, mode, gid, origin) == 0) { - /* create indexes immediately with the dbox header */ - if (index_storage_mailbox_open(box) < 0) - return -1; - mbox->creating = TRUE; - ret = dbox_write_index_header(box, update); - mbox->creating = FALSE; - if (ret < 0) - return -1; - } else if (errno != EEXIST) { - if (!mail_storage_set_error_from_errno(box->storage)) { - mail_storage_set_critical(box->storage, - "mkdir(%s) failed: %m", box->path); - } - return -1; - } - return 0; -} - -static bool -dbox_cleanup_if_exists(struct mailbox_list *list, const char *path) -{ - struct stat st; - - if (stat(path, &st) < 0) - return FALSE; - - /* check once in a while if there are temp files to clean up */ - if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { - /* there haven't been any changes to this directory since we - last checked it. */ - } else if (st.st_atime < ioloop_time - DBOX_TMP_SCAN_SECS) { - /* time to scan */ - const char *prefix = - mailbox_list_get_global_temp_prefix(list); - - (void)unlink_old_files(path, prefix, - ioloop_time - DBOX_TMP_DELETE_SECS); - } - return TRUE; -} - -int dbox_mailbox_open(struct mailbox *box) -{ - if (box->input != NULL) { - mail_storage_set_critical(box->storage, - "dbox doesn't support streamed mailboxes"); - return -1; - } - - if (dbox_cleanup_if_exists(box->list, box->path)) { - return index_storage_mailbox_open(box); - } else if (errno == ENOENT) { - if (strcmp(box->name, "INBOX") == 0 && - (box->list->ns->flags & NAMESPACE_FLAG_INBOX) != 0) { - /* INBOX always exists, create it */ - if (dbox_mailbox_create_indexes(box, NULL) < 0) - return -1; - return box->opened ? 0 : - index_storage_mailbox_open(box); - } - - mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, - T_MAIL_ERR_MAILBOX_NOT_FOUND(box->name)); - return -1; - } else if (errno == EACCES) { - mail_storage_set_critical(box->storage, "%s", - mail_error_eacces_msg("stat", box->path)); - return -1; - } else { - mail_storage_set_critical(box->storage, - "stat(%s) failed: %m", box->path); - return -1; - } -} - -static void dbox_mailbox_close(struct mailbox *box) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; - - maildir_uidlist_deinit(&mbox->maildir_uidlist); - index_storage_mailbox_close(box); -} - -static void dbox_storage_get_status_guid(struct mailbox *box, - struct mailbox_status *status_r) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; - struct dbox_index_header hdr; - - if (dbox_read_header(mbox, &hdr) < 0) - memset(&hdr, 0, sizeof(hdr)); - - if (mail_guid_128_is_empty(hdr.mailbox_guid)) { - /* regenerate it */ - if (dbox_write_index_header(box, NULL) < 0 || - dbox_read_header(mbox, &hdr) < 0) - return; - } - memcpy(status_r->mailbox_guid, hdr.mailbox_guid, - sizeof(status_r->mailbox_guid)); -} - -static void -dbox_storage_get_status(struct mailbox *box, enum mailbox_status_items items, - struct mailbox_status *status_r) -{ - index_storage_get_status(box, items, status_r); - - if ((items & STATUS_GUID) != 0) - dbox_storage_get_status_guid(box, status_r); -} - -static int -dbox_mailbox_create(struct mailbox *box, const struct mailbox_update *update, - bool directory) -{ - const char *path, *alt_path, *origin; - struct stat st; - - path = mailbox_list_get_path(box->list, box->name, - directory ? MAILBOX_LIST_PATH_TYPE_DIR : - MAILBOX_LIST_PATH_TYPE_MAILBOX); - if (stat(path, &st) == 0) { - mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, - "Mailbox already exists"); - return -1; - } - - if (directory) { - mode_t mode; - gid_t gid; - - mailbox_list_get_dir_permissions(box->list, NULL, &mode, - &gid, &origin); - if (mkdir_parents_chgrp(path, mode, gid, origin) == 0) - return 0; - else if (errno == EEXIST) { - mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, - "Mailbox already exists"); - } else if (!mail_storage_set_error_from_errno(box->storage)) { - mail_storage_set_critical(box->storage, - "mkdir(%s) failed: %m", path); - } - return -1; - } - - /* make sure the alt path doesn't exist yet. it shouldn't (except with - race conditions with RENAME/DELETE), but if something crashed and - left it lying around we don't want to start overwriting files in - it. */ - alt_path = dbox_get_alt_path(box->list, path); - if (alt_path != NULL && stat(alt_path, &st) == 0) { - mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, - "Mailbox already exists"); - return -1; - } - - return dbox_mailbox_create_indexes(box, update); -} - -static int -dbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) -{ - if (!box->opened) { - if (index_storage_mailbox_open(box) < 0) - return -1; - } - return dbox_write_index_header(box, update); -} - -static int -dbox_mailbox_unref_mails(struct mailbox_list *list, const char *path) -{ - struct dbox_storage *storage = (struct dbox_storage *)list->ns->storage; - const struct mail_storage_settings *old_set; - struct mail_storage_settings tmp_set; - struct mailbox *box; - struct dbox_mailbox *mbox; - const struct mail_index_header *hdr; - const struct dbox_mail_index_record *dbox_rec; - struct dbox_map_transaction_context *map_trans; - ARRAY_TYPE(uint32_t) map_uids; - const void *data; - bool expunged; - uint32_t seq; - int ret; - - old_set = list->mail_set; - tmp_set = *list->mail_set; - tmp_set.mail_full_filesystem_access = TRUE; - list->mail_set = &tmp_set; - box = dbox_mailbox_alloc(&storage->storage, list, path, NULL, - MAILBOX_FLAG_IGNORE_ACLS | - MAILBOX_FLAG_KEEP_RECENT); - ret = mailbox_open(box); - list->mail_set = old_set; - if (ret < 0) { - mailbox_close(&box); - return -1; - } - mbox = (struct dbox_mailbox *)box; - - /* get a list of all map_uids in this mailbox */ - i_array_init(&map_uids, 128); - hdr = mail_index_get_header(mbox->ibox.view); - for (seq = 1; seq <= hdr->messages_count; seq++) { - mail_index_lookup_ext(mbox->ibox.view, seq, mbox->dbox_ext_id, - &data, &expunged); - dbox_rec = data; - if (dbox_rec == NULL) { - /* no multi-mails */ - break; - } - if (dbox_rec->map_uid != 0) - array_append(&map_uids, &dbox_rec->map_uid, 1); - } - - /* unreference the map_uids */ - map_trans = dbox_map_transaction_begin(storage->map, FALSE); - ret = dbox_map_update_refcounts(map_trans, &map_uids, -1); - if (ret == 0) - ret = dbox_map_transaction_commit(map_trans); - dbox_map_transaction_free(&map_trans); - array_free(&map_uids); - mailbox_close(&box); - return ret; -} - -static const char *dbox_get_trash_dest(const char *trash_dir) -{ - const char *path; - unsigned char randbuf[16]; - struct stat st; - - do { - random_fill_weak(randbuf, sizeof(randbuf)); - path = t_strconcat(trash_dir, "/", - binary_to_hex(randbuf, sizeof(randbuf)), NULL); - } while (lstat(path, &st) == 0); - return path; -} - -static int -dbox_list_delete_mailbox(struct mailbox_list *list, const char *name) -{ - struct dbox_mailbox_list *mlist = DBOX_LIST_CONTEXT(list); - struct stat st; - const char *path, *alt_path, *trash_dir, *trash_dest; - bool deleted = FALSE; - int ret; - - /* Make sure the indexes are closed before trying to delete the - directory that contains them. It can still fail with some NFS - implementations if indexes are opened by another session, but - that can't really be helped. */ - index_storage_destroy_unrefed(); - - /* delete the index and control directories */ - if (mlist->module_ctx.super.delete_mailbox(list, name) < 0) - return -1; - - path = mailbox_list_get_path(list, name, - MAILBOX_LIST_PATH_TYPE_MAILBOX); - trash_dir = mailbox_list_get_path(list, NULL, - MAILBOX_LIST_PATH_TYPE_DIR); - trash_dir = t_strconcat(trash_dir, "/"DBOX_TRASH_DIR_NAME, NULL); - trash_dest = dbox_get_trash_dest(trash_dir); - - /* first try renaming the actual mailbox to trash directory */ - ret = rename(path, trash_dest); - if (ret < 0 && errno == ENOENT) { - /* either source mailbox doesn't exist or trash directory - doesn't exist. try creating the trash and retrying. */ - const char *origin; - mode_t mode; - gid_t gid; - - mailbox_list_get_dir_permissions(list, NULL, &mode, - &gid, &origin); - if (mkdir_parents_chgrp(trash_dir, mode, gid, origin) < 0 && - errno != EEXIST) { - mailbox_list_set_critical(list, - "mkdir(%s) failed: %m", trash_dir); - return -1; - } - ret = rename(path, trash_dest); - } - if (ret == 0) { - if (dbox_mailbox_unref_mails(list, trash_dest) < 0) { - /* we've already renamed it. there's no going back. */ - mailbox_list_set_internal_error(list); - ret = -1; - } - if (unlink_directory(trash_dest, TRUE) < 0) { - mailbox_list_set_critical(list, - "unlink_directory(%s) failed: %m", trash_dest); - ret = -1; - } - /* if there's an alt path, delete it too */ - alt_path = dbox_get_alt_path(list, path); - if (alt_path != NULL) { - if (unlink_directory(alt_path, TRUE) < 0) { - mailbox_list_set_critical(list, - "unlink_directory(%s) failed: %m", alt_path); - ret = -1; - } - } - /* try to delete the parent directory also */ - deleted = TRUE; - path = mailbox_list_get_path(list, name, - MAILBOX_LIST_PATH_TYPE_DIR); - } else if (errno != ENOENT) { - mailbox_list_set_critical(list, "stat(%s) failed: %m", path); - return -1; - } else { - /* mailbox not found - what about the directory? */ - path = mailbox_list_get_path(list, name, - MAILBOX_LIST_PATH_TYPE_DIR); - if (stat(path, &st) == 0) { - /* delete the directory */ - } else if (errno == ENOENT) { - mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND, - T_MAIL_ERR_MAILBOX_NOT_FOUND(name)); - return -1; - } else if (!mailbox_list_set_error_from_errno(list)) { - mailbox_list_set_critical(list, "stat(%s) failed: %m", - path); - return -1; - } - ret = 0; - } - - alt_path = dbox_get_alt_path(list, path); - if (alt_path != NULL) - (void)rmdir(alt_path); - - if (rmdir(path) == 0) - return ret; - else if (errno == ENOTEMPTY) { - if (deleted) - return ret; - mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND, - t_strdup_printf("Directory %s isn't empty, " - "can't delete it.", name)); - } else if (!mailbox_list_set_error_from_errno(list)) { - mailbox_list_set_critical(list, "rmdir() failed for %s: %m", - path); - } - return -1; -} - -static int -dbox_list_rename_get_alt_paths(struct mailbox_list *oldlist, - const char *oldname, - struct mailbox_list *newlist, - const char *newname, - enum mailbox_list_path_type path_type, - const char **oldpath_r, const char **newpath_r) -{ - const char *path; - - path = mailbox_list_get_path(oldlist, oldname, path_type); - *oldpath_r = dbox_get_alt_path(oldlist, path); - if (*oldpath_r == NULL) - return 0; - - path = mailbox_list_get_path(newlist, newname, path_type); - *newpath_r = dbox_get_alt_path(newlist, path); - if (*newpath_r == NULL) { - /* destination dbox storage doesn't have alt-path defined. - we can't do the rename easily. */ - mailbox_list_set_error(oldlist, MAIL_ERROR_NOTPOSSIBLE, - "Can't rename mailboxes across specified storages."); - return -1; - } - return 1; -} - -static int -dbox_list_rename_mailbox_pre(struct mailbox_list *oldlist, - const char *oldname, - struct mailbox_list *newlist, - const char *newname) -{ - const char *alt_oldpath, *alt_newpath; - struct stat st; - int ret; - - ret = dbox_list_rename_get_alt_paths(oldlist, oldname, newlist, newname, - MAILBOX_LIST_PATH_TYPE_DIR, - &alt_oldpath, &alt_newpath); - if (ret <= 0) - return ret; - - if (stat(alt_newpath, &st) == 0) { - /* race condition or a directory left there lying around? - safest to just report error. */ - mailbox_list_set_error(oldlist, MAIL_ERROR_EXISTS, - "Target mailbox already exists"); - return -1; - } else if (errno != ENOENT) { - mailbox_list_set_critical(oldlist, "stat(%s) failed: %m", - alt_newpath); - return -1; - } - return 0; -} - -static int -dbox_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname, - struct mailbox_list *newlist, const char *newname, - bool rename_children) -{ - struct dbox_mailbox_list *oldmlist = DBOX_LIST_CONTEXT(oldlist); - enum mailbox_list_path_type path_type; - const char *alt_oldpath, *alt_newpath, *path; - int ret; - - ret = oldmlist->module_ctx.super. - rename_mailbox(oldlist, oldname, newlist, newname, - rename_children); - if (ret < 0) - return -1; - - path_type = rename_children ? MAILBOX_LIST_PATH_TYPE_DIR : - MAILBOX_LIST_PATH_TYPE_MAILBOX; - ret = dbox_list_rename_get_alt_paths(oldlist, oldname, newlist, newname, - path_type, &alt_oldpath, - &alt_newpath); - if (ret <= 0) - return ret; - - if (rename(alt_oldpath, alt_newpath) == 0) { - /* ok */ - if (!rename_children) { - path = mailbox_list_get_path(oldlist, oldname, - MAILBOX_LIST_PATH_TYPE_DIR); - if (rmdir(path) < 0 && - errno != ENOENT && errno != ENOTEMPTY) { - mailbox_list_set_critical(oldlist, - "rmdir(%s) failed: %m", path); - } - } - } else if (errno != ENOENT) { - /* renaming is done already, so just log the error */ - mailbox_list_set_critical(oldlist, "rename(%s, %s) failed: %m", - alt_oldpath, alt_newpath); - } - return 0; -} - -static void dbox_notify_changes(struct mailbox *box) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; - const char *path; - - if (box->notify_callback == NULL) - index_mailbox_check_remove_all(&mbox->ibox); - else { - path = t_strdup_printf("%s/"DBOX_INDEX_PREFIX".log", - mbox->ibox.box.path); - index_mailbox_check_add(&mbox->ibox, path); - } -} - -static int dbox_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx - ATTR_UNUSED, - const char *dir, const char *fname, - const char *mailbox_name ATTR_UNUSED, - enum mailbox_list_file_type type, - enum mailbox_info_flags *flags) -{ - const char *path, *maildir_path; - struct stat st, st2; - int ret = 1; - - /* try to avoid stat() with these checks */ - if (type != MAILBOX_LIST_FILE_TYPE_DIR && - type != MAILBOX_LIST_FILE_TYPE_SYMLINK && - type != MAILBOX_LIST_FILE_TYPE_UNKNOWN) { - /* it's a file */ - *flags |= MAILBOX_NOSELECT | MAILBOX_NOINFERIORS; - return 0; - } - - /* need to stat() then */ - path = t_strconcat(dir, "/", fname, NULL); - if (stat(path, &st) == 0) { - if (!S_ISDIR(st.st_mode)) { - /* non-directory */ - *flags |= MAILBOX_NOSELECT | MAILBOX_NOINFERIORS; - ret = 0; - } else if (st.st_nlink == 2) { - /* no subdirectories */ - *flags |= MAILBOX_NOCHILDREN; - } else if (*ctx->list->set.maildir_name != '\0') { - /* default configuration: we have one directory - containing the mailboxes. if there are 3 links, - either this is a selectable mailbox without children - or non-selectable mailbox with children */ - if (st.st_nlink > 3) - *flags |= MAILBOX_CHILDREN; - } else { - /* non-default configuration: all subdirectories are - child mailboxes. */ - if (st.st_nlink > 2) - *flags |= MAILBOX_CHILDREN; - } - } else if (errno == ENOENT) { - /* doesn't exist - probably a non-existing subscribed mailbox */ - *flags |= MAILBOX_NONEXISTENT; - } else { - /* non-selectable. probably either access denied, or symlink - destination not found. don't bother logging errors. */ - *flags |= MAILBOX_NOSELECT; - } - if ((*flags & (MAILBOX_NOSELECT | MAILBOX_NONEXISTENT)) == 0) { - /* make sure it's a selectable mailbox */ - maildir_path = t_strconcat(path, "/"DBOX_MAILDIR_NAME, NULL); - if (stat(maildir_path, &st2) < 0 || !S_ISDIR(st2.st_mode)) - *flags |= MAILBOX_NOSELECT; - if (st.st_nlink == 3 && *ctx->list->set.maildir_name != '\0') { - /* now we know what link count 3 means. */ - if ((*flags & MAILBOX_NOSELECT) != 0) - *flags |= MAILBOX_CHILDREN; - else - *flags |= MAILBOX_NOCHILDREN; - } - } - return ret; -} - -static void dbox_storage_add_list(struct mail_storage *storage, - struct mailbox_list *list) -{ - struct dbox_mailbox_list *mlist; - - mlist = p_new(list->pool, struct dbox_mailbox_list, 1); - mlist->module_ctx.super = list->v; - mlist->set = mail_storage_get_driver_settings(storage); - - list->v.iter_is_mailbox = dbox_list_iter_is_mailbox; - list->v.delete_mailbox = dbox_list_delete_mailbox; - list->v.rename_mailbox = dbox_list_rename_mailbox; - list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre; - - MODULE_CONTEXT_SET(list, dbox_mailbox_list_module, mlist); -} - -struct mail_storage dbox_storage = { - MEMBER(name) DBOX_STORAGE_NAME, - MEMBER(class_flags) MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT, /* FIXME: for multi-dbox only.. */ - - { - dbox_get_setting_parser_info, - dbox_storage_alloc, - dbox_storage_create, - dbox_storage_destroy, - dbox_storage_add_list, - dbox_storage_get_list_settings, - NULL, - dbox_mailbox_alloc, - dbox_sync_purge - } -}; - -struct mailbox dbox_mailbox = { - MEMBER(name) NULL, - MEMBER(storage) NULL, - MEMBER(list) NULL, - - { - index_storage_is_readonly, - index_storage_allow_new_keywords, - index_storage_mailbox_enable, - dbox_mailbox_open, - dbox_mailbox_close, - dbox_mailbox_create, - dbox_mailbox_update, - dbox_storage_get_status, - NULL, - NULL, - dbox_storage_sync_init, - index_mailbox_sync_next, - index_mailbox_sync_deinit, - NULL, - dbox_notify_changes, - index_transaction_begin, - index_transaction_commit, - index_transaction_rollback, - index_transaction_set_max_modseq, - index_keywords_create, - index_keywords_create_from_indexes, - index_keywords_ref, - index_keywords_unref, - index_keyword_is_valid, - index_storage_get_seq_range, - index_storage_get_uid_range, - index_storage_get_expunges, - NULL, - NULL, - NULL, - dbox_mail_alloc, - index_header_lookup_init, - index_header_lookup_deinit, - index_storage_search_init, - index_storage_search_deinit, - index_storage_search_next_nonblock, - index_storage_search_next_update_seq, - dbox_save_alloc, - dbox_save_begin, - dbox_save_continue, - dbox_save_finish, - dbox_save_cancel, - dbox_copy, - index_storage_is_inconsistent - } -};
--- a/src/lib-storage/index/dbox/dbox-storage.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,123 +0,0 @@ -#ifndef DBOX_STORAGE_H -#define DBOX_STORAGE_H - -#include "index-storage.h" -#include "mailbox-list-private.h" -#include "dbox-settings.h" - -#define DBOX_STORAGE_NAME "dbox" -#define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions" -#define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity" -#define DBOX_INDEX_PREFIX "dovecot.index" -#define DBOX_DIR_GUID_FILE_NAME "dbox-GUID" - -#define DBOX_MAILBOX_DIR_NAME "mailboxes" -#define DBOX_TRASH_DIR_NAME "trash" -#define DBOX_MAILDIR_NAME "dbox-Mails" -#define DBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index" -#define DBOX_GLOBAL_DIR_NAME "storage" -#define DBOX_MAIL_FILE_MULTI_PREFIX "m." -#define DBOX_MAIL_FILE_UID_PREFIX "u." -#define DBOX_MAIL_FILE_MULTI_FORMAT DBOX_MAIL_FILE_MULTI_PREFIX"%u" -#define DBOX_MAIL_FILE_UID_FORMAT DBOX_MAIL_FILE_UID_PREFIX"%u" -#define DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX ".broken" - -/* How often to scan for stale temp files (based on dir's atime) */ -#define DBOX_TMP_SCAN_SECS (8*60*60) -/* Delete temp files having ctime older than this. */ -#define DBOX_TMP_DELETE_SECS (36*60*60) - -/* Flag specifies if the message should be in primary or alternative storage */ -#define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND - -#define DBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t)) -struct dbox_index_header { - uint32_t map_uid_validity; - uint32_t highest_maildir_uid; - uint8_t mailbox_guid[MAIL_GUID_128_SIZE]; -}; - -struct dbox_storage { - struct mail_storage storage; - union mailbox_list_module_context list_module_ctx; - const struct dbox_settings *set; - - /* root path for alt directory */ - const char *alt_dir; - /* paths for storage directories */ - const char *storage_dir, *alt_storage_dir; - struct dbox_map *map; - - /* mode/gid to use for new dbox storage files */ - mode_t dir_create_mode; - gid_t create_gid; - const char *create_gid_origin; - - ARRAY_DEFINE(open_files, struct dbox_file *); - - unsigned int sync_rebuild:1; - unsigned int have_multi_msgs:1; -}; - -struct dbox_mail_index_record { - uint32_t map_uid; - /* UNIX timestamp of when the message was saved/copied to this - mailbox */ - uint32_t save_date; -}; - -struct dbox_mailbox { - struct index_mailbox ibox; - struct dbox_storage *storage; - - struct maildir_uidlist *maildir_uidlist; - uint32_t highest_maildir_uid; - uint32_t map_uid_validity; - - uint32_t dbox_ext_id, dbox_hdr_ext_id, guid_ext_id; - - const char *alt_path; - - unsigned int creating:1; -}; - -extern struct mail_vfuncs dbox_mail_vfuncs; - -struct mailbox * -dbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, - const char *name, struct istream *input, - enum mailbox_flags flags); -int dbox_mailbox_open(struct mailbox *box); - -struct mail * -dbox_mail_alloc(struct mailbox_transaction_context *t, - enum mail_fetch_field wanted_fields, - struct mailbox_header_lookup_ctx *wanted_headers); - -/* Get map_uid for wanted message. */ -int dbox_mail_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view, - uint32_t seq, uint32_t *map_uid_r); -uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); -int dbox_read_header(struct dbox_mailbox *mbox, struct dbox_index_header *hdr); -void dbox_update_header(struct dbox_mailbox *mbox, - struct mail_index_transaction *trans, - const struct mailbox_update *update); - -struct mail_save_context * -dbox_save_alloc(struct mailbox_transaction_context *_t); -int dbox_save_begin(struct mail_save_context *ctx, struct istream *input); -int dbox_save_continue(struct mail_save_context *ctx); -int dbox_save_finish(struct mail_save_context *ctx); -void dbox_save_cancel(struct mail_save_context *ctx); - -struct dbox_file * -dbox_save_file_get_file(struct mailbox_transaction_context *t, - uint32_t seq, uoff_t *offset_r); - -int dbox_transaction_save_commit_pre(struct mail_save_context *ctx); -void dbox_transaction_save_commit_post(struct mail_save_context *ctx); -void dbox_transaction_save_rollback(struct mail_save_context *ctx); - -int dbox_copy(struct mail_save_context *ctx, struct mail *mail); - -#endif
--- a/src/lib-storage/index/dbox/dbox-sync-file.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,368 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "istream.h" -#include "ostream.h" -#include "str.h" -#include "hex-binary.h" -#include "dbox-storage.h" -#include "dbox-file.h" -#include "dbox-map.h" -#include "dbox-sync.h" - -#include <stdlib.h> - -struct dbox_mail_move { - struct dbox_file *file; - uint32_t offset; -}; -ARRAY_DEFINE_TYPE(dbox_mail_move, struct dbox_mail_move); - -static int dbox_sync_file_unlink(struct dbox_file *file) -{ - const char *path, *primary_path; - bool alt = FALSE; - - path = primary_path = dbox_file_get_primary_path(file); - while (unlink(path) < 0) { - if (errno != ENOENT) { - mail_storage_set_critical(&file->storage->storage, - "unlink(%s) failed: %m", path); - return -1; - } - if (file->storage->alt_storage_dir == NULL || alt) { - /* not found */ - i_warning("dbox: File unexpectedly lost: %s/%s", - primary_path, file->fname); - return 0; - } - - /* try the alternative path */ - path = dbox_file_get_alt_path(file); - alt = TRUE; - } - return 1; -} - -static int dbox_map_file_msg_offset_cmp(const void *p1, const void *p2) -{ - const struct dbox_map_file_msg *m1 = p1, *m2 = p2; - - if (m1->offset < m2->offset) - return -1; - else if (m1->offset > m2->offset) - return 1; - else - return 0; -} - -static int -dbox_sync_file_copy_metadata(struct dbox_file *file, struct ostream *output) -{ - struct dbox_metadata_header meta_hdr; - const char *line; - const unsigned char *data; - size_t size; - int ret; - - ret = i_stream_read_data(file->input, &data, &size, - sizeof(meta_hdr)); - if (ret <= 0) { - i_assert(ret == -1); - if (file->input->stream_errno == 0) { - dbox_file_set_corrupted(file, "missing metadata"); - return 0; - } - mail_storage_set_critical(&file->storage->storage, - "read(%s) failed: %m", file->current_path); - return -1; - } - - memcpy(&meta_hdr, data, sizeof(meta_hdr)); - if (memcmp(meta_hdr.magic_post, DBOX_MAGIC_POST, - sizeof(meta_hdr.magic_post)) != 0) { - dbox_file_set_corrupted(file, "invalid metadata magic"); - return 0; - } - i_stream_skip(file->input, sizeof(meta_hdr)); - if (output != NULL) - o_stream_send(output, &meta_hdr, sizeof(meta_hdr)); - while ((line = i_stream_read_next_line(file->input)) != NULL) { - if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { - /* end of metadata */ - break; - } - if (output != NULL) { - o_stream_send_str(output, line); - o_stream_send(output, "\n", 1); - } - } - if (line == NULL) { - dbox_file_set_corrupted(file, "missing end-of-metadata line"); - return 0; - } - if (output != NULL) - o_stream_send(output, "\n", 1); - return 1; -} - -int dbox_sync_file_purge(struct dbox_file *file) -{ - struct mail_storage *storage = &file->storage->storage; - struct dbox_file *out_file; - struct stat st; - struct istream *input; - struct ostream *output = NULL; - struct dbox_map_append_context *append_ctx; - ARRAY_TYPE(dbox_map_file_msg) msgs_arr; - const struct dbox_map_file_msg *msgs; - ARRAY_TYPE(seq_range) expunged_map_uids; - ARRAY_TYPE(uint32_t) copied_map_uids; - unsigned int i, count; - uoff_t offset, physical_size, msg_size; - bool expunged; - int ret; - - if ((ret = dbox_file_try_lock(file)) <= 0) - return ret; - - /* make sure the file still exists. another process may have already - deleted it. */ - if (stat(file->current_path, &st) < 0) { - dbox_file_unlock(file); - if (errno == ENOENT) - return 0; - - mail_storage_set_critical(storage, - "stat(%s) failed: %m", file->current_path); - return -1; - } - - i_array_init(&msgs_arr, 128); - if (dbox_map_get_file_msgs(file->storage->map, file->file_id, - &msgs_arr) < 0) { - array_free(&msgs_arr); - dbox_file_unlock(file); - return -1; - } - /* sort messages by their offset */ - array_sort(&msgs_arr, dbox_map_file_msg_offset_cmp); - - msgs = array_get(&msgs_arr, &count); - append_ctx = dbox_map_append_begin_storage(file->storage); - i_array_init(&copied_map_uids, I_MIN(count, 1)); - i_array_init(&expunged_map_uids, I_MIN(count, 1)); - offset = file->file_header_size; - for (i = 0; i < count; i++) { - if ((ret = dbox_file_get_mail_stream(file, offset, - &physical_size, - NULL, &expunged)) <= 0) - break; - msg_size = file->msg_header_size + physical_size; - - if (msgs[i].offset != offset) { - /* map doesn't match file's actual contents */ - dbox_file_set_corrupted(file, - "purging found mismatched offsets " - "(%"PRIuUOFF_T" vs %u, %u/%u)", - offset, msgs[i].offset, i, count); - ret = 0; - break; - } - - if (msgs[i].refcount == 0) { - seq_range_array_add(&expunged_map_uids, 0, - msgs[i].map_uid); - output = NULL; - } else { - /* non-expunged message. write it to output file. */ - if (dbox_map_append_next(append_ctx, physical_size, - &out_file, &output) < 0) { - ret = -1; - break; - } - i_assert(file->file_id != out_file->file_id); - - i_stream_seek(file->input, offset); - input = i_stream_create_limit(file->input, msg_size); - ret = o_stream_send_istream(output, input); - if (input->stream_errno != 0) { - errno = input->stream_errno; - mail_storage_set_critical(storage, - "read(%s) failed: %m", - file->current_path); - i_stream_unref(&input); - break; - } - i_stream_unref(&input); - if (output->stream_errno != 0) { - errno = output->stream_errno; - mail_storage_set_critical(storage, - "write(%s) failed: %m", - out_file->current_path); - break; - } - i_assert(ret == (off_t)msg_size); - } - - /* copy/skip metadata */ - i_stream_seek(file->input, offset + msg_size); - if ((ret = dbox_sync_file_copy_metadata(file, output)) <= 0) - break; - - if (output != NULL) { - dbox_map_append_finish_multi_mail(append_ctx); - array_append(&copied_map_uids, &msgs[i].map_uid, 1); - } - offset = file->input->v_offset; - } - if (offset != (uoff_t)st.st_size && ret > 0) { - /* file has more messages than what map tells us */ - dbox_file_set_corrupted(file, - "more messages available than in map " - "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", offset, st.st_size); - ret = 0; - } - array_free(&msgs_arr); msgs = NULL; - - if (ret <= 0) { - dbox_map_append_free(&append_ctx); - dbox_file_unlock(file); - ret = -1; - } else if (array_count(&copied_map_uids) == 0) { - /* everything expunged in this file, unlink it */ - ret = dbox_sync_file_unlink(file); - dbox_map_append_free(&append_ctx); - } else { - /* assign new file_id + offset to moved messages */ - if (dbox_map_append_move(append_ctx, &copied_map_uids, - &expunged_map_uids) < 0 || - dbox_map_append_commit(append_ctx) < 0) { - dbox_file_unlock(file); - ret = -1; - } else { - ret = 1; - (void)dbox_sync_file_unlink(file); - } - dbox_map_append_free(&append_ctx); - } - array_free(&copied_map_uids); - array_free(&expunged_map_uids); - return ret; -} - -static void -dbox_sync_file_move_if_needed(struct dbox_file *file, - const struct dbox_sync_file_entry *entry) -{ - if (!entry->move_to_alt && !entry->move_from_alt) - return; - - if (entry->move_to_alt != file->alt_path) { - /* move the file. if it fails, nothing broke so - don't worry about it. */ - if (dbox_file_try_lock(file) > 0) { - (void)dbox_file_move(file, !file->alt_path); - dbox_file_unlock(file); - } - } -} - -static int -dbox_sync_verify_expunge_guid(struct dbox_sync_context *ctx, - const struct dbox_sync_expunge *expunge) -{ - const void *data; - uint32_t uid; - - mail_index_lookup_uid(ctx->sync_view, expunge->seq, &uid); - mail_index_lookup_ext(ctx->sync_view, expunge->seq, - ctx->mbox->guid_ext_id, &data, NULL); - if (mail_guid_128_is_empty(expunge->guid_128) || - memcmp(data, expunge->guid_128, MAIL_GUID_128_SIZE) == 0) - return 0; - - mail_storage_set_critical(&ctx->mbox->storage->storage, - "Mailbox %s: Expunged GUID mismatch for UID %u: %s vs %s", - ctx->mbox->ibox.box.vname, uid, - binary_to_hex(data, MAIL_GUID_128_SIZE), - binary_to_hex(expunge->guid_128, MAIL_GUID_128_SIZE)); - return -1; -} - -static int -dbox_sync_verify_expunge_guids(struct dbox_sync_context *ctx, - const struct dbox_sync_file_entry *entry) -{ - const struct dbox_sync_expunge *expunges; - unsigned int i, count; - - expunges = array_get(&entry->expunges, &count); - for (i = 0; i < count; i++) { - if (dbox_sync_verify_expunge_guid(ctx, &expunges[i]) < 0) - return -1; - } - return 0; -} - -static void -dbox_sync_mark_expunges(struct dbox_sync_context *ctx, - const struct dbox_sync_file_entry *entry) -{ - struct mailbox *box = &ctx->mbox->ibox.box; - const struct dbox_sync_expunge *expunges; - unsigned int i, count; - const void *data; - uint32_t uid; - - expunges = array_get(&entry->expunges, &count); - for (i = 0; i < count; i++) { - mail_index_lookup_uid(ctx->sync_view, expunges[i].seq, &uid); - mail_index_lookup_ext(ctx->sync_view, expunges[i].seq, - ctx->mbox->guid_ext_id, &data, NULL); - mail_index_expunge_guid(ctx->trans, expunges[i].seq, data); - - if (box->v.sync_notify != NULL) - box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE); - } -} - -int dbox_sync_file(struct dbox_sync_context *ctx, - const struct dbox_sync_file_entry *entry) -{ - struct dbox_mailbox *mbox = ctx->mbox; - struct dbox_file *file; - int ret = 1; - - file = entry->file_id != 0 ? - dbox_file_init_multi(mbox->storage, entry->file_id) : - dbox_file_init_single(mbox, entry->uid); - if (!array_is_created(&entry->expunges)) { - /* no expunges - we want to move it */ - dbox_sync_file_move_if_needed(file, entry); - } else if (dbox_sync_verify_expunge_guids(ctx, entry) < 0) { - /* guid mismatches, see if index rebuilding helps */ - ret = 0; - } else if (entry->uid != 0) { - /* single-message file, we can unlink it */ - if ((ret = dbox_sync_file_unlink(file)) == 0) { - /* file was lost, delete it */ - dbox_sync_mark_expunges(ctx, entry); - ret = 1; - } - } else { - if (ctx->map_trans == NULL) { - ctx->map_trans = - dbox_map_transaction_begin(mbox->storage->map, - FALSE); - } - if (dbox_map_update_refcounts(ctx->map_trans, - (void *)&entry->expunges, -1) < 0) - ret = -1; - else - dbox_sync_mark_expunges(ctx, entry); - } - dbox_file_unref(&file); - return ret; -}
--- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,470 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "dbox-storage.h" -#include "maildir/maildir-uidlist.h" -#include "maildir/maildir-keywords.h" -#include "maildir/maildir-filename.h" -#include "dbox-map.h" -#include "dbox-file.h" -#include "dbox-sync.h" - -#include <stdlib.h> -#include <dirent.h> - -struct dbox_sync_rebuild_context { - struct dbox_mailbox *mbox; - - struct mail_index_view *view; - struct mail_index_transaction *trans; - uint32_t cache_ext_id; - uint32_t cache_reset_id; - - struct mail_index *backup_index; - struct mail_index_view *backup_view; - - struct maildir_uidlist_sync_ctx *maildir_sync_ctx; - struct maildir_keywords *mk; - struct maildir_keywords_sync_ctx *maildir_sync_keywords; - - uint32_t highest_uid; - - unsigned int cache_used:1; - unsigned int storage_rebuild:1; -}; - -static void dbox_sync_set_uidvalidity(struct dbox_sync_rebuild_context *ctx) -{ - struct mailbox *box = &ctx->mbox->ibox.box; - uint32_t uid_validity; - - /* if uidvalidity is set in the old index, use it */ - uid_validity = mail_index_get_header(ctx->view)->uid_validity; - if (uid_validity == 0) - uid_validity = dbox_get_uidvalidity_next(box->list); - - mail_index_update_header(ctx->trans, - offsetof(struct mail_index_header, uid_validity), - &uid_validity, sizeof(uid_validity), TRUE); -} - -static void -dbox_sync_index_copy_cache(struct dbox_sync_rebuild_context *ctx, - struct mail_index_view *view, - uint32_t old_seq, uint32_t new_seq) -{ - struct mail_index_map *map; - const void *data; - uint32_t reset_id; - bool expunged; - - if (ctx->cache_ext_id == (uint32_t)-1) - return; - - mail_index_lookup_ext_full(view, old_seq, ctx->cache_ext_id, - &map, &data, &expunged); - if (expunged) - return; - - if (!mail_index_ext_get_reset_id(view, map, ctx->cache_ext_id, - &reset_id) || reset_id == 0) - return; - - if (!ctx->cache_used) { - /* set reset id */ - ctx->cache_used = TRUE; - ctx->cache_reset_id = reset_id; - mail_index_ext_reset(ctx->trans, ctx->cache_ext_id, - ctx->cache_reset_id, TRUE); - } - if (ctx->cache_reset_id == reset_id) { - mail_index_update_ext(ctx->trans, new_seq, - ctx->cache_ext_id, data, NULL); - } -} - -static void -dbox_sync_index_copy_from_old(struct dbox_sync_rebuild_context *ctx, - struct mail_index_view *view, - uint32_t old_seq, uint32_t new_seq) -{ - struct mail_index *index = mail_index_view_get_index(view); - const struct mail_index_record *rec; - ARRAY_TYPE(keyword_indexes) old_keywords; - struct mail_keywords *kw; - - /* copy flags */ - rec = mail_index_lookup(view, old_seq); - mail_index_update_flags(ctx->trans, new_seq, - MODIFY_REPLACE, rec->flags); - - /* copy keywords */ - t_array_init(&old_keywords, 32); - mail_index_lookup_keywords(view, old_seq, &old_keywords); - kw = mail_index_keywords_create_from_indexes(index, &old_keywords); - mail_index_update_keywords(ctx->trans, new_seq, MODIFY_REPLACE, kw); - mail_index_keywords_unref(&kw); - - dbox_sync_index_copy_cache(ctx, view, old_seq, new_seq); -} - -static void -dbox_sync_index_copy_from_maildir(struct dbox_sync_rebuild_context *ctx, - struct dbox_file *file, uint32_t seq) -{ - ARRAY_TYPE(keyword_indexes) keyword_indexes; - struct mail_keywords *keywords; - enum mail_flags flags; - - t_array_init(&keyword_indexes, 32); - maildir_filename_get_flags(ctx->maildir_sync_keywords, - file->fname, &flags, &keyword_indexes); - mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags); - - keywords = mail_index_keywords_create_from_indexes(ctx->mbox->ibox.index, - &keyword_indexes); - mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, keywords); - mail_index_keywords_unref(&keywords); -} - -void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx, - struct dbox_file *file, - uint32_t new_seq, uint32_t uid) -{ - uint32_t old_seq; - - if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) { - /* the message exists in the old index. - copy the metadata from it. */ - dbox_sync_index_copy_from_old(ctx, ctx->view, old_seq, new_seq); - } else if (ctx->backup_view != NULL && - mail_index_lookup_seq(ctx->backup_view, uid, &old_seq)) { - /* copy the metadata from backup index. */ - dbox_sync_index_copy_from_old(ctx, ctx->backup_view, - old_seq, new_seq); - } else if (file != NULL && file->maildir_file) { - /* we're probably doing initial sync after migration from - maildir. preserve the old flags. */ - dbox_sync_index_copy_from_maildir(ctx, file, new_seq); - } -} - -static int dbox_sync_add_file_index(struct dbox_sync_rebuild_context *ctx, - struct dbox_file *file) -{ - uint32_t seq; - uoff_t size; - bool expunged; - int ret; - - ret = dbox_file_get_mail_stream(file, 0, &size, NULL, &expunged); - if (ret <= 0) { - if (ret < 0) - return -1; - - i_warning("dbox: Ignoring broken file: %s", file->current_path); - return 0; - } - if (expunged) { - /* the file just got deleted? */ - return 0; - } - - mail_index_append(ctx->trans, file->uid, &seq); - T_BEGIN { - dbox_sync_rebuild_index_metadata(ctx, file, seq, file->uid); - } T_END; - return 0; -} - -static int -dbox_sync_add_uid_file(struct dbox_sync_rebuild_context *ctx, - const char *dir, const char *fname) -{ - struct dbox_file *file; - unsigned long uid; - char *p; - int ret; - - fname += sizeof(DBOX_MAIL_FILE_UID_PREFIX)-1; - uid = strtoul(fname, &p, 10); - if (*p != '\0' || uid == 0 || uid >= (uint32_t)-1) { - i_warning("dbox %s: Ignoring invalid filename %s", - ctx->mbox->ibox.box.path, fname); - return 0; - } - - if (ctx->highest_uid < uid) - ctx->highest_uid = uid; - - file = dbox_file_init_single(ctx->mbox, uid); - file->current_path = i_strdup_printf("%s/%s", dir, fname); - - ret = dbox_sync_add_file_index(ctx, file); - dbox_file_unref(&file); - return ret; -} - -static int -dbox_sync_add_maildir_file(struct dbox_sync_rebuild_context *ctx, - const char *fname) -{ - int ret; - - if (ctx->maildir_sync_ctx == NULL) { - i_assert(ctx->mk == NULL); - - ctx->mk = maildir_keywords_init_readonly(&ctx->mbox->ibox.box); - ctx->maildir_sync_keywords = - maildir_keywords_sync_init(ctx->mk, - ctx->mbox->ibox.index); - - ret = maildir_uidlist_sync_init(ctx->mbox->maildir_uidlist, - MAILDIR_UIDLIST_SYNC_NOLOCK, - &ctx->maildir_sync_ctx); - if (ret <= 0) { - i_assert(ret < 0); - return -1; - } - } - - /* sync all maildir files first and let maildir uidlist code assign - UIDs for unseen files. */ - ret = maildir_uidlist_sync_next(ctx->maildir_sync_ctx, fname, 0); - if (ret == 0) { - i_warning("%s: Ignoring duplicate maildir file: %s", - ctx->mbox->ibox.box.path, fname); - } - return ret; -} - -static int -dbox_sync_add_file(struct dbox_sync_rebuild_context *ctx, - const char *path, const char *fname, bool primary) -{ - if (strncmp(fname, DBOX_MAIL_FILE_UID_PREFIX, - sizeof(DBOX_MAIL_FILE_UID_PREFIX)-1) == 0) - return dbox_sync_add_uid_file(ctx, path, fname); - - if (primary && strstr(fname, ":2,") != NULL) - return dbox_sync_add_maildir_file(ctx, fname); - return 0; -} - -static int dbox_sync_index_rebuild_dir(struct dbox_sync_rebuild_context *ctx, - const char *path, bool primary) -{ - struct mail_storage *storage = ctx->mbox->ibox.box.storage; - DIR *dir; - struct dirent *d; - int ret = 0; - - dir = opendir(path); - if (dir == NULL) { - if (errno == ENOENT) { - if (!primary) { - /* alt directory doesn't exist, ignore */ - return 0; - } - mailbox_set_deleted(&ctx->mbox->ibox.box); - return -1; - } - mail_storage_set_critical(storage, - "opendir(%s) failed: %m", path); - return -1; - } - do { - errno = 0; - if ((d = readdir(dir)) == NULL) - break; - - T_BEGIN { - ret = dbox_sync_add_file(ctx, path, d->d_name, primary); - } T_END; - } while (ret >= 0); - if (errno != 0) { - mail_storage_set_critical(storage, - "readdir(%s) failed: %m", path); - ret = -1; - } - - if (closedir(dir) < 0) { - mail_storage_set_critical(storage, - "closedir(%s) failed: %m", path); - ret = -1; - } - return ret; -} - -static int dbox_sync_maildir_finish(struct dbox_sync_rebuild_context *ctx) -{ - struct dbox_mailbox *mbox = ctx->mbox; - struct maildir_uidlist_iter_ctx *iter; - struct mail_index_view *trans_view; - struct dbox_file *file; - const char *fname; - enum maildir_uidlist_rec_flag flags; - uint32_t uid, next_uid; - int ret = 0; - - if (ctx->maildir_sync_ctx == NULL) - return 0; - - /* we'll need the uidlist to contain the latest filenames. - since there's no easy way to figure out if they changed, just - recreate the uidlist always. */ - maildir_uidlist_sync_recreate(ctx->maildir_sync_ctx); - - /* update the maildir uidlist's next_uid if we have seen higher - dbox UIDs */ - trans_view = mail_index_transaction_open_updated_view(ctx->trans); - next_uid = mail_index_get_header(trans_view)->next_uid; - mail_index_view_close(&trans_view); - maildir_uidlist_set_next_uid(mbox->maildir_uidlist, next_uid, FALSE); - maildir_uidlist_set_next_uid(mbox->maildir_uidlist, - ctx->highest_uid + 1, FALSE); - /* assign UIDs for new maildir mails before iterating */ - maildir_uidlist_sync_finish(ctx->maildir_sync_ctx); - - mbox->highest_maildir_uid = - maildir_uidlist_get_next_uid(mbox->maildir_uidlist); - - iter = maildir_uidlist_iter_init(mbox->maildir_uidlist); - while (maildir_uidlist_iter_next(iter, &uid, &flags, &fname)) { - file = dbox_file_init_single(mbox, uid); - file->current_path = - i_strdup_printf("%s/%s", ctx->mbox->ibox.box.path, - fname); - - ret = dbox_sync_add_file_index(ctx, file); - dbox_file_unref(&file); - if (ret < 0) - break; - } - maildir_uidlist_iter_deinit(&iter); - return ret < 0 ? -1 : 0; -} - -static void dbox_sync_update_header(struct dbox_sync_rebuild_context *ctx) -{ - struct dbox_index_header hdr; - - if (dbox_read_header(ctx->mbox, &hdr) < 0) - memset(&hdr, 0, sizeof(hdr)); - if (!mail_guid_128_is_empty(hdr.mailbox_guid)) - mail_generate_guid_128(hdr.mailbox_guid); - if (hdr.highest_maildir_uid < ctx->mbox->highest_maildir_uid) - hdr.highest_maildir_uid = ctx->mbox->highest_maildir_uid; - hdr.map_uid_validity = !ctx->storage_rebuild ? 0 : - dbox_map_get_uid_validity(ctx->mbox->storage->map); - mail_index_update_header_ext(ctx->trans, ctx->mbox->dbox_hdr_ext_id, 0, - &hdr, sizeof(hdr)); -} - -struct dbox_sync_rebuild_context * -dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox, - struct mail_index_view *view, - struct mail_index_transaction *trans, - bool storage_rebuild) -{ - struct mailbox *box = &mbox->ibox.box; - struct dbox_sync_rebuild_context *ctx; - const char *index_dir; - enum mail_index_open_flags open_flags = MAIL_INDEX_OPEN_FLAG_READONLY; - - ctx = i_new(struct dbox_sync_rebuild_context, 1); - ctx->mbox = mbox; - ctx->view = view; - ctx->trans = trans; - ctx->storage_rebuild = storage_rebuild; - mail_index_reset(ctx->trans); - index_mailbox_reset_uidvalidity(&mbox->ibox); - mail_index_ext_lookup(mbox->ibox.index, "cache", &ctx->cache_ext_id); - - /* if backup index file exists, try to use it */ - index_dir = mailbox_list_get_path(box->list, box->name, - MAILBOX_LIST_PATH_TYPE_INDEX); - ctx->backup_index = - mail_index_alloc(index_dir, DBOX_INDEX_PREFIX".backup"); - -#ifndef MMAP_CONFLICTS_WRITE - if (box->storage->set->mmap_disable) -#endif - open_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE; - if (mail_index_open(ctx->backup_index, open_flags, - box->storage->set->parsed_lock_method) <= 0) - mail_index_free(&ctx->backup_index); - else - ctx->backup_view = mail_index_view_open(ctx->backup_index); - return ctx; -} - -int dbox_sync_index_rebuild_singles(struct dbox_sync_rebuild_context *ctx) -{ - int ret = 0; - - dbox_sync_set_uidvalidity(ctx); - if (dbox_sync_index_rebuild_dir(ctx, ctx->mbox->ibox.box.path, TRUE) < 0) - ret = -1; - else if (ctx->mbox->alt_path != NULL) { - if (dbox_sync_index_rebuild_dir(ctx, ctx->mbox->alt_path, - FALSE) < 0) - ret = -1; - } - - if (ret == 0) { - if (dbox_sync_maildir_finish(ctx) < 0) - ret = -1; - } - - if (ctx->maildir_sync_ctx != NULL) { - if (maildir_uidlist_sync_deinit(&ctx->maildir_sync_ctx, - ret == 0) < 0) - ret = -1; - } - if (ctx->maildir_sync_keywords != NULL) - maildir_keywords_sync_deinit(&ctx->maildir_sync_keywords); - if (ctx->mk != NULL) - maildir_keywords_deinit(&ctx->mk); - return ret; -} - -void dbox_sync_index_rebuild_deinit(struct dbox_sync_rebuild_context **_ctx) -{ - struct dbox_sync_rebuild_context *ctx = *_ctx; - - *_ctx = NULL; - if (ctx->backup_index != NULL) { - mail_index_view_close(&ctx->backup_view); - mail_index_free(&ctx->backup_index); - } - dbox_sync_update_header(ctx); - i_free(ctx); -} - -int dbox_sync_index_rebuild(struct dbox_mailbox *mbox) -{ - struct dbox_sync_rebuild_context *ctx; - struct mail_index_view *view; - struct mail_index_transaction *trans; - int ret; - - view = mail_index_view_open(mbox->ibox.index); - trans = mail_index_transaction_begin(view, - MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); - - ctx = dbox_sync_index_rebuild_init(mbox, view, trans, FALSE); - ret = dbox_sync_index_rebuild_singles(ctx); - dbox_sync_index_rebuild_deinit(&ctx); - - if (ret < 0) - mail_index_transaction_rollback(&trans); - else - ret = mail_index_transaction_commit(&trans); - mail_index_view_close(&view); - - if (ret == 0) - mbox->storage->sync_rebuild = FALSE; - return ret; -}
--- a/src/lib-storage/index/dbox/dbox-sync.c Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,402 +0,0 @@ -/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "ioloop.h" -#include "str.h" -#include "hash.h" -#include "dbox-storage.h" -#include "dbox-storage-rebuild.h" -#include "dbox-map.h" -#include "dbox-file.h" -#include "dbox-sync.h" - -#define DBOX_REBUILD_COUNT 3 - -static unsigned int dbox_sync_file_entry_hash(const void *p) -{ - const struct dbox_sync_file_entry *entry = p; - - if (entry->file_id != 0) - return entry->file_id | 0x80000000; - else - return entry->uid; -} - -static int dbox_sync_file_entry_cmp(const void *p1, const void *p2) -{ - const struct dbox_sync_file_entry *entry1 = p1, *entry2 = p2; - - /* this is only for hashing, don't bother ever returning 1. */ - if (entry1->file_id != entry2->file_id) - return -1; - if (entry1->uid != entry2->uid) - return -1; - return 0; -} - -static int dbox_sync_add_seq(struct dbox_sync_context *ctx, - const struct mail_index_sync_rec *sync_rec, - uint32_t seq) -{ - struct dbox_sync_file_entry *entry, lookup_entry; - struct dbox_sync_expunge *expunge; - uint32_t map_uid; - uoff_t offset; - int ret; - - i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE || - sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); - - memset(&lookup_entry, 0, sizeof(lookup_entry)); - if (dbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0) - return ctx->mbox->storage->sync_rebuild ? 0 : -1; - if (map_uid == 0) - mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid); - else { - ret = dbox_map_lookup(ctx->mbox->storage->map, map_uid, - &lookup_entry.file_id, &offset); - if (ret <= 0) { - if (ret < 0) - return -1; - /* mailbox is locked while syncing, so if ret=0 the - message got expunged from storage before it was - expunged from mailbox. that shouldn't happen. */ - dbox_map_set_corrupted(ctx->mbox->storage->map, - "unexpectedly lost map_uid=%u", map_uid); - return 0; - } - } - - entry = hash_table_lookup(ctx->syncs, &lookup_entry); - if (entry == NULL) { - entry = p_new(ctx->pool, struct dbox_sync_file_entry, 1); - *entry = lookup_entry; - hash_table_insert(ctx->syncs, entry, entry); - } - - if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { - if (!array_is_created(&entry->expunges)) { - p_array_init(&entry->expunges, ctx->pool, - lookup_entry.uid != 0 ? 1 : 3); - } - - expunge = array_append_space(&entry->expunges); - expunge->map_uid = map_uid; - expunge->seq = seq; - memcpy(expunge->guid_128, sync_rec->guid_128, - sizeof(expunge->guid_128)); - if (entry->file_id != 0) - ctx->have_storage_expunges = TRUE; - } else { - if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0) - entry->move_to_alt = TRUE; - else - entry->move_from_alt = TRUE; - } - return 1; -} - -static int dbox_sync_add(struct dbox_sync_context *ctx, - const struct mail_index_sync_rec *sync_rec) -{ - uint32_t seq, seq1, seq2; - int ret; - - if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { - /* we're interested */ - } else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) { - /* we care only about alt flag changes */ - if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) == 0 && - (sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) == 0) - return 1; - } else { - /* not interested */ - return 1; - } - - if (!mail_index_lookup_seq_range(ctx->sync_view, - sync_rec->uid1, sync_rec->uid2, - &seq1, &seq2)) { - /* already expunged everything. nothing to do. */ - return 1; - } - - for (seq = seq1; seq <= seq2; seq++) { - if ((ret = dbox_sync_add_seq(ctx, sync_rec, seq)) <= 0) - return ret; - } - return 1; -} - -static int dbox_sync_index(struct dbox_sync_context *ctx) -{ - struct mailbox *box = &ctx->mbox->ibox.box; - const struct mail_index_header *hdr; - struct mail_index_sync_rec sync_rec; - struct hash_iterate_context *iter; - void *key, *value; - uint32_t seq1, seq2; - int ret = 1; - - hdr = mail_index_get_header(ctx->sync_view); - if (hdr->uid_validity == 0) { - /* newly created index file */ - return 0; - } - - /* mark the newly seen messages as recent */ - if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, - hdr->next_uid, &seq1, &seq2)) { - index_mailbox_set_recent_seq(&ctx->mbox->ibox, ctx->sync_view, - seq1, seq2); - } - - /* read all changes and group changes to same file_id together */ - ctx->pool = pool_alloconly_create("dbox sync pool", 1024*32); - ctx->syncs = hash_table_create(default_pool, ctx->pool, 0, - dbox_sync_file_entry_hash, - dbox_sync_file_entry_cmp); - - while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) { - if ((ret = dbox_sync_add(ctx, &sync_rec)) <= 0) - break; - } - - if (ret > 0) { - if (ctx->have_storage_expunges) { - /* prevent a user from saving + expunging messages - all the time and using lots of disk space. - but avoid doing this in situations where a user - simply expunges a lot of mail for the first time. - that's why we do this calculation before current - sync: the purging is triggered only after the - second expunge. */ - if ((ctx->flags & DBOX_SYNC_FLAG_NO_PURGE) == 0 && - dbox_map_want_purge(ctx->mbox->storage->map)) - ctx->purge = TRUE; - } - - /* now sync each file separately */ - iter = hash_table_iterate_init(ctx->syncs); - while (hash_table_iterate(iter, &key, &value)) { - const struct dbox_sync_file_entry *entry = value; - - if ((ret = dbox_sync_file(ctx, entry)) <= 0) - break; - } - hash_table_iterate_deinit(&iter); - } - - if (ret > 0 && ctx->map_trans != NULL) { - if (dbox_map_transaction_commit(ctx->map_trans) < 0) - ret = -1; - dbox_map_transaction_free(&ctx->map_trans); - } - - if (box->v.sync_notify != NULL) - box->v.sync_notify(box, 0, 0); - - hash_table_destroy(&ctx->syncs); - pool_unref(&ctx->pool); - return ret; -} - -static int dbox_refresh_header(struct dbox_mailbox *mbox, bool retry) -{ - struct mail_index_view *view; - struct dbox_index_header hdr; - int ret; - - view = mail_index_view_open(mbox->ibox.index); - ret = dbox_read_header(mbox, &hdr); - mail_index_view_close(&view); - - if (ret == 0) { - mbox->highest_maildir_uid = hdr.highest_maildir_uid; - ret = mbox->storage->sync_rebuild ? -1 : 0; - } else if (retry) { - (void)mail_index_refresh(mbox->ibox.index); - return dbox_refresh_header(mbox, FALSE); - } - return ret; -} - -int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags, - struct dbox_sync_context **ctx_r) -{ - struct mail_storage *storage = mbox->ibox.box.storage; - struct dbox_sync_context *ctx; - enum mail_index_sync_flags sync_flags = 0; - unsigned int i; - int ret; - bool rebuild, storage_rebuilt = FALSE; - - rebuild = dbox_refresh_header(mbox, TRUE) < 0 || - (flags & DBOX_SYNC_FLAG_FORCE_REBUILD) != 0; - if (rebuild) { - if (dbox_storage_rebuild(mbox->storage) < 0) - return -1; - index_mailbox_reset_uidvalidity(&mbox->ibox); - storage_rebuilt = TRUE; - } - - ctx = i_new(struct dbox_sync_context, 1); - ctx->mbox = mbox; - ctx->flags = flags; - - if ((mbox->ibox.box.flags & MAILBOX_FLAG_KEEP_RECENT) == 0) - sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; - if (!rebuild && (flags & DBOX_SYNC_FLAG_FORCE) == 0) - sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; - if ((flags & DBOX_SYNC_FLAG_FSYNC) != 0) - sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC; - /* don't write unnecessary dirty flag updates */ - sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; - - for (i = 0;; i++) { - ret = mail_index_sync_begin(mbox->ibox.index, - &ctx->index_sync_ctx, - &ctx->sync_view, &ctx->trans, - sync_flags); - if (ret <= 0) { - if (ret < 0) - mail_storage_set_index_error(&mbox->ibox); - i_free(ctx); - *ctx_r = NULL; - return ret; - } - - /* now that we're locked, check again if we want to rebuild */ - if (dbox_refresh_header(mbox, FALSE) < 0) - ret = 0; - else { - if ((ret = dbox_sync_index(ctx)) > 0) - break; - } - - /* failure. keep the index locked while we're doing a - rebuild. */ - if (ret == 0) { - if (!storage_rebuilt) { - /* we'll need to rebuild storage too. - try again from the beginning. */ - mbox->storage->sync_rebuild = TRUE; - mail_index_sync_rollback(&ctx->index_sync_ctx); - i_free(ctx); - return dbox_sync_begin(mbox, flags, ctx_r); - } - if (mbox->storage->have_multi_msgs) { - mail_storage_set_critical(storage, - "dbox %s: Storage keeps breaking", - ctx->mbox->ibox.box.path); - ret = -1; - } else if (i >= DBOX_REBUILD_COUNT) { - mail_storage_set_critical(storage, - "dbox %s: Index keeps breaking", - ctx->mbox->ibox.box.path); - ret = -1; - } else { - /* do a full resync and try again. */ - i_warning("dbox %s: Rebuilding index", - ctx->mbox->ibox.box.path); - ret = dbox_sync_index_rebuild(mbox); - } - } - mail_index_sync_rollback(&ctx->index_sync_ctx); - if (ret < 0) { - i_free(ctx); - return -1; - } - } - - *ctx_r = ctx; - return 0; -} - -int dbox_sync_finish(struct dbox_sync_context **_ctx, bool success) -{ - struct dbox_sync_context *ctx = *_ctx; - int ret = success ? 0 : -1; - - *_ctx = NULL; - - if (ctx->map_trans != NULL) - dbox_map_transaction_free(&ctx->map_trans); - - if (success) { - if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { - mail_storage_set_index_error(&ctx->mbox->ibox); - ret = -1; - } - } else { - mail_index_sync_rollback(&ctx->index_sync_ctx); - } - if (ctx->path != NULL) - str_free(&ctx->path); - - if (ctx->purge) - (void)dbox_sync_purge(&ctx->mbox->storage->storage); - i_free(ctx); - return ret; -} - -int dbox_sync(struct dbox_mailbox *mbox, enum dbox_sync_flags flags) -{ - struct dbox_sync_context *sync_ctx; - - if (dbox_sync_begin(mbox, flags, &sync_ctx) < 0) - return -1; - - if (sync_ctx == NULL) - return 0; - return dbox_sync_finish(&sync_ctx, TRUE); -} - -struct mailbox_sync_context * -dbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) -{ - struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; - enum dbox_sync_flags dbox_sync_flags = 0; - int ret = 0; - - if (!box->opened) { - if (mailbox_open(box) < 0) - ret = -1; - } - - if (ret == 0 && (index_mailbox_want_full_sync(&mbox->ibox, flags) || - mbox->storage->sync_rebuild)) { - if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) - dbox_sync_flags |= DBOX_SYNC_FLAG_FORCE_REBUILD; - ret = dbox_sync(mbox, dbox_sync_flags); - } - - return index_mailbox_sync_init(box, flags, ret < 0); -} - -int dbox_sync_purge(struct mail_storage *_storage) -{ - struct dbox_storage *storage = (struct dbox_storage *)_storage; - const ARRAY_TYPE(seq_range) *ref0_file_ids; - struct dbox_file *file; - struct seq_range_iter iter; - unsigned int i = 0; - uint32_t file_id; - bool deleted; - int ret = 0; - - ref0_file_ids = dbox_map_get_zero_ref_files(storage->map); - seq_range_array_iter_init(&iter, ref0_file_ids); i = 0; - while (seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN { - file = dbox_file_init_multi(storage, file_id); - if (dbox_file_open_or_create(file, &deleted) > 0 && !deleted) { - if (dbox_sync_file_purge(file) < 0) - ret = -1; - } else { - dbox_map_remove_file_id(storage->map, file_id); - } - dbox_file_unref(&file); - } T_END; - return ret; -}
--- a/src/lib-storage/index/dbox/dbox-sync.h Tue Oct 06 13:24:01 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,74 +0,0 @@ -#ifndef DBOX_SYNC_H -#define DBOX_SYNC_H - -struct mailbox; -struct dbox_mailbox; - -enum dbox_sync_flags { - DBOX_SYNC_FLAG_FORCE = 0x01, - DBOX_SYNC_FLAG_FSYNC = 0x02, - DBOX_SYNC_FLAG_FORCE_REBUILD = 0x04, - DBOX_SYNC_FLAG_NO_PURGE = 0x08 -}; - -struct dbox_sync_expunge { - /* keep map_uid first, so we can just cast it to - dbox_map_update_refcounts() */ - uint32_t map_uid; - uint32_t seq; - uint8_t guid_128[MAIL_GUID_128_SIZE]; -}; - -struct dbox_sync_file_entry { - uint32_t uid, file_id; - - unsigned int move_from_alt:1; - unsigned int move_to_alt:1; - ARRAY_DEFINE(expunges, struct dbox_sync_expunge); -}; - -struct dbox_sync_context { - struct dbox_mailbox *mbox; - struct mail_index_sync_ctx *index_sync_ctx; - struct mail_index_view *sync_view; - struct mail_index_transaction *trans; - struct dbox_map_transaction_context *map_trans; - enum dbox_sync_flags flags; - - string_t *path; - unsigned int path_dir_prefix_len; - - pool_t pool; - struct hash_table *syncs; /* struct dbox_sync_file_entry */ - - unsigned int have_storage_expunges:1; - unsigned int purge:1; -}; - -int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags, - struct dbox_sync_context **ctx_r); -int dbox_sync_finish(struct dbox_sync_context **ctx, bool success); -int dbox_sync(struct dbox_mailbox *mbox, enum dbox_sync_flags flags); - -int dbox_sync_purge(struct mail_storage *storage); -int dbox_sync_file(struct dbox_sync_context *ctx, - const struct dbox_sync_file_entry *entry); -int dbox_sync_file_purge(struct dbox_file *file); - -struct dbox_sync_rebuild_context * -dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox, - struct mail_index_view *view, - struct mail_index_transaction *trans, - bool storage_rebuild); -int dbox_sync_index_rebuild_singles(struct dbox_sync_rebuild_context *ctx); -void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx, - struct dbox_file *file, - uint32_t new_seq, uint32_t uid); -void dbox_sync_index_rebuild_deinit(struct dbox_sync_rebuild_context **ctx); - -int dbox_sync_index_rebuild(struct dbox_mailbox *mbox); - -struct mailbox_sync_context * -dbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); - -#endif