dovecot/core-2.2: src/lib-index/mail-cache.c annotate

annotate src/lib-index/mail-cache.c @ 22576:707ae9de3812

lib: istream-multiplex - Minor code cleanup Avoid propagating the error twice, and avoid any confusion about what "got" actually contains.

author	Timo Sirainen <timo.sirainen@dovecot.fi>
date	Thu, 05 Oct 2017 20:24:11 +0300
parents	03b5a6dfd3ba
children	65ada9976a7f

rev	line source
21390 2e2563132d5f Updated copyright notices to include the year 2017. Stephan Bosch <stephan.bosch@dovecot.fi> parents: 21351 diff changeset	1 /* Copyright (c) 2003-2017 Dovecot authors, see the included COPYING file */
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	2
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
3470 346a494c2feb Moved array declaration to array-decl.h and include it in lib.h. So array.h Timo Sirainen <tss@iki.fi> parents: 3380 diff changeset	4 #include "array.h"
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	5 #include "buffer.h"
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	6 #include "hash.h"
4247 9c3f19110538 Don't compress cache file if it was just compressed by another process. Timo Sirainen <tss@iki.fi> parents: 4023 diff changeset	7 #include "nfs-workarounds.h"
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	8 #include "file-cache.h"
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	9 #include "mmap-util.h"
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	10 #include "read-full.h"
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	11 #include "write-full.h"
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	12 #include "mail-cache-private.h"
21186 34074cc72a15 lib-index: Limit mmap errors in index cache to 1/s Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21168 diff changeset	13 #include "ioloop.h"
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	14
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	15 #include <unistd.h>
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	16
15333 0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	17 #define MAIL_CACHE_MIN_HEADER_READ_SIZE 4096
0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	18
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	19 void mail_cache_set_syscall_error(struct mail_cache *cache,
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	20 const char *function)
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	21 {
10686 442c980d6221 lib-index: Fixed logging file creation EACCES failures. Timo Sirainen <tss@iki.fi> parents: 10582 diff changeset	22 mail_index_file_set_syscall_error(cache->index, cache->filepath,
442c980d6221 lib-index: Fixed logging file creation EACCES failures. Timo Sirainen <tss@iki.fi> parents: 10582 diff changeset	23 function);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	24 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	25
10006 a425ba46a64c lib-index: Take MAIL_INDEX_OPEN_FLAG_READONLY more seriously. Timo Sirainen <tss@iki.fi> parents: 9696 diff changeset	26 static void mail_cache_unlink(struct mail_cache *cache)
a425ba46a64c lib-index: Take MAIL_INDEX_OPEN_FLAG_READONLY more seriously. Timo Sirainen <tss@iki.fi> parents: 9696 diff changeset	27 {
21422 aa57a51c8ce6 index: don't try to unlink in-memory cache file Josef 'Jeff' Sipek <jeff.sipek@dovecot.fi> parents: 21390 diff changeset	28 if (!cache->index->readonly && !MAIL_INDEX_IS_IN_MEMORY(cache->index))
21447 de3c25669608 lib-index: Don't log error on dovecot.index.cache delete if it doesn't exist Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21422 diff changeset	29 i_unlink_if_exists(cache->filepath);
10006 a425ba46a64c lib-index: Take MAIL_INDEX_OPEN_FLAG_READONLY more seriously. Timo Sirainen <tss@iki.fi> parents: 9696 diff changeset	30 }
a425ba46a64c lib-index: Take MAIL_INDEX_OPEN_FLAG_READONLY more seriously. Timo Sirainen <tss@iki.fi> parents: 9696 diff changeset	31
11988 117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	32 void mail_cache_reset(struct mail_cache *cache)
117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	33 {
117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	34 mail_cache_unlink(cache);
117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	35 /* mark the cache as unusable */
117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	36 cache->hdr = NULL;
117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	37 }
117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	38
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	39 void mail_cache_set_corrupted(struct mail_cache cache, const char fmt, ...)
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	40 {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	41 va_list va;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	42
11988 117053f20528 lib-index: Added mail_cache_reset(). Timo Sirainen <tss@iki.fi> parents: 11634 diff changeset	43 mail_cache_reset(cache);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	44
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	45 va_start(va, fmt);
7226 e6693a0ec8e1 Renamed T_FRAME_BEGIN/END to T_BEGIN/END. Removed T_FRAME() macro and Timo Sirainen <tss@iki.fi> parents: 7205 diff changeset	46 T_BEGIN {
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6857 diff changeset	47 mail_index_set_error(cache->index,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6857 diff changeset	48 "Corrupted index cache file %s: %s",
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6857 diff changeset	49 cache->filepath,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6857 diff changeset	50 t_strdup_vprintf(fmt, va));
7226 e6693a0ec8e1 Renamed T_FRAME_BEGIN/END to T_BEGIN/END. Removed T_FRAME() macro and Timo Sirainen <tss@iki.fi> parents: 7205 diff changeset	51 } T_END;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	52 va_end(va);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	53 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	54
22033 03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	55 void mail_cache_set_seq_corrupted_reason(struct mail_cache_view *cache_view,
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	56 uint32_t seq, const char *reason)
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	57 {
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	58 uint32_t empty = 0;
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	59 struct mail_cache *cache = cache_view->cache;
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	60 struct mail_index_view *view = cache_view->view;
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	61
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	62 mail_index_set_error(cache->index,
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	63 "Corrupted record in index cache file %s: %s",
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	64 cache->filepath, reason);
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	65
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	66 /* drop cache pointer */
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	67 struct mail_index_transaction *t =
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	68 mail_index_transaction_begin(view, MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	69 mail_index_update_ext(t, seq, cache->ext_id, &empty, NULL);
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	70
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	71 if (mail_index_transaction_commit(&t) < 0)
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	72 mail_cache_reset(cache);
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	73 else
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	74 mail_cache_expunge_count(cache, 1);
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	75 }
03b5a6dfd3ba lib-index: Add mail_cache_set_seq_corrupted_reason Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21537 diff changeset	76
2247 2d51bb58a070 Added some smartness for deciding what to cache. Cache compression code compiles, but untested. Timo Sirainen <tss@iki.fi> parents: 2220 diff changeset	77 void mail_cache_file_close(struct mail_cache *cache)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	78 {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	79 if (cache->mmap_base != NULL) {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	80 if (munmap(cache->mmap_base, cache->mmap_length) < 0)
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	81 mail_cache_set_syscall_error(cache, "munmap()");
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	82 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	83
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	84 if (cache->file_cache != NULL)
bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	85 file_cache_set_fd(cache->file_cache, -1);
19917 1ef6401c749d lib-index: After MAIL_INDEX_OPEN_FLAG_SAVEONLY access cache's read_buf wasn't always reset. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19552 diff changeset	86 if (cache->read_buf != NULL)
1ef6401c749d lib-index: After MAIL_INDEX_OPEN_FLAG_SAVEONLY access cache's read_buf wasn't always reset. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19552 diff changeset	87 buffer_set_used_size(cache->read_buf, 0);
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	88
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	89 cache->mmap_base = NULL;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	90 cache->hdr = NULL;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	91 cache->mmap_length = 0;
6705 68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6703 diff changeset	92 cache->last_field_header_offset = 0;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	93
4876 f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	94 if (cache->file_lock != NULL)
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	95 file_lock_free(&cache->file_lock);
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	96 cache->locked = FALSE;
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	97
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	98 if (cache->fd != -1) {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	99 if (close(cache->fd) < 0)
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	100 mail_cache_set_syscall_error(cache, "close()");
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	101 cache->fd = -1;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	102 }
20041 4d83654e61d6 lib-index: If opening a cache file fails, try again later. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20040 diff changeset	103 cache->opened = FALSE;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	104 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	105
5145 3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	106 static void mail_cache_init_file_cache(struct mail_cache *cache)
3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	107 {
3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	108 struct stat st;
3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	109
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	110 if (cache->file_cache != NULL)
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	111 file_cache_set_fd(cache->file_cache, cache->fd);
5145 3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	112
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	113 if (fstat(cache->fd, &st) == 0) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	114 if (cache->file_cache != NULL)
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	115 (void)file_cache_set_size(cache->file_cache, st.st_size);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	116 } else if (!ESTALE_FSTAT(errno)) {
5145 3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	117 mail_cache_set_syscall_error(cache, "fstat()");
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	118 }
6815 c991d1b132d8 Fixed cache file's NFS attribute cache flushing. Timo Sirainen <tss@iki.fi> parents: 6772 diff changeset	119
c991d1b132d8 Fixed cache file's NFS attribute cache flushing. Timo Sirainen <tss@iki.fi> parents: 6772 diff changeset	120 cache->st_ino = st.st_ino;
c991d1b132d8 Fixed cache file's NFS attribute cache flushing. Timo Sirainen <tss@iki.fi> parents: 6772 diff changeset	121 cache->st_dev = st.st_dev;
5145 3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	122 }
3eecde9cbea4 Allocate cache file's size amount of memory for the file cache so it doesn't Timo Sirainen <tss@iki.fi> parents: 5014 diff changeset	123
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	124 static int mail_cache_try_open(struct mail_cache *cache)
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	125 {
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	126 const void *data;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	127
20041 4d83654e61d6 lib-index: If opening a cache file fails, try again later. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20040 diff changeset	128 i_assert(!cache->opened);
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	129 cache->opened = TRUE;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	130
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	131 if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	132 return 0;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	133
20041 4d83654e61d6 lib-index: If opening a cache file fails, try again later. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20040 diff changeset	134 i_assert(cache->fd == -1);
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	135 cache->fd = nfs_safe_open(cache->filepath,
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	136 cache->index->readonly ? O_RDONLY : O_RDWR);
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	137 if (cache->fd == -1) {
20041 4d83654e61d6 lib-index: If opening a cache file fails, try again later. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20040 diff changeset	138 mail_cache_file_close(cache);
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	139 if (errno == ENOENT) {
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	140 cache->need_compress_file_seq = 0;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	141 return 0;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	142 }
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	143
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	144 mail_cache_set_syscall_error(cache, "open()");
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	145 return -1;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	146 }
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	147
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	148 mail_cache_init_file_cache(cache);
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	149
20041 4d83654e61d6 lib-index: If opening a cache file fails, try again later. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20040 diff changeset	150 if (mail_cache_map(cache, 0, 0, &data) < 0) {
4d83654e61d6 lib-index: If opening a cache file fails, try again later. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20040 diff changeset	151 mail_cache_file_close(cache);
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	152 return -1;
20041 4d83654e61d6 lib-index: If opening a cache file fails, try again later. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20040 diff changeset	153 }
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	154 return 1;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	155 }
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	156
6701 c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	157 static bool mail_cache_need_reopen(struct mail_cache *cache)
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	158 {
6815 c991d1b132d8 Fixed cache file's NFS attribute cache flushing. Timo Sirainen <tss@iki.fi> parents: 6772 diff changeset	159 struct stat st;
6701 c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	160
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	161 if (MAIL_CACHE_IS_UNUSABLE(cache)) {
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	162 if (cache->need_compress_file_seq != 0) {
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	163 /* we're waiting for compression */
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	164 return FALSE;
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	165 }
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	166 if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) {
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	167 /* disabled */
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	168 return FALSE;
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	169 }
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	170 }
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	171
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	172 if (cache->fd == -1)
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	173 return TRUE;
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	174
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	175 /* see if the file has changed */
9696 126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	176 if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
6829 dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	177 i_assert(!cache->locked);
6857 41911abe6fa7 NFS cache flushing updates. Timo Sirainen <tss@iki.fi> parents: 6841 diff changeset	178 nfs_flush_file_handle_cache(cache->filepath);
6829 dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	179 }
6815 c991d1b132d8 Fixed cache file's NFS attribute cache flushing. Timo Sirainen <tss@iki.fi> parents: 6772 diff changeset	180 if (nfs_safe_stat(cache->filepath, &st) < 0) {
6701 c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	181 mail_cache_set_syscall_error(cache, "stat()");
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	182 return TRUE;
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	183 }
6829 dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	184
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	185 if (st.st_ino != cache->st_ino \|\|
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	186 !CMP_DEV_T(st.st_dev, cache->st_dev)) {
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	187 /* file changed */
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	188 return TRUE;
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	189 }
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	190
9696 126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	191 if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
6815 c991d1b132d8 Fixed cache file's NFS attribute cache flushing. Timo Sirainen <tss@iki.fi> parents: 6772 diff changeset	192 /* if the old file has been deleted, the new file may have
6829 dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	193 the same inode as the old one. we'll catch this here by
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	194 checking if fstat() fails with ESTALE */
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	195 if (fstat(cache->fd, &st) < 0) {
12936 d14b0fd0a423 Linux NFS: fstat() may return ENOENT instead of ESTALE in some kernel versions. Timo Sirainen <tss@iki.fi> parents: 12782 diff changeset	196 if (ESTALE_FSTAT(errno))
6829 dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	197 return TRUE;
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	198 mail_cache_set_syscall_error(cache, "fstat()");
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	199 return FALSE;
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	200 }
6815 c991d1b132d8 Fixed cache file's NFS attribute cache flushing. Timo Sirainen <tss@iki.fi> parents: 6772 diff changeset	201 }
6829 dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	202 return FALSE;
6701 c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	203 }
c601e8cd0ffc mail_cache_reopen(): Don't bother reopening if file is still the same. Timo Sirainen <tss@iki.fi> parents: 6700 diff changeset	204
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	205 static int mail_cache_reopen_now(struct mail_cache *cache)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	206 {
2853 512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	207 struct mail_index_view *view;
512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	208 const struct mail_index_ext *ext;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	209
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	210 mail_cache_file_close(cache);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	211
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	212 if (mail_cache_try_open(cache) <= 0)
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	213 return -1;
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	214
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	215 if (mail_cache_header_fields_read(cache) < 0)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	216 return -1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	217
2853 512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	218 view = mail_index_view_open(cache->index);
512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	219 ext = mail_index_view_get_ext(view, cache->ext_id);
512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	220 if (ext == NULL \|\| cache->hdr->file_seq != ext->reset_id) {
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	221 /* still different - maybe a race condition or maybe the
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	222 file_seq really is corrupted. either way, this shouldn't
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	223 happen often so we'll just mark cache to be compressed
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	224 later which fixes this. */
4247 9c3f19110538 Don't compress cache file if it was just compressed by another process. Timo Sirainen <tss@iki.fi> parents: 4023 diff changeset	225 cache->need_compress_file_seq = cache->hdr->file_seq;
3879 928229f8b3e6 deinit, unref, destroy, close, free, etc. functions now take a pointer to Timo Sirainen <tss@iki.fi> parents: 3863 diff changeset	226 mail_index_view_close(&view);
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	227 return 0;
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	228 }
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	229
3879 928229f8b3e6 deinit, unref, destroy, close, free, etc. functions now take a pointer to Timo Sirainen <tss@iki.fi> parents: 3863 diff changeset	230 mail_index_view_close(&view);
5736 b3ed52bbeca7 Changed mail_cache_foreach() to be an iterator: mail_cache_lookup_iter_() Timo Sirainen <tss@iki.fi>* parents: 5734 diff changeset	231 i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	232 return 1;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	233 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	234
17901 a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	235 int mail_cache_reopen(struct mail_cache *cache)
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	236 {
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	237 i_assert(!cache->locked);
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	238
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	239 if (!mail_cache_need_reopen(cache)) {
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	240 /* reopening does no good */
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	241 return 0;
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	242 }
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	243 return mail_cache_reopen_now(cache);
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	244 }
a10eea380073 lib-index: Code cleanup - avoid code duplication. Timo Sirainen <tss@iki.fi> parents: 17900 diff changeset	245
8292 9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	246 static void mail_cache_update_need_compress(struct mail_cache *cache)
9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	247 {
9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	248 const struct mail_cache_header *hdr = cache->hdr;
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	249 struct stat st;
15374 debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	250 unsigned int msg_count;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	251 unsigned int records_count, cont_percentage, delete_percentage;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	252 bool want_compress = FALSE;
8292 9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	253
15374 debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	254 if (hdr->minor_version == 0) {
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	255 /* compress to get ourself into the new header version */
8292 9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	256 cache->need_compress_file_seq = hdr->file_seq;
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	257 return;
8292 9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	258 }
9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	259
15374 debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	260 msg_count = cache->index->map->rec_map->records_count;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	261 if (msg_count == 0)
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	262 records_count = 1;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	263 else if (hdr->record_count == 0 \|\| hdr->record_count > msg_count*2) {
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	264 /* probably not the real record_count, but hole offset that
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	265 Dovecot <=v2.1 versions used to use in this position.
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	266 we already checked that minor_version>0, but this could
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	267 happen if old Dovecot was used to access mailbox after
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	268 it had been updated. */
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	269 records_count = I_MAX(msg_count, 1);
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	270 cache->hdr_copy.record_count = msg_count;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	271 cache->hdr_modified = TRUE;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	272 } else {
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	273 records_count = hdr->record_count;
8292 9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	274 }
9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	275
15374 debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	276 cont_percentage = hdr->continued_record_count * 100 / records_count;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	277 if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE) {
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	278 /* too many continued rows, compress */
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	279 want_compress = TRUE;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	280 }
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	281
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	282 delete_percentage = hdr->deleted_record_count * 100 /
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	283 (records_count + hdr->deleted_record_count);
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	284 if (delete_percentage >= MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE) {
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	285 /* too many deleted records, compress */
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	286 want_compress = TRUE;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	287 }
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	288
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	289 if (want_compress) {
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	290 if (fstat(cache->fd, &st) < 0) {
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	291 if (!ESTALE_FSTAT(errno))
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	292 mail_cache_set_syscall_error(cache, "fstat()");
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	293 return;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	294 }
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	295 if (st.st_size >= MAIL_CACHE_COMPRESS_MIN_SIZE)
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	296 cache->need_compress_file_seq = hdr->file_seq;
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	297 }
debecd057f9c lib-index: Replaced cache's deleted_space with deleted_record_count. Timo Sirainen <tss@iki.fi> parents: 15373 diff changeset	298
8292 9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	299 }
9ed4ecd4a866 Fixes to handling "out of disk space/quota" write failures. Timo Sirainen <tss@iki.fi> parents: 8190 diff changeset	300
15312 0d1de37ad9d8 lib-index: Code cleanup: Removed mail_cache.data Timo Sirainen <tss@iki.fi> parents: 15311 diff changeset	301 static bool mail_cache_verify_header(struct mail_cache *cache,
0d1de37ad9d8 lib-index: Code cleanup: Removed mail_cache.data Timo Sirainen <tss@iki.fi> parents: 15311 diff changeset	302 const struct mail_cache_header *hdr)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	303 {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	304 /* check that the header is still ok */
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	305 if (cache->mmap_length < sizeof(struct mail_cache_header)) {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	306 mail_cache_set_corrupted(cache, "File too small");
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	307 return FALSE;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	308 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	309
15373 8323e81785e2 lib-index: Added minor version to dovecot.index.cache file header. Timo Sirainen <tss@iki.fi> parents: 15371 diff changeset	310 if (hdr->major_version != MAIL_CACHE_MAJOR_VERSION) {
2276 5f374049abdb Caching fixes and optimizations. Removed all network byte ordering code - Timo Sirainen <tss@iki.fi> parents: 2275 diff changeset	311 /* version changed - upgrade silently */
10006 a425ba46a64c lib-index: Take MAIL_INDEX_OPEN_FLAG_READONLY more seriously. Timo Sirainen <tss@iki.fi> parents: 9696 diff changeset	312 mail_cache_unlink(cache);
2276 5f374049abdb Caching fixes and optimizations. Removed all network byte ordering code - Timo Sirainen <tss@iki.fi> parents: 2275 diff changeset	313 return FALSE;
5f374049abdb Caching fixes and optimizations. Removed all network byte ordering code - Timo Sirainen <tss@iki.fi> parents: 2275 diff changeset	314 }
4657 a2a9b6a92831 Don't use time_t anymore in cache file. uint32_t is large enough and makes Timo Sirainen <tss@iki.fi> parents: 4655 diff changeset	315 if (hdr->compat_sizeof_uoff_t != sizeof(uoff_t)) {
a2a9b6a92831 Don't use time_t anymore in cache file. uint32_t is large enough and makes Timo Sirainen <tss@iki.fi> parents: 4655 diff changeset	316 /* architecture change - handle silently(?) */
10006 a425ba46a64c lib-index: Take MAIL_INDEX_OPEN_FLAG_READONLY more seriously. Timo Sirainen <tss@iki.fi> parents: 9696 diff changeset	317 mail_cache_unlink(cache);
4657 a2a9b6a92831 Don't use time_t anymore in cache file. uint32_t is large enough and makes Timo Sirainen <tss@iki.fi> parents: 4655 diff changeset	318 return FALSE;
3819 61f0e1fe9b55 Moved uoff_t and time_t compatibility checks to cache file, since they're Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	319 }
2276 5f374049abdb Caching fixes and optimizations. Removed all network byte ordering code - Timo Sirainen <tss@iki.fi> parents: 2275 diff changeset	320
6703 75c48f171ad3 Keep a separate copy of mail_cache_header when mmap_disable=yes, otherwise Timo Sirainen <tss@iki.fi> parents: 6702 diff changeset	321 if (hdr->indexid != cache->index->indexid) {
6407 1deb160e5fbb Don't bother complaining about indexid changes. The real reason for the Timo Sirainen <tss@iki.fi> parents: 6360 diff changeset	322 /* index id changed - handle silently */
10006 a425ba46a64c lib-index: Take MAIL_INDEX_OPEN_FLAG_READONLY more seriously. Timo Sirainen <tss@iki.fi> parents: 9696 diff changeset	323 mail_cache_unlink(cache);
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	324 return FALSE;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	325 }
6703 75c48f171ad3 Keep a separate copy of mail_cache_header when mmap_disable=yes, otherwise Timo Sirainen <tss@iki.fi> parents: 6702 diff changeset	326 if (hdr->file_seq == 0) {
5014 59b04ed146a9 Make sure file_seq isn't zero in cache file header. Timo Sirainen <tss@iki.fi> parents: 4968 diff changeset	327 mail_cache_set_corrupted(cache, "file_seq is 0");
59b04ed146a9 Make sure file_seq isn't zero in cache file header. Timo Sirainen <tss@iki.fi> parents: 4968 diff changeset	328 return FALSE;
59b04ed146a9 Make sure file_seq isn't zero in cache file header. Timo Sirainen <tss@iki.fi> parents: 4968 diff changeset	329 }
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	330 return TRUE;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	331 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	332
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	333 static int
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	334 mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size,
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	335 const void *hdr_data, bool copy_hdr)
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	336 {
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	337 const struct mail_cache_header *hdr = hdr_data;
3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	338
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	339 if (offset == 0) {
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	340 /* verify the header validity only with offset=0. this way
3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	341 we won't waste time re-verifying it all the time */
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	342 if (!mail_cache_verify_header(cache, hdr)) {
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	343 cache->need_compress_file_seq =
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	344 !MAIL_CACHE_IS_UNUSABLE(cache) &&
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	345 cache->hdr->file_seq != 0 ?
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	346 cache->hdr->file_seq : 0;
15537 348e9bad84f0 lib-index: Fixed assert-crash caused by recent cache file changes. Timo Sirainen <tss@iki.fi> parents: 15536 diff changeset	347 cache->hdr = NULL;
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	348 return -1;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	349 }
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	350 }
3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	351 if (hdr_data != NULL) {
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	352 if (!copy_hdr)
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	353 cache->hdr = hdr;
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	354 else {
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	355 memcpy(&cache->hdr_ro_copy, hdr,
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	356 sizeof(cache->hdr_ro_copy));
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	357 cache->hdr = &cache->hdr_ro_copy;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	358 }
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	359 mail_cache_update_need_compress(cache);
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	360 } else {
3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	361 i_assert(cache->hdr != NULL);
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	362 }
15537 348e9bad84f0 lib-index: Fixed assert-crash caused by recent cache file changes. Timo Sirainen <tss@iki.fi> parents: 15536 diff changeset	363 i_assert(cache->hdr->file_seq != 0);
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	364
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	365 if (offset + size > cache->mmap_length)
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	366 return 0;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	367 return 1;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	368 }
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	369
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	370 static int
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	371 mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size,
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	372 const void **data_r)
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	373 {
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	374 const void *hdr_data;
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	375 void *data;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	376 ssize_t ret;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	377
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	378 if (cache->read_buf == NULL) {
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	379 cache->read_buf =
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	380 buffer_create_dynamic(default_pool, size);
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	381 } else if (cache->read_offset <= offset &&
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	382 cache->read_offset + cache->read_buf->used >= offset+size) {
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	383 /* already mapped */
15318 99aec8d4d6c6 lib-index: Crashfix for MAIL_INDEX_OPEN_FLAG_SAVEONLY change. Timo Sirainen <tss@iki.fi> parents: 15313 diff changeset	384 *data_r = CONST_PTR_OFFSET(cache->read_buf->data,
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	385 offset - cache->read_offset);
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	386 hdr_data = offset == 0 ? *data_r : NULL;
3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	387 return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE);
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	388 } else {
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	389 buffer_set_used_size(cache->read_buf, 0);
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	390 }
15333 0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	391 if (offset == 0 && size < MAIL_CACHE_MIN_HEADER_READ_SIZE) {
0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	392 /* we can usually read the fields header after the cache
0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	393 header. we need them both, so try to read them all with one
0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	394 pread() call. */
0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	395 size = MAIL_CACHE_MIN_HEADER_READ_SIZE;
0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	396 }
0efb98659a1f lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15332 diff changeset	397
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	398 data = buffer_append_space_unsafe(cache->read_buf, size);
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	399 ret = pread(cache->fd, data, size, offset);
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	400 if (ret < 0) {
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	401 if (errno != ESTALE)
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	402 mail_cache_set_syscall_error(cache, "read()");
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	403
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	404 buffer_set_used_size(cache->read_buf, 0);
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	405 cache->hdr = NULL;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	406 cache->mmap_length = 0;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	407 return -1;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	408 }
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	409 buffer_set_used_size(cache->read_buf, ret);
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	410
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	411 cache->read_offset = offset;
15565 bded819417d9 lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY was buggy when reading data near end of dovecot.index.cache. Timo Sirainen <tss@iki.fi> parents: 15537 diff changeset	412 cache->mmap_length = offset + cache->read_buf->used;
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	413
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	414 *data_r = data;
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	415 hdr_data = offset == 0 ? *data_r : NULL;
15565 bded819417d9 lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY was buggy when reading data near end of dovecot.index.cache. Timo Sirainen <tss@iki.fi> parents: 15537 diff changeset	416 return mail_cache_map_finish(cache, offset,
bded819417d9 lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY was buggy when reading data near end of dovecot.index.cache. Timo Sirainen <tss@iki.fi> parents: 15537 diff changeset	417 cache->read_buf->used, hdr_data, TRUE);
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	418 }
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	419
15311 022d0d21e56d lib-index: mail_cache_map() API cleanup Timo Sirainen <tss@iki.fi> parents: 14133 diff changeset	420 int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
022d0d21e56d lib-index: mail_cache_map() API cleanup Timo Sirainen <tss@iki.fi> parents: 14133 diff changeset	421 const void **data_r)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	422 {
15792 99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	423 struct stat st;
15311 022d0d21e56d lib-index: mail_cache_map() API cleanup Timo Sirainen <tss@iki.fi> parents: 14133 diff changeset	424 const void *data;
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	425 ssize_t ret;
bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	426
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	427 if (size == 0)
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	428 size = sizeof(struct mail_cache_header);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	429
15792 99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	430 /* verify offset + size before trying to allocate a huge amount of
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	431 memory due to them. note that we may be prefetching more than we
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	432 actually need, so don't fail too early. */
15805 443ff272317f lib-index: Bug in cache file size verification caused the whole cache file to be mapped. Timo Sirainen <tss@iki.fi> parents: 15793 diff changeset	433 if ((size > cache->mmap_length \|\| offset + size > cache->mmap_length) &&
443ff272317f lib-index: Bug in cache file size verification caused the whole cache file to be mapped. Timo Sirainen <tss@iki.fi> parents: 15793 diff changeset	434 (offset > 0 \|\| size > sizeof(struct mail_cache_header))) {
15792 99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	435 if (fstat(cache->fd, &st) < 0) {
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	436 i_error("fstat(%s) failed: %m", cache->filepath);
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	437 return -1;
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	438 }
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	439 if (offset >= (uoff_t)st.st_size) {
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	440 *data_r = NULL;
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	441 return 0;
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	442 }
15805 443ff272317f lib-index: Bug in cache file size verification caused the whole cache file to be mapped. Timo Sirainen <tss@iki.fi> parents: 15793 diff changeset	443 if (offset + size > (uoff_t)st.st_size)
443ff272317f lib-index: Bug in cache file size verification caused the whole cache file to be mapped. Timo Sirainen <tss@iki.fi> parents: 15793 diff changeset	444 size = st.st_size - offset;
15792 99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	445 }
99b7be36631b lib-index: Fixes to handling broken cache records that point outside file. Timo Sirainen <tss@iki.fi> parents: 15567 diff changeset	446
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	447 cache->remap_counter++;
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	448 if (cache->map_with_read)
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	449 return mail_cache_map_with_read(cache, offset, size, data_r);
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	450
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	451 if (cache->file_cache != NULL) {
bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	452 ret = file_cache_read(cache->file_cache, offset, size);
bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	453 if (ret < 0) {
4023 b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	454 /* In case of ESTALE we'll simply fail without error
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	455 messages. The caller will then just have to
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	456 fallback to generating the value itself.
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	457
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	458 We can't simply reopen the cache flie, because
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	459 using it requires also having updated file
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	460 offsets. */
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	461 if (errno != ESTALE)
b19ccbaa3802 Try to handle ESTALE NFS errors the best way we can. Timo Sirainen <timo.sirainen@movial.fi> parents: 3981 diff changeset	462 mail_cache_set_syscall_error(cache, "read()");
15537 348e9bad84f0 lib-index: Fixed assert-crash caused by recent cache file changes. Timo Sirainen <tss@iki.fi> parents: 15536 diff changeset	463 cache->hdr = NULL;
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	464 return -1;
bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	465 }
bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	466
15312 0d1de37ad9d8 lib-index: Code cleanup: Removed mail_cache.data Timo Sirainen <tss@iki.fi> parents: 15311 diff changeset	467 data = file_cache_get_map(cache->file_cache,
0d1de37ad9d8 lib-index: Code cleanup: Removed mail_cache.data Timo Sirainen <tss@iki.fi> parents: 15311 diff changeset	468 &cache->mmap_length);
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	469 *data_r = offset > cache->mmap_length ? NULL :
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	470 CONST_PTR_OFFSET(data, offset);
15537 348e9bad84f0 lib-index: Fixed assert-crash caused by recent cache file changes. Timo Sirainen <tss@iki.fi> parents: 15536 diff changeset	471 return mail_cache_map_finish(cache, offset, size,
348e9bad84f0 lib-index: Fixed assert-crash caused by recent cache file changes. Timo Sirainen <tss@iki.fi> parents: 15536 diff changeset	472 offset == 0 ? data : NULL, TRUE);
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	473 }
bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	474
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	475 if (offset < cache->mmap_length &&
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	476 size <= cache->mmap_length - offset) {
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	477 /* already mapped */
15567 32ce915e046a lib-index: If map_with_read gets disabled, reset the cache buffers properly. Timo Sirainen <tss@iki.fi> parents: 15565 diff changeset	478 i_assert(cache->mmap_base != NULL);
15311 022d0d21e56d lib-index: mail_cache_map() API cleanup Timo Sirainen <tss@iki.fi> parents: 14133 diff changeset	479 *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset);
022d0d21e56d lib-index: mail_cache_map() API cleanup Timo Sirainen <tss@iki.fi> parents: 14133 diff changeset	480 return 1;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	481 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	482
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	483 if (cache->mmap_base != NULL) {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	484 if (munmap(cache->mmap_base, cache->mmap_length) < 0)
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	485 mail_cache_set_syscall_error(cache, "munmap()");
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	486 } else {
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	487 if (cache->fd == -1) {
3243 40b4ba3c55b8 In-memory indexes work again. Just pass dir as NULL to mail_index_alloc(). Timo Sirainen <tss@iki.fi> parents: 3224 diff changeset	488 /* unusable, waiting for compression or
40b4ba3c55b8 In-memory indexes work again. Just pass dir as NULL to mail_index_alloc(). Timo Sirainen <tss@iki.fi> parents: 3224 diff changeset	489 index is in memory */
4247 9c3f19110538 Don't compress cache file if it was just compressed by another process. Timo Sirainen <tss@iki.fi> parents: 4023 diff changeset	490 i_assert(cache->need_compress_file_seq != 0 \|\|
3243 40b4ba3c55b8 In-memory indexes work again. Just pass dir as NULL to mail_index_alloc(). Timo Sirainen <tss@iki.fi> parents: 3224 diff changeset	491 MAIL_INDEX_IS_IN_MEMORY(cache->index));
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	492 return -1;
c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	493 }
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	494 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	495
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	496 /* map the whole file */
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	497 cache->hdr = NULL;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	498 cache->mmap_length = 0;
19917 1ef6401c749d lib-index: After MAIL_INDEX_OPEN_FLAG_SAVEONLY access cache's read_buf wasn't always reset. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19552 diff changeset	499 if (cache->read_buf != NULL)
1ef6401c749d lib-index: After MAIL_INDEX_OPEN_FLAG_SAVEONLY access cache's read_buf wasn't always reset. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19552 diff changeset	500 buffer_set_used_size(cache->read_buf, 0);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	501
2277 41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	502 cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	503 if (cache->mmap_base == MAP_FAILED) {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	504 cache->mmap_base = NULL;
21186 34074cc72a15 lib-index: Limit mmap errors in index cache to 1/s Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21168 diff changeset	505 if (ioloop_time != cache->last_mmap_error_time) {
34074cc72a15 lib-index: Limit mmap errors in index cache to 1/s Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21168 diff changeset	506 cache->last_mmap_error_time = ioloop_time;
34074cc72a15 lib-index: Limit mmap errors in index cache to 1/s Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21168 diff changeset	507 mail_cache_set_syscall_error(cache, t_strdup_printf(
34074cc72a15 lib-index: Limit mmap errors in index cache to 1/s Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21168 diff changeset	508 "mmap(size=%"PRIuSIZE_T")", cache->mmap_length));
34074cc72a15 lib-index: Limit mmap errors in index cache to 1/s Aki Tuomi <aki.tuomi@dovecot.fi> parents: 21168 diff changeset	509 }
21537 8ebf69bd381b lib-index: zero mmap_length after logging error Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21447 diff changeset	510 cache->mmap_length = 0;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	511 return -1;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	512 }
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	513 *data_r = offset > cache->mmap_length ? NULL :
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	514 CONST_PTR_OFFSET(cache->mmap_base, offset);
15332 3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	515 return mail_cache_map_finish(cache, offset, size,
3700fb6f8a42 lib-index: After recent cache changes, cache was often wrongly being thought of as unusable Timo Sirainen <tss@iki.fi> parents: 15318 diff changeset	516 cache->mmap_base, FALSE);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	517 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	518
4655 df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	519 int mail_cache_open_and_verify(struct mail_cache *cache)
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	520 {
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	521 int ret;
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	522
20040 4da8d743d98f lib-index: Fixed calling mail_cache_open_and_verify() on an already open cache. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19917 diff changeset	523 if (cache->opened)
4da8d743d98f lib-index: Fixed calling mail_cache_open_and_verify() on an already open cache. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19917 diff changeset	524 return 0;
4655 df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	525 ret = mail_cache_try_open(cache);
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	526 if (ret > 0)
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	527 ret = mail_cache_header_fields_read(cache);
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	528 if (ret < 0) {
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	529 /* failed for some reason - doesn't really matter,
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	530 it's disabled for now. */
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	531 mail_cache_file_close(cache);
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	532 }
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	533 return ret;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	534 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	535
3129 1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	536 static struct mail_cache mail_cache_alloc(struct mail_index index)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	537 {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	538 struct mail_cache *cache;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	539
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	540 cache = i_new(struct mail_cache, 1);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	541 cache->index = index;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	542 cache->fd = -1;
3129 1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	543 cache->filepath =
3316 62d31be5ae60 s/PREFIX/SUFFIX/ for .cache and .log filename suffixes.. Timo Sirainen <tss@iki.fi> parents: 3271 diff changeset	544 i_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL);
14078 f6102f0af71d Increase initial memory pool sizes. Timo Sirainen <tss@iki.fi> parents: 13339 diff changeset	545 cache->field_pool = pool_alloconly_create("Cache fields", 2048);
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	546 hash_table_create(&cache->field_name_hash, cache->field_pool, 0,
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	547 strcase_hash, strcasecmp);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	548
9696 126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	549 cache->dotlock_settings.use_excl_lock =
126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	550 (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	551 cache->dotlock_settings.nfs_flush =
126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	552 (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
11623 c3ee4421e86a Added mail_max_lock_timeout setting. Timo Sirainen <tss@iki.fi> parents: 11438 diff changeset	553 cache->dotlock_settings.timeout =
c3ee4421e86a Added mail_max_lock_timeout setting. Timo Sirainen <tss@iki.fi> parents: 11438 diff changeset	554 I_MIN(MAIL_CACHE_LOCK_TIMEOUT, index->max_lock_timeout_secs);
3106 9c4aa309dbac Changed dotlocking API. Timo Sirainen <tss@iki.fi> parents: 3103 diff changeset	555 cache->dotlock_settings.stale_timeout = MAIL_CACHE_LOCK_CHANGE_TIMEOUT;
9c4aa309dbac Changed dotlocking API. Timo Sirainen <tss@iki.fi> parents: 3103 diff changeset	556
9696 126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	557 if (!MAIL_INDEX_IS_IN_MEMORY(index) &&
126132cb1c19 index: Removed duplication of mail_index_open_flags from struct mail_index. Timo Sirainen <tss@iki.fi> parents: 8973 diff changeset	558 (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0)
21307 3a8ad59caa7d lib-index: Use file_cache_new_path() for dovecot.index.cache Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21186 diff changeset	559 cache->file_cache = file_cache_new_path(-1, cache->filepath);
15313 51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	560 cache->map_with_read =
51f2be9aa8ad lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. Timo Sirainen <tss@iki.fi> parents: 15312 diff changeset	561 (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	562
2853 512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	563 cache->ext_id =
512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	564 mail_index_ext_register(index, "cache", 0,
512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	565 sizeof(uint32_t), sizeof(uint32_t));
4903 204d7edc7cdc Added context parameter type safety checks for most callback APIs. Timo Sirainen <tss@iki.fi> parents: 4876 diff changeset	566 mail_index_register_expunge_handler(index, cache->ext_id, FALSE,
204d7edc7cdc Added context parameter type safety checks for most callback APIs. Timo Sirainen <tss@iki.fi> parents: 4876 diff changeset	567 mail_cache_expunge_handler, cache);
2200 97bb7b127617 Beginnings of getting cache file working again. Easy to break currently, but Timo Sirainen <tss@iki.fi> parents: 1956 diff changeset	568 return cache;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	569 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	570
3129 1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	571 struct mail_cache mail_cache_open_or_create(struct mail_index index)
1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	572 {
1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	573 struct mail_cache *cache;
1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	574
1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	575 cache = mail_cache_alloc(index);
1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	576 return cache;
1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	577 }
1ea614abead3 Don't complain about cache indexid changing when recreating index. Timo Sirainen <tss@iki.fi> parents: 3106 diff changeset	578
3879 928229f8b3e6 deinit, unref, destroy, close, free, etc. functions now take a pointer to Timo Sirainen <tss@iki.fi> parents: 3863 diff changeset	579 void mail_cache_free(struct mail_cache **_cache)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	580 {
3879 928229f8b3e6 deinit, unref, destroy, close, free, etc. functions now take a pointer to Timo Sirainen <tss@iki.fi> parents: 3863 diff changeset	581 struct mail_cache cache = _cache;
928229f8b3e6 deinit, unref, destroy, close, free, etc. functions now take a pointer to Timo Sirainen <tss@iki.fi> parents: 3863 diff changeset	582
928229f8b3e6 deinit, unref, destroy, close, free, etc. functions now take a pointer to Timo Sirainen <tss@iki.fi> parents: 3863 diff changeset	583 *_cache = NULL;
15371 67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	584 if (cache->file_cache != NULL)
3879 928229f8b3e6 deinit, unref, destroy, close, free, etc. functions now take a pointer to Timo Sirainen <tss@iki.fi> parents: 3863 diff changeset	585 file_cache_free(&cache->file_cache);
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	586
3224 f6e0d1826645 Added sync/sync_lost/expunge handler unregistering, and also do that. Fixes Timo Sirainen <tss@iki.fi> parents: 3191 diff changeset	587 mail_index_unregister_expunge_handler(cache->index, cache->ext_id);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	588 mail_cache_file_close(cache);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	589
15536 e62938129955 lib-index: Fixed memory leak with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15505 diff changeset	590 if (cache->read_buf != NULL)
e62938129955 lib-index: Fixed memory leak with MAIL_INDEX_OPEN_FLAG_SAVEONLY Timo Sirainen <tss@iki.fi> parents: 15505 diff changeset	591 buffer_free(&cache->read_buf);
8573 f9166a09423a Renamed hash_() to hash_table_() to avoid conflicts with OSX's strhash.h Timo Sirainen <tss@iki.fi> parents: 8292 diff changeset	592 hash_table_destroy(&cache->field_name_hash);
6428 7cad076906eb pool_unref() now takes ** pointer. Timo Sirainen <tss@iki.fi> parents: 6417 diff changeset	593 pool_unref(&cache->field_pool);
2362 f2acbe281ac1 Allocate cache->fields and cache->fields_file_map with malloc rather than Timo Sirainen <tss@iki.fi> parents: 2343 diff changeset	594 i_free(cache->field_file_map);
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	595 i_free(cache->file_field_map);
2362 f2acbe281ac1 Allocate cache->fields and cache->fields_file_map with malloc rather than Timo Sirainen <tss@iki.fi> parents: 2343 diff changeset	596 i_free(cache->fields);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	597 i_free(cache->filepath);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	598 i_free(cache);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	599 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	600
8188 2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	601 static int mail_cache_lock_file(struct mail_cache *cache, bool nonblock)
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	602 {
11623 c3ee4421e86a Added mail_max_lock_timeout setting. Timo Sirainen <tss@iki.fi> parents: 11438 diff changeset	603 unsigned int timeout_secs;
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	604 int ret;
9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	605
8190 9902de77194e Mail cache: If locking failed, the next time try to lock it only once. Timo Sirainen <tss@iki.fi> parents: 8189 diff changeset	606 if (cache->last_lock_failed) {
9902de77194e Mail cache: If locking failed, the next time try to lock it only once. Timo Sirainen <tss@iki.fi> parents: 8189 diff changeset	607 /* previous locking failed. don't waste time waiting on it
9902de77194e Mail cache: If locking failed, the next time try to lock it only once. Timo Sirainen <tss@iki.fi> parents: 8189 diff changeset	608 again, just try once to see if it's available now. */
9902de77194e Mail cache: If locking failed, the next time try to lock it only once. Timo Sirainen <tss@iki.fi> parents: 8189 diff changeset	609 nonblock = TRUE;
9902de77194e Mail cache: If locking failed, the next time try to lock it only once. Timo Sirainen <tss@iki.fi> parents: 8189 diff changeset	610 }
9902de77194e Mail cache: If locking failed, the next time try to lock it only once. Timo Sirainen <tss@iki.fi> parents: 8189 diff changeset	611
4876 f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	612 if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	613 i_assert(cache->file_lock == NULL);
11623 c3ee4421e86a Added mail_max_lock_timeout setting. Timo Sirainen <tss@iki.fi> parents: 11438 diff changeset	614 timeout_secs = I_MIN(MAIL_CACHE_LOCK_TIMEOUT,
c3ee4421e86a Added mail_max_lock_timeout setting. Timo Sirainen <tss@iki.fi> parents: 11438 diff changeset	615 cache->index->max_lock_timeout_secs);
c3ee4421e86a Added mail_max_lock_timeout setting. Timo Sirainen <tss@iki.fi> parents: 11438 diff changeset	616
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	617 ret = mail_index_lock_fd(cache->index, cache->filepath,
9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	618 cache->fd, F_WRLCK,
11623 c3ee4421e86a Added mail_max_lock_timeout setting. Timo Sirainen <tss@iki.fi> parents: 11438 diff changeset	619 nonblock ? 0 : timeout_secs,
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	620 &cache->file_lock);
4876 f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	621 } else {
8188 2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	622 enum dotlock_create_flags flags =
2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	623 nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0;
2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	624
4876 f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	625 i_assert(cache->dotlock == NULL);
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	626 ret = file_dotlock_create(&cache->dotlock_settings,
8188 2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	627 cache->filepath, flags,
2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	628 &cache->dotlock);
8189 7528e7256a1a Mail cache: Don't warn if dotlocking fails due to timeout (non-dotlocking already didn't). Timo Sirainen <tss@iki.fi> parents: 8188 diff changeset	629 if (ret < 0) {
6772 b78cfc60f6a1 Added missing error logging for file_dotlock_create/open calls. Timo Sirainen <tss@iki.fi> parents: 6708 diff changeset	630 mail_cache_set_syscall_error(cache,
b78cfc60f6a1 Added missing error logging for file_dotlock_create/open calls. Timo Sirainen <tss@iki.fi> parents: 6708 diff changeset	631 "file_dotlock_create()");
b78cfc60f6a1 Added missing error logging for file_dotlock_create/open calls. Timo Sirainen <tss@iki.fi> parents: 6708 diff changeset	632 }
4876 f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	633 }
8190 9902de77194e Mail cache: If locking failed, the next time try to lock it only once. Timo Sirainen <tss@iki.fi> parents: 8189 diff changeset	634 cache->last_lock_failed = ret <= 0;
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	635
8189 7528e7256a1a Mail cache: Don't warn if dotlocking fails due to timeout (non-dotlocking already didn't). Timo Sirainen <tss@iki.fi> parents: 8188 diff changeset	636 /* don't bother warning if locking failed due to a timeout. since cache
7528e7256a1a Mail cache: Don't warn if dotlocking fails due to timeout (non-dotlocking already didn't). Timo Sirainen <tss@iki.fi> parents: 8188 diff changeset	637 updating isn't all that important we're using a very short timeout
7528e7256a1a Mail cache: Don't warn if dotlocking fails due to timeout (non-dotlocking already didn't). Timo Sirainen <tss@iki.fi> parents: 8188 diff changeset	638 so it can be triggered sometimes on heavy load */
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	639 if (ret <= 0)
9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	640 return ret;
9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	641
6829 dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	642 mail_index_flush_read_cache(cache->index, cache->filepath, cache->fd,
dbab5e592577 NFS cache flushing APIs changed and backend implements it a bit differently. Timo Sirainen <tss@iki.fi> parents: 6827 diff changeset	643 TRUE);
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	644 return 1;
4876 f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	645 }
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	646
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	647 static void mail_cache_unlock_file(struct mail_cache *cache)
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	648 {
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	649 if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK)
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	650 file_unlock(&cache->file_lock);
f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	651 else
14637 a77ad2346cf0 Added file_dotlock_delete_verified() and changed file_dotlock_delete() to return void. Timo Sirainen <tss@iki.fi> parents: 14133 diff changeset	652 file_dotlock_delete(&cache->dotlock);
2941 89ab5c72430d lock_method=dotlock doesn't crash anymore while trying to modify cache file. Timo Sirainen <tss@iki.fi> parents: 2936 diff changeset	653 }
89ab5c72430d lock_method=dotlock doesn't crash anymore while trying to modify cache file. Timo Sirainen <tss@iki.fi> parents: 2936 diff changeset	654
8188 2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	655 static int
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	656 mail_cache_lock_full(struct mail_cache *cache, bool nonblock)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	657 {
2853 512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	658 const struct mail_index_ext *ext;
15311 022d0d21e56d lib-index: mail_cache_map() API cleanup Timo Sirainen <tss@iki.fi> parents: 14133 diff changeset	659 const void *data;
5947 7ebe0593f488 Don't create cache file until something is actually being added to it. Timo Sirainen <tss@iki.fi> parents: 5736 diff changeset	660 struct mail_index_view *iview;
7ebe0593f488 Don't create cache file until something is actually being added to it. Timo Sirainen <tss@iki.fi> parents: 5736 diff changeset	661 uint32_t reset_id;
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	662 int i;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	663
2277 41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	664 i_assert(!cache->locked);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	665
4655 df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	666 if (!cache->opened)
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	667 (void)mail_cache_open_and_verify(cache);
df8de2e47fb5 Delay opening the cache file until it's actually needed. Timo Sirainen <tss@iki.fi> parents: 4653 diff changeset	668
3833 318c70c1d4ec Beginnings of fallbacking to in-memory indexes when write fails with "out of Timo Sirainen <tss@iki.fi> parents: 3819 diff changeset	669 if (MAIL_CACHE_IS_UNUSABLE(cache) \|\|
11438 7adc45a6da2e lib-index: If index is read-only, always fail when trying to lock cache file. Timo Sirainen <tss@iki.fi> parents: 10686 diff changeset	670 MAIL_INDEX_IS_IN_MEMORY(cache->index) \|\|
7adc45a6da2e lib-index: If index is read-only, always fail when trying to lock cache file. Timo Sirainen <tss@iki.fi> parents: 10686 diff changeset	671 cache->index->readonly)
2220 357bc14b6d05 Disable cache with mmap_disabled=yes, for now. Timo Sirainen <tss@iki.fi> parents: 2204 diff changeset	672 return 0;
357bc14b6d05 Disable cache with mmap_disabled=yes, for now. Timo Sirainen <tss@iki.fi> parents: 2204 diff changeset	673
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	674 for (;;) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	675 if (mail_cache_lock_file(cache, nonblock) <= 0)
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	676 return -1;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	677 i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	678 if (!mail_cache_need_reopen(cache)) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	679 /* locked the latest file */
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	680 break;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	681 }
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	682 if (mail_cache_reopen_now(cache) <= 0) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	683 i_assert(cache->file_lock == NULL);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	684 return -1;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	685 }
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	686 i_assert(cache->file_lock == NULL);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	687 /* okay, so it was just compressed. try again. */
2853 512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	688 }
512dd7d76cdc Removed cache_offset from mail_index_record and changed it to use extension Timo Sirainen <tss@iki.fi> parents: 2828 diff changeset	689
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	690 /* now verify that the index reset_id matches the cache's file_seq */
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	691 for (i = 0; ; i++) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	692 iview = mail_index_view_open(cache->index);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	693 ext = mail_index_view_get_ext(iview, cache->ext_id);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	694 reset_id = ext == NULL ? 0 : ext->reset_id;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	695 mail_index_view_close(&iview);
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	696
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	697 if (cache->hdr->file_seq == reset_id)
2275 c68a3c9f6d73 Cache file compression works now and compressed cache file is reopened. Timo Sirainen <tss@iki.fi> parents: 2251 diff changeset	698 break;
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	699 /* mismatch. try refreshing index once. if that doesn't help,
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	700 we can't use the cache. */
17907 51233ae4af26 lib-index: Fixed assert-crash in some cache locking race conditions. Timo Sirainen <tss@iki.fi> parents: 17902 diff changeset	701 if (i > 0 \|\| cache->index->mapping) {
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	702 mail_cache_unlock_file(cache);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	703 return 0;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	704 }
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	705 if (mail_index_refresh(cache->index) < 0) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	706 mail_cache_unlock_file(cache);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	707 return -1;
2929 ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2920 diff changeset	708 }
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	709 }
2277 41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	710
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	711 /* successfully locked - make sure our header is up to date */
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	712 cache->locked = TRUE;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	713 cache->hdr_modified = FALSE;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	714
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	715 if (cache->file_cache != NULL) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	716 file_cache_invalidate(cache->file_cache, 0,
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	717 sizeof(struct mail_cache_header));
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	718 }
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	719 if (cache->read_buf != NULL)
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	720 buffer_set_used_size(cache->read_buf, 0);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	721 if (mail_cache_map(cache, 0, 0, &data) <= 0) {
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	722 (void)mail_cache_unlock(cache);
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	723 return -1;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	724 }
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	725 cache->hdr_copy = *cache->hdr;
8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	726 return 1;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	727 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	728
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	729 int mail_cache_lock(struct mail_cache *cache)
8188 2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	730 {
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	731 return mail_cache_lock_full(cache, FALSE);
8188 2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	732 }
2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	733
2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	734 int mail_cache_try_lock(struct mail_cache *cache)
2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	735 {
17902 8c84b69e7f69 lib-index: mail_cache_lock() partial rewrite. Timo Sirainen <tss@iki.fi> parents: 17901 diff changeset	736 return mail_cache_lock_full(cache, TRUE);
8188 2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	737 }
2db48458d73b Mail cache compression: If we can't get lock immediately, do it later. Timo Sirainen <tss@iki.fi> parents: 8187 diff changeset	738
3627 c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	739 int mail_cache_unlock(struct mail_cache *cache)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	740 {
3627 c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	741 int ret = 0;
c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	742
2277 41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	743 i_assert(cache->locked);
41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	744
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	745 if (cache->field_header_write_pending)
3627 c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	746 ret = mail_cache_header_fields_update(cache);
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	747
2920 9b772db4503d Don't write to closed cache file Timo Sirainen <tss@iki.fi> parents: 2908 diff changeset	748 if (MAIL_CACHE_IS_UNUSABLE(cache)) {
9b772db4503d Don't write to closed cache file Timo Sirainen <tss@iki.fi> parents: 2908 diff changeset	749 /* we found it to be broken during the lock. just clean up. */
9b772db4503d Don't write to closed cache file Timo Sirainen <tss@iki.fi> parents: 2908 diff changeset	750 cache->hdr_modified = FALSE;
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	751 cache->locked = FALSE;
3627 c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	752 return -1;
2920 9b772db4503d Don't write to closed cache file Timo Sirainen <tss@iki.fi> parents: 2908 diff changeset	753 }
9b772db4503d Don't write to closed cache file Timo Sirainen <tss@iki.fi> parents: 2908 diff changeset	754
2277 41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	755 if (cache->hdr_modified) {
41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	756 cache->hdr_modified = FALSE;
3627 c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	757 if (mail_cache_write(cache, &cache->hdr_copy,
c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	758 sizeof(cache->hdr_copy), 0) < 0)
c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	759 ret = -1;
6703 75c48f171ad3 Keep a separate copy of mail_cache_header when mmap_disable=yes, otherwise Timo Sirainen <tss@iki.fi> parents: 6702 diff changeset	760 cache->hdr_ro_copy = cache->hdr_copy;
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2853 diff changeset	761 mail_cache_update_need_compress(cache);
2277 41e56f28d085 Cache updating is done now by first reserving space where to write, and then Timo Sirainen <tss@iki.fi> parents: 2276 diff changeset	762 }
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	763
11634 7f19062f58fd Renamed fsync_disable to mail_fsync=optimized\|always\|never. Timo Sirainen <tss@iki.fi> parents: 11623 diff changeset	764 if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
5972 9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	765 if (fdatasync(cache->fd) < 0)
9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	766 mail_cache_set_syscall_error(cache, "fdatasync()");
9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	767 }
9fb9dc4d8df8 Flush NFS caches when needed if MAIL_INDEX_OPEN_FLAG_NFS_FLUSH is enabled. Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	768
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	769 cache->locked = FALSE;
4876 f1d77064884c Lock handling changes. Everything goes through file-lock API now and there's Timo Sirainen <tss@iki.fi> parents: 4657 diff changeset	770 mail_cache_unlock_file(cache);
3627 c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3553 diff changeset	771 return ret;
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	772 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	773
3375 9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	774 int mail_cache_write(struct mail_cache cache, const void data, size_t size,
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	775 uoff_t offset)
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	776 {
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	777 i_assert(cache->locked);
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	778
3375 9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	779 if (pwrite_full(cache->fd, data, size, offset) < 0) {
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	780 mail_cache_set_syscall_error(cache, "pwrite_full()");
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	781 return -1;
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	782 }
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	783
15312 0d1de37ad9d8 lib-index: Code cleanup: Removed mail_cache.data Timo Sirainen <tss@iki.fi> parents: 15311 diff changeset	784 if (cache->file_cache != NULL)
3375 9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	785 file_cache_write(cache->file_cache, data, size, offset);
15505 8252bcd7c489 lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY may have caused stale data to be read from cache. Timo Sirainen <tss@iki.fi> parents: 15504 diff changeset	786 if (cache->read_buf != NULL)
8252bcd7c489 lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY may have caused stale data to be read from cache. Timo Sirainen <tss@iki.fi> parents: 15504 diff changeset	787 buffer_set_used_size(cache->read_buf, 0);
3375 9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	788 return 0;
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	789 }
9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3316 diff changeset	790
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	791 int mail_cache_append(struct mail_cache cache, const void data, size_t size,
15371 67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	792 uint32_t *offset)
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	793 {
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	794 struct stat st;
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	795
15371 67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	796 if (*offset == 0) {
67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	797 if (fstat(cache->fd, &st) < 0) {
67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	798 if (!ESTALE_FSTAT(errno))
67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	799 mail_cache_set_syscall_error(cache, "fstat()");
67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	800 return -1;
67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	801 }
21351 10034ed9ee46 lib-index: Fix checking if cache file becomes >4GB Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21307 diff changeset	802 if (st.st_size > (uint32_t)-1) {
10034ed9ee46 lib-index: Fix checking if cache file becomes >4GB Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21307 diff changeset	803 mail_cache_set_corrupted(cache, "Cache file too large");
10034ed9ee46 lib-index: Fix checking if cache file becomes >4GB Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21307 diff changeset	804 return -1;
10034ed9ee46 lib-index: Fix checking if cache file becomes >4GB Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21307 diff changeset	805 }
15371 67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	806 *offset = st.st_size;
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	807 }
21351 10034ed9ee46 lib-index: Fix checking if cache file becomes >4GB Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21307 diff changeset	808 if ((uint32_t)-1 - *offset < size) {
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	809 mail_cache_set_corrupted(cache, "Cache file too large");
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	810 return -1;
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	811 }
15371 67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	812 if (mail_cache_write(cache, data, size, *offset) < 0)
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	813 return -1;
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	814
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	815 /* FIXME: this is updated only so that older Dovecot versions (<=v2.1)
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	816 can read this file. we can remove this later. */
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	817 cache->hdr_modified = TRUE;
15371 67afcb730109 lib-index: Cache record linking is now while writing new records, not later with pwrite(). Timo Sirainen <tss@iki.fi> parents: 15369 diff changeset	818 cache->hdr_copy.backwards_compat_used_file_size = *offset + size;
15190 60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	819 return 0;
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	820 }
60c4815778fb lib-index: Simplified writing to dovecot.index.cache file. Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	821
13335 11ff6a7eb576 lib-index: Added mail_cache_exists() Timo Sirainen <tss@iki.fi> parents: 13328 diff changeset	822 bool mail_cache_exists(struct mail_cache *cache)
11ff6a7eb576 lib-index: Added mail_cache_exists() Timo Sirainen <tss@iki.fi> parents: 13328 diff changeset	823 {
11ff6a7eb576 lib-index: Added mail_cache_exists() Timo Sirainen <tss@iki.fi> parents: 13328 diff changeset	824 return !MAIL_CACHE_IS_UNUSABLE(cache);
11ff6a7eb576 lib-index: Added mail_cache_exists() Timo Sirainen <tss@iki.fi> parents: 13328 diff changeset	825 }
11ff6a7eb576 lib-index: Added mail_cache_exists() Timo Sirainen <tss@iki.fi> parents: 13328 diff changeset	826
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	827 struct mail_cache_view *
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	828 mail_cache_view_open(struct mail_cache cache, struct mail_index_view iview)
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	829 {
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	830 struct mail_cache_view *view;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	831
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	832 view = i_new(struct mail_cache_view, 1);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	833 view->cache = cache;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	834 view->view = iview;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	835 view->cached_exists_buf =
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: 2307 diff changeset	836 buffer_create_dynamic(default_pool,
2708 f1e9f3ec8135 Buffer API change: we no longer support limited sized buffers where Timo Sirainen <tss@iki.fi> parents: 2632 diff changeset	837 cache->file_fields_count + 10);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	838 return view;
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	839 }
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	840
13339 b355309955f4 lib-index: Changed mail_cache_view_close() API to take pointer-to-pointer. Timo Sirainen <tss@iki.fi> parents: 13335 diff changeset	841 void mail_cache_view_close(struct mail_cache_view **_view)
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	842 {
13339 b355309955f4 lib-index: Changed mail_cache_view_close() API to take pointer-to-pointer. Timo Sirainen <tss@iki.fi> parents: 13335 diff changeset	843 struct mail_cache_view view = _view;
b355309955f4 lib-index: Changed mail_cache_view_close() API to take pointer-to-pointer. Timo Sirainen <tss@iki.fi> parents: 13335 diff changeset	844
6455 5ad7a36ca8c3 mail_cache_transaction_commit()/rollback() API is now public and takes a ** Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	845 i_assert(view->trans_view == NULL);
5ad7a36ca8c3 mail_cache_transaction_commit()/rollback() API is now public and takes a ** Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	846
13339 b355309955f4 lib-index: Changed mail_cache_view_close() API to take pointer-to-pointer. Timo Sirainen <tss@iki.fi> parents: 13335 diff changeset	847 *_view = NULL;
8187 f1203dce2d8c Mail cache: lock_method=dotlock could have caused a process to deadlock with itself. Timo Sirainen <tss@iki.fi> parents: 8183 diff changeset	848 if (view->cache->field_header_write_pending &&
f1203dce2d8c Mail cache: lock_method=dotlock could have caused a process to deadlock with itself. Timo Sirainen <tss@iki.fi> parents: 8183 diff changeset	849 !view->cache->compressing)
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	850 (void)mail_cache_header_fields_update(view->cache);
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	851
6414 a6a49d5efc59 Changed buffer_free() and buffer_free_without_data() APIs to take ** pointer Timo Sirainen <tss@iki.fi> parents: 6407 diff changeset	852 buffer_free(&view->cached_exists_buf);
1915 79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	853 i_free(view);
79790750c349 importing new index code. mbox still broken. Timo Sirainen <tss@iki.fi> parents: diff changeset	854 }
6841 0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	855
13328 4a19aea565f7 lib-index: Added mail_cache_view_update_cache_decisions() Timo Sirainen <tss@iki.fi> parents: 12936 diff changeset	856 void mail_cache_view_update_cache_decisions(struct mail_cache_view *view,
4a19aea565f7 lib-index: Added mail_cache_view_update_cache_decisions() Timo Sirainen <tss@iki.fi> parents: 12936 diff changeset	857 bool update)
4a19aea565f7 lib-index: Added mail_cache_view_update_cache_decisions() Timo Sirainen <tss@iki.fi> parents: 12936 diff changeset	858 {
4a19aea565f7 lib-index: Added mail_cache_view_update_cache_decisions() Timo Sirainen <tss@iki.fi> parents: 12936 diff changeset	859 view->no_decision_updates = !update;
4a19aea565f7 lib-index: Added mail_cache_view_update_cache_decisions() Timo Sirainen <tss@iki.fi> parents: 12936 diff changeset	860 }
4a19aea565f7 lib-index: Added mail_cache_view_update_cache_decisions() Timo Sirainen <tss@iki.fi> parents: 12936 diff changeset	861
6841 0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	862 uint32_t mail_cache_get_first_new_seq(struct mail_index_view *view)
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	863 {
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	864 const struct mail_index_header *idx_hdr;
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	865 uint32_t first_new_seq, message_count;
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	866
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	867 idx_hdr = mail_index_get_header(view);
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	868 if (idx_hdr->day_first_uid[7] == 0)
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	869 return 1;
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	870
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	871 if (!mail_index_lookup_seq_range(view, idx_hdr->day_first_uid[7],
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	872 (uint32_t)-1, &first_new_seq,
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	873 &message_count)) {
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	874 /* all messages are too old */
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	875 return message_count+1;
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	876 }
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	877 return first_new_seq;
0c970b3493ac mail_cache_field_want_add(): Return TRUE for temp fields only if we're Timo Sirainen <tss@iki.fi> parents: 6830 diff changeset	878 }

Mercurial > dovecot > core-2.2

annotate src/lib-index/mail-cache.c @ 22576:707ae9de3812