Mercurial > dovecot > core-2.2
annotate src/lib-index/mail-hash.c @ 29:e9375147c0cb HEAD
Added write_full() which is a simple wrapper around write() meant for
writing into files.
When there's too much deleted data in index files, they're now compressed
when the index is being opened.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 26 Aug 2002 02:46:59 +0300 |
parents | 1b34ec11fff8 |
children | 1ab58549429d |
rev | line source |
---|---|
0 | 1 /* Copyright (C) 2002 Timo Sirainen */ |
2 | |
3 #include "lib.h" | |
4 #include "primes.h" | |
5 #include "mmap-util.h" | |
29
e9375147c0cb
Added write_full() which is a simple wrapper around write() meant for
Timo Sirainen <tss@iki.fi>
parents:
5
diff
changeset
|
6 #include "write-full.h" |
0 | 7 #include "mail-index.h" |
8 #include "mail-index-util.h" | |
9 #include "mail-hash.h" | |
10 | |
11 #include <stdio.h> | |
12 #include <fcntl.h> | |
13 | |
14 /* Minimum record count for a hash file. By default, the hash file size is | |
15 the number of messages * 3, and it's rebuilt after the file is 3/4 full. | |
16 Use only primes as hash file sizes. */ | |
17 #define MIN_HASH_SIZE 109 | |
18 | |
19 /* When rebuilding hash, make it 30% full */ | |
20 #define MIN_PERCENTAGE 30 | |
21 | |
22 /* Try rebuilding hash sometimes soon after it's 60% full */ | |
23 #define REBUILD_PERCENTAGE 60 | |
24 | |
25 /* Force a rebuild when hash is 80% full */ | |
26 #define FORCED_REBUILD_PERCENTAGE 80 | |
27 | |
28 #define HASH_FUNC(uid) (uid * 2) | |
29 | |
30 struct _MailHash { | |
31 MailIndex *index; | |
32 | |
33 unsigned int size; | |
34 | |
35 int fd; | |
36 char *filepath; | |
37 | |
38 void *mmap_base; | |
39 size_t mmap_length; | |
40 | |
41 MailHashHeader *header; | |
42 unsigned int dirty_mmap:1; | |
43 unsigned int modified:1; | |
44 }; | |
45 | |
46 static int mmap_update(MailHash *hash) | |
47 { | |
48 if (!hash->dirty_mmap) | |
49 return TRUE; | |
50 | |
51 if (hash->mmap_base != NULL) | |
52 (void)munmap(hash->mmap_base, hash->mmap_length); | |
53 | |
54 hash->mmap_base = mmap_rw_file(hash->fd, &hash->mmap_length); | |
55 if (hash->mmap_base == MAP_FAILED) { | |
56 hash->mmap_base = NULL; | |
57 hash->header = NULL; | |
58 index_set_error(hash->index, | |
59 "hash: mmap() failed with file %s: %m", | |
60 hash->filepath); | |
61 return FALSE; | |
62 } | |
63 | |
64 if (hash->mmap_length <= sizeof(MailHashHeader) || | |
65 (hash->mmap_length - sizeof(MailHashHeader)) % | |
66 sizeof(MailHashRecord) != 0) { | |
67 /* hash must be corrupted, rebuilding should have noticed | |
68 if it was only partially written. */ | |
69 hash->header = NULL; | |
70 index_set_error(hash->index, "Corrupted hash file %s: %m", | |
71 hash->filepath); | |
72 return FALSE; | |
73 } | |
74 | |
75 hash->dirty_mmap = FALSE; | |
76 hash->header = hash->mmap_base; | |
77 return TRUE; | |
78 } | |
79 | |
80 static MailHash *mail_hash_new(MailIndex *index) | |
81 { | |
82 MailHash *hash; | |
83 | |
84 hash = i_new(MailHash, 1); | |
85 hash->fd = -1; | |
86 hash->index = index; | |
87 hash->filepath = i_strconcat(index->filepath, ".hash", NULL); | |
88 hash->dirty_mmap = TRUE; | |
89 | |
90 index->hash = hash; | |
91 return hash; | |
92 } | |
93 | |
94 int mail_hash_create(MailIndex *index) | |
95 { | |
96 i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); | |
97 | |
98 return mail_hash_rebuild(mail_hash_new(index)); | |
99 } | |
100 | |
101 static int mail_hash_lock_and_rebuild(MailHash *hash) | |
102 { | |
103 if (!hash->index->set_lock(hash->index, MAIL_LOCK_EXCLUSIVE)) | |
104 return FALSE; | |
105 return mail_hash_rebuild(hash); | |
106 } | |
107 | |
108 int mail_hash_open_or_create(MailIndex *index) | |
109 { | |
110 MailHash *hash; | |
111 | |
112 hash = mail_hash_new(index); | |
113 | |
114 hash->fd = open(hash->filepath, O_RDWR); | |
115 if (hash->fd == -1) | |
116 return mail_hash_lock_and_rebuild(hash); | |
117 | |
118 if (!mmap_update(hash)) { | |
119 /* mmap() failure is fatal */ | |
120 mail_hash_free(hash); | |
121 return FALSE; | |
122 } | |
123 | |
124 /* verify that this really is the hash file for wanted index */ | |
125 if (hash->header->indexid != index->indexid) { | |
126 /* mismatch - just recreate it */ | |
127 (void)munmap(hash->mmap_base, hash->mmap_length); | |
128 hash->mmap_base = NULL; | |
129 hash->dirty_mmap = TRUE; | |
130 | |
131 return mail_hash_lock_and_rebuild(hash); | |
132 } | |
133 | |
134 hash->size = (hash->mmap_length - sizeof(MailHashHeader)) / | |
135 sizeof(MailHashRecord); | |
136 return TRUE; | |
137 } | |
138 | |
139 void mail_hash_free(MailHash *hash) | |
140 { | |
141 hash->index->hash = NULL; | |
142 | |
143 if (hash->mmap_base != NULL) { | |
144 (void)munmap(hash->mmap_base, hash->mmap_length); | |
145 hash->mmap_base = NULL; | |
146 } | |
147 | |
148 (void)close(hash->fd); | |
149 i_free(hash->filepath); | |
150 i_free(hash); | |
151 } | |
152 | |
153 static int file_set_size(int fd, off_t size) | |
154 { | |
155 char block[1024]; | |
156 unsigned int i, full_blocks; | |
157 off_t pos; | |
158 | |
159 /* try truncating it to the size we want. if this succeeds, the written | |
160 area is full of zeros - exactly what we want. however, this may not | |
161 work at all, in which case we fallback to write()ing the zeros. */ | |
162 if (ftruncate(fd, size) != -1 && lseek(fd, 0, SEEK_END) == size) | |
163 return lseek(fd, 0, SEEK_SET) == 0; | |
164 | |
165 /* skip the existing data in file */ | |
166 pos = lseek(fd, 0, SEEK_END); | |
5
1b34ec11fff8
Message data is parsed in blocks (no longer entirely mmap()ed). Several
Timo Sirainen <tss@iki.fi>
parents:
0
diff
changeset
|
167 if (pos == -1) |
0 | 168 return FALSE; |
169 size -= pos; | |
170 | |
171 memset(block, 0, sizeof(block)); | |
172 | |
173 /* write in 1kb blocks */ | |
174 full_blocks = size / sizeof(block); | |
175 for (i = 0; i < full_blocks; i++) { | |
29
e9375147c0cb
Added write_full() which is a simple wrapper around write() meant for
Timo Sirainen <tss@iki.fi>
parents:
5
diff
changeset
|
176 if (write_full(fd, block, sizeof(block)) < 0) |
0 | 177 return FALSE; |
178 } | |
179 | |
180 /* write the remainder */ | |
181 i = size % sizeof(block); | |
29
e9375147c0cb
Added write_full() which is a simple wrapper around write() meant for
Timo Sirainen <tss@iki.fi>
parents:
5
diff
changeset
|
182 return i == 0 ? TRUE : write_full(fd, block, i) == 0; |
0 | 183 } |
184 | |
185 static int hash_rebuild_to_file(MailIndex *index, int fd, | |
186 unsigned int hash_size, | |
187 unsigned int messages_count) | |
188 { | |
189 MailHashHeader *hdr; | |
190 MailHashRecord *rec; | |
191 MailIndexRecord *idx_rec; | |
192 void *mmap_base; | |
193 unsigned int i, count; | |
194 size_t mmap_length; | |
195 size_t new_size; | |
196 | |
197 /* fill the file with zeros */ | |
198 new_size = sizeof(MailHashHeader) + hash_size * sizeof(MailHashRecord); | |
199 if (!file_set_size(fd, (off_t) new_size)) { | |
200 index_set_error(index, | |
201 "Failed to fill temp hash to size %lu: %m", | |
202 (unsigned long) new_size); | |
203 return FALSE; | |
204 } | |
205 | |
206 /* now, mmap() it */ | |
207 mmap_base = mmap_rw_file(fd, &mmap_length); | |
208 if (mmap_base == MAP_FAILED) { | |
209 index_set_error(index, "mmap()ing temp hash failed: %m"); | |
210 return FALSE; | |
211 } | |
212 | |
213 i_assert(mmap_length == new_size); | |
214 | |
215 /* we have empty hash file mmap()ed now. fill it by reading the | |
216 messages from index. */ | |
217 rec = (MailHashRecord *) ((char *) mmap_base + sizeof(MailHashHeader)); | |
218 idx_rec = index->lookup(index, 1); | |
219 for (count = 0; idx_rec != NULL; count++) { | |
220 i = HASH_FUNC(idx_rec->uid) % hash_size; | |
221 rec[i].uid = idx_rec->uid; | |
222 rec[i].position = INDEX_FILE_POSITION(index, idx_rec); | |
223 idx_rec = index->next(index, idx_rec); | |
224 } | |
225 | |
226 if (count != messages_count) { | |
227 /* mark this as an error but don't fail because of it. */ | |
228 INDEX_MARK_CORRUPTED(index); | |
229 index_set_error(index, "Missing messages while rebuilding " | |
230 "hash file %s - %u found, header says %u", | |
231 index->filepath, count, messages_count); | |
232 } | |
233 | |
234 /* setup header */ | |
235 hdr = mmap_base; | |
236 hdr->indexid = index->indexid; | |
237 hdr->used_records = count; | |
238 | |
239 return munmap(mmap_base, mmap_length) == 0; | |
240 } | |
241 | |
242 int mail_hash_sync_file(MailHash *hash) | |
243 { | |
244 if (!hash->modified) | |
245 return TRUE; | |
246 hash->modified = FALSE; | |
247 | |
248 if (msync(hash->mmap_base, hash->mmap_length, MS_SYNC) == 0) | |
249 return TRUE; | |
250 else { | |
251 index_set_error(hash->index, "msync() failed for %s: %m", | |
252 hash->filepath); | |
253 return FALSE; | |
254 } | |
255 } | |
256 | |
257 int mail_hash_rebuild(MailHash *hash) | |
258 { | |
259 MailIndexHeader *index_header; | |
260 const char *path; | |
261 unsigned int hash_size; | |
262 int fd; | |
263 | |
264 i_assert(hash->index->lock_type == MAIL_LOCK_EXCLUSIVE); | |
265 | |
266 /* first get the number of messages in index */ | |
267 index_header = hash->index->get_header(hash->index); | |
268 | |
269 /* then figure out size for our hash */ | |
270 hash_size = primes_closest(index_header->messages_count * 100 / MIN_PERCENTAGE); | |
271 if (hash_size < MIN_HASH_SIZE) | |
272 hash_size = MIN_HASH_SIZE; | |
273 | |
274 /* create the hash in a new temp file */ | |
275 fd = mail_index_create_temp_file(hash->index, &path); | |
276 if (fd == -1) | |
277 return FALSE; | |
278 | |
279 if (!hash_rebuild_to_file(hash->index, fd, hash_size, | |
280 index_header->messages_count)) { | |
281 (void)close(fd); | |
282 (void)unlink(path); | |
283 return FALSE; | |
284 } | |
285 | |
286 if (fsync(fd) == -1) { | |
287 index_set_error(hash->index, | |
288 "fsync() failed with temp hash %s: %m", path); | |
289 (void)close(fd); | |
290 (void)unlink(path); | |
291 return FALSE; | |
292 } | |
293 | |
294 /* replace old hash file with this new one */ | |
295 if (rename(path, hash->filepath) == -1) { | |
296 index_set_error(hash->index, "rename(%s, %s) failed: %m", | |
297 path, hash->filepath); | |
298 | |
299 (void)close(fd); | |
300 (void)unlink(path); | |
301 return FALSE; | |
302 } | |
303 | |
304 /* switch fds */ | |
305 if (hash->fd != -1) | |
306 (void)close(hash->fd); | |
307 hash->size = hash_size; | |
308 hash->fd = fd; | |
309 hash->dirty_mmap = TRUE; | |
310 return TRUE; | |
311 } | |
312 | |
313 off_t mail_hash_lookup_uid(MailHash *hash, unsigned int uid) | |
314 { | |
315 MailHashRecord *rec; | |
316 unsigned int hashidx, idx; | |
317 | |
318 i_assert(uid > 0); | |
319 i_assert(hash->index->lock_type != MAIL_LOCK_UNLOCK); | |
320 | |
321 if (hash->fd == -1 || !mmap_update(hash)) | |
322 return 0; | |
323 | |
324 /* our hashing function is simple - UID*2. The *2 is there because | |
325 UIDs are normally contiguous, so once the UIDs wrap around, we | |
326 don't want to go through lots of records just to find an empty | |
327 spot */ | |
328 hashidx = HASH_FUNC(uid) % hash->size; | |
329 rec = (MailHashRecord *) ((char *) hash->mmap_base + | |
330 sizeof(MailHashHeader)); | |
331 | |
332 /* check from hash index to end of file */ | |
333 for (idx = hashidx; idx < hash->size; idx++) { | |
334 if (rec[idx].uid == uid) | |
335 return rec[idx].position; | |
336 | |
337 if (rec[idx].uid == 0) { | |
338 /* empty hash record - not found. */ | |
339 return 0; | |
340 } | |
341 } | |
342 | |
343 /* check from beginning of file to hash index */ | |
344 for (idx = 0; idx < hashidx; idx++) { | |
345 if (rec[idx].uid == uid) | |
346 return rec[idx].position; | |
347 | |
348 if (rec[idx].uid == 0) { | |
349 /* empty hash record - not found. */ | |
350 return 0; | |
351 } | |
352 } | |
353 | |
354 /* checked through the whole hash file. this really shouldn't happen, | |
355 we should have rebuilt it long time ago.. */ | |
356 return 0; | |
357 } | |
358 | |
359 static MailHashRecord *hash_find_uid_or_free(MailHash *hash, unsigned int uid) | |
360 { | |
361 MailHashRecord *rec; | |
362 unsigned int hashidx, idx; | |
363 | |
364 hashidx = HASH_FUNC(uid) % hash->size; | |
365 rec = (MailHashRecord *) ((char *) hash->mmap_base + | |
366 sizeof(MailHashHeader)); | |
367 | |
368 /* check from hash index to end of file */ | |
369 for (idx = hashidx; idx < hash->size; idx++) { | |
370 if (rec[idx].uid == 0 || rec[idx].uid == uid) | |
371 return rec+idx; | |
372 } | |
373 | |
374 /* check from beginning of file to hash index */ | |
375 for (idx = 0; idx < hashidx; idx++) { | |
376 if (rec[idx].uid == 0 || rec[idx].uid == uid) | |
377 return rec+idx; | |
378 } | |
379 | |
380 /* hash file is full */ | |
381 return NULL; | |
382 } | |
383 | |
384 void mail_hash_update(MailHash *hash, unsigned int uid, off_t pos) | |
385 { | |
386 MailHashRecord *rec; | |
387 unsigned int max_used; | |
388 | |
389 i_assert(uid > 0); | |
390 i_assert(hash->index->lock_type == MAIL_LOCK_EXCLUSIVE); | |
391 | |
392 if (hash->fd == -1 || !mmap_update(hash)) | |
393 return; | |
394 | |
395 if (hash->header->used_records > | |
396 hash->size * FORCED_REBUILD_PERCENTAGE / 100) { | |
397 /* we really need a rebuild. */ | |
398 mail_hash_rebuild(hash); | |
399 } | |
400 | |
401 /* place the hash into first free record after wanted position */ | |
402 rec = hash_find_uid_or_free(hash, uid); | |
403 | |
404 if (rec == NULL) { | |
405 /* this should never happen, we had already checked that | |
406 at least 1/5 of hash was empty. except, if the header | |
407 contained invalid record count for some reason. rebuild.. */ | |
408 mail_hash_rebuild(hash); | |
409 | |
410 rec = hash_find_uid_or_free(hash, uid); | |
411 i_assert(rec != NULL); | |
412 } | |
413 | |
414 if (pos != 0) { | |
415 /* insert/update record */ | |
416 if (rec->uid == 0) { | |
417 /* update records count, and see if hash is | |
418 getting full */ | |
419 max_used = hash->size / 100 * REBUILD_PERCENTAGE; | |
420 if (++hash->header->used_records > max_used) { | |
421 hash->index->set_flags |= | |
422 MAIL_INDEX_FLAG_REBUILD_HASH; | |
423 } | |
424 } | |
425 rec->uid = uid; | |
426 rec->position = pos; | |
427 } else { | |
428 /* delete record */ | |
429 rec->uid = 0; | |
430 rec->position = 0; | |
431 hash->header->used_records--; | |
432 } | |
433 | |
434 hash->modified = FALSE; | |
435 } |