Mercurial > dovecot > original-hg > dovecot-1.2

comparison src/lib-storage/index/maildir/maildir-sync.c @ 1915:79790750c349 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
importing new index code. mbox still broken.
author Timo Sirainen <tss@iki.fi>
date Tue, 27 Apr 2004 23:25:52 +0300
parents
children 68938dccbc45
comparison
equal deleted inserted replaced
1914:1f156d653f4a 1915:79790750c349
1 /*
2 1. read files in maildir
3 2. see if they're all found in uidlist read in memory
4 3. if not, check if uidlist's mtime has changed and read it if so
5 4. if still not, lock uidlist, sync it once more and generate UIDs for new
6 files
7 5. apply changes in transaction log
8 6. apply changes in maildir to index
9 */
10
11 /* Copyright (C) 2004 Timo Sirainen */
12
13 #include "lib.h"
14 #include "ioloop.h"
15 #include "buffer.h"
16 #include "hash.h"
17 #include "str.h"
18 #include "maildir-storage.h"
19 #include "maildir-uidlist.h"
20
21 #include <stdio.h>
22 #include <unistd.h>
23 #include <dirent.h>
24 #include <sys/stat.h>
25
26 #define MAILDIR_SYNC_SECS 1
27
28 #define MAILDIR_FILENAME_FLAG_FOUND 128
29
30 struct maildir_sync_context {
31 struct index_mailbox *ibox;
32 const char *new_dir, *cur_dir;
33
34 struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
35 };
36
37 static int maildir_expunge(struct index_mailbox *ibox, const char *path,
38 void *context __attr_unused__)
39 {
40 if (unlink(path) == 0)
41 return 1;
42 if (errno == ENOENT)
43 return 0;
44
45 mail_storage_set_critical(ibox->box.storage,
46 "unlink(%s) failed: %m", path);
47 return -1;
48 }
49
50 static int maildir_sync_flags(struct index_mailbox *ibox, const char *path,
51 void *context)
52 {
53 struct mail_index_sync_rec *syncrec = context;
54 const char *newpath;
55 enum mail_flags flags;
56 uint8_t flags8;
57 custom_flags_mask_t custom_flags;
58
59 (void)maildir_filename_get_flags(path, &flags, custom_flags);
60
61 flags8 = flags;
62 mail_index_sync_flags_apply(syncrec, &flags8, custom_flags);
63
64 newpath = maildir_filename_set_flags(path, flags8, custom_flags);
65 if (rename(path, newpath) == 0)
66 return 1;
67 if (errno == ENOENT)
68 return 0;
69
70 mail_storage_set_critical(ibox->box.storage,
71 "rename(%s, %s) failed: %m", path, newpath);
72 return -1;
73 }
74
75 static int maildir_sync_record(struct index_mailbox *ibox,
76 struct mail_index_view *view,
77 struct mail_index_sync_rec *syncrec)
78 {
79 const struct mail_index_record *rec;
80 uint32_t seq, uid;
81
82 switch (syncrec->type) {
83 case MAIL_INDEX_SYNC_TYPE_APPEND:
84 break;
85 case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
86 for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) {
87 if (mail_index_lookup_uid(view, seq, &uid) < 0)
88 return -1;
89 if (maildir_file_do(ibox, uid, maildir_expunge,
90 NULL) < 0)
91 return -1;
92 }
93 break;
94 case MAIL_INDEX_SYNC_TYPE_FLAGS:
95 for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) {
96 if (mail_index_lookup_uid(view, seq, &uid) < 0)
97 return -1;
98 if (maildir_file_do(ibox, uid, maildir_sync_flags,
99 syncrec) < 0)
100 return -1;
101 }
102 break;
103 }
104
105 return 0;
106 }
107
108 int maildir_sync_last_commit(struct index_mailbox *ibox)
109 {
110 struct mail_index_view *view;
111 struct mail_index_sync_ctx *sync_ctx;
112 struct mail_index_sync_rec sync_rec;
113 int ret;
114
115 if (ibox->commit_log_file_seq == 0)
116 return 0;
117
118 ret = mail_index_sync_begin(ibox->index, &sync_ctx, &view,
119 ibox->commit_log_file_seq,
120 ibox->commit_log_file_offset);
121 if (ret > 0) {
122 while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) {
123 if (maildir_sync_record(ibox, view, &sync_rec) < 0) {
124 ret = -1;
125 break;
126 }
127 }
128 if (mail_index_sync_end(sync_ctx) < 0)
129 ret = -1;
130 }
131
132 if (ret == 0) {
133 ibox->commit_log_file_seq = 0;
134 ibox->commit_log_file_offset = 0;
135 } else {
136 // FIXME: this is bad - we have to fix this in some way
137 mail_storage_set_index_error(ibox);
138 }
139 return ret;
140 }
141
142 static struct maildir_sync_context *
143 maildir_sync_context_new(struct index_mailbox *ibox)
144 {
145 struct maildir_sync_context *ctx;
146
147 ctx = t_new(struct maildir_sync_context, 1);
148 ctx->ibox = ibox;
149 ctx->new_dir = t_strconcat(ibox->path, "/new", NULL);
150 ctx->cur_dir = t_strconcat(ibox->path, "/cur", NULL);
151 return ctx;
152 }
153
154 static void maildir_sync_deinit(struct maildir_sync_context *ctx)
155 {
156 if (ctx->uidlist_sync_ctx != NULL)
157 (void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
158 }
159
160 static int maildir_fix_duplicate(struct index_mailbox *ibox, const char *dir,
161 const char *old_fname)
162 {
163 const char *new_fname, *old_path, *new_path;
164 int ret = 0;
165
166 t_push();
167
168 old_path = t_strconcat(dir, "/", old_fname, NULL);
169 new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
170 new_path = t_strconcat(ibox->path, "/new/", new_fname, NULL);
171
172 if (rename(old_path, new_path) == 0) {
173 i_warning("Fixed duplicate in %s: %s -> %s",
174 ibox->path, old_fname, new_fname);
175 } else if (errno != ENOENT) {
176 mail_storage_set_critical(ibox->box.storage,
177 "rename(%s, %s) failed: %m", old_path, new_path);
178 ret = -1;
179 }
180 t_pop();
181
182 return ret;
183 }
184
185 static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
186 {
187 struct mail_storage *storage = ctx->ibox->box.storage;
188 const char *dir;
189 DIR *dirp;
190 string_t *src, *dest;
191 struct dirent *dp;
192 int move_new, this_new, ret = 1;
193
194 src = t_str_new(1024);
195 dest = t_str_new(1024);
196
197 dir = new_dir ? ctx->new_dir : ctx->cur_dir;
198 dirp = opendir(dir);
199 if (dirp == NULL) {
200 mail_storage_set_critical(storage,
201 "opendir(%s) failed: %m", dir);
202 return -1;
203 }
204
205 move_new = new_dir;
206 while ((dp = readdir(dirp)) != NULL) {
207 if (dp->d_name[0] == '.')
208 continue;
209
210 this_new = new_dir;
211 if (move_new) {
212 str_truncate(src, 0);
213 str_truncate(dest, 0);
214 str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
215 str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
216 if (rename(str_c(src), str_c(dest)) == 0 ||
217 ENOTFOUND(errno)) {
218 /* moved - we'll look at it later in cur/ dir */
219 this_new = FALSE;
220 continue;
221 } else if (ENOSPACE(errno)) {
222 /* not enough disk space, leave here */
223 move_new = FALSE;
224 } else {
225 mail_storage_set_critical(storage,
226 "rename(%s, %s) failed: %m",
227 str_c(src), str_c(dest));
228 }
229 }
230
231 ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
232 dp->d_name, this_new);
233 if (ret <= 0) {
234 if (ret < 0)
235 break;
236
237 /* possibly duplicate - try fixing it */
238 if (maildir_fix_duplicate(ctx->ibox,
239 dir, dp->d_name) < 0) {
240 ret = -1;
241 break;
242 }
243 }
244 }
245
246 if (closedir(dirp) < 0) {
247 mail_storage_set_critical(storage,
248 "closedir(%s) failed: %m", dir);
249 }
250 return ret < 0 ? -1 : 0;
251 }
252
253 static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
254 int *new_changed_r, int *cur_changed_r)
255 {
256 struct index_mailbox *ibox = ctx->ibox;
257 struct stat st;
258 time_t new_mtime, cur_mtime;
259
260 *new_changed_r = *cur_changed_r = FALSE;
261
262 if (stat(ctx->new_dir, &st) < 0) {
263 mail_storage_set_critical(ibox->box.storage,
264 "stat(%s) failed: %m", ctx->new_dir);
265 return -1;
266 }
267 new_mtime = st.st_mtime;
268
269 if (stat(ctx->cur_dir, &st) < 0) {
270 mail_storage_set_critical(ibox->box.storage,
271 "stat(%s) failed: %m", ctx->cur_dir);
272 return -1;
273 }
274 cur_mtime = st.st_mtime;
275
276 if (new_mtime != ibox->last_new_mtime ||
277 new_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS) {
278 *new_changed_r = TRUE;
279 ibox->last_new_mtime = new_mtime;
280 }
281 if (cur_mtime != ibox->last_cur_mtime ||
282 (cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS &&
283 ioloop_time - ibox->last_sync > MAILDIR_SYNC_SECS)) {
284 /* cur/ changed, or delayed cur/ check */
285 *cur_changed_r = TRUE;
286 ibox->last_cur_mtime = cur_mtime;
287 }
288 ibox->last_sync = ioloop_time;
289
290 return 0;
291 }
292
293 static int maildir_sync_index(struct maildir_sync_context *ctx)
294 {
295 struct index_mailbox *ibox = ctx->ibox;
296 struct mail_index_sync_ctx *sync_ctx;
297 struct mail_index_sync_rec sync_rec;
298 struct maildir_uidlist_iter_ctx *iter;
299 struct mail_index_transaction *trans;
300 struct mail_index_view *view;
301 const struct mail_index_header *hdr;
302 const struct mail_index_record *rec;
303 uint32_t seq, uid, uflags;
304 const char *filename;
305 enum mail_flags flags;
306 custom_flags_mask_t custom_flags;
307 int ret = 0;
308
309 if (mail_index_sync_begin(ibox->index, &sync_ctx, &view,
310 (uint32_t)-1, (uoff_t)-1) <= 0) {
311 // FIXME: ?
312 return -1;
313 }
314
315 hdr = mail_index_get_header(view);
316 trans = mail_index_transaction_begin(view, FALSE);
317
318 seq = 0;
319 iter = maildir_uidlist_iter_init(ibox->uidlist);
320 while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
321 maildir_filename_get_flags(filename, &flags, custom_flags);
322
323 __again:
324 seq++;
325 if (seq > hdr->messages_count) {
326 mail_index_append(trans, uid, &seq);
327 mail_index_update_flags(trans, seq, MODIFY_REPLACE,
328 flags, custom_flags);
329 continue;
330 }
331
332 if (mail_index_lookup(view, seq, &rec) < 0) {
333 mail_storage_set_index_error(ibox);
334 ret = -1;
335 break;
336 }
337
338 if (rec->uid < uid) {
339 /* expunged */
340 mail_index_expunge(trans, seq);
341 goto __again;
342 }
343
344 if (rec->uid > uid) {
345 /* new UID in the middle of the mailbox -
346 shouldn't happen */
347 mail_storage_set_critical(ibox->box.storage,
348 "Maildir sync: UID inserted in the middle "
349 "of mailbox (%u > %u)", rec->uid, uid);
350 (void)mail_index_reset(ibox->index);
351 ret = -1;
352 break;
353 }
354
355 maildir_filename_get_flags(filename, &flags, custom_flags);
356 if (rec->flags & MAIL_RECENT)
357 flags |= MAIL_RECENT;
358 if ((uint8_t)flags != (rec->flags & MAIL_FLAGS_MASK) ||
359 memcmp(custom_flags, rec->custom_flags,
360 INDEX_CUSTOM_FLAGS_BYTE_COUNT) != 0) {
361 mail_index_update_flags(trans, seq, MODIFY_REPLACE,
362 flags, custom_flags);
363 }
364 }
365 maildir_uidlist_iter_deinit(iter);
366
367 if (ret < 0)
368 mail_index_transaction_rollback(trans);
369 else {
370 uint32_t seq;
371 uoff_t offset;
372
373 if (mail_index_transaction_commit(trans, &seq, &offset) < 0)
374 mail_storage_set_index_error(ibox);
375 else {
376 ibox->commit_log_file_seq = seq;
377 ibox->commit_log_file_offset = offset;
378 }
379 }
380
381 /* now, sync the index */
382 while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) {
383 if (maildir_sync_record(ibox, view, &sync_rec) < 0) {
384 ret = -1;
385 break;
386 }
387 }
388 if (mail_index_sync_end(sync_ctx) < 0)
389 ret = -1;
390
391 if (ret == 0) {
392 ibox->commit_log_file_seq = 0;
393 ibox->commit_log_file_offset = 0;
394 } else {
395 // FIXME: this is bad - we have to fix this in some way
396 mail_storage_set_index_error(ibox);
397 }
398
399 return ret;
400 }
401
402 static int maildir_sync_context(struct maildir_sync_context *ctx,
403 int *changes_r)
404 {
405 int ret, new_changed, cur_changed;
406
407 if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0)
408 return -1;
409
410 ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
411
412 if (maildir_scan_dir(ctx, TRUE) < 0)
413 return -1;
414 if (maildir_scan_dir(ctx, FALSE) < 0)
415 return -1;
416
417 ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
418 ctx->uidlist_sync_ctx = NULL;
419
420 if (ret == 0)
421 ret = maildir_sync_index(ctx);
422 return ret;
423 }
424
425 static int maildir_sync_context_readonly(struct maildir_sync_context *ctx)
426 {
427 int ret;
428
429 ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
430
431 if (maildir_scan_dir(ctx, TRUE) < 0)
432 return -1;
433 if (maildir_scan_dir(ctx, FALSE) < 0)
434 return -1;
435
436 ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
437 ctx->uidlist_sync_ctx = NULL;
438
439 return ret;
440 }
441
442 int maildir_storage_sync_readonly(struct index_mailbox *ibox)
443 {
444 struct maildir_sync_context *ctx;
445 int ret;
446
447 ctx = maildir_sync_context_new(ibox);
448 ret = maildir_sync_context_readonly(ctx);
449 maildir_sync_deinit(ctx);
450 return ret;
451 }
452
453 int maildir_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
454 {
455 struct index_mailbox *ibox = (struct index_mailbox *)box;
456 struct maildir_sync_context *ctx;
457 int changes, ret;
458
459 if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
460 ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
461 ibox->sync_last_check = ioloop_time;
462
463 ctx = maildir_sync_context_new(ibox);
464 ret = maildir_sync_context(ctx, &changes);
465 maildir_sync_deinit(ctx);
466
467 if (ret < 0)
468 return -1;
469 }
470
471 return index_storage_sync(box, flags);
472 }