Mercurial > dovecot > original-hg > dovecot-1.2

comparison src/lib-storage/index/maildir/maildir-sync.c @ 5920:00c5e3cbeaf0 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Moved index syncing code to its own file.
author Timo Sirainen <tss@iki.fi>
date Sun, 08 Jul 2007 23:28:22 +0300
parents 126b74419f52
children 095b3adc537b
comparison
equal deleted inserted replaced
5919:cc439e4e99cb 5920:00c5e3cbeaf0
175 #include "array.h" 175 #include "array.h"
176 #include "buffer.h" 176 #include "buffer.h"
177 #include "hash.h" 177 #include "hash.h"
178 #include "str.h" 178 #include "str.h"
179 #include "maildir-storage.h" 179 #include "maildir-storage.h"
180 #include "index-sync-changes.h"
181 #include "maildir-uidlist.h" 180 #include "maildir-uidlist.h"
182 #include "maildir-keywords.h"
183 #include "maildir-filename.h" 181 #include "maildir-filename.h"
184 #include "maildir-sync.h" 182 #include "maildir-sync.h"
185 183
186 #include <stdio.h> 184 #include <stdio.h>
187 #include <stddef.h> 185 #include <stddef.h>
198 left. This value is just an optimization to avoid checking the directory 196 left. This value is just an optimization to avoid checking the directory
199 twice unneededly. usually only NFS is the problem case. 1 is the safest 197 twice unneededly. usually only NFS is the problem case. 1 is the safest
200 bet here, but I guess 5 will do just fine too. */ 198 bet here, but I guess 5 will do just fine too. */
201 #define MAILDIR_RENAME_RESCAN_COUNT 5 199 #define MAILDIR_RENAME_RESCAN_COUNT 5
202 200
203 /* After moving 100 mails from new/ to cur/, check if we need to touch the
204 uidlist lock. */
205 #define MAILDIR_SLOW_MOVE_COUNT 100
206 /* readdir() should be pretty fast to do, but check anyway every 10000 mails
207 to see if we need to touch the uidlist lock. */
208 #define MAILDIR_SLOW_CHECK_COUNT 10000
209
210 /* This is mostly to avoid infinite looping when rename() destination already 201 /* This is mostly to avoid infinite looping when rename() destination already
211 exists as the hard link of the file itself. */ 202 exists as the hard link of the file itself. */
212 #define MAILDIR_SCAN_DIR_MAX_COUNT 5 203 #define MAILDIR_SCAN_DIR_MAX_COUNT 5
213 204
214 #define DUPE_LINKS_DELETE_SECS 30 205 #define DUPE_LINKS_DELETE_SECS 30
222 213
223 struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; 214 struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
224 struct maildir_index_sync_context *index_sync_ctx; 215 struct maildir_index_sync_context *index_sync_ctx;
225 }; 216 };
226 217
227 struct maildir_index_sync_context { 218 void maildir_sync_notify(struct maildir_sync_context *ctx)
228 struct maildir_mailbox *mbox;
229 struct maildir_sync_context *maildir_sync_ctx;
230
231 struct mail_index_view *view;
232 struct mail_index_sync_ctx *sync_ctx;
233 struct maildir_keywords_sync_ctx *keywords_sync_ctx;
234 struct mail_index_transaction *trans;
235
236 struct index_sync_changes_context *sync_changes;
237 enum mail_flags flags;
238 ARRAY_TYPE(keyword_indexes) keywords;
239
240 uint32_t seq, uid;
241
242 bool changed;
243 };
244
245 struct maildir_keywords_sync_ctx *
246 maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx)
247 {
248 return ctx->keywords_sync_ctx;
249 }
250
251 static int maildir_expunge(struct maildir_mailbox *mbox, const char *path,
252 struct maildir_index_sync_context *ctx)
253 {
254 struct mailbox *box = &mbox->ibox.box;
255
256 if (unlink(path) == 0) {
257 if (box->v.sync_notify != NULL) {
258 box->v.sync_notify(box, ctx->uid,
259 MAILBOX_SYNC_TYPE_EXPUNGE);
260 }
261 mail_index_expunge(ctx->trans, ctx->seq);
262 ctx->changed = TRUE;
263 return 1;
264 }
265 if (errno == ENOENT)
266 return 0;
267
268 mail_storage_set_critical(&mbox->storage->storage,
269 "unlink(%s) failed: %m", path);
270 return -1;
271 }
272
273 static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path,
274 struct maildir_index_sync_context *ctx)
275 {
276 struct mailbox *box = &mbox->ibox.box;
277 const char *dir, *fname, *newfname, *newpath;
278 enum mailbox_sync_type sync_type = 0;
279 uint8_t flags8;
280
281 fname = strrchr(path, '/');
282 i_assert(fname != NULL);
283 fname++;
284 dir = t_strdup_until(path, fname);
285
286 /* get the current flags and keywords */
287 maildir_filename_get_flags(ctx->keywords_sync_ctx,
288 fname, &ctx->flags, &ctx->keywords);
289
290 /* apply changes */
291 flags8 = ctx->flags;
292 index_sync_changes_apply(ctx->sync_changes, NULL,
293 &flags8, &ctx->keywords, &sync_type);
294 ctx->flags = flags8;
295
296 /* and try renaming with the new name */
297 newfname = maildir_filename_set_flags(ctx->keywords_sync_ctx, fname,
298 ctx->flags, &ctx->keywords);
299 newpath = t_strconcat(dir, newfname, NULL);
300 if (rename(path, newpath) == 0) {
301 if (box->v.sync_notify != NULL)
302 box->v.sync_notify(box, ctx->uid, sync_type);
303
304 ctx->changed = TRUE;
305 return 1;
306 }
307 if (errno == ENOENT)
308 return 0;
309
310 if (!ENOSPACE(errno) && errno != EACCES) {
311 mail_storage_set_critical(&mbox->storage->storage,
312 "rename(%s, %s) failed: %m", path, newpath);
313 }
314 return -1;
315 }
316
317 static void maildir_sync_notify(struct maildir_sync_context *ctx)
318 { 219 {
319 time_t now; 220 time_t now;
320 221
321 if (ctx == NULL) { 222 if (ctx == NULL) {
322 /* we got here from maildir-save.c. it has no 223 /* we got here from maildir-save.c. it has no
661 } 562 }
662 563
663 return 0; 564 return 0;
664 } 565 }
665 566
666 int maildir_sync_index_begin(struct maildir_mailbox *mbox,
667 struct maildir_index_sync_context **ctx_r)
668 {
669 struct maildir_index_sync_context *ctx;
670 struct mail_index_sync_ctx *sync_ctx;
671 struct mail_index_view *view;
672 struct mail_index_transaction *trans;
673
674 if (mail_index_sync_begin(mbox->ibox.index, &sync_ctx, &view, &trans,
675 (uint32_t)-1, (uoff_t)-1, 0) <= 0) {
676 mail_storage_set_index_error(&mbox->ibox);
677 return -1;
678 }
679
680 ctx = i_new(struct maildir_index_sync_context, 1);
681 ctx->mbox = mbox;
682 ctx->sync_ctx = sync_ctx;
683 ctx->view = view;
684 ctx->trans = trans;
685 ctx->keywords_sync_ctx =
686 maildir_keywords_sync_init(mbox->keywords, mbox->ibox.index);
687
688 ctx->sync_changes = index_sync_changes_init(&mbox->ibox, ctx->sync_ctx,
689 ctx->view, ctx->trans,
690 mbox->ibox.readonly);
691
692 *ctx_r = ctx;
693 return 0;
694 }
695
696 int maildir_sync_index_finish(struct maildir_index_sync_context **_ctx,
697 bool failed, bool cancel)
698 {
699 struct maildir_index_sync_context *ctx = *_ctx;
700 struct maildir_mailbox *mbox = ctx->mbox;
701 int ret = failed ? -1 : 0;
702
703 *_ctx = NULL;
704
705 if (ret < 0 || cancel)
706 mail_index_sync_rollback(&ctx->sync_ctx);
707 else {
708 /* Set syncing_commit=TRUE so that if any sync callbacks try
709 to access mails which got lost (eg. expunge callback trying
710 to open the file which was just unlinked) we don't try to
711 start a second index sync and crash. */
712 mbox->syncing_commit = TRUE;
713 if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
714 mail_storage_set_index_error(&mbox->ibox);
715 ret = -1;
716 } else {
717 mbox->ibox.commit_log_file_seq = 0;
718 mbox->ibox.commit_log_file_offset = 0;
719 }
720 mbox->syncing_commit = FALSE;
721 }
722
723 maildir_keywords_sync_deinit(ctx->keywords_sync_ctx);
724 ctx->keywords_sync_ctx = NULL;
725
726 index_sync_changes_deinit(&ctx->sync_changes);
727 i_free(ctx);
728 return ret;
729 }
730
731 int maildir_sync_index(struct maildir_index_sync_context *ctx,
732 bool partial)
733 {
734 struct maildir_mailbox *mbox = ctx->mbox;
735 struct mail_index_view *view = ctx->view;
736 struct maildir_uidlist_iter_ctx *iter;
737 struct mail_index_transaction *trans = ctx->trans;
738 const struct mail_index_header *hdr;
739 struct mail_index_header empty_hdr;
740 const struct mail_index_record *rec;
741 uint32_t seq, uid, prev_uid;
742 enum maildir_uidlist_rec_flag uflags;
743 const char *filename;
744 ARRAY_TYPE(keyword_indexes) idx_keywords;
745 uint32_t uid_validity, next_uid;
746 uint64_t value;
747 unsigned int changes = 0;
748 int ret = 0;
749 bool expunged, full_rescan = FALSE;
750
751 i_assert(!mbox->syncing_commit);
752 i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist));
753
754 hdr = mail_index_get_header(view);
755 uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
756 if (uid_validity != hdr->uid_validity &&
757 uid_validity != 0 && hdr->uid_validity != 0) {
758 /* uidvalidity changed and mailbox isn't being initialized,
759 reset mailbox so we can add all messages as new */
760 i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)",
761 mbox->path, hdr->uid_validity, uid_validity);
762 mail_index_reset(trans);
763
764 memset(&empty_hdr, 0, sizeof(empty_hdr));
765 empty_hdr.next_uid = 1;
766 hdr = &empty_hdr;
767 }
768
769 mbox->syncing_commit = TRUE;
770 seq = prev_uid = 0;
771 t_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS);
772 t_array_init(&idx_keywords, MAILDIR_MAX_KEYWORDS);
773 iter = maildir_uidlist_iter_init(mbox->uidlist);
774 while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
775 maildir_filename_get_flags(ctx->keywords_sync_ctx, filename,
776 &ctx->flags, &ctx->keywords);
777
778 i_assert(uid > prev_uid);
779 prev_uid = uid;
780
781 /* the private flags are kept only in indexes. don't use them
782 at all even for newly seen mails */
783 ctx->flags &= ~mbox->private_flags_mask;
784
785 if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 &&
786 (uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 &&
787 (uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) {
788 /* mail is recent for next session as well */
789 ctx->flags |= MAIL_RECENT;
790 }
791
792 __again:
793 ctx->seq = ++seq;
794 ctx->uid = uid;
795
796 if (seq > hdr->messages_count) {
797 if (uid < hdr->next_uid) {
798 /* most likely a race condition: we read the
799 maildir, then someone else expunged messages
800 and committed changes to index. so, this
801 message shouldn't actually exist. mark it
802 racy and check in next sync.
803
804 the difference between this and the later
805 check is that this one happens when messages
806 are expunged from the end */
807 if ((uflags &
808 MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
809 /* partial syncing */
810 continue;
811 }
812 if ((uflags &
813 MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {
814 mail_storage_set_critical(
815 &mbox->storage->storage,
816 "Maildir %s sync: "
817 "UID < next_uid "
818 "(%u < %u, file = %s)",
819 mbox->path, uid, hdr->next_uid,
820 filename);
821 mail_index_mark_corrupted(
822 mbox->ibox.index);
823 ret = -1;
824 break;
825 }
826 mbox->dirty_cur_time = ioloop_time;
827 maildir_uidlist_add_flags(mbox->uidlist,
828 filename,
829 MAILDIR_UIDLIST_REC_FLAG_RACING);
830
831 seq--;
832 continue;
833 }
834
835 mail_index_append(trans, uid, &seq);
836 mail_index_update_flags(trans, seq, MODIFY_REPLACE,
837 ctx->flags);
838
839 if (array_count(&ctx->keywords) > 0) {
840 struct mail_keywords *kw;
841
842 kw = mail_index_keywords_create_from_indexes(
843 trans, &ctx->keywords);
844 mail_index_update_keywords(trans, seq,
845 MODIFY_REPLACE, kw);
846 mail_index_keywords_free(&kw);
847 }
848 continue;
849 }
850
851 if (mail_index_lookup(view, seq, &rec) < 0) {
852 mail_storage_set_index_error(&mbox->ibox);
853 ret = -1;
854 break;
855 }
856
857 if (rec->uid < uid) {
858 /* expunged */
859 mail_index_expunge(trans, seq);
860 goto __again;
861 }
862
863 if (rec->uid > uid) {
864 /* most likely a race condition: we read the
865 maildir, then someone else expunged messages and
866 committed changes to index. so, this message
867 shouldn't actually exist. mark it racy and check
868 in next sync. */
869 if ((uflags &
870 MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
871 /* partial syncing */
872 seq--;
873 continue;
874 }
875 if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {
876 mail_storage_set_critical(
877 &mbox->storage->storage,
878 "Maildir %s sync: "
879 "UID inserted in the middle of mailbox "
880 "(%u > %u, file = %s)",
881 mbox->path, rec->uid, uid, filename);
882 mail_index_mark_corrupted(mbox->ibox.index);
883 ret = -1;
884 break;
885 }
886
887 mbox->dirty_cur_time = ioloop_time;
888 maildir_uidlist_add_flags(mbox->uidlist, filename,
889 MAILDIR_UIDLIST_REC_FLAG_RACING);
890
891 seq--;
892 continue;
893 }
894
895 if (index_sync_changes_read(ctx->sync_changes, rec->uid,
896 &expunged) < 0) {
897 ret = -1;
898 break;
899 }
900
901 if (expunged) {
902 if (maildir_file_do(ctx->mbox, ctx->uid,
903 maildir_expunge, ctx) >= 0) {
904 /* successful expunge */
905 mail_index_expunge(trans, ctx->seq);
906 }
907 if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
908 maildir_sync_notify(ctx->maildir_sync_ctx);
909 continue;
910 }
911
912 /* the private flags are stored only in indexes, keep them */
913 ctx->flags |= rec->flags & mbox->private_flags_mask;
914
915 if ((rec->flags & MAIL_RECENT) != 0) {
916 index_mailbox_set_recent(&mbox->ibox, seq);
917 if (mbox->ibox.keep_recent) {
918 ctx->flags |= MAIL_RECENT;
919 } else {
920 mail_index_update_flags(trans, seq,
921 MODIFY_REMOVE,
922 MAIL_RECENT);
923 }
924 }
925
926 if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
927 /* partial syncing */
928 if ((ctx->flags & MAIL_RECENT) != 0) {
929 /* we last saw this mail in new/, but it's
930 not there anymore. possibly expunged,
931 make sure. */
932 full_rescan = TRUE;
933 }
934 continue;
935 }
936
937 if (index_sync_changes_have(ctx->sync_changes)) {
938 /* apply flag changes to maildir */
939 if (maildir_file_do(ctx->mbox, ctx->uid,
940 maildir_sync_flags, ctx) < 0)
941 ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
942 if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
943 maildir_sync_notify(ctx->maildir_sync_ctx);
944 }
945
946 if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
947 /* we haven't been able to update maildir with this
948 record's flag changes. don't sync them. */
949 continue;
950 }
951
952 if ((ctx->flags & ~MAIL_RECENT) !=
953 (rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT))) {
954 /* FIXME: this is wrong if there's pending changes in
955 transaction log already. it gets fixed in next sync
956 however.. */
957 mail_index_update_flags(trans, seq, MODIFY_REPLACE,
958 ctx->flags);
959 } else if ((ctx->flags & MAIL_RECENT) == 0 &&
960 (rec->flags & MAIL_RECENT) != 0) {
961 /* just remove recent flag */
962 mail_index_update_flags(trans, seq, MODIFY_REMOVE,
963 MAIL_RECENT);
964 }
965
966 /* update keywords if they have changed */
967 if (mail_index_lookup_keywords(view, seq, &idx_keywords) < 0) {
968 mail_storage_set_index_error(&mbox->ibox);
969 ret = -1;
970 break;
971 }
972 if (!index_keyword_array_cmp(&ctx->keywords, &idx_keywords)) {
973 struct mail_keywords *kw;
974
975 kw = mail_index_keywords_create_from_indexes(
976 trans, &ctx->keywords);
977 mail_index_update_keywords(trans, seq,
978 MODIFY_REPLACE, kw);
979 mail_index_keywords_free(&kw);
980 }
981 }
982 maildir_uidlist_iter_deinit(iter);
983 mbox->syncing_commit = FALSE;
984
985 if (mbox->ibox.box.v.sync_notify != NULL)
986 mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0);
987
988 if (!partial) {
989 /* expunge the rest */
990 for (seq++; seq <= hdr->messages_count; seq++)
991 mail_index_expunge(trans, seq);
992
993 /* next_uid must be updated only in non-partial syncs since
994 partial syncs don't add the new mails to index. also we'll
995 have to do it here before syncing index records, since after
996 that the uidlist's next_uid value may have changed. */
997 next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
998 i_assert(next_uid > prev_uid);
999 if (hdr->next_uid < next_uid) {
1000 mail_index_update_header(trans,
1001 offsetof(struct mail_index_header, next_uid),
1002 &next_uid, sizeof(next_uid), FALSE);
1003 }
1004 }
1005
1006 if (ctx->changed)
1007 mbox->dirty_cur_time = ioloop_time;
1008 if (mbox->dirty_cur_time != 0)
1009 mbox->last_dirty_flags |= MAILDIR_DIRTY_CUR;
1010
1011 if (mbox->last_cur_mtime != (time_t)hdr->sync_stamp) {
1012 uint32_t sync_stamp = mbox->last_cur_mtime;
1013
1014 mail_index_update_header(trans,
1015 offsetof(struct mail_index_header, sync_stamp),
1016 &sync_stamp, sizeof(sync_stamp), TRUE);
1017 }
1018
1019 /* FIXME: use a header extension instead of sync_size.. */
1020 value = mbox->last_new_mtime |
1021 ((uint64_t)mbox->last_dirty_flags << 32);
1022 if (value != hdr->sync_size) {
1023 mail_index_update_header(trans,
1024 offsetof(struct mail_index_header, sync_size),
1025 &value, sizeof(value), TRUE);
1026 }
1027
1028 if (hdr->uid_validity == 0) {
1029 /* get the initial uidvalidity */
1030 if (maildir_uidlist_refresh(mbox->uidlist) < 0)
1031 ret = -1;
1032 uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
1033 if (uid_validity == 0) {
1034 uid_validity = ioloop_time;
1035 maildir_uidlist_set_uid_validity(mbox->uidlist,
1036 uid_validity, 0);
1037 }
1038 } else if (uid_validity == 0) {
1039 maildir_uidlist_set_uid_validity(mbox->uidlist,
1040 hdr->uid_validity,
1041 hdr->next_uid);
1042 }
1043
1044 if (uid_validity != hdr->uid_validity && uid_validity != 0) {
1045 mail_index_update_header(trans,
1046 offsetof(struct mail_index_header, uid_validity),
1047 &uid_validity, sizeof(uid_validity), TRUE);
1048 }
1049
1050 return ret < 0 ? -1 : (full_rescan ? 0 : 1);
1051 }
1052
1053 static int maildir_sync_context(struct maildir_sync_context *ctx, bool forced, 567 static int maildir_sync_context(struct maildir_sync_context *ctx, bool forced,
1054 bool sync_last_commit) 568 bool sync_last_commit)
1055 { 569 {
1056 bool new_changed, cur_changed, full_rescan = FALSE; 570 bool new_changed, cur_changed, full_rescan = FALSE;
1057 int ret; 571 int ret;
1124 the trouble? .. */ 638 the trouble? .. */
1125 return ret; 639 return ret;
1126 } 640 }
1127 641
1128 if (!ctx->mbox->syncing_commit) { 642 if (!ctx->mbox->syncing_commit) {
1129 if (maildir_sync_index_begin(ctx->mbox, 643 if (maildir_sync_index_begin(ctx->mbox, ctx,
1130 &ctx->index_sync_ctx) < 0) 644 &ctx->index_sync_ctx) < 0)
1131 return -1; 645 return -1;
1132 ctx->index_sync_ctx->maildir_sync_ctx = ctx;
1133 } 646 }
1134 647
1135 if (new_changed || cur_changed) { 648 if (new_changed || cur_changed) {
1136 /* if we're going to check cur/ dir our current logic requires 649 /* if we're going to check cur/ dir our current logic requires
1137 that new/ dir is checked as well. it's a good idea anyway. */ 650 that new/ dir is checked as well. it's a good idea anyway. */