Mercurial > dovecot > original-hg > dovecot-1.2
changeset 2151:16287320d080 HEAD
Several fixes in space/offset logic. Should be much more robust now.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Fri, 18 Jun 2004 00:29:20 +0300 |
parents | 347ca1cf6372 |
children | c7b77b11c89c |
files | src/lib-storage/index/mbox/mbox-sync-parse.c src/lib-storage/index/mbox/mbox-sync-private.h src/lib-storage/index/mbox/mbox-sync-rewrite.c src/lib-storage/index/mbox/mbox-sync-update.c src/lib-storage/index/mbox/mbox-sync.c |
diffstat | 5 files changed, 197 insertions(+), 134 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/mbox/mbox-sync-parse.c Fri Jun 18 00:28:23 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c Fri Jun 18 00:29:20 2004 +0300 @@ -33,6 +33,32 @@ struct message_header_line *hdr); }; +static void parse_trailing_whitespace(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + size_t i, space = 0; + + /* the value may contain newlines. we can't count whitespace before + and after it as a single contiguous whitespace block, as that may + get us into situation where removing whitespace goes eg. + " \n \n" -> " \n\n" which would then be treated as end of headers. + + that could probably be avoided by being careful, but as newlines + should never be there (we don't generate them), it's not worth the + trouble. */ + + for (i = hdr->full_value_len; i > 0; i--) { + if (!IS_LWSP(hdr->full_value[i-1])) + break; + space++; + } + + if (space > ctx->mail.space) { + ctx->mail.offset = hdr->full_value_offset + i; + ctx->mail.space = space; + } +} + static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr) { int i; @@ -121,6 +147,7 @@ // FIXME: save keywords + parse_trailing_whitespace(ctx, hdr); return TRUE; } @@ -130,7 +157,8 @@ if (!parse_x_imap_base(ctx, hdr)) return FALSE; - /* this is the UW-IMAP style "FOLDER INTERNAL DATA" message. skip it. */ + /* this is the c-client style "FOLDER INTERNAL DATA" message. + skip it. */ ctx->pseudo = TRUE; return TRUE; } @@ -138,22 +166,10 @@ static int parse_x_keywords(struct mbox_sync_mail_context *ctx, struct message_header_line *hdr) { - size_t i, space = 0; - - for (i = hdr->full_value_len; i > 0; i--) { - if (!IS_LWSP_LF(hdr->full_value[i-1])) - break; - space++; - } - - if (space > ctx->mail.space) { - ctx->mail.offset = hdr->full_value_offset + i; - ctx->mail.space = space; - } - // FIXME: parse them ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); + parse_trailing_whitespace(ctx, hdr); return TRUE; } @@ -161,7 +177,7 @@ struct message_header_line *hdr) { uint32_t value = 0; - size_t i, space_pos, extra_space = 0; + size_t i; if (ctx->mail.uid != 0) { /* duplicate */ @@ -174,15 +190,6 @@ value = value*10 + (hdr->full_value[i] - '0'); } - space_pos = i; - for (; i < hdr->full_value_len; i++) { - if (!IS_LWSP_LF(hdr->full_value[i])) { - /* broken value */ - return FALSE; - } - extra_space++; - } - if (value >= ctx->sync_ctx->next_uid) { /* next_uid broken - fix it */ ctx->sync_ctx->next_uid = value+1; @@ -192,17 +199,20 @@ /* broken - UIDs must be growing */ return FALSE; } - ctx->sync_ctx->prev_msg_uid = value; - - ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); ctx->mail.uid = value; - if (extra_space != 0 && ctx->mail.space == 0) { - /* set it only if X-Keywords hasn't been seen. spaces in X-UID - should be removed when writing X-Keywords. */ - ctx->mail.offset = hdr->full_value_offset + space_pos; - ctx->mail.space = extra_space; + ctx->sync_ctx->prev_msg_uid = value; + + if (ctx->sync_ctx->dest_first_mail && !ctx->seen_imapbase) { + /* everything was good, except we can't have X-UID before + X-IMAPbase header (to keep c-client compatibility). keep + the UID, but when we're rewriting this makes sure the + X-UID is appended after X-IMAPbase. */ + return FALSE; } + + ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); + parse_trailing_whitespace(ctx, hdr); return TRUE; } @@ -279,7 +289,7 @@ str_truncate(ctx->header, 0); line_start_pos = 0; - hdr_ctx = message_parse_header_init(input, NULL); + hdr_ctx = message_parse_header_init(input, NULL, FALSE); while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) { if (hdr->eoh) { ctx->have_eoh = TRUE; @@ -292,6 +302,18 @@ str_append(ctx->header, ": "); } + if (ctx->header_first_change == (size_t)-1 && + hdr->full_value_offset != str_len(ctx->header)) { + /* whitespaces around ':' are non-standard. either + there's whitespace before ':' or none after. + if we're going to rewrite this message, we can't + do it partially from here after as offsets won't + match. this shouldn't happen pretty much ever, so + don't try to optimize this - just rewrite the whole + thing. */ + ctx->no_partial_rewrite = TRUE; + } + func = header_func_find(hdr->name); if (func != NULL) { if (hdr->continues) @@ -318,7 +340,7 @@ message_parse_header_deinit(hdr_ctx); if ((ctx->seq == 1 && sync_ctx->base_uid_validity == 0) || - (ctx->seq > 1 && sync_ctx->first_uid == 0)) { + (ctx->seq > 1 && sync_ctx->dest_first_mail)) { /* missing X-IMAPbase */ ctx->need_rewrite = TRUE; }
--- a/src/lib-storage/index/mbox/mbox-sync-private.h Fri Jun 18 00:28:23 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-private.h Fri Jun 18 00:29:20 2004 +0300 @@ -31,9 +31,19 @@ keywords_mask_t keywords; uoff_t from_offset; - uoff_t offset; /* if space <= 0, points to beginning */ + uoff_t body_size; + + /* following variables have a bit overloaded functionality: + + a) space <= 0 : offset points to beginning of headers. space is the + amount of space missing that is required to be able to rewrite + the headers + b) space > 0 : offset points to beginning of whitespace that can + be removed. space is the amount of data that can be removed from + there. note that the message may contain more whitespace + elsewhere. */ + uoff_t offset; off_t space; - uoff_t body_size; }; struct mbox_sync_mail_context { @@ -52,6 +62,7 @@ unsigned int have_eoh:1; unsigned int need_rewrite:1; + unsigned int no_partial_rewrite:1; unsigned int seen_imapbase:1; unsigned int pseudo:1; unsigned int updated:1; @@ -78,9 +89,11 @@ buffer_t *mails, *syncs; struct mail_index_sync_rec sync_rec; - uint32_t prev_msg_uid, next_uid, first_uid; + uint32_t prev_msg_uid, next_uid; uint32_t seq, idx_seq, need_space_seq; off_t expunged_space, space_diff; + + unsigned int dest_first_mail:1; }; int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock);
--- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c Fri Jun 18 00:28:23 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c Fri Jun 18 00:29:20 2004 +0300 @@ -16,6 +16,8 @@ struct ostream *output; off_t ret; + i_assert(size < OFF_T_MAX); + if (size == 0 || source == dest) return 0; @@ -50,7 +52,7 @@ static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, size_t size) { - size_t data_size, pos; + size_t data_size, pos, start_pos; const unsigned char *data; void *p; @@ -58,24 +60,33 @@ /* Append at the end of X-Keywords header, or X-UID if it doesn't exist */ - pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ? + start_pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ? ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] : ctx->hdr_pos[MBOX_HDR_X_UID]; - data = buffer_get_data(ctx->header, &data_size); - while (pos < data_size && data[pos] != '\n') - pos++; + data = str_data(ctx->header); + data_size = str_len(ctx->header); + for (pos = start_pos; pos < data_size; pos++) { + if (data[pos] == '\n') { + /* possibly continues in next line */ + if (pos+1 == data_size || !IS_LWSP(data[pos+1])) + break; + start_pos = pos+1; + } else if (!IS_LWSP(data[pos])) { + start_pos = pos+1; + } + } + + /* pos points to end of headers now, and start_pos to beginning of + whitespace. */ buffer_copy(ctx->header, pos + size, ctx->header, pos, (size_t)-1); p = buffer_get_space_unsafe(ctx->header, pos, size); memset(p, ' ', size); - ctx->mail.offset = ctx->hdr_offset + pos; - if (ctx->mail.space < 0) - ctx->mail.space = size; - else - ctx->mail.space += size; + ctx->mail.offset = ctx->hdr_offset + start_pos; + ctx->mail.space = (pos - start_pos) + size; if (ctx->header_first_change > pos) ctx->header_first_change = pos; @@ -83,39 +94,56 @@ } static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx, - size_t pos, size_t *size) + size_t start_pos, size_t *size) { const unsigned char *data; - size_t data_size, end, nonspace; + size_t data_size, pos, last_line_pos; - /* find the end of the lwsp */ - nonspace = pos-1; + /* find the end of the LWSP */ data = str_data(ctx->header); data_size = str_len(ctx->header); - for (end = pos; end < data_size; end++) { - if (data[end] == '\n') { - if (end+1 == data_size || !IS_LWSP(data[end+1])) + + for (pos = last_line_pos = start_pos; pos < data_size; pos++) { + if (data[pos] == '\n') { + /* possibly continues in next line */ + if (pos+1 == data_size || !IS_LWSP(data[pos+1])) { + data_size = pos; break; - } else { - if (!IS_LWSP(data[end])) - nonspace = end; + } + last_line_pos = pos+1; + } else if (!IS_LWSP(data[pos])) { + start_pos = last_line_pos = pos+1; } } + if (start_pos == data_size) + return; + /* and remove what we can */ - nonspace++; - if (end-nonspace < *size) { - str_delete(ctx->header, nonspace, end-nonspace); - *size -= end-nonspace; - } else { - str_delete(ctx->header, nonspace, *size); - end -= *size; - *size = 0; + if (ctx->header_first_change > start_pos) + ctx->header_first_change = start_pos; + ctx->header_last_change = (size_t)-1; + + if (data_size - start_pos <= *size) { + /* remove it all */ + str_delete(ctx->header, start_pos, data_size - start_pos); + *size -= data_size - start_pos; + return; + } - if (ctx->mail.space < end-nonspace) { - ctx->mail.space = end-nonspace; - ctx->mail.offset = ctx->hdr_offset + nonspace; - } + /* we have more space than needed. since we're removing from + the beginning of header instead of end, we don't have to + worry about multiline-headers. */ + str_delete(ctx->header, start_pos, *size); + last_line_pos = last_line_pos <= *size ? + start_pos : last_line_pos - *size; + + data_size -= *size; + *size = 0; + + if (ctx->mail.space < data_size - last_line_pos) { + ctx->mail.space = data_size - last_line_pos; + ctx->mail.offset = ctx->hdr_offset + last_line_pos; } } @@ -123,35 +151,28 @@ size_t size) { static enum header_position space_positions[] = { + MBOX_HDR_X_UID, MBOX_HDR_X_KEYWORDS, - MBOX_HDR_X_UID, MBOX_HDR_X_IMAPBASE }; enum header_position pos; int i; - ctx->header_last_change = (size_t)-1; - ctx->mail.space = 0; ctx->mail.offset = ctx->hdr_offset; for (i = 0; i < 3 && size > 0; i++) { pos = space_positions[i]; if (ctx->hdr_pos[pos] != (size_t)-1) { - if (ctx->header_first_change > ctx->hdr_pos[pos]) - ctx->header_first_change = ctx->hdr_pos[pos]; mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos], &size); } } - - i_assert(size == 0); } int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff) { size_t old_hdr_size, new_hdr_size; - const unsigned char *data; i_assert(ctx->sync_ctx->ibox->mbox_lock_type == F_WRLCK); @@ -162,43 +183,53 @@ if (new_hdr_size < old_hdr_size) { mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size); } else if (new_hdr_size > old_hdr_size) { - size_t needed = new_hdr_size - old_hdr_size; + /* try removing the space where we can */ + mbox_sync_headers_remove_space(ctx, + new_hdr_size - old_hdr_size); + new_hdr_size = str_len(ctx->header); - if (ctx->mail.space >= 0) - mbox_sync_headers_remove_space(ctx, needed); - else if (move_diff < 0 && needed <= -move_diff) { + if (new_hdr_size <= old_hdr_size) { + /* good, we removed enough. */ + i_assert(new_hdr_size == old_hdr_size); + } else if (move_diff < 0 && + new_hdr_size - old_hdr_size <= -move_diff) { /* moving backwards - we can use the extra space from it, just update expunged_space accordingly */ - i_assert(ctx->sync_ctx->expunged_space >= needed); - ctx->sync_ctx->expunged_space -= needed; + i_assert(ctx->mail.space == 0); + i_assert(ctx->sync_ctx->expunged_space >= + new_hdr_size - old_hdr_size); + ctx->sync_ctx->expunged_space -= + new_hdr_size - old_hdr_size; } else { + /* couldn't get enough space */ + i_assert(ctx->mail.space == 0); + ctx->mail.space = + -(ssize_t)(new_hdr_size - old_hdr_size); return 0; } } + i_assert(ctx->mail.space >= 0); + if (ctx->header_first_change == (size_t)-1 && move_diff == 0) { /* no changes actually. we get here if index sync record told us to do something that was already there */ return 1; } - if (move_diff != 0) { - /* we're moving the header, forget about partial write - optimizations */ + if (move_diff != 0 || ctx->no_partial_rewrite) { + /* forget about partial write optimizations */ ctx->header_first_change = 0; ctx->header_last_change = 0; } - /* FIXME: last_change should rather just tell if we want to truncate - to beginning of extra whitespace */ if (ctx->header_last_change != (size_t)-1 && ctx->header_last_change != 0) str_truncate(ctx->header, ctx->header_last_change); - data = str_data(ctx->header); - new_hdr_size = str_len(ctx->header); - if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change, - new_hdr_size - ctx->header_first_change, + if (pwrite_full(ctx->sync_ctx->fd, + str_data(ctx->header) + ctx->header_first_change, + str_len(ctx->header) - ctx->header_first_change, ctx->hdr_offset + move_diff + ctx->header_first_change) < 0) { mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()"); @@ -232,6 +263,7 @@ so we have to fool it. */ old_prev_msg_uid = sync_ctx->prev_msg_uid; sync_ctx->prev_msg_uid = mails[idx].uid-1; + sync_ctx->dest_first_mail = seq == 1; mbox_sync_parse_next_mail(sync_ctx->input, &mail_ctx, TRUE); if (mails[idx].space != 0) @@ -243,8 +275,13 @@ str_append_c(mail_ctx.header, '\n'); } + sync_ctx->prev_msg_uid = old_prev_msg_uid; + sync_ctx->dest_first_mail = FALSE; + + mail_ctx.mail.space = + -(ssize_t)(str_len(mail_ctx.header) - + (mail_ctx.body_offset - mail_ctx.hdr_offset)); i_assert(mail_ctx.mail.space == mails[idx].space); - sync_ctx->prev_msg_uid = old_prev_msg_uid; if (mail_ctx.mail.space <= 0) mbox_sync_headers_add_space(&mail_ctx, extra_per_mail); @@ -301,11 +338,13 @@ so we have to fool it. */ old_prev_msg_uid = sync_ctx->prev_msg_uid; sync_ctx->prev_msg_uid = mails[idx].uid-1; + sync_ctx->dest_first_mail = seq == 1; mbox_sync_parse_next_mail(sync_ctx->input, &mail_ctx, TRUE); mbox_sync_update_header_from(&mail_ctx, &mails[idx]); sync_ctx->prev_msg_uid = old_prev_msg_uid; + sync_ctx->dest_first_mail = FALSE; mbox_sync_headers_add_space(&mail_ctx,end_offset - start_offset); @@ -347,13 +386,11 @@ i_assert(mails[idx].space >= 0); end_offset = mails[idx].offset + mails[idx].space; + /* after expunge the next mail must have been missing space, or we + would have moved it backwards already */ i_assert(mails[0].space < 0 || mails[0].uid == 0); start_offset = mails[0].offset; - /* after expunge the next mail must have been missing space, or we - would have moved it backwards already */ - i_assert(mails[0].uid != 0 || mails[1].space < 0); - /* start moving backwards */ do { idx--;
--- a/src/lib-storage/index/mbox/mbox-sync-update.c Fri Jun 18 00:28:23 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-update.c Fri Jun 18 00:29:20 2004 +0300 @@ -19,28 +19,30 @@ static void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx, size_t pos, size_t need, size_t have) { + ssize_t diff = (ssize_t)need - (ssize_t)have; int i; - if (need == have) { + if (diff == 0) { if (ctx->header_last_change < pos + have || ctx->header_last_change == (size_t)-1) ctx->header_last_change = pos + have; } else { + /* FIXME: if (diff < ctx->space && pos < ctx->offset) then + move the data only up to space offset and give/take the + space from there. update header_last_change accordingly. */ ctx->header_last_change = (size_t)-1; for (i = 0; i < MBOX_HDR_COUNT; i++) { if (ctx->hdr_pos[i] > pos && ctx->hdr_pos[i] != (size_t)-1) - ctx->hdr_pos[i] += need - have; + ctx->hdr_pos[i] += diff; } - if (need < have) { - str_delete(ctx->header, pos, have-need); - ctx->mail.space += have - need; - } else { + if (diff < 0) + str_delete(ctx->header, pos, -diff); + else { ctx->header_last_change = (size_t)-1; - buffer_copy(ctx->header, pos + (need-have), + buffer_copy(ctx->header, pos + diff, ctx->header, pos, (size_t)-1); - ctx->mail.space -= need - have; } } } @@ -111,7 +113,7 @@ str_append_c(ctx->header, '\n'); } - if (ctx->mail.uid == ctx->sync_ctx->first_uid && + if (ctx->sync_ctx->dest_first_mail && ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1) { if (ctx->sync_ctx->base_uid_validity == 0) { ctx->sync_ctx->base_uid_validity = @@ -174,16 +176,6 @@ if (ctx->header_first_change == (size_t)-1) ctx->header_first_change = new_hdr_size; ctx->header_last_change = (size_t)-1; - ctx->mail.space -= str_len(ctx->header) - new_hdr_size; - if (ctx->mail.space > 0) { - /* we should rewrite this header, so offset - must be broken if it's used anymore. */ - ctx->mail.offset = (uoff_t)-1; - } else { - /* we don't have enough space for this header, change - offset to point back to beginning of headers */ - ctx->mail.offset = ctx->hdr_offset; - } } if (ctx->have_eoh) @@ -216,7 +208,7 @@ const char *p, *hdr; size_t pos; - if (ctx->mail.uid != ctx->sync_ctx->first_uid || + if (ctx->sync_ctx->dest_first_mail || ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1 || ctx->sync_ctx->update_base_uid_last == 0 || ctx->sync_ctx->update_base_uid_last < ctx->sync_ctx->base_uid_last)
--- a/src/lib-storage/index/mbox/mbox-sync.c Fri Jun 18 00:28:23 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync.c Fri Jun 18 00:29:20 2004 +0300 @@ -188,7 +188,7 @@ mail_ctx->mail.offset = istream_raw_mbox_get_header_offset(sync_ctx->input); - if (mail_ctx->seq > 1 && sync_ctx->first_uid == mail_ctx->mail.uid) { + if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) { /* First message was expunged and this is the next one. Skip \n header */ mail_ctx->from_offset++; @@ -489,7 +489,7 @@ mail_ctx->mail.body_size; mail_ctx->mail.body_size = 0; - if (mail_ctx->sync_ctx->seq == 1) { + if (mail_ctx->sync_ctx->dest_first_mail) { /* expunging first message, fix space to contain next message's \n header too since it will be removed. */ mail_ctx->mail.space++; @@ -505,9 +505,6 @@ off_t move_diff; int ret; - if (sync_ctx->first_uid == 0) - sync_ctx->first_uid = mail_ctx->mail.uid; - if (sync_ctx->ibox->mbox_readonly) return 0; @@ -627,6 +624,7 @@ /* set to -1, since they're always increased later */ sync_ctx->seq = sync_ctx->idx_seq = seq-1; + sync_ctx->dest_first_mail = sync_ctx->seq == 0; if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) { mail_storage_set_critical(sync_ctx->ibox->box.storage, "Cached message offset %s is invalid for mbox file %s", @@ -654,31 +652,31 @@ "Mailbox isn't a valid mbox file"); return -1; } + sync_ctx->dest_first_mail = TRUE; } else { /* we sync only what we need to. jump to first record that needs updating */ const struct mail_index_sync_rec *sync_rec; + size_t size; - if (buffer_get_used_size(sync_ctx->syncs) == 0) { + if (buffer_get_used_size(sync_ctx->syncs) == 0 && + sync_ctx->sync_rec.uid1 == 0) { if (mbox_sync_read_index_syncs(sync_ctx, 1, &expunged) < 0) return -1; - if (buffer_get_used_size(sync_ctx->syncs) == 0) { + if (buffer_get_used_size(sync_ctx->syncs) == 0 && + sync_ctx->sync_rec.uid1 == 0) { /* nothing to do */ return 0; } } - sync_rec = buffer_get_data(sync_ctx->syncs, NULL); + sync_rec = buffer_get_data(sync_ctx->syncs, &size); + if (size == 0) + sync_rec = &sync_ctx->sync_rec; if (mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1) < 0) return -1; - - if (sync_ctx->seq > 0) { - if (mail_index_lookup_uid(sync_ctx->sync_view, 1, - &sync_ctx->first_uid) < 0) - return -1; - } } while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) { @@ -688,9 +686,10 @@ if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0) return -1; - if (!expunged) + if (!expunged) { ret = mbox_sync_handle_header(mail_ctx); - else { + sync_ctx->dest_first_mail = FALSE; + } else { mail_ctx->mail.uid = 0; ret = mbox_sync_handle_expunge(mail_ctx); } @@ -880,7 +879,7 @@ sync_ctx->base_uid_last = 0; sync_ctx->next_uid = 1; - sync_ctx->prev_msg_uid = sync_ctx->first_uid = 0; + sync_ctx->prev_msg_uid = 0; sync_ctx->seq = sync_ctx->idx_seq = 0; }