Mercurial > dovecot > original-hg > dovecot-1.2
changeset 3254:a2943c050571 HEAD
Keywords are now stored in X-Keywords headers in mbox. Did several related
API changes to get better performance.
line wrap: on
line diff
--- a/src/imap/client.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/imap/client.h Sun Apr 03 00:08:56 2005 +0300 @@ -11,8 +11,7 @@ struct mailbox_keywords { pool_t pool; /* will be p_clear()ed when changed */ - char **keywords; - unsigned int keywords_count; + array_t ARRAY_DEFINE(keywords, const char *); }; struct client_command_context {
--- a/src/imap/cmd-append.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/imap/cmd-append.c Sun Apr 03 00:08:56 2005 +0300 @@ -393,8 +393,8 @@ mailbox_close(ctx->box); ctx->box = NULL; } else { - client_save_keywords(&client->keywords, status.keywords, - status.keywords_count); + client_save_keywords(&client->keywords, + status.keywords); } ctx->t = ctx->box == NULL ? NULL : mailbox_transaction_begin(ctx->box,
--- a/src/imap/cmd-select.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/imap/cmd-select.c Sun Apr 03 00:08:56 2005 +0300 @@ -51,8 +51,7 @@ return TRUE; } - client_save_keywords(&client->keywords, - status.keywords, status.keywords_count); + client_save_keywords(&client->keywords, status.keywords); client->messages_count = status.messages; client->recent_count = status.recent; @@ -61,8 +60,7 @@ client->mailbox = box; client->select_counter++; - client_send_mailbox_flags(client, box, status.keywords, - status.keywords_count); + client_send_mailbox_flags(client, box, status.keywords); client_send_line(client, t_strdup_printf("* %u EXISTS", status.messages));
--- a/src/imap/commands-util.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/imap/commands-util.c Sun Apr 03 00:08:56 2005 +0300 @@ -154,13 +154,13 @@ static int is_valid_keyword(struct client_command_context *cmd, const char *keyword) { - struct mailbox_keywords *keywords = &cmd->client->keywords; - size_t i; + const char *const *names; + unsigned int i, count; /* if it already exists, skip validity checks */ - for (i = 0; i < keywords->keywords_count; i++) { - if (keywords->keywords[i] != NULL && - strcasecmp(keywords->keywords[i], keyword) == 0) + names = array_get(&cmd->client->keywords.keywords, &count); + for (i = 0; i < count; i++) { + if (strcasecmp(names[i], keyword) == 0) return TRUE; } @@ -241,27 +241,21 @@ return TRUE; } -static const char * -get_keywords_string(const char *const keywords[], unsigned int keywords_count) +static const char *get_keywords_string(const array_t *keywords) { + ARRAY_SET_TYPE(keywords, const char *); string_t *str; - unsigned int i; + const char *const *names; + unsigned int i, count; - /* first see if there even is keywords */ - for (i = 0; i < keywords_count; i++) { - if (keywords[i] != NULL) - break; - } - - if (i == keywords_count) + if (array_count(keywords) == 0) return ""; str = t_str_new(256); - for (; i < keywords_count; i++) { - if (keywords[i] != NULL) { - str_append_c(str, ' '); - str_append(str, keywords[i]); - } + names = array_get(keywords, &count); + for (i = 0; i < count; i++) { + str_append_c(str, ' '); + str_append(str, names[i]); } return str_c(str); } @@ -269,12 +263,11 @@ #define SYSTEM_FLAGS "\\Answered \\Flagged \\Deleted \\Seen \\Draft" void client_send_mailbox_flags(struct client *client, struct mailbox *box, - const char *const keywords[], - unsigned int keywords_count) + const array_t *keywords) { const char *str; - str = get_keywords_string(keywords, keywords_count); + str = get_keywords_string(keywords); client_send_line(client, t_strconcat("* FLAGS ("SYSTEM_FLAGS, str, ")", NULL)); @@ -290,24 +283,22 @@ } void client_save_keywords(struct mailbox_keywords *dest, - const char *const keywords[], - unsigned int keywords_count) + const array_t *keywords) { - unsigned int i; + ARRAY_SET_TYPE(keywords, const char *); + const char *const *names; + unsigned int i, count; p_clear(dest->pool); + ARRAY_CREATE(&dest->keywords, dest->pool, + const char *, array_count(keywords)); - if (keywords_count == 0) { - dest->keywords = NULL; - dest->keywords_count = 0; - return; + names = array_get(keywords, &count); + for (i = 0; i < count; i++) { + const char *name = p_strdup(dest->pool, names[i]); + + array_append(&dest->keywords, &name, 1); } - - dest->keywords = p_new(dest->pool, char *, keywords_count); - dest->keywords_count = keywords_count; - - for (i = 0; i < keywords_count; i++) - dest->keywords[i] = p_strdup(dest->pool, keywords[i]); } int mailbox_equals(struct mailbox *box1, struct mail_storage *storage2,
--- a/src/imap/commands-util.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/imap/commands-util.h Sun Apr 03 00:08:56 2005 +0300 @@ -44,13 +44,11 @@ /* Send FLAGS + PERMANENTFLAGS to client. */ void client_send_mailbox_flags(struct client *client, struct mailbox *box, - const char *const keywords[], - unsigned int keywords_count); + const array_t *keywords); /* Copy keywords into dest. dest must have been initialized. */ void client_save_keywords(struct mailbox_keywords *dest, - const char *const keywords[], - unsigned int keywords_count); + const array_t *keywords); int mailbox_equals(struct mailbox *box1, struct mail_storage *storage2, const char *name2);
--- a/src/lib-index/mail-index-private.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index-private.h Sun Apr 03 00:08:56 2005 +0300 @@ -62,14 +62,6 @@ /* unsigned char name[] */ }; -struct mail_keywords { - struct mail_index *index; - unsigned int count; - - /* variable sized list of keyword indexes */ - uint32_t idx[1]; -}; - struct mail_index_keyword_header { uint32_t keywords_count; /* struct mail_index_keyword_header_rec[] */ @@ -99,9 +91,7 @@ buffer_t *buffer; buffer_t *hdr_copy_buf; - pool_t keywords_pool; - const char *const *keywords; - unsigned int keywords_count; + array_t ARRAY_DEFINE(keyword_idx_map, unsigned int); /* file -> index */ unsigned int write_to_disk:1; }; @@ -142,8 +132,8 @@ uoff_t sync_log_file_offset; pool_t keywords_pool; - array_t ARRAY_DEFINE(keywords_arr, const char *); - const char *const *keywords; + array_t ARRAY_DEFINE(keywords, const char *); + struct hash_table *keywords_hash; /* name -> idx */ uint32_t keywords_ext_id; unsigned int last_grow_count;
--- a/src/lib-index/mail-index-sync-keywords.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index-sync-keywords.c Sun Apr 03 00:08:56 2005 +0300 @@ -11,18 +11,24 @@ const char *keyword_name, unsigned int *idx_r) { struct mail_index_map *map = ctx->view->map; - unsigned int i; + const unsigned int *idx_map; + unsigned int i, count, keyword_idx; if (!ctx->keywords_read) { if (mail_index_map_read_keywords(ctx->view->index, map) < 0) return -1; ctx->keywords_read = TRUE; } - - for (i = 0; i < map->keywords_count; i++) { - if (strcmp(map->keywords[i], keyword_name) == 0) { - *idx_r = i; - return 1; + if (mail_index_keyword_lookup(ctx->view->index, keyword_name, + FALSE, &keyword_idx) && + array_is_created(&map->keyword_idx_map)) { + /* FIXME: slow. maybe create index -> file mapping as well */ + idx_map = array_get(&map->keyword_idx_map, &count); + for (i = 0; i < count; i++) { + if (idx_map[i] == keyword_idx) { + *idx_r = i; + return 1; + } } }
--- a/src/lib-index/mail-index-sync-private.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index-sync-private.h Sun Apr 03 00:08:56 2005 +0300 @@ -26,7 +26,8 @@ struct mail_index_sync_list { const array_t *ARRAY_DEFINE_PTR(array, struct uid_range); unsigned int idx; - unsigned int keyword_num; + unsigned int keyword_idx:31; + unsigned int keyword_remove:1; }; struct mail_index_expunge_handler {
--- a/src/lib-index/mail-index-sync.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index-sync.c Sun Apr 03 00:08:56 2005 +0300 @@ -185,7 +185,8 @@ mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx, int *seen_external_r) { - struct mail_index_sync_list *synclist; + struct mail_index_sync_list *synclist; + const struct mail_index_transaction_keyword_update *keyword_updates; unsigned int i, keyword_count; int ret; @@ -236,10 +237,20 @@ synclist->array = &ctx->trans->keyword_resets; } + keyword_updates = keyword_count == 0 ? NULL : + array_get(&ctx->trans->keyword_updates, NULL); for (i = 0; i < keyword_count; i++) { - synclist = array_modifyable_append(&ctx->sync_list); - synclist->array = array_idx(&ctx->trans->keyword_updates, i); - synclist->keyword_num = i; + if (array_is_created(&keyword_updates[i].add_seq)) { + synclist = array_modifyable_append(&ctx->sync_list); + synclist->array = &keyword_updates[i].add_seq; + synclist->keyword_idx = i; + } + if (array_is_created(&keyword_updates[i].remove_seq)) { + synclist = array_modifyable_append(&ctx->sync_list); + synclist->array = &keyword_updates[i].remove_seq; + synclist->keyword_idx = i; + synclist->keyword_remove = TRUE; + } } return ret; @@ -430,16 +441,17 @@ rec->remove_flags = update->remove_flags; } -static void mail_index_sync_get_keyword_update(struct mail_index_sync_rec *rec, - const struct uid_range *range, - unsigned int num) +static void +mail_index_sync_get_keyword_update(struct mail_index_sync_rec *rec, + const struct uid_range *range, + struct mail_index_sync_list *sync_list) { - rec->type = num % 2 == 0 ? + rec->type = !sync_list->keyword_remove ? MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD : MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE; rec->uid1 = range->uid1; rec->uid2 = range->uid2; - rec->keyword_idx = num / 2; + rec->keyword_idx = sync_list->keyword_idx; } static void mail_index_sync_get_keyword_reset(struct mail_index_sync_rec *rec, @@ -530,7 +542,7 @@ mail_index_sync_get_keyword_reset(sync_rec, uid_range); } else { mail_index_sync_get_keyword_update(sync_rec, uid_range, - sync_list[i].keyword_num); + &sync_list[i]); } sync_list[i].idx++; @@ -624,12 +636,6 @@ mail_index_sync_end(ctx); } -const char *const *const * -mail_index_sync_get_keywords(struct mail_index_sync_ctx *ctx) -{ - return &ctx->index->keywords; -} - void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec, uint8_t *flags) { @@ -637,3 +643,41 @@ *flags = (*flags & ~sync_rec->remove_flags) | sync_rec->add_flags; } + +int mail_index_sync_keywords_apply(const struct mail_index_sync_rec *sync_rec, + array_t *keywords) +{ + ARRAY_SET_TYPE(keywords, unsigned int); + const unsigned int *keyword_indexes; + unsigned int idx = sync_rec->keyword_idx; + unsigned int i, count; + + keyword_indexes = array_get(keywords, &count); + switch (sync_rec->type) { + case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + for (i = 0; i < count; i++) { + if (keyword_indexes[i] == idx) + return FALSE; + } + + array_append(keywords, &idx, 1); + return TRUE; + case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + for (i = 0; i < count; i++) { + if (keyword_indexes[i] == idx) { + array_delete(keywords, i, 1); + return TRUE; + } + } + return FALSE; + case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET: + if (array_count(keywords) == 0) + return FALSE; + + array_clear(keywords); + return TRUE; + default: + i_unreached(); + return FALSE; + } +}
--- a/src/lib-index/mail-index-transaction-private.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index-transaction-private.h Sun Apr 03 00:08:56 2005 +0300 @@ -3,6 +3,11 @@ #include "mail-transaction-log.h" +struct mail_index_transaction_keyword_update { + array_t ARRAY_DEFINE(add_seq, uint32_t); + array_t ARRAY_DEFINE(remove_seq, uint32_t); +}; + struct mail_index_transaction { int refcount; struct mail_index_view *view; @@ -21,7 +26,8 @@ array_t ARRAY_DEFINE(ext_resizes, struct mail_transaction_ext_intro); array_t ARRAY_DEFINE(ext_resets, uint32_t); - array_t ARRAY_DEFINE(keyword_updates, array_t); + array_t ARRAY_DEFINE(keyword_updates, + struct mail_index_transaction_keyword_update); array_t ARRAY_DEFINE(keyword_resets, struct seq_range); struct mail_cache_transaction_ctx *cache_trans_ctx;
--- a/src/lib-index/mail-index-transaction.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index-transaction.c Sun Apr 03 00:08:56 2005 +0300 @@ -56,11 +56,15 @@ } if (array_is_created(&t->keyword_updates)) { - recs = array_get_modifyable(&t->keyword_updates, &count); + struct mail_index_transaction_keyword_update *u; + + u = array_get_modifyable(&t->keyword_updates, &count); for (i = 0; i < count; i++) { - if (array_is_created(&recs[i])) - array_free(&recs[i]); + if (array_is_created(&u[i].add_seq)) + array_free(&u[i].add_seq); + if (array_is_created(&u[i].remove_seq)) + array_free(&u[i].remove_seq); } array_free(&t->keyword_updates); } @@ -830,11 +834,9 @@ mail_index_keywords_create(struct mail_index_transaction *t, const char *const keywords[]) { - /* @UNSAFE */ struct mail_index *index = t->view->index; struct mail_keywords *k; - const char **missing_keywords, *keyword; - unsigned int count, i, j, k_pos = 0, missing_count = 0; + unsigned int i, count; if (keywords == NULL) { k = i_new(struct mail_keywords, 1); @@ -843,48 +845,39 @@ } count = strarray_length(keywords); + /* @UNSAFE */ + k = i_malloc(sizeof(struct mail_keywords) + + (sizeof(k->idx) * (count-1))); + k->index = index; + k->count = count; + + /* look up the keywords from index. they're never removed from there + so we can permanently store indexes to them. */ + for (i = 0; i < count; i++) { + (void)mail_index_keyword_lookup(index, keywords[i], + TRUE, &k->idx[i]); + } + return k; +} + +struct mail_keywords * +mail_index_keywords_create_from_indexes(struct mail_index_transaction *t, + const array_t *keyword_indexes) +{ + ARRAY_SET_TYPE(keyword_indexes, unsigned int); + struct mail_keywords *k; + unsigned int count; + + count = array_count(keyword_indexes); + + /* @UNSAFE */ k = i_malloc(sizeof(struct mail_keywords) + (sizeof(k->idx) * (count-1))); k->index = t->view->index; k->count = count; - t_push(); - missing_keywords = t_new(const char *, count + 1); - - /* look up the keywords from index. they're never removed from there - so we can permanently store indexes to them. */ - for (i = 0; i < count; i++) { - for (j = 0; index->keywords[j] != NULL; j++) { - if (strcasecmp(keywords[i], index->keywords[j]) == 0) - break; - } - - if (index->keywords[j] != NULL) - k->idx[k_pos++] = j; - else - missing_keywords[missing_count++] = keywords[i]; - } - - if (missing_count > 0) { - /* add missing keywords. first drop the trailing NULL. */ - array_delete(&index->keywords_arr, - array_count(&index->keywords_arr) - 1, 1); - - j = array_count(&index->keywords_arr); - for (; *missing_keywords != NULL; missing_keywords++, j++) { - keyword = p_strdup(index->keywords_pool, - *missing_keywords); - array_append(&index->keywords_arr, &keyword, 1); - - k->idx[k_pos++] = j; - } - - (void)array_modifyable_append(&index->keywords_arr); - index->keywords = array_idx(&index->keywords_arr, 0); - } - i_assert(k_pos == count); - - t_pop(); + memcpy(k->idx, array_get(keyword_indexes, NULL), + count * sizeof(k->idx[0])); return k; } @@ -897,8 +890,8 @@ enum modify_type modify_type, struct mail_keywords *keywords) { - array_t *arr; - unsigned int i, idx; + struct mail_index_transaction_keyword_update *u; + unsigned int i; i_assert(seq > 0 && (seq <= mail_index_view_get_messages_count(t->view) || @@ -906,48 +899,40 @@ i_assert(keywords->count > 0 || modify_type == MODIFY_REPLACE); i_assert(keywords->index == t->view->index); - /* keyword_updates is an array of - { buffer_t *add_seq; buffer_t *remove_seq; } - which is why there's the multiplication by 2. - - If t->keyword_resets is set for the sequence, there's no need to - update remove_seq as it will remove all keywords. */ - if (!array_is_created(&t->keyword_updates)) { uint32_t max_idx = keywords->idx[keywords->count-1]; ARRAY_CREATE(&t->keyword_updates, default_pool, - array_t, max_idx * 2); + struct mail_index_transaction_keyword_update, + max_idx); } switch (modify_type) { case MODIFY_ADD: for (i = 0; i < keywords->count; i++) { - idx = keywords->idx[i] * 2; - arr = array_modifyable_idx(&t->keyword_updates, idx); - mail_index_seq_range_array_add(arr, 16, seq); - - arr = array_modifyable_idx(&t->keyword_updates, idx+1); - mail_index_seq_range_array_remove(arr, seq); + u = array_modifyable_idx(&t->keyword_updates, + keywords->idx[i]); + mail_index_seq_range_array_add(&u->add_seq, 16, seq); + mail_index_seq_range_array_remove(&u->remove_seq, seq); } break; case MODIFY_REMOVE: for (i = 0; i < keywords->count; i++) { - idx = keywords->idx[i] * 2; - arr = array_modifyable_idx(&t->keyword_updates, idx); - mail_index_seq_range_array_remove(arr, seq); - - arr = array_modifyable_idx(&t->keyword_updates, idx+1); - mail_index_seq_range_array_add(arr, 16, seq); + u = array_modifyable_idx(&t->keyword_updates, + keywords->idx[i]); + mail_index_seq_range_array_remove(&u->add_seq, seq); + mail_index_seq_range_array_add(&u->remove_seq, 16, seq); } break; case MODIFY_REPLACE: for (i = 0; i < keywords->count; i++) { - idx = keywords->idx[i] * 2; - arr = array_modifyable_idx(&t->keyword_updates, idx); - mail_index_seq_range_array_add(arr, 16, seq); + u = array_modifyable_idx(&t->keyword_updates, + keywords->idx[i]); + mail_index_seq_range_array_add(&u->add_seq, 16, seq); } + /* If t->keyword_resets is set for a sequence, there's no + need to update remove_seq as it will remove all keywords. */ mail_index_seq_range_array_add(&t->keyword_resets, 16, seq); break; }
--- a/src/lib-index/mail-index-view.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index-view.c Sun Apr 03 00:08:56 2005 +0300 @@ -458,58 +458,71 @@ } int mail_index_lookup_keywords(struct mail_index_view *view, uint32_t seq, - buffer_t *buf, const char *const **keywords_r) + array_t *keyword_idx) { + ARRAY_SET_TYPE(keyword_idx, unsigned int); struct mail_index_map *map; const struct mail_index_ext *ext; const void *data; - unsigned int i, j; + const unsigned int *keyword_idx_map; + unsigned int i, j, keyword_count, index_idx; uint32_t ext_id, idx; int ret; - *keywords_r = NULL; - buffer_set_used_size(buf, 0); + array_clear(keyword_idx); ext_id = view->index->keywords_ext_id; ret = mail_index_lookup_ext_full(view, seq, ext_id, &map, &data); if (ret < 0) return -1; - if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) { - buffer_append_zero(buf, sizeof(const char *)); - *keywords_r = buf->data; + if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) return ret; - } ext = array_idx(&map->extensions, idx); + if (!array_is_created(&map->keyword_idx_map)) { + keyword_idx_map = NULL; + keyword_count = 0; + } else { + keyword_idx_map = array_get(&map->keyword_idx_map, + &keyword_count); + } + for (i = 0, idx = 0; i < ext->record_size; i++) { - if (((const char *)data)[i] == 0) + if (((const unsigned char *)data)[i] == 0) continue; + idx = i * CHAR_BIT; for (j = 0; j < CHAR_BIT; j++, idx++) { - if ((((const char *)data)[i] & (1 << j)) == 0) + if ((((const unsigned char *)data)[i] & (1 << j)) == 0) continue; - if (idx >= map->keywords_count) { + if (idx >= keyword_count) { /* keyword header is updated, re-read it so we know what this one is called */ if (mail_index_map_read_keywords(view->index, map) < 0) return -1; - if (idx >= map->keywords_count) { + + if (!array_is_created(&map->keyword_idx_map)) + return ret; + + keyword_idx_map = + array_get(&map->keyword_idx_map, + &keyword_count); + + if (idx >= keyword_count) { /* extra bits set in keyword bytes. shouldn't happen, but just ignore. */ break; } } - buffer_append(buf, &map->keywords[idx], - sizeof(const char *)); + + index_idx = keyword_idx_map[idx]; + array_append(keyword_idx, &index_idx, 1); } } - buffer_append_zero(buf, sizeof(const char *)); - *keywords_r = buf->data; - return ret; }
--- a/src/lib-index/mail-index.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index.c Sun Apr 03 00:08:56 2005 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "buffer.h" +#include "hash.h" #include "mmap-util.h" #include "read-full.h" #include "write-full.h" @@ -45,23 +46,25 @@ index->keywords_ext_id = mail_index_ext_register(index, "keywords", 128, 2, 1); index->keywords_pool = pool_alloconly_create("keywords", 512); - ARRAY_CREATE(&index->keywords_arr, default_pool, - const char *, 16); - (void)array_modifyable_append(&index->keywords_arr); - index->keywords = array_idx(&index->keywords_arr, 0); + ARRAY_CREATE(&index->keywords, default_pool, const char *, 16); + index->keywords_hash = + hash_create(default_pool, index->keywords_pool, 0, + strcase_hash, (hash_cmp_callback_t *)strcasecmp); return index; } void mail_index_free(struct mail_index *index) { mail_index_close(index); + + hash_destroy(index->keywords_hash); pool_unref(index->extension_pool); pool_unref(index->keywords_pool); array_free(&index->sync_handlers); array_free(&index->sync_lost_handlers); array_free(&index->expunge_handlers); - array_free(&index->keywords_arr); + array_free(&index->keywords); i_free(index->error); i_free(index->dir); @@ -333,20 +336,45 @@ return 1; } +int mail_index_keyword_lookup(struct mail_index *index, + const char *keyword, int autocreate, + unsigned int *idx_r) +{ + char *keyword_dup; + void *value; + + if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) { + *idx_r = POINTER_CAST_TO(value, unsigned int); + return TRUE; + } + + if (!autocreate) { + *idx_r = (unsigned int)-1; + return FALSE; + } + + keyword = keyword_dup = p_strdup(index->keywords_pool, keyword); + *idx_r = array_count(&index->keywords); + + hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r)); + array_append(&index->keywords, &keyword, 1); + return TRUE; +} + int mail_index_map_read_keywords(struct mail_index *index, struct mail_index_map *map) { const struct mail_index_ext *ext; const struct mail_index_keyword_header *kw_hdr; const struct mail_index_keyword_header_rec *kw_rec; - const char *name, **keywords_list; - unsigned int i, name_len; + const char *name; + unsigned int i, name_len, old_count; uint32_t ext_id; ext_id = mail_index_map_lookup_ext(map, "keywords"); if (ext_id == (uint32_t)-1) { - map->keywords = NULL; - map->keywords_count = 0; + if (array_is_created(&map->keyword_idx_map)) + array_clear(&map->keyword_idx_map); return 0; } @@ -356,6 +384,23 @@ kw_rec = (const void *)(kw_hdr + 1); name = (const char *)(kw_rec + kw_hdr->keywords_count); + old_count = !array_is_created(&map->keyword_idx_map) ? 0 : + array_count(&map->keyword_idx_map); + + /* Keywords can only be added in mapping. */ + if (kw_hdr->keywords_count == old_count) { + /* nothing changed */ + return 0; + } + + /* make sure the header is valid */ + if (kw_hdr->keywords_count < old_count) { + mail_index_set_error(index, "Corrupted index file %s: " + "Keywords removed unexpectedly", + index->filepath); + return -1; + } + if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) { mail_index_set_error(index, "Corrupted index file %s: " "keywords_count larger than header size", @@ -363,7 +408,6 @@ return -1; } - /* make sure the header is valid */ name_len = (const char *)kw_hdr + ext->hdr_size - name; for (i = 0; i < kw_hdr->keywords_count; i++) { if (kw_rec[i].name_offset > name_len) { @@ -380,42 +424,43 @@ return -1; } - if (map->keywords_pool == NULL) - map->keywords_pool = pool_alloconly_create("keywords", 1024); - - /* Save keywords in memory. Only new keywords should come into the - mapping, so keep the existing keyword strings in memory to allow - mail_index_lookup_keywords() to safely return direct pointers - into them. */ - if (kw_hdr->keywords_count < map->keywords_count) { - mail_index_set_error(index, "Corrupted index file %s: " - "Keywords removed unexpectedly", - index->filepath); - return -1; - } - if (kw_hdr->keywords_count == map->keywords_count) { - /* nothing changed */ - return 0; + /* create file -> index mapping */ + if (!array_is_created(&map->keyword_idx_map)) { + ARRAY_CREATE(&map->keyword_idx_map, default_pool, + unsigned int, kw_hdr->keywords_count); } - /* @UNSAFE */ - keywords_list = p_new(map->keywords_pool, - const char *, kw_hdr->keywords_count + 1); - for (i = 0; i < map->keywords_count; i++) - keywords_list[i] = map->keywords[i]; +#ifdef DEBUG + for (i = 0; i < array_count(&map->keyword_idx_map); i++) { + const char *keyword = name + kw_rec[i].name_offset; + const unsigned int *old_idx; + unsigned int idx; + + old_idx = array_idx(&map->keyword_idx_map, i); + if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) || + idx != *old_idx) { + mail_index_set_error(index, "Corrupted index file %s: " + "Keywords changed unexpectedly", + index->filepath); + return -1; + } + } +#endif + i = array_count(&map->keyword_idx_map); for (; i < kw_hdr->keywords_count; i++) { - keywords_list[i] = p_strdup(map->keywords_pool, - name + kw_rec[i].name_offset); + const char *keyword = name + kw_rec[i].name_offset; + unsigned int idx; + + (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx); + array_append(&map->keyword_idx_map, &idx, 1); } - map->keywords = keywords_list; - map->keywords_count = kw_hdr->keywords_count; return 0; } -const char *const *mail_index_get_keywords(struct mail_index *index) +const array_t *mail_index_get_keywords(struct mail_index *index) { (void)mail_index_map_read_keywords(index, index->map); - return index->map->keywords; + return &index->keywords; } static int mail_index_check_header(struct mail_index *index, @@ -508,8 +553,8 @@ mail_index_map_clear(index, map); if (map->extension_pool != NULL) pool_unref(map->extension_pool); - if (map->keywords_pool != NULL) - pool_unref(map->keywords_pool); + if (array_is_created(&map->keyword_idx_map)) + array_free(&map->keyword_idx_map); buffer_free(map->hdr_copy_buf); i_free(map); }
--- a/src/lib-index/mail-index.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-index.h Sun Apr 03 00:08:56 2005 +0300 @@ -106,6 +106,14 @@ uint8_t flags; /* enum mail_flags | enum mail_index_mail_flags */ }; +struct mail_keywords { + struct mail_index *index; + unsigned int count; + + /* variable sized list of keyword indexes */ + unsigned int idx[1]; +}; + enum mail_index_sync_type { MAIL_INDEX_SYNC_TYPE_APPEND = 0x01, MAIL_INDEX_SYNC_TYPE_EXPUNGE = 0x02, @@ -124,11 +132,10 @@ uint8_t add_flags; uint8_t remove_flags; - /* MAIL_INDEX_SYNC_TYPE_KEYWORDS: */ + /* MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD, .._REMOVE: */ unsigned int keyword_idx; }; -struct mail_keywords; struct mail_index; struct mail_index_map; struct mail_index_view; @@ -256,7 +263,7 @@ struct mail_index_map **map_r, const struct mail_index_record **rec_r); int mail_index_lookup_keywords(struct mail_index_view *view, uint32_t seq, - buffer_t *buf, const char *const **keywords_r); + array_t *keyword_idx); /* Returns the UID for given message. May be slightly faster than mail_index_lookup()->uid. */ int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq, @@ -291,13 +298,24 @@ enum modify_type modify_type, enum mail_flags flags); -/* Return a NULL-terminated list of all existing keywords. */ -const char *const *mail_index_get_keywords(struct mail_index *index); +/* Lookup a keyword, returns TRUE if found, FALSE if not. If autocreate is + TRUE, the keyword is automatically created and TRUE is always returned. */ +int mail_index_keyword_lookup(struct mail_index *index, + const char *keyword, int autocreate, + unsigned int *idx_r); +/* Return a pointer to array of NULL-terminated list of keywords. Note that + the array contents (and thus pointers inside it) may change after calling + mail_index_keywords_create() or mail_index_sync_begin(). */ +const array_t *mail_index_get_keywords(struct mail_index *index); + /* Create a keyword list structure. It's freed automatically at the end of the transaction. */ struct mail_keywords * mail_index_keywords_create(struct mail_index_transaction *t, const char *const keywords[]); +struct mail_keywords * +mail_index_keywords_create_from_indexes(struct mail_index_transaction *t, + const array_t *keyword_indexes); /* Free the keywords. */ void mail_index_keywords_free(struct mail_keywords *keywords); /* Update keywords for given message. */ @@ -317,15 +335,14 @@ /* Reset the error message. */ void mail_index_reset_error(struct mail_index *index); -/* Return a pointer to NULL-terminated list of keywords which are referenced - in mail_index_sync_rec->keyword_idx. Note tat the pointer may change after - calling mail_index_keywords_create(). */ -const char *const *const * -mail_index_sync_get_keywords(struct mail_index_sync_ctx *ctx); /* Apply changes in MAIL_INDEX_SYNC_TYPE_FLAGS typed sync records to given - flags variables. */ + flags variable. */ void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec, uint8_t *flags); +/* Apply changes in MAIL_INDEX_SYNC_TYPE_KEYWORD_* typed sync records to given + keywords array. Returns TRUE If something was changed. */ +int mail_index_sync_keywords_apply(const struct mail_index_sync_rec *sync_rec, + array_t *keywords); /* register index extension. name is a unique identifier for the extension. returns unique identifier for the name. */
--- a/src/lib-index/mail-transaction-log-append.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-index/mail-transaction-log-append.c Sun Apr 03 00:08:56 2005 +0300 @@ -272,37 +272,56 @@ return 0; } +static int +log_append_keyword_update(struct mail_transaction_log_file *file, + struct mail_index_transaction *t, + buffer_t *hdr_buf, enum modify_type modify_type, + const char *keyword, const buffer_t *buffer) +{ + struct mail_transaction_keyword_update kt_hdr; + + memset(&kt_hdr, 0, sizeof(kt_hdr)); + kt_hdr.modify_type = modify_type; + kt_hdr.name_size = strlen(keyword); + + buffer_set_used_size(hdr_buf, 0); + buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr)); + buffer_append(hdr_buf, keyword, kt_hdr.name_size); + if ((hdr_buf->used % 4) != 0) + buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4)); + + return log_append_buffer(file, buffer, hdr_buf, + MAIL_TRANSACTION_KEYWORD_UPDATE, t->external); +} + static int log_append_keyword_updates(struct mail_transaction_log_file *file, struct mail_index_transaction *t) { - struct mail_index *index = t->view->index; - struct mail_transaction_keyword_update kt_hdr; + const struct mail_index_transaction_keyword_update *updates; + const char *const *keywords; buffer_t *hdr_buf; - array_t *updates; - unsigned int i, count; + unsigned int i, count, keywords_count; hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64); + keywords = array_get_modifyable(&t->view->index->keywords, + &keywords_count); updates = array_get_modifyable(&t->keyword_updates, &count); - for (i = 0; i < count; i++) { - if (!array_is_created(&updates[i])) - continue; - - buffer_set_used_size(hdr_buf, 0); + i_assert(count <= keywords_count); - memset(&kt_hdr, 0, sizeof(kt_hdr)); - kt_hdr.modify_type = (i & 1) == 0 ? MODIFY_ADD : MODIFY_REMOVE; - kt_hdr.name_size = strlen(index->keywords[i / 2]); - buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr)); - buffer_append(hdr_buf, index->keywords[i / 2], - kt_hdr.name_size); - if ((hdr_buf->used % 4) != 0) - buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4)); - - if (log_append_buffer(file, updates[i].buffer, hdr_buf, - MAIL_TRANSACTION_KEYWORD_UPDATE, - t->external) < 0) - return -1; + for (i = 0; i < count; i++) { + if (array_is_created(&updates[i].add_seq)) { + if (log_append_keyword_update(file, t, hdr_buf, + MODIFY_ADD, keywords[i], + updates[i].add_seq.buffer) < 0) + return -1; + } + if (array_is_created(&updates[i].remove_seq)) { + if (log_append_keyword_update(file, t, hdr_buf, + MODIFY_REMOVE, keywords[i], + updates[i].remove_seq.buffer) < 0) + return -1; + } } return 0;
--- a/src/lib-storage/index/index-mail.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/index-mail.c Sun Apr 03 00:08:56 2005 +0300 @@ -170,20 +170,41 @@ { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; - const char *const *keywords; + array_t ARRAY_DEFINE(keyword_indexes_arr, unsigned int); + const char *const *names; + const unsigned int *keyword_indexes; + unsigned int i, count, names_count; + + if (array_is_created(&data->keywords)) + return array_get(&data->keywords, NULL); - if (data->keywords_buf == NULL) { - data->keywords_buf = - buffer_create_dynamic(mail->data_pool, 128); - } - + t_push(); + ARRAY_CREATE(&keyword_indexes_arr, pool_datastack_create(), + unsigned int, 128); if (mail_index_lookup_keywords(mail->ibox->view, mail->data.seq, - data->keywords_buf, &keywords) < 0) { + &keyword_indexes_arr) < 0) { mail_storage_set_index_error(mail->ibox); + t_pop(); return NULL; } - return keywords; + keyword_indexes = array_get(&keyword_indexes_arr, &count); + names = array_get(mail->ibox->keyword_names, &names_count); + + ARRAY_CREATE(&data->keywords, mail->data_pool, const char *, count); + for (i = 0; i < count; i++) { + const char *name; + i_assert(keyword_indexes[i] < names_count); + + name = names[keyword_indexes[i]]; + array_append(&data->keywords, &name, 1); + } + + /* end with NULL */ + (void)array_modifyable_append(&data->keywords); + + t_pop(); + return array_get(&data->keywords, NULL); } const struct message_part *index_mail_get_parts(struct mail *_mail)
--- a/src/lib-storage/index/index-mail.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/index-mail.h Sun Apr 03 00:08:56 2005 +0300 @@ -53,7 +53,6 @@ struct index_mail_data { enum mail_flags flags; - const char *const *keywords; time_t date, received_date; uoff_t virtual_size, physical_size; @@ -73,7 +72,7 @@ struct message_size hdr_size, body_size; struct message_parser_ctx *parser_ctx; int parsing_count; - buffer_t *keywords_buf; + array_t ARRAY_DEFINE(keywords, const char *); unsigned int parse_header:1; unsigned int save_envelope:1;
--- a/src/lib-storage/index/index-status.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/index-status.c Sun Apr 03 00:08:56 2005 +0300 @@ -29,10 +29,8 @@ } } - if (items & STATUS_KEYWORDS) { + if (items & STATUS_KEYWORDS) status_r->keywords = mail_index_get_keywords(ibox->index); - status_r->keywords_count = strarray_length(status_r->keywords); - } return 0; }
--- a/src/lib-storage/index/index-storage.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/index-storage.c Sun Apr 03 00:08:56 2005 +0300 @@ -332,6 +332,7 @@ ibox->cache = mail_index_get_cache(index); index_cache_register_defaults(ibox); ibox->view = mail_index_view_open(index); + ibox->keyword_names = mail_index_get_keywords(index); return 0; } while (0);
--- a/src/lib-storage/index/index-storage.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/index-storage.h Sun Apr 03 00:08:56 2005 +0300 @@ -62,6 +62,7 @@ uint32_t commit_log_file_seq; uoff_t commit_log_file_offset; + const array_t *ARRAY_DEFINE_PTR(keyword_names, const char *); struct mail_cache_field *cache_fields; buffer_t *recent_flags; uint32_t recent_flags_start_seq, recent_flags_count;
--- a/src/lib-storage/index/mbox/mbox-save.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/mbox/mbox-save.c Sun Apr 03 00:08:56 2005 +0300 @@ -232,13 +232,21 @@ { unsigned char space[MBOX_HEADER_PADDING+1 + sizeof("Content-Length: \n")-1 + MAX_INT_STRLEN]; - unsigned int i; + const array_t *keyword_names_list; + ARRAY_SET_TYPE(keyword_names_list, const char *); + const char *const *keyword_names; + unsigned int i, keyword_names_count; + + keyword_names_list = mail_index_get_keywords(ctx->ibox->index); + keyword_names = array_get(keyword_names_list, &keyword_names_count); str_append(ctx->headers, "X-Keywords:"); - /*FIXME:for (i = 0; i < count; i++) { + for (i = 0; i < keywords->count; i++) { + i_assert(keywords->idx[i] < keyword_names_count); + str_append_c(ctx->headers, ' '); - str_append(ctx->headers, keywords[i]); - }*/ + str_append(ctx->headers, keyword_names[keywords->idx[i]]); + } memset(space, ' ', sizeof(space)); str_append_n(ctx->headers, space, sizeof(space));
--- a/src/lib-storage/index/mbox/mbox-sync-parse.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c Sun Apr 03 00:08:56 2005 +0300 @@ -107,6 +107,46 @@ return TRUE; } +static void +parse_imap_keywords_list(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr, size_t pos) +{ + const char *keyword; + size_t keyword_start; + unsigned int idx, count; + + count = 0; + while (pos < hdr->full_value_len) { + if (IS_LWSP_LF(hdr->full_value[pos])) { + pos++; + continue; + } + + /* read the keyword */ + keyword_start = pos; + for (; pos < hdr->full_value_len; pos++) { + if (IS_LWSP_LF(hdr->full_value[pos])) + break; + } + + /* add it to index's keyword list if it's not there already */ + t_push(); + keyword = t_strndup(hdr->full_value + keyword_start, + pos - keyword_start); + (void)mail_index_keyword_lookup(ctx->sync_ctx->ibox->index, + keyword, TRUE, &idx); + t_pop(); + + count++; + } + + if (count != array_count(ctx->sync_ctx->ibox->keyword_names)) { + /* need to update this list */ + ctx->update_imapbase_keywords = TRUE; + ctx->need_rewrite = TRUE; + } +} + static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, struct message_header_line *hdr) { @@ -128,17 +168,16 @@ pos = end - str; t_pop(); - while (pos < hdr->full_value_len && IS_LWSP_LF(hdr->full_value[pos])) - pos++; - if (uid_validity == 0) { /* broken */ return FALSE; } if (ctx->sync_ctx->base_uid_validity == 0) { + /* first time parsing this. save the values. */ ctx->sync_ctx->base_uid_validity = uid_validity; ctx->sync_ctx->base_uid_last = uid_last; + if (ctx->sync_ctx->next_uid-1 <= uid_last) ctx->sync_ctx->next_uid = uid_last+1; else { @@ -156,12 +195,8 @@ ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); ctx->seen_imapbase = TRUE; - if (pos == hdr->full_value_len) - return TRUE; - - // FIXME: save keywords - - parse_trailing_whitespace(ctx, hdr); + parse_imap_keywords_list(ctx, hdr, pos); + parse_trailing_whitespace(ctx, hdr); return TRUE; } @@ -180,10 +215,73 @@ static int parse_x_keywords(struct mbox_sync_mail_context *ctx, struct message_header_line *hdr) { - // FIXME: parse them + array_t ARRAY_DEFINE(keyword_list, unsigned int); + string_t *keyword; + size_t keyword_start; + unsigned int idx; + size_t pos; + + if (array_is_created(&ctx->mail.keywords)) + return FALSE; /* duplicate header, delete */ + + /* read keyword indexes to temporary array first */ + t_push(); + keyword = t_str_new(128); + ARRAY_CREATE(&keyword_list, pool_datastack_create(), unsigned int, 16); + + for (pos = 0; pos < hdr->full_value_len; ) { + if (IS_LWSP_LF(hdr->full_value[pos])) { + pos++; + continue; + } + + /* read the keyword string */ + keyword_start = pos; + for (; pos < hdr->full_value_len; pos++) { + if (IS_LWSP_LF(hdr->full_value[pos])) + break; + } + + str_truncate(keyword, 0); + str_append_n(keyword, hdr->full_value + keyword_start, + pos - keyword_start); + if (!mail_index_keyword_lookup(ctx->sync_ctx->ibox->index, + str_c(keyword), FALSE, &idx)) { + if (ctx->sync_ctx->ibox->mbox_sync_dirty && + !ctx->sync_ctx->dest_first_mail && + !ctx->sync_ctx->seen_first_mail) { + /* we'll have to read the X-IMAP header to + make sure we have the latest list of + keywords */ + i_assert(!ctx->sync_ctx->sync_restart); + ctx->sync_ctx->sync_restart = TRUE; + t_pop(); + return FALSE; + } + + /* index is fully up-to-date and the keyword wasn't + found. that means the sent mail originally + contained X-Keywords header. Delete it. */ + t_pop(); + return FALSE; + } + + array_append(&keyword_list, &idx, 1); + } + + /* once we know how many keywords there are, we can allocate the array + from mail_keyword_pool without wasting memory. */ + if (array_count(&keyword_list) > 0) { + ARRAY_CREATE(&ctx->mail.keywords, + ctx->sync_ctx->mail_keyword_pool, + unsigned int, array_count(&keyword_list)); + array_append_array(&ctx->mail.keywords, &keyword_list); + } ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); parse_trailing_whitespace(ctx, hdr); + + t_pop(); return TRUE; }
--- a/src/lib-storage/index/mbox/mbox-sync-private.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-private.h Sun Apr 03 00:08:56 2005 +0300 @@ -42,7 +42,7 @@ uint32_t uid; uint32_t idx_seq; uint8_t flags; - uint32_t keywords_idx; /* +1 */ + array_t ARRAY_DEFINE(keywords, unsigned int); uoff_t from_offset; uoff_t body_size; @@ -85,6 +85,7 @@ unsigned int recent:1; unsigned int dirty:1; unsigned int uid_broken:1; + unsigned int update_imapbase_keywords:1; }; struct mbox_sync_context { @@ -109,6 +110,8 @@ array_t ARRAY_DEFINE(syncs, struct mail_index_sync_rec); struct mail_index_sync_rec sync_rec; + pool_t mail_keyword_pool; + uint32_t prev_msg_uid, next_uid; uint32_t seq, idx_seq, need_space_seq; off_t expunged_space, space_diff; @@ -129,8 +132,7 @@ int mbox_sync_parse_match_mail(struct index_mailbox *ibox, struct mail_index_view *view, uint32_t seq); -void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, - array_t *syncs_arr); +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx); void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, const struct mbox_sync_mail *mail); int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff); @@ -138,11 +140,15 @@ uoff_t end_offset, off_t move_diff, uoff_t extra_space, uint32_t first_seq, uint32_t last_seq); -void mbox_sync_apply_index_syncs(array_t *syncs_arr, uint8_t *flags); +void mbox_sync_apply_index_syncs(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail *mail, + int *keywords_changed_r); int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset); int mbox_move(struct mbox_sync_context *sync_ctx, uoff_t dest, uoff_t source, uoff_t size); void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx, size_t pos, size_t need, size_t have); +void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size); #endif
--- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c Sun Apr 03 00:08:56 2005 +0300 @@ -73,8 +73,8 @@ return 0; } -static void -mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, size_t size) +void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size) { size_t data_size, pos, start_pos; const unsigned char *data; @@ -322,9 +322,13 @@ } mbox_sync_parse_next_mail(sync_ctx->input, &mail_ctx); - if (mails[idx].space != 0) + if (mails[idx].space != 0) { + if (mails[idx].space < 0) { + /* remove all possible spacing before updating */ + mbox_sync_headers_remove_space(&mail_ctx, (size_t)-1); + } mbox_sync_update_header_from(&mail_ctx, &mails[idx]); - else { + } else { /* updating might just try to add headers and mess up our calculations completely. so only add the EOH here. */ if (mail_ctx.have_eoh)
--- a/src/lib-storage/index/mbox/mbox-sync-update.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-update.c Sun Apr 03 00:08:56 2005 +0300 @@ -103,6 +103,68 @@ } } +static void keywords_append(struct mbox_sync_context *sync_ctx, string_t *dest, + const array_t *keyword_indexes_arr) +{ + ARRAY_SET_TYPE(keyword_indexes_arr, unsigned int); + const char *const *keyword_names; + const unsigned int *keyword_indexes; + unsigned int i, idx_count, keywords_count; + size_t last_break; + + keyword_names = array_get(sync_ctx->ibox->keyword_names, + &keywords_count); + keyword_indexes = array_get(keyword_indexes_arr, &idx_count); + + for (i = 0, last_break = 0; i < idx_count; i++) { + i_assert(keyword_indexes[i] < keywords_count); + + /* try avoid overly long lines but cutting them + every 70 chars or so */ + if (str_len(dest) - last_break < 70) { + if (i > 0) + str_append_c(dest, ' '); + } else { + str_append(dest, "\n\t"); + last_break = str_len(dest); + } + str_append(dest, keyword_names[keyword_indexes[i]]); + } +} + +static void +keywords_append_all(struct mbox_sync_mail_context *ctx, string_t *dest) +{ + const char *const *names; + const unsigned char *p; + unsigned int i, count; + size_t last_break; + + p = str_data(dest); + if (str_len(dest) < 70) + last_break = 0; + else { + /* set last_break to beginning of line */ + for (last_break = str_len(dest); last_break > 0; last_break--) { + if (p[last_break-1] == '\n') + break; + } + } + + names = array_get(ctx->sync_ctx->ibox->keyword_names, &count); + for (i = 0; i < count; i++) { + /* try avoid overly long lines but cutting them + every 70 chars or so */ + if (str_len(dest) - last_break < 70) + str_append_c(dest, ' '); + else { + str_append(dest, "\n\t"); + last_break = str_len(dest); + } + str_append(dest, names[i]); + } +} + static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) { size_t old_hdr_size, new_hdr_size; @@ -130,7 +192,7 @@ str_printfa(ctx->header, "%u %010u", ctx->sync_ctx->base_uid_validity, ctx->sync_ctx->next_uid-1); - //FIXME:keywords_append(ctx, all_keywords); + keywords_append_all(ctx, ctx->header); str_append_c(ctx->header, '\n'); } @@ -157,10 +219,12 @@ } if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1 && - ctx->mail.keywords_idx != 0) { + array_is_created(&ctx->mail.keywords) && + array_count(&ctx->mail.keywords) > 0) { str_append(ctx->header, "X-Keywords: "); ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); - //keywords_append(ctx, ctx->mail.keywords); + keywords_append(ctx->sync_ctx, ctx->header, + &ctx->mail.keywords); str_append_c(ctx->header, '\n'); } @@ -196,14 +260,11 @@ } } -static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx) -{ -} - static void mbox_sync_update_line(struct mbox_sync_mail_context *ctx, size_t pos, string_t *new_line) { const char *hdr, *p; + uoff_t file_pos; if (ctx->header_first_change > pos) ctx->header_first_change = pos; @@ -213,32 +274,61 @@ if (p == NULL) { /* shouldn't really happen, but allow anyway.. */ - ctx->header_last_change = (size_t)-1; - str_truncate(ctx->header, pos); - str_append_str(ctx->header, new_line); - } else { - mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1); - buffer_copy(ctx->header, pos, new_line, 0, (size_t)-1); + p = hdr + strlen(hdr); + } + + file_pos = pos + ctx->hdr_offset; + if (ctx->mail.space > 0 && ctx->mail.offset >= file_pos && + ctx->mail.offset < file_pos + (p - hdr)) { + /* extra space points to this line. remove it. */ + ctx->mail.offset = ctx->hdr_offset; + ctx->mail.space = 0; } + + mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1); + buffer_copy(ctx->header, pos, new_line, 0, (size_t)-1); +} + +static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx) +{ + string_t *str; + + if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1) + return; + + t_push(); + str = t_str_new(256); + keywords_append(ctx->sync_ctx, str, &ctx->mail.keywords); + str_append_c(str, '\n'); + mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_KEYWORDS], str); + t_pop(); } static void mbox_sync_update_x_imap_base(struct mbox_sync_mail_context *ctx) { + struct mbox_sync_context *sync_ctx = ctx->sync_ctx; string_t *str; - if (!ctx->sync_ctx->dest_first_mail || - ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1 || - ctx->sync_ctx->update_base_uid_last == 0 || - ctx->sync_ctx->update_base_uid_last < ctx->sync_ctx->base_uid_last) + if (!sync_ctx->dest_first_mail || + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1) + return; + + if (sync_ctx->update_base_uid_last <= sync_ctx->base_uid_last) + sync_ctx->update_base_uid_last = 0; + + /* see if anything changed */ + if (!(ctx->update_imapbase_keywords || + sync_ctx->update_base_uid_last != 0)) return; /* update uid-last field in X-IMAPbase */ t_push(); str = t_str_new(200); - str_printfa(str, "%u %010u", ctx->sync_ctx->base_uid_validity, - ctx->sync_ctx->update_base_uid_last); - //FIXME:keywords_append(ctx, all_keywords); + str_printfa(str, "%u %010u", sync_ctx->base_uid_validity, + sync_ctx->update_base_uid_last != 0 ? + sync_ctx->update_base_uid_last : sync_ctx->base_uid_last); + keywords_append_all(ctx, str); str_append_c(str, '\n'); mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_IMAPBASE], str); @@ -260,30 +350,24 @@ t_pop(); } -void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, - array_t *syncs_arr) +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx) { - ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec); - const struct mail_index_sync_rec *syncs; uint8_t old_flags; - uint32_t old_keywords_idx; - unsigned int i, count; + int keywords_changed; i_assert(ctx->mail.uid != 0 || ctx->pseudo); - syncs = array_get(syncs_arr, &count); old_flags = ctx->mail.flags; - if (count != 0) { - old_keywords_idx = ctx->mail.keywords_idx; - mbox_sync_apply_index_syncs(syncs_arr, &ctx->mail.flags); + if (array_count(&ctx->sync_ctx->syncs) > 0) { + mbox_sync_apply_index_syncs(ctx->sync_ctx, &ctx->mail, + &keywords_changed); if ((old_flags & XSTATUS_FLAGS_MASK) != (ctx->mail.flags & XSTATUS_FLAGS_MASK)) mbox_sync_update_xstatus(ctx); - /*FIXME:if (memcmp(old_keywords, ctx->mail.keywords, - INDEX_KEYWORDS_BYTE_COUNT) != 0) - mbox_sync_update_xkeywords(ctx);*/ + if (keywords_changed) + mbox_sync_update_xkeywords(ctx); } if (!ctx->sync_ctx->ibox->keep_recent) @@ -317,12 +401,30 @@ (mail->flags & XSTATUS_FLAGS_MASK); mbox_sync_update_xstatus(ctx); } - /*FIXME:if (memcmp(ctx->mail.keywords, mail->keywords, - INDEX_KEYWORDS_BYTE_COUNT) != 0) { - memcpy(ctx->mail.keywords, mail->keywords, - INDEX_KEYWORDS_BYTE_COUNT); + if (!array_is_created(&mail->keywords) || + array_count(&mail->keywords) == 0) { + /* no keywords for this mail */ + if (array_is_created(&ctx->mail.keywords)) { + array_clear(&ctx->mail.keywords); + mbox_sync_update_xkeywords(ctx); + } + } else if (!array_is_created(&ctx->mail.keywords)) { + /* adding first keywords */ + ARRAY_CREATE(&ctx->mail.keywords, + ctx->sync_ctx->mail_keyword_pool, + unsigned int, + array_count(&mail->keywords)); + array_append_array(&ctx->mail.keywords, + &mail->keywords); mbox_sync_update_xkeywords(ctx); - }*/ + } else if (!buffer_cmp(ctx->mail.keywords.buffer, + mail->keywords.buffer)) { + /* keywords changed. */ + array_clear(&ctx->mail.keywords); + array_append_array(&ctx->mail.keywords, + &mail->keywords); + mbox_sync_update_xkeywords(ctx); + } i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid); ctx->mail.uid = mail->uid;
--- a/src/lib-storage/index/mbox/mbox-sync.c Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync.c Sun Apr 03 00:08:56 2005 +0300 @@ -205,16 +205,43 @@ return 0; } -void mbox_sync_apply_index_syncs(array_t *syncs_arr, uint8_t *flags) +void mbox_sync_apply_index_syncs(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail *mail, + int *keywords_changed_r) { - ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec); const struct mail_index_sync_rec *syncs; unsigned int i, count; - syncs = array_get(syncs_arr, &count); + *keywords_changed_r = FALSE; + + syncs = array_get(&sync_ctx->syncs, &count); for (i = 0; i < count; i++) { - if (syncs[i].type == MAIL_INDEX_SYNC_TYPE_FLAGS) - mail_index_sync_flags_apply(&syncs[i], flags); + switch (syncs[i].type) { + case MAIL_INDEX_SYNC_TYPE_FLAGS: + mail_index_sync_flags_apply(&syncs[i], &mail->flags); + break; + case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET: + if (!array_is_created(&mail->keywords)) { + /* no existing keywords */ + if (syncs[i].type != + MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD) + break; + + /* adding, create the array */ + ARRAY_CREATE(&mail->keywords, + sync_ctx->mail_keyword_pool, + unsigned int, + I_MIN(10, count - i)); + } + if (mail_index_sync_keywords_apply(&syncs[i], + &mail->keywords)) + *keywords_changed_r = TRUE; + break; + default: + break; + } } } @@ -335,12 +362,27 @@ return 0; } -static int mbox_sync_update_index(struct mbox_sync_context *sync_ctx, - struct mbox_sync_mail_context *mail_ctx, +static void +mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mail_keywords *keywords; + + keywords = !array_is_created(&mail_ctx->mail.keywords) ? + mail_index_keywords_create(sync_ctx->t, NULL) : + mail_index_keywords_create_from_indexes(sync_ctx->t, + &mail_ctx->mail.keywords); + mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, keywords); + mail_index_keywords_free(keywords); +} + +static int mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx, const struct mail_index_record *rec) { + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; struct mbox_sync_mail *mail = &mail_ctx->mail; - uint8_t idx_flags, mbox_flags; + uint8_t mbox_flags; mbox_flags = mail->flags & MAIL_FLAGS_MASK; @@ -354,6 +396,7 @@ mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq); mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, MODIFY_REPLACE, mbox_flags); + mbox_sync_update_index_keywords(mail_ctx); if (sync_ctx->ibox->mbox_save_md5 != 0) { mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, @@ -364,24 +407,41 @@ /* see if we need to update flags in index file. the flags in sync records are automatically applied to rec->flags at the end of index syncing, so calculate those new flags first */ - idx_flags = rec->flags; - mbox_sync_apply_index_syncs(&sync_ctx->syncs, &idx_flags); + struct mbox_sync_mail idx_mail; + int keywords_changed; + + memset(&idx_mail, 0, sizeof(idx_mail)); + idx_mail.flags = rec->flags; - if ((idx_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* get old keywords */ + t_push(); + ARRAY_CREATE(&idx_mail.keywords, pool_datastack_create(), + unsigned int, 32); + if (mail_index_lookup_keywords(sync_ctx->sync_view, + sync_ctx->idx_seq, + &idx_mail.keywords) < 0) { + mail_storage_set_index_error(sync_ctx->ibox); + t_pop(); + return -1; + } + mbox_sync_apply_index_syncs(sync_ctx, &idx_mail, + &keywords_changed); + + if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { /* flags are dirty. ignore whatever was in the mbox, but update recent flag state if needed. */ mbox_flags &= MAIL_RECENT; - mbox_flags |= idx_flags & ~MAIL_RECENT; + mbox_flags |= idx_mail.flags & ~MAIL_RECENT; } else { /* keep index's internal flags */ mbox_flags &= MAIL_FLAGS_MASK; - mbox_flags |= idx_flags & ~MAIL_FLAGS_MASK; + mbox_flags |= idx_mail.flags & ~MAIL_FLAGS_MASK; } - if ((idx_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) == + if ((idx_mail.flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) == (mbox_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY)) { /* all flags are same, except possibly dirty flag */ - if (idx_flags != mbox_flags) { + if (idx_mail.flags != mbox_flags) { /* dirty flag state changed */ int dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0; @@ -390,18 +450,21 @@ dirty ? MODIFY_ADD : MODIFY_REMOVE, MAIL_INDEX_MAIL_FLAG_DIRTY); } - } else if ((idx_flags & ~MAIL_RECENT) != + } else if ((idx_mail.flags & ~MAIL_RECENT) != (mbox_flags & ~MAIL_RECENT)) { /* flags other than MAIL_RECENT have changed */ mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, MODIFY_REPLACE, mbox_flags); - } else if (((idx_flags ^ mbox_flags) & MAIL_RECENT) != 0) { + } else if (((idx_mail.flags ^ mbox_flags) & MAIL_RECENT) != 0) { /* drop recent flag */ mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, MODIFY_REMOVE, MAIL_RECENT); } - // FIXME: keywords + if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 && + !array_cmp(&idx_mail.keywords, &mail_ctx->mail.keywords)) + mbox_sync_update_index_keywords(mail_ctx); + t_pop(); } if (mail_ctx->recent && @@ -523,7 +586,7 @@ if (mbox_read_from_line(mail_ctx) < 0) return -1; - mbox_sync_update_header(mail_ctx, &sync_ctx->syncs); + mbox_sync_update_header(mail_ctx); ret = mbox_sync_try_rewrite(mail_ctx, move_diff); if (ret < 0) return -1; @@ -546,7 +609,7 @@ array_count(&sync_ctx->syncs) != 0 || (mail_ctx->seq == 1 && sync_ctx->update_base_uid_last != 0)) { - mbox_sync_update_header(mail_ctx, &sync_ctx->syncs); + mbox_sync_update_header(mail_ctx); if (sync_ctx->delay_writes) { /* mark it dirty and do it later */ mail_ctx->dirty = TRUE; @@ -887,8 +950,7 @@ if (!mail_ctx->pseudo) { if (!expunged) { - if (mbox_sync_update_index(sync_ctx, mail_ctx, - rec) < 0) + if (mbox_sync_update_index(mail_ctx, rec) < 0) return -1; } sync_ctx->idx_seq++; @@ -1424,6 +1486,10 @@ sync_ctx.index_sync_ctx = index_sync_ctx; sync_ctx.sync_view = sync_view; sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE, TRUE); + sync_ctx.mail_keyword_pool = pool_alloconly_create("keywords", 4096); + + /* make sure we've read the latest keywords in index */ + (void)mail_index_get_keywords(ibox->index); ARRAY_CREATE(&sync_ctx.mails, default_pool, struct mbox_sync_mail, 64); @@ -1523,6 +1589,7 @@ ret = -1; } + pool_unref(sync_ctx.mail_keyword_pool); str_free(sync_ctx.header); str_free(sync_ctx.from_line); array_free(&sync_ctx.mails);
--- a/src/lib-storage/mail-storage.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib-storage/mail-storage.h Sun Apr 03 00:08:56 2005 +0300 @@ -153,9 +153,7 @@ unsigned int diskspace_full:1; - /* may be allocated from data stack */ - unsigned int keywords_count; - const char *const *keywords; + const array_t *ARRAY_DEFINE_PTR(keywords, const char *); }; struct mailbox_sync_rec {
--- a/src/lib/array.h Sat Apr 02 22:31:26 2005 +0300 +++ b/src/lib/array.h Sun Apr 03 00:08:56 2005 +0300 @@ -281,4 +281,16 @@ return array->buffer->used / array->element_size; } +static inline int +array_cmp(const array_t *array1, const array_t *array2) +{ + if (!array_is_created(array1) || array1->buffer->used == 0) + return !array_is_created(array2) || array2->buffer->used == 0; + + if (!array_is_created(array2)) + return FALSE; + + return buffer_cmp(array1->buffer, array2->buffer); +} + #endif