Mercurial > dovecot > core-2.2

changeset 12990:c9b7e829c6a9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
pop3: Added support for showing messages in "pop3 order".
author Timo Sirainen <tss@iki.fi>
date Wed, 04 May 2011 11:43:59 +0200
parents fac2d4fe86b1
children d0f54521bb3b
files src/pop3/pop3-client.c src/pop3/pop3-client.h src/pop3/pop3-commands.c
diffstat 3 files changed, 148 insertions(+), 58 deletions(-) [+]
line wrap: on
line diff
--- a/src/pop3/pop3-client.c	Wed May 04 11:43:16 2011 +0200
+++ b/src/pop3/pop3-client.c	Wed May 04 11:43:59 2011 +0200
@@ -39,6 +39,11 @@
 struct client *pop3_clients;
 unsigned int pop3_client_count;
 
+static enum mail_sort_type pop3_sort_program[] = {
+	MAIL_SORT_POP3_ORDER,
+	MAIL_SORT_END
+};
+
 static void client_input(struct client *client);
 static int client_output(struct client *client);
 
@@ -103,6 +108,28 @@
 	return mail_get_virtual_size(mail, size_r);
 }
 
+static void
+msgnum_to_seq_map_add(ARRAY_TYPE(uint32_t) *msgnum_to_seq_map,
+		      struct client *client, struct mail *mail,
+		      unsigned int msgnum)
+{
+	uint32_t seq;
+
+	if (mail->seq == msgnum+1)
+		return;
+
+	if (!array_is_created(msgnum_to_seq_map))
+		i_array_init(msgnum_to_seq_map, client->messages_count);
+	else {
+		/* add any messages between this and the previous one that had
+		   a POP3 order defined */
+		seq = array_count(msgnum_to_seq_map) + 1;
+		for (; seq <= msgnum; seq++)
+			array_append(msgnum_to_seq_map, &seq, 1);
+	}
+	array_append(msgnum_to_seq_map, &mail->seq, 1);
+}
+
 static int read_mailbox(struct client *client, uint32_t *failed_uid_r)
 {
         struct mailbox_status status;
@@ -112,6 +139,8 @@
 	struct mail *mail;
 	uoff_t size;
 	ARRAY_DEFINE(message_sizes, uoff_t);
+	ARRAY_TYPE(uint32_t) msgnum_to_seq_map = ARRAY_INIT;
+	unsigned int msgnum;
 	int ret = 1;
 
 	*failed_uid_r = 0;
@@ -124,13 +153,14 @@
 
 	search_args = mail_search_build_init();
 	mail_search_build_add_all(search_args);
-	ctx = mailbox_search_init(t, search_args, NULL);
+	ctx = mailbox_search_init(t, search_args, pop3_sort_program);
 	mail_search_args_unref(&search_args);
 
-	client->last_seen = 0;
+	client->last_seen_pop3_msn = 0;
 	client->total_size = 0;
 	i_array_init(&message_sizes, client->messages_count);
 
+	msgnum = 0;
 	mail = mail_alloc(t, MAIL_FETCH_VIRTUAL_SIZE, NULL);
 	while (mailbox_search_next(ctx, mail)) {
 		if (pop3_mail_get_size(client, mail, &size) < 0) {
@@ -138,29 +168,40 @@
 			*failed_uid_r = mail->uid;
 			break;
 		}
+		msgnum_to_seq_map_add(&msgnum_to_seq_map, client, mail, msgnum);
 
 		if ((mail_get_flags(mail) & MAIL_SEEN) != 0)
-			client->last_seen = mail->seq;
+			client->last_seen_pop3_msn = msgnum + 1;
 		client->total_size += size;
 
 		array_append(&message_sizes, &size, 1);
+		msgnum++;
 	}
 	mail_free(&mail);
 
 	if (mailbox_search_deinit(&ctx) < 0)
 		ret = -1;
 
-	if (ret > 0) {
-		client->trans = t;
-		client->message_sizes =
-			buffer_free_without_data(&message_sizes.arr.buffer);
-	} else {
+	if (ret <= 0) {
 		/* commit the transaction instead of rollbacking to make sure
 		   we don't lose data (virtual sizes) added to cache file */
 		(void)mailbox_transaction_commit(&t);
 		array_free(&message_sizes);
+		if (array_is_created(&msgnum_to_seq_map))
+			array_free(&msgnum_to_seq_map);
+		return ret;
 	}
-	return ret;
+
+	client->trans = t;
+	client->message_sizes =
+		buffer_free_without_data(&message_sizes.arr.buffer);
+	if (array_is_created(&msgnum_to_seq_map)) {
+		client->msgnum_to_seq_map_count =
+			array_count(&msgnum_to_seq_map);
+		client->msgnum_to_seq_map =
+			buffer_free_without_data(&msgnum_to_seq_map.arr.buffer);
+	}
+	return 1;
 }
 
 static int init_mailbox(struct client *client, const char **error_r)
@@ -349,8 +390,8 @@
 	}
 
 	/* 1..new-1 were probably left to mailbox by previous POP3 session */
-	old_msg_count = client->lowest_retr > 0 ?
-		client->lowest_retr - 1 : client->messages_count;
+	old_msg_count = client->lowest_retr_pop3_msn > 0 ?
+		client->lowest_retr_pop3_msn - 1 : client->messages_count;
 	for (i = 0, old_hash = 0; i < old_msg_count; i++)
 		old_hash ^= client->message_uidl_hashes[i];
 
@@ -464,6 +505,7 @@
 	i_free(client->message_uidl_hashes);
 	i_free(client->deleted_bitmask);
 	i_free(client->seen_bitmask);
+	i_free(client->msgnum_to_seq_map);
 
 	if (client->io != NULL)
 		io_remove(&client->io);
--- a/src/pop3/pop3-client.h	Wed May 04 11:43:16 2011 +0200
+++ b/src/pop3/pop3-client.h	Wed May 04 11:43:59 2011 +0200
@@ -9,6 +9,12 @@
 #define MSGS_BITMASK_SIZE(client) \
 	(((client)->messages_count + (CHAR_BIT-1)) / CHAR_BIT)
 
+/*
+   pop3_msn = 1..n in the POP3 protocol
+   msgnum = 0..n-1 = pop3_msn-1
+   seq = 1..n = mail's sequence number in lib-storage. when pop3 sort ordering
+     is used, msgnum_to_seq_map[] can be used for translation.
+*/
 struct client {
 	struct client *prev, *next;
 
@@ -34,17 +40,23 @@
 	unsigned int uid_validity;
 	unsigned int messages_count;
 	unsigned int deleted_count, expunged_count, seen_change_count;
-	uint32_t *message_uidl_hashes;
-	uoff_t *message_sizes;
 	uoff_t total_size;
 	uoff_t deleted_size;
-	uint32_t last_seen, lowest_retr;
+	uint32_t last_seen_pop3_msn, lowest_retr_pop3_msn;
+
+	/* [msgnum] contains mail seq. anything after it has seq = msgnum+1 */
+	uint32_t *msgnum_to_seq_map;
+	uint32_t msgnum_to_seq_map_count;
 
 	uoff_t top_bytes;
 	uoff_t retr_bytes;
 	unsigned int top_count;
 	unsigned int retr_count;
 
+	/* [msgnum] */
+	uint32_t *message_uidl_hashes;
+	uoff_t *message_sizes;
+	/* [msgnum/8] & msgnum%8 */
 	unsigned char *deleted_bitmask;
 	unsigned char *seen_bitmask;
 
--- a/src/pop3/pop3-commands.c	Wed May 04 11:43:16 2011 +0200
+++ b/src/pop3/pop3-commands.c	Wed May 04 11:43:59 2011 +0200
@@ -14,6 +14,17 @@
 #include "pop3-capability.h"
 #include "pop3-commands.h"
 
+static enum mail_sort_type pop3_sort_program[] = {
+	MAIL_SORT_POP3_ORDER,
+	MAIL_SORT_END
+};
+
+static uint32_t msgnum_to_seq(struct client *client, uint32_t msgnum)
+{
+	return msgnum < client->msgnum_to_seq_map_count ?
+		client->msgnum_to_seq_map[msgnum] : msgnum+1;
+}
+
 static const char *get_msgnum(struct client *client, const char *args,
 			      unsigned int *msgnum)
 {
@@ -132,8 +143,8 @@
 			    (1 << (ctx->msgnum % CHAR_BIT)))
 				continue;
 		}
-		ret = client_send_line(client, "%u %"PRIuUOFF_T,
-				       ctx->msgnum+1,
+
+		ret = client_send_line(client, "%u %"PRIuUOFF_T, ctx->msgnum+1,
 				       client->message_sizes[ctx->msgnum]);
 		if (ret < 0)
 			break;
@@ -172,7 +183,7 @@
 
 static int cmd_last(struct client *client, const char *args ATTR_UNUSED)
 {
-	client_send_line(client, "+OK %u", client->last_seen);
+	client_send_line(client, "+OK %u", client->last_seen_pop3_msn);
 	return 1;
 }
 
@@ -197,12 +208,28 @@
 	return search_args;
 }
 
+static int client_verify_ordering(struct client *client,
+				  struct mail *mail, uint32_t msgnum)
+{
+	uint32_t seq;
+
+	seq = msgnum_to_seq(client, msgnum);
+	if (seq != mail->seq) {
+		i_error("Message ordering changed unexpectedly "
+			"(msg #%u: storage seq %u -> %u)",
+			msgnum+1, seq, mail->seq);
+		return -1;
+	}
+	return 0;
+}
+
 bool client_update_mails(struct client *client)
 {
 	struct mail_search_args *search_args;
 	struct mail_search_context *ctx;
 	struct mail *mail;
-	uint32_t idx, bit;
+	uint32_t msgnum, bit;
+	bool ret = TRUE;
 
 	if (mailbox_is_readonly(client->mailbox)) {
 		/* silently ignore */
@@ -210,26 +237,35 @@
 	}
 
 	search_args = pop3_search_build(client, 0);
-	ctx = mailbox_search_init(client->trans, search_args, NULL);
+	ctx = mailbox_search_init(client->trans, search_args,
+				  pop3_sort_program);
 	mail_search_args_unref(&search_args);
 
+	msgnum = 0;
 	mail = mail_alloc(client->trans, 0, NULL);
 	while (mailbox_search_next(ctx, mail)) {
-		idx = mail->seq - 1;
-		bit = 1 << (idx % CHAR_BIT);
+		if (client_verify_ordering(client, mail, msgnum) < 0) {
+			ret = FALSE;
+			break;
+		}
+
+		bit = 1 << (msgnum % CHAR_BIT);
 		if (client->deleted_bitmask != NULL &&
-		    (client->deleted_bitmask[idx / CHAR_BIT] & bit) != 0) {
+		    (client->deleted_bitmask[msgnum / CHAR_BIT] & bit) != 0) {
 			mail_expunge(mail);
 			client->expunged_count++;
 		} else if (client->seen_bitmask != NULL &&
-			   (client->seen_bitmask[idx / CHAR_BIT] & bit) != 0) {
+			   (client->seen_bitmask[msgnum / CHAR_BIT] & bit) != 0) {
 			mail_update_flags(mail, MODIFY_ADD, MAIL_SEEN);
 		}
+		msgnum++;
 	}
 	mail_free(&mail);
 
 	client->seen_change_count = 0;
-	return mailbox_search_deinit(&ctx) == 0;
+	if (mailbox_search_deinit(&ctx) < 0)
+		ret = FALSE;
+	return ret;
 }
 
 static int cmd_quit(struct client *client, const char *args ATTR_UNUSED)
@@ -260,7 +296,6 @@
 }
 
 struct fetch_context {
-	struct mail_search_context *search_ctx;
 	struct mail *mail;
 	struct istream *stream;
 	uoff_t body_lines;
@@ -274,7 +309,6 @@
 
 static void fetch_deinit(struct fetch_context *ctx)
 {
-	(void)mailbox_search_deinit(&ctx->search_ctx);
 	mail_free(&ctx->mail);
 	i_free(ctx);
 }
@@ -393,23 +427,17 @@
 		 uoff_t *byte_counter)
 {
         struct fetch_context *ctx;
-	struct mail_search_args *search_args;
 	int ret;
 
-	search_args = pop3_search_build(client, msgnum+1);
-
 	ctx = i_new(struct fetch_context, 1);
-	ctx->search_ctx = mailbox_search_init(client->trans, search_args, NULL);
-	mail_search_args_unref(&search_args);
-
 	ctx->byte_counter = byte_counter;
 	ctx->byte_counter_offset = client->output->offset;
 
 	ctx->mail = mail_alloc(client->trans, MAIL_FETCH_STREAM_HEADER |
 			       MAIL_FETCH_STREAM_BODY, NULL);
+	mail_set_seq(ctx->mail, msgnum_to_seq(client, msgnum));
 
-	if (!mailbox_search_next(ctx->search_ctx, ctx->mail) ||
-	    mail_get_stream(ctx->mail, NULL, NULL, &ctx->stream) < 0) {
+	if (mail_get_stream(ctx->mail, NULL, NULL, &ctx->stream) < 0) {
 		ret = client_reply_msg_expunged(client, msgnum);
 		fetch_deinit(ctx);
 		return ret;
@@ -418,7 +446,8 @@
 	if (body_lines == (uoff_t)-1 && client->seen_bitmask != NULL) {
 		if ((mail_get_flags(ctx->mail) & MAIL_SEEN) == 0) {
 			/* mark the message seen with RETR command */
-			client->seen_bitmask[msgnum / CHAR_BIT] |= 1 << (msgnum % CHAR_BIT);
+			client->seen_bitmask[msgnum / CHAR_BIT] |=
+				1 << (msgnum % CHAR_BIT);
 			client->seen_change_count++;
 		}
 	}
@@ -445,10 +474,11 @@
 	if (get_msgnum(client, args, &msgnum) == NULL)
 		return -1;
 
-	if (client->lowest_retr > msgnum+1 || client->lowest_retr == 0)
-		client->lowest_retr = msgnum+1;
-	if (client->last_seen <= msgnum)
-		client->last_seen = msgnum+1;
+	if (client->lowest_retr_pop3_msn > msgnum+1 ||
+	    client->lowest_retr_pop3_msn == 0)
+		client->lowest_retr_pop3_msn = msgnum+1;
+	if (client->last_seen_pop3_msn < msgnum+1)
+		client->last_seen_pop3_msn = msgnum+1;
 
 	client->retr_count++;
 	return fetch(client, msgnum, (uoff_t)-1, &client->retr_bytes);
@@ -460,7 +490,7 @@
 	struct mail *mail;
 	struct mail_search_args *search_args;
 
-	client->last_seen = 0;
+	client->last_seen_pop3_msn = 0;
 
 	if (client->deleted) {
 		client->deleted = FALSE;
@@ -496,8 +526,8 @@
 
 static int cmd_stat(struct client *client, const char *args ATTR_UNUSED)
 {
-	client_send_line(client, "+OK %u %"PRIuUOFF_T, client->
-			 messages_count - client->deleted_count,
+	client_send_line(client, "+OK %u %"PRIuUOFF_T,
+			 client->messages_count - client->deleted_count,
 			 client->total_size - client->deleted_size);
 	return 1;
 }
@@ -520,7 +550,8 @@
 struct cmd_uidl_context {
 	struct mail_search_context *search_ctx;
 	struct mail *mail;
-	unsigned int message;
+	uint32_t msgnum;
+	bool list_all;
 };
 
 static bool pop3_get_uid(struct client *client, struct cmd_uidl_context *ctx,
@@ -595,7 +626,7 @@
 	memcpy(tab, static_tab, sizeof(static_tab));
 	tab[0].value = t_strdup_printf("%u", client->uid_validity);
 
-	save_hashes = client->message_uidl_hashes_save && ctx->message == 0;
+	save_hashes = client->message_uidl_hashes_save && ctx->list_all;
 	if (save_hashes && client->message_uidl_hashes == NULL) {
 		client->message_uidl_hashes =
 			i_new(uint32_t, client->messages_count);
@@ -603,32 +634,33 @@
 
 	str = t_str_new(128);
 	while (mailbox_search_next(ctx->search_ctx, ctx->mail)) {
-		uint32_t idx = ctx->mail->seq - 1;
+		uint32_t msgnum = ctx->msgnum++;
 
+		if (client_verify_ordering(client, ctx->mail, msgnum) < 0)
+			i_fatal("Can't finish POP3 UIDL command");
 		if (client->deleted) {
-			if (client->deleted_bitmask[idx / CHAR_BIT] &
-			    (1 << (idx % CHAR_BIT)))
+			if (client->deleted_bitmask[msgnum / CHAR_BIT] &
+			    (1 << (msgnum % CHAR_BIT)))
 				continue;
 		}
 		found = TRUE;
 
 		str_truncate(str, 0);
-		str_printfa(str, ctx->message == 0 ? "%u " : "+OK %u ",
-			    ctx->mail->seq);
+		str_printfa(str, ctx->list_all ? "%u " : "+OK %u ", msgnum+1);
 		uidl_pos = str_len(str);
 		if (!pop3_get_uid(client, ctx, tab, str) &&
 		    client->set->pop3_save_uidl)
 			mail_update_pop3_uidl(ctx->mail, str_c(str) + uidl_pos);
 
 		if (save_hashes) {
-			client->message_uidl_hashes[idx] =
+			client->message_uidl_hashes[msgnum] =
 				crc32_str(str_c(str) + uidl_pos);
 		}
 
 		ret = client_send_line(client, "%s", str_c(str));
 		if (ret < 0)
 			break;
-		if (ret == 0 && ctx->message == 0) {
+		if (ret == 0 && ctx->list_all) {
 			/* output is being buffered, continue when there's
 			   more space */
 			return FALSE;
@@ -643,7 +675,7 @@
 
 	if (save_hashes)
 		client->message_uidl_hashes_save = FALSE;
-	if (ctx->message == 0)
+	if (ctx->list_all)
 		client_send_line(client, ".");
 	i_free(ctx);
 	return found;
@@ -657,26 +689,27 @@
 }
 
 static struct cmd_uidl_context *
-cmd_uidl_init(struct client *client, unsigned int message)
+cmd_uidl_init(struct client *client, uint32_t seq)
 {
         struct cmd_uidl_context *ctx;
 	struct mail_search_args *search_args;
 	enum mail_fetch_field wanted_fields;
 
-	search_args = pop3_search_build(client, message);
+	search_args = pop3_search_build(client, seq);
 
 	ctx = i_new(struct cmd_uidl_context, 1);
-	ctx->message = message;
+	ctx->list_all = seq == 0;
 
 	wanted_fields = 0;
 	if ((client->uidl_keymask & UIDL_MD5) != 0)
 		wanted_fields |= MAIL_FETCH_HEADER_MD5;
 
-	ctx->search_ctx = mailbox_search_init(client->trans, search_args, NULL);
+	ctx->search_ctx = mailbox_search_init(client->trans, search_args,
+					      pop3_sort_program);
 	mail_search_args_unref(&search_args);
 
 	ctx->mail = mail_alloc(client->trans, wanted_fields, NULL);
-	if (message == 0) {
+	if (seq == 0) {
 		client->cmd = cmd_uidl_callback;
 		client->cmd_context = ctx;
 	}
@@ -686,6 +719,7 @@
 static int cmd_uidl(struct client *client, const char *args)
 {
         struct cmd_uidl_context *ctx;
+	uint32_t seq;
 
 	if (*args == '\0') {
 		client_send_line(client, "+OK");
@@ -697,7 +731,9 @@
 		if (get_msgnum(client, args, &msgnum) == NULL)
 			return -1;
 
-		ctx = cmd_uidl_init(client, msgnum+1);
+		seq = msgnum_to_seq(client, msgnum);
+		ctx = cmd_uidl_init(client, seq);
+		ctx->msgnum = msgnum;
 		if (!list_uids_iter(client, ctx))
 			return client_reply_msg_expunged(client, msgnum);
 	}