Mercurial > dovecot > core-2.2
changeset 18973:58f65e129b4d
fts-solr: Flush Solr indexing HTTP requests every 1000 mails.
This avoids the HTTP request from becoming too huge.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Tue, 18 Aug 2015 23:20:35 +0300 |
parents | ba4ac54e79a7 |
children | 0f442376beae |
files | src/plugins/fts-solr/fts-backend-solr.c |
diffstat | 1 files changed, 11 insertions(+), 5 deletions(-) [+] |
line wrap: on
line diff
--- a/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:20:07 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:20:35 2015 +0300 @@ -27,6 +27,8 @@ #define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024 #define SOLR_QUERY_MAX_MAILBOX_COUNT 10 +/* How often to flush indexing request to Solr before beginning a new one. */ +#define SOLR_MAIL_FLUSH_INTERVAL 1000 struct solr_fts_backend { struct fts_backend backend; @@ -51,6 +53,7 @@ ARRAY(struct solr_fts_field) fields; uint32_t last_indexed_uid; + unsigned int mails_since_flush; unsigned int tokenized_input:1; unsigned int last_indexed_uid_set:1; @@ -330,13 +333,14 @@ } static int -fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx) +fts_backed_solr_build_flush(struct solr_fts_backend_update_context *ctx) { if (ctx->post == NULL) return 0; fts_backend_solr_doc_close(ctx); str_append(ctx->cmd, "</add>"); + ctx->mails_since_flush = 0; solr_connection_post_more(ctx->post, str_data(ctx->cmd), str_len(ctx->cmd)); @@ -367,7 +371,7 @@ const char *str; int ret = _ctx->failed ? -1 : 0; - if (fts_backed_solr_build_commit(ctx) < 0) + if (fts_backed_solr_build_flush(ctx) < 0) ret = -1; if (ctx->documents_added || ctx->expunges) { @@ -405,7 +409,7 @@ if (ctx->prev_uid != 0) { /* flush solr between mailboxes, so we don't wrongly update last_uid before we know it has succeeded */ - if (fts_backed_solr_build_commit(ctx) < 0) + if (fts_backed_solr_build_flush(ctx) < 0) _ctx->failed = TRUE; else if (!_ctx->failed) fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid); @@ -466,9 +470,11 @@ struct solr_fts_backend *backend = (struct solr_fts_backend *)ctx->ctx.backend; + if (ctx->mails_since_flush++ >= SOLR_MAIL_FLUSH_INTERVAL) { + if (fts_backed_solr_build_flush(ctx) < 0) + ctx->ctx.failed = TRUE; + } if (ctx->post == NULL) { - i_assert(ctx->prev_uid == 0); - if (ctx->cmd == NULL) ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); ctx->post = solr_connection_post_begin(backend->solr_conn);