Mercurial > dovecot > original-hg > dovecot-1.2
changeset 8001:3fbfdda3e5d3 HEAD
fts-solr: Send data to Solr in larger packets. Commit after expunges.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 13 Jul 2008 15:05:52 +0300 |
parents | 61be6a4722c5 |
children | e814614ffb65 |
files | src/plugins/fts-solr/fts-backend-solr.c src/plugins/fts-solr/solr-connection.c |
diffstat | 2 files changed, 11 insertions(+), 10 deletions(-) [+] |
line wrap: on
line diff
--- a/src/plugins/fts-solr/fts-backend-solr.c Sun Jul 13 14:44:03 2008 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Sun Jul 13 15:05:52 2008 +0300 @@ -10,6 +10,8 @@ #include <stdlib.h> #include <curl/curl.h> +#define SOLR_CMDBUF_SIZE (1024*64) + struct solr_fts_backend_build_context { struct fts_backend_build_context ctx; @@ -119,7 +121,7 @@ ctx = i_new(struct solr_fts_backend_build_context, 1); ctx->ctx.backend = backend; ctx->post = solr_connection_post_begin(solr_conn); - ctx->cmd = str_new(default_pool, 256); + ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status); ctx->uid_validity = status.uidvalidity; @@ -141,7 +143,6 @@ /* body comes first, then headers */ if (ctx->prev_uid != uid) { /* uid changed */ - str_truncate(cmd, 0); if (ctx->prev_uid == 0) str_append(cmd, "<add>"); else @@ -171,20 +172,18 @@ } else { str_append(cmd, "<field name=\"body\">"); } - solr_connection_post_more(ctx->post, str_data(cmd), - str_len(cmd)); } else if (headers && !ctx->headers) { - str_truncate(cmd, 0); str_append(cmd, "</field><field name=\"hdr\">"); - solr_connection_post_more(ctx->post, str_data(cmd), - str_len(cmd)); } else { i_assert(!(!headers && ctx->headers)); } - str_truncate(cmd, 0); xml_encode_data(cmd, data, size); - solr_connection_post_more(ctx->post, str_data(cmd), str_len(cmd)); + if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) { + solr_connection_post_more(ctx->post, str_data(cmd), + str_len(cmd)); + str_truncate(cmd, 0); + } return 0; } @@ -196,7 +195,6 @@ int ret = 0; if (ctx->prev_uid != 0) { - str_truncate(ctx->cmd, 0); str_append(ctx->cmd, "</field></doc></add>"); solr_connection_post_more(ctx->post, str_data(ctx->cmd), str_len(ctx->cmd)); @@ -241,6 +239,8 @@ struct mailbox *box ATTR_UNUSED, bool committed ATTR_UNUSED) { + solr_connection_post(solr_conn, + "<commit waitFlush=\"false\" waitSearcher=\"false\"/>"); } static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
--- a/src/plugins/fts-solr/solr-connection.c Sun Jul 13 14:44:03 2008 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Sun Jul 13 15:05:52 2008 +0300 @@ -319,6 +319,7 @@ i_free_and_null(conn->http_failure); conn->xml_failed = FALSE; + XML_ParserReset(conn->xml_parser, "UTF-8"); XML_SetElementHandler(conn->xml_parser, solr_lookup_xml_start, solr_lookup_xml_end); XML_SetCharacterDataHandler(conn->xml_parser, solr_lookup_xml_data);