Mercurial > dovecot > core-2.2
changeset 18583:1963690280b7
lib-fts: Removed "simple" normalizer.
It translated input to titlecase, which wasn't suitable for snowball
stemming that wanted lowercase input. Since that doesn't work, there's
probably no good for the existence of this (perhaps in future it's replaced
by unicode-aware lowercaser).
| author | Timo Sirainen <tss@iki.fi> |
|---|---|
| date | Sat, 09 May 2015 14:09:37 +0300 |
| parents | b2f9e4cf17db |
| children | 75b4b312ea09 |
| files | src/lib-fts/Makefile.am src/lib-fts/fts-filter-normalizer-simple.c src/lib-fts/fts-filter.c src/lib-fts/fts-filter.h |
| diffstat | 4 files changed, 0 insertions(+), 85 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-fts/Makefile.am Sat May 09 14:19:48 2015 +0300 +++ b/src/lib-fts/Makefile.am Sat May 09 14:09:37 2015 +0300 @@ -62,7 +62,6 @@ libfts_la_SOURCES = \ fts-filter.c \ fts-filter-normalizer-icu.c \ - fts-filter-normalizer-simple.c \ fts-filter-stopwords.c \ fts-filter-stemmer-snowball.c \ fts-language.c \
--- a/src/lib-fts/fts-filter-normalizer-simple.c Sat May 09 14:19:48 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,79 +0,0 @@ -/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "unichar.h" -#include "str.h" -#include "fts-filter.h" -#include "fts-filter-private.h" -#include "fts-language.h" - -struct fts_filter_normalizer_simple { - struct fts_filter filter; - string_t *str; -}; - -static bool -fts_filter_normalizer_simple_supports(const struct fts_language *lang ATTR_UNUSED) -{ - return TRUE; -} - -static void -fts_filter_normalizer_simple_destroy(struct fts_filter *_filter) -{ - struct fts_filter_normalizer_simple *filter = - (struct fts_filter_normalizer_simple *)_filter; - - str_free(&filter->str); - i_free(filter); -} - -static int -fts_filter_normalizer_simple_create(const struct fts_language *lang ATTR_UNUSED, - const char *const *settings, - struct fts_filter **filter_r, - const char **error_r) -{ - struct fts_filter_normalizer_simple *filter; - - if (settings[0] != NULL) { - *error_r = t_strdup_printf("Unknown setting: %s", settings[0]); - return -1; - } - filter = i_new(struct fts_filter_normalizer_simple, 1); - filter->filter = *fts_filter_normalizer_simple; - filter->str = str_new(default_pool, 128); - - *filter_r = &filter->filter; - return 0; -} - -static int -fts_filter_normalizer_simple_filter(struct fts_filter *_filter, - const char **token, - const char **error_r ATTR_UNUSED) -{ - struct fts_filter_normalizer_simple *filter = - (struct fts_filter_normalizer_simple *)_filter; - - str_truncate(filter->str, 0); - if (uni_utf8_to_decomposed_titlecase(*token, strlen(*token), - filter->str) < 0) - i_panic("fts-normalizer-simple: Token is not valid UTF-8: %s", *token); - *token = str_c(filter->str); - return 1; -} - -static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { - fts_filter_normalizer_simple_supports, - fts_filter_normalizer_simple_create, - fts_filter_normalizer_simple_filter, - fts_filter_normalizer_simple_destroy -}; - -static const struct fts_filter fts_filter_normalizer_simple_real = { - .class_name = SIMPLE_NORMALIZER_FILTER_NAME, - .v = &normalizer_filter_vfuncs -}; - -const struct fts_filter *fts_filter_normalizer_simple = &fts_filter_normalizer_simple_real;
--- a/src/lib-fts/fts-filter.c Sat May 09 14:19:48 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 14:09:37 2015 +0300 @@ -15,7 +15,6 @@ fts_filter_register(fts_filter_stopwords); fts_filter_register(fts_filter_stemmer_snowball); fts_filter_register(fts_filter_normalizer_icu); - fts_filter_register(fts_filter_normalizer_simple); } void fts_filters_deinit(void)
--- a/src/lib-fts/fts-filter.h Sat May 09 14:19:48 2015 +0300 +++ b/src/lib-fts/fts-filter.h Sat May 09 14:09:37 2015 +0300 @@ -33,10 +33,6 @@ extern const struct fts_filter *fts_filter_normalizer_icu; #define ICU_NORMALIZER_FILTER_NAME "normalizer-icu" -/* Normalization using i;unicode-casemap (RFC 5051) */ -extern const struct fts_filter *fts_filter_normalizer_simple; -#define SIMPLE_NORMALIZER_FILTER_NAME "normalizer-simple" - /* Register all built-in filters. */ void fts_filters_init(void); void fts_filters_deinit(void);