Mercurial > dovecot > core-2.2

changeset 18590:12aeb1ac8f0d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lib-fts: Removed tokenizer name macros from fts-tokenizer.h The tokenizers can be directly accessed via their class structs.
author Timo Sirainen <tss@iki.fi>
date Sat, 09 May 2015 14:56:33 +0300
parents 5937f4d505c5
children e2a0b43e73ef
files src/lib-fts/fts-tokenizer-address.c src/lib-fts/fts-tokenizer-generic.c src/lib-fts/fts-tokenizer.h src/lib-fts/test-fts-tokenizer.c
diffstat 4 files changed, 16 insertions(+), 19 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-fts/fts-tokenizer-address.c	Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/fts-tokenizer-address.c	Sat May 09 14:56:33 2015 +0300
@@ -305,7 +305,7 @@
 };
 
 static const struct fts_tokenizer fts_tokenizer_email_address_real = {
-	.name = FTS_TOKENIZER_EMAIL_ADDRESS_NAME,
+	.name = "email-address",
 	.v = &email_address_tokenizer_vfuncs
 };
 const struct fts_tokenizer *fts_tokenizer_email_address =
--- a/src/lib-fts/fts-tokenizer-generic.c	Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/fts-tokenizer-generic.c	Sat May 09 14:56:33 2015 +0300
@@ -596,7 +596,7 @@
 };
 
 static const struct fts_tokenizer fts_tokenizer_generic_real = {
-	.name = FTS_TOKENIZER_GENERIC_NAME,
+	.name = "generic",
 	.v = &generic_tokenizer_vfuncs
 };
 const struct fts_tokenizer *fts_tokenizer_generic = &fts_tokenizer_generic_real;
--- a/src/lib-fts/fts-tokenizer.h	Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/fts-tokenizer.h	Sat May 09 14:56:33 2015 +0300
@@ -23,7 +23,6 @@
    further. Defaults to disabled. Enable by defining the keyword (and any
    value). */
 extern const struct fts_tokenizer *fts_tokenizer_email_address;
-#define FTS_TOKENIZER_EMAIL_ADDRESS_NAME "email-address"
 
 /* Generic email content tokenizer. Cuts text into tokens. */
 /* Settings: 
@@ -41,7 +40,6 @@
    differ in some details, e.g. simple will cut "a.b" and tr29 will
    not. The default is "simple" */
 extern const struct fts_tokenizer *fts_tokenizer_generic;
-#define FTS_TOKENIZER_GENERIC_NAME "generic"
 
 /*
  Tokenizing workflow, find --> create --> filter --> destroy.
--- a/src/lib-fts/test-fts-tokenizer.c	Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/test-fts-tokenizer.c	Sat May 09 14:56:33 2015 +0300
@@ -11,6 +11,14 @@
 
 #include <stdlib.h>
 
+static void test_fts_tokenizer_find(void)
+{
+	test_begin("fts tokenizer find");
+	test_assert(fts_tokenizer_find("email-address") == fts_tokenizer_email_address);
+	test_assert(fts_tokenizer_find("generic") == fts_tokenizer_generic);
+	test_end();
+}
+
 static void test_fts_tokenizer_generic_only(void)
 {
 	static const unsigned char input[] =
@@ -22,14 +30,12 @@
 		"and", "longlonglongabcdefghijklmnopqr",
 		"more", "Hello", "world", "last", NULL
 	};
-	const struct fts_tokenizer *tok_class;
 	struct fts_tokenizer *tok;
 	const char * const *eopp = expected_output;
 	const char *token, *error;
 
 	test_begin("fts tokenizer generic simple");
-	tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
-	test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0);
+	test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0);
 /*TODO: Uncomment when fts-tokenizer-generic-private.h inclusion is fixed */
 /*test_assert(((struct generic_fts_tokenizer *) tok)->algorithm ==  BOUNDARY_ALGORITHM_SIMPLE);*/
 	while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
@@ -57,15 +63,13 @@
 		"there", "was", "text", "galore",
 		"and", "more", NULL
 	};
-	const struct fts_tokenizer *tok_class;
 	struct fts_tokenizer *tok;
 	const char * const *eopp = expected_output;
 	const char *token, *error;
 
 	test_begin("fts tokenizer generic simple with Unicode whitespace");
 	fts_tokenizer_register(fts_tokenizer_generic);
-	tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
-	test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0);
+	test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0);
 	while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
 		test_assert(strcmp(token, *eopp) == 0);
 		eopp++;
@@ -132,15 +136,13 @@
 		"and", "more", "Hello", "world", "3.14",
 		"3,14", "last", "longlonglongabcdefghijklmnopqr", "1", NULL
 	};
-	const struct fts_tokenizer *tok_class;
 	struct fts_tokenizer *tok;
 	const char * const *eopp = expected_output;
 	const char *token, *error;
 
 	test_begin("fts tokenizer generic TR29");
 	fts_tokenizer_register(fts_tokenizer_generic);
-	tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
-	test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);
+	test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0);
 	while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
 		test_assert(strcmp(token, *eopp) == 0);
 		eopp++;
@@ -169,15 +171,13 @@
 		"there", "was", "text", "galore",
 		"and", "more", NULL
 	};
-	const struct fts_tokenizer *tok_class;
 	struct fts_tokenizer *tok;
 	const char * const *eopp = expected_output;
 	const char *token, *error;
 
 	test_begin("fts tokenizer generic TR29 with Unicode whitespace");
 	fts_tokenizer_register(fts_tokenizer_generic);
-	tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
-	test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);
+	test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0);
 	while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
 		test_assert(strcmp(token, *eopp) == 0);
 		eopp++;
@@ -200,15 +200,13 @@
 	static const char *const expected_output[] = {
 		"hello", "world", NULL
 	};
-	const struct fts_tokenizer *tok_class;
 	struct fts_tokenizer *tok;
 	const char * const *eopp = expected_output;
 	const char *token, *error;
 
 	test_begin("fts tokenizer generic TR29 with MinNumLet U+FF0E at end");
 	fts_tokenizer_register(fts_tokenizer_generic);
-	tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
-	test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);
+	test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0);
 	while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
 		test_assert(null_strcmp(token, *eopp) == 0);
 		eopp++;
@@ -581,6 +579,7 @@
 int main(void)
 {
 	static void (*test_functions[])(void) = {
+		test_fts_tokenizer_find,
 		test_fts_tokenizer_generic_only,
 		test_fts_tokenizer_generic_unicode_whitespace,
 		test_fts_tokenizer_char_generic_only,