Kouhei Sutou
null+****@clear*****
Wed Nov 21 17:05:14 JST 2012
Kouhei Sutou 2012-11-21 17:05:14 +0900 (Wed, 21 Nov 2012) New Revision: 3d44c6476d47f9571389691e226572a91ef2988c https://github.com/groonga/groonga/commit/3d44c6476d47f9571389691e226572a91ef2988c Log: Disable tokenized delimiter for now TODO: We should handle it just on query expander -> tokenizer phase not all phases. Modified files: lib/tokenizer.c test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/default.test test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/normalize.test test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/default.test test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/normalize.test test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/default.test test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/normalize.test test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/default.test test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/normalize.test Modified: lib/tokenizer.c (+5 -0) =================================================================== --- lib/tokenizer.c 2012-11-21 19:26:35 +0900 (6ec9887) +++ lib/tokenizer.c 2012-11-21 17:05:14 +0900 (6a377fb) @@ -112,6 +112,11 @@ grn_tokenizer_have_tokenized_delimiter(grn_ctx *ctx, const char *current = str_ptr; const char *end = str_ptr + str_length; + /* TODO: disabled tokenized delimiter for now. + We should handle it just on query expander -> tokenizer phase not + all phases. */ + return GRN_FALSE; + if (encoding != GRN_ENC_UTF8) { return GRN_FALSE; } Modified: test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/default.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/default.test 2012-11-21 19:26:35 +0900 (98bc30b) +++ test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/default.test 2012-11-21 17:05:14 +0900 (cde81c7) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + table_create Memos TABLE_NO_KEY column_create Memos content COLUMN_SCALAR Text Modified: test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/normalize.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/normalize.test 2012-11-21 19:26:35 +0900 (e2ae5b4) +++ test/command/suite/table_create/default_tokenizer/bigram/tokenized_delimiter/normalize.test 2012-11-21 17:05:14 +0900 (6393fb9) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + table_create Memos TABLE_NO_KEY column_create Memos content COLUMN_SCALAR Text Modified: test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/default.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/default.test 2012-11-21 19:26:35 +0900 (63d061f) +++ test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/default.test 2012-11-21 17:05:14 +0900 (114bdf0) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + table_create Tags TABLE_PAT_KEY ShortText \ --default_tokenizer TokenDelimit Modified: test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/normalize.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/normalize.test 2012-11-21 19:26:35 +0900 (d4ebc41) +++ test/command/suite/table_create/default_tokenizer/delimit/tokenized_delimiter/normalize.test 2012-11-21 17:05:14 +0900 (97af50c) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + table_create Tags TABLE_PAT_KEY|KEY_NORMALIZE ShortText \ --default_tokenizer TokenDelimit Modified: test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/default.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/default.test 2012-11-21 19:26:35 +0900 (765b083) +++ test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/default.test 2012-11-21 17:05:14 +0900 (8bd6967) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + #@on-error omit register tokenizers/kytea #@on-error default Modified: test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/normalize.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/normalize.test 2012-11-21 19:26:35 +0900 (35a7ec4) +++ test/command/suite/table_create/default_tokenizer/kytea/tokenized_delimiter/normalize.test 2012-11-21 17:05:14 +0900 (d2e977b) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + #@on-error omit register tokenizers/kytea #@on-error default Modified: test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/default.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/default.test 2012-11-21 19:26:35 +0900 (156b383) +++ test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/default.test 2012-11-21 17:05:14 +0900 (9f315c9) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + table_create Memos TABLE_NO_KEY column_create Memos content COLUMN_SCALAR Text Modified: test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/normalize.test (+2 -0) =================================================================== --- test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/normalize.test 2012-11-21 19:26:35 +0900 (ec0d2db) +++ test/command/suite/table_create/default_tokenizer/mecab/tokenized_delimiter/normalize.test 2012-11-21 17:05:14 +0900 (1b2e6b1) @@ -1,3 +1,5 @@ +#@omit "tokenized delimiter is disabled for now." + table_create Memos TABLE_NO_KEY column_create Memos content COLUMN_SCALAR Text -------------- next part -------------- HTML����������������������������...Download