Kouhei Sutou
null+****@clear*****
Thu Oct 31 18:44:45 JST 2013
Kouhei Sutou 2013-10-31 18:44:45 +0900 (Thu, 31 Oct 2013) New Revision: cf906932ceb0a48c43000778ca9989e354f8a947 https://github.com/groonga/groonga/commit/cf906932ceb0a48c43000778ca9989e354f8a947 Message: Show tokenizer and original text in empty token warning message Added files: test/command/suite/tokenize/empty_token.expected test/command/suite/tokenize/empty_token.test Modified files: lib/token.c Modified: lib/token.c (+9 -1) =================================================================== --- lib/token.c 2013-10-30 19:46:31 +0900 (189517c) +++ lib/token.c 2013-10-31 18:44:45 +0900 (429a711) @@ -576,7 +576,15 @@ grn_token_next(grn_ctx *ctx, grn_token *token) ? GRN_TOKEN_DONE : GRN_TOKEN_DOING; token->force_prefix = 0; if (token->curr_size == 0) { - GRN_LOG(ctx, GRN_WARN, "[token_next] ignore an empty token."); + char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE]; + int tokenizer_name_length; + tokenizer_name_length = + grn_obj_name(ctx, token->tokenizer, + tokenizer_name, GRN_TABLE_MAX_KEY_SIZE); + GRN_LOG(ctx, GRN_WARN, + "[token_next] ignore an empty token: <%.*s>: <%.*s>", + tokenizer_name_length, tokenizer_name, + token->orig_blen, token->orig); continue; } if (token->curr_size > GRN_TABLE_MAX_KEY_SIZE) { Added: test/command/suite/tokenize/empty_token.expected (+3 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/tokenize/empty_token.expected 2013-10-31 18:44:45 +0900 (275fbb7) @@ -0,0 +1,3 @@ +tokenize TokenDelimit "aBc" NormalizerAuto ENABLE_TOKENIZED_DELIMITER +[[0,0.0,0.0],[{"value":"ab","position":0},{"value":"c","position":1}]] +#|w| [token_next] ignore an empty token: <TokenDelimit>: <aBc> Added: test/command/suite/tokenize/empty_token.test (+1 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/tokenize/empty_token.test 2013-10-31 18:44:45 +0900 (c2c6db5) @@ -0,0 +1 @@ +tokenize TokenDelimit "aBc" NormalizerAuto ENABLE_TOKENIZED_DELIMITER -------------- next part -------------- HTML����������������������������...Download