naoa
null+****@clear*****
Fri Aug 15 08:59:00 JST 2014
naoa 2014-08-15 08:59:00 +0900 (Fri, 15 Aug 2014) New Revision: c5ec116da1c3276ea2a39c6581eeeabea49d9f01 https://github.com/groonga/groonga/commit/c5ec116da1c3276ea2a39c6581eeeabea49d9f01 Merged f87c462: Merge pull request #193 from naoa/add-token-position-skip Message: token: add tokenizer token skip Modified files: include/groonga/tokenizer.h lib/token.c Modified: include/groonga/tokenizer.h (+2 -0) =================================================================== --- include/groonga/tokenizer.h 2014-08-15 08:15:03 +0900 (5865ae4) +++ include/groonga/tokenizer.h 2014-08-15 08:59:00 +0900 (4107eb9) @@ -161,6 +161,8 @@ typedef unsigned int grn_tokenizer_status; #define GRN_TOKENIZER_TOKEN_UNMATURED (0x01L<<2) /* GRN_TOKENIZER_TOKEN_REACH_END means that ... */ #define GRN_TOKENIZER_TOKEN_REACH_END (0x01L<<3) +/* GRN_TOKENIZER_TOKEN_SKIP means that the token is skipped */ +#define GRN_TOKENIZER_TOKEN_SKIP (0x01L<<4) /* * GRN_TOKENIZER_CONTINUE and GRN_TOKENIZER_LAST are deprecated. They Modified: lib/token.c (+4 -0) =================================================================== --- lib/token.c 2014-08-15 08:15:03 +0900 (596221c) +++ lib/token.c 2014-08-15 08:59:00 +0900 (54a1f0c) @@ -579,6 +579,10 @@ grn_token_next(grn_ctx *ctx, grn_token *token) (status & GRN_TOKENIZER_TOKEN_REACH_END))) ? GRN_TOKEN_DONE : GRN_TOKEN_DOING; token->force_prefix = 0; + if (status & GRN_TOKENIZER_TOKEN_SKIP) { + token->pos++; + continue; + } if (token->curr_size == 0) { char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE]; int tokenizer_name_length; -------------- next part -------------- HTML����������������������������...Download