Kouhei Sutou
null+****@clear*****
Fri Dec 21 16:44:23 JST 2012
Kouhei Sutou 2012-12-21 16:44:23 +0900 (Fri, 21 Dec 2012) New Revision: a24b2fd1c6af53d715aeb433ccba592f102d4bdc https://github.com/groonga/groonga/commit/a24b2fd1c6af53d715aeb433ccba592f102d4bdc Log: Make GRN_TOKEN_* public GRN_TOKEN_* -> GRN_TOKENIZER_TOKEN_* Modified files: include/groonga/tokenizer.h lib/token.c lib/token.h lib/tokenizer.c Modified: include/groonga/tokenizer.h (+25 -10) =================================================================== --- include/groonga/tokenizer.h 2012-12-21 16:38:55 +0900 (5d0155f) +++ include/groonga/tokenizer.h 2012-12-21 16:44:23 +0900 (7684701) @@ -136,16 +136,31 @@ GRN_PLUGIN_EXPORT void grn_tokenizer_token_init(grn_ctx *ctx, grn_tokenizer_toke GRN_PLUGIN_EXPORT void grn_tokenizer_token_fin(grn_ctx *ctx, grn_tokenizer_token *token); /* - grn_tokenizer_status provides a list of tokenizer status codes. - GRN_TOKENIZER_CONTINUE means that the next token is not the last one and - GRN_TOKENIZER_LAST means that the next token is the last one. If a document - or query contains no tokens, please push an empty string with - GRN_TOKENIZER_LAST as a token. - */ -typedef enum { - GRN_TOKENIZER_CONTINUE = 0, - GRN_TOKENIZER_LAST = 1 -} grn_tokenizer_status; + * grn_tokenizer_status is a flag set for tokenizer status codes. + * If a document or query contains no tokens, push an empty string with + * GRN_TOKENIZER_TOKEN_LAST as a token. + */ +typedef unsigned int grn_tokenizer_status; + +/* GRN_TOKENIZER_TOKEN_CONTINUE means that the next token is not the last one. */ +#define GRN_TOKENIZER_TOKEN_CONTINUE (0) +/* GRN_TOKENIZER_TOKEN_LAST means that the next token is the last one. */ +#define GRN_TOKENIZER_TOKEN_LAST (0x01L<<0) +/* GRN_TOKENIZER_TOKEN_OVERLAP means that ... */ +#define GRN_TOKENIZER_TOKEN_OVERLAP (0x01L<<1) +/* GRN_TOKENIZER_TOKEN_UNMATURED means that ... */ +#define GRN_TOKENIZER_TOKEN_UNMATURED (0x01L<<2) +/* GRN_TOKENIZER_TOKEN_REACH_END means that ... */ +#define GRN_TOKENIZER_TOKEN_REACH_END (0x01L<<3) + +/* + * GRN_TOKENIZER_CONTINUE and GRN_TOKENIZER_LAST are deprecated. They + * are just for backward compatibility. Use + * GRN_TOKENIZER_TOKEN_CONTINUE and GRN_TOKENIZER_TOKEN_LAST + * instead. + */ +#define GRN_TOKENIZER_CONTINUE GRN_TOKENIZER_TOKEN_CONTINUE +#define GRN_TOKENIZER_LAST GRN_TOKENIZER_TOKEN_LAST /* grn_tokenizer_token_push() pushes the next token into `token'. Note that Modified: lib/token.c (+13 -11) =================================================================== --- lib/token.c 2012-12-21 16:38:55 +0900 (54d831e) +++ lib/token.c 2012-12-21 16:44:23 +0900 (03fc31b) @@ -62,11 +62,12 @@ uvector_next(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) byte *p = token->curr + token->unit; if (token->tail < p) { GRN_TEXT_SET_REF(&token->curr_, token->curr, 0); - GRN_UINT32_SET(ctx, &token->stat_, GRN_TOKEN_LAST); + GRN_UINT32_SET(ctx, &token->stat_, GRN_TOKENIZER_TOKEN_LAST); } else { GRN_TEXT_SET_REF(&token->curr_, token->curr, token->unit); token->curr = p; - GRN_UINT32_SET(ctx, &token->stat_, token->tail == p ? GRN_TOKEN_LAST : 0); + GRN_UINT32_SET(ctx, &token->stat_, + token->tail == p ? GRN_TOKENIZER_TOKEN_LAST : 0); } grn_ctx_push(ctx, &token->curr_); grn_ctx_push(ctx, &token->stat_); @@ -375,8 +376,8 @@ ngram_next(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) len++; r += cl; } - if (token->overlap) { status |= GRN_TOKEN_OVERLAP; } - if (len < token->ngram_unit) { status |= GRN_TOKEN_UNMATURED; } + if (token->overlap) { status |= GRN_TOKENIZER_TOKEN_OVERLAP; } + if (len < token->ngram_unit) { status |= GRN_TOKENIZER_TOKEN_UNMATURED; } token->overlap = (len > 1) ? 1 : 0; } } @@ -385,11 +386,11 @@ ngram_next(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) token->tail = pos + len - 1; if (p == r || token->next == e) { token->skip = 0; - status |= GRN_TOKEN_LAST; + status |= GRN_TOKENIZER_TOKEN_LAST; } else { token->skip = token->overlap ? 1 : len; } - if (r == e) { status |= GRN_TOKEN_REACH_END; } + if (r == e) { status |= GRN_TOKENIZER_TOKEN_REACH_END; } GRN_TEXT_SET_REF(&token->curr_, p, r - p); GRN_UINT32_SET(ctx, &token->stat_, status); grn_ctx_push(ctx, &token->curr_); @@ -504,15 +505,16 @@ grn_token_next(grn_ctx *ctx, grn_token *token) token->curr = GRN_TEXT_VALUE(curr_); token->curr_size = GRN_TEXT_LEN(curr_); status = GRN_UINT32_VALUE(stat_); - token->status = ((status & GRN_TOKEN_LAST) || - (token->mode == GRN_TOKEN_GET && (status & GRN_TOKEN_REACH_END))) + token->status = ((status & GRN_TOKENIZER_TOKEN_LAST) || + (token->mode == GRN_TOKEN_GET && + (status & GRN_TOKENIZER_TOKEN_REACH_END))) ? GRN_TOKEN_DONE : GRN_TOKEN_DOING; token->force_prefix = 0; - if (status & GRN_TOKEN_UNMATURED) { - if (status & GRN_TOKEN_OVERLAP) { + if (status & GRN_TOKENIZER_TOKEN_UNMATURED) { + if (status & GRN_TOKENIZER_TOKEN_OVERLAP) { if (token->mode == GRN_TOKEN_GET) { token->pos++; continue; } } else { - if (status & GRN_TOKEN_LAST) { token->force_prefix = 1; } + if (status & GRN_TOKENIZER_TOKEN_LAST) { token->force_prefix = 1; } } } } else { Modified: lib/token.h (+0 -5) =================================================================== --- lib/token.h 2012-12-21 16:38:55 +0900 (139f139) +++ lib/token.h 2012-12-21 16:44:23 +0900 (33586bb) @@ -67,11 +67,6 @@ typedef struct { grn_obj *nstr; } grn_token; -#define GRN_TOKEN_LAST (0x01L<<0) -#define GRN_TOKEN_OVERLAP (0x01L<<1) -#define GRN_TOKEN_UNMATURED (0x01L<<2) -#define GRN_TOKEN_REACH_END (0x01L<<3) - extern grn_obj *grn_uvector_tokenizer; grn_rc grn_token_init(void); Modified: lib/tokenizer.c (+1 -9) =================================================================== --- lib/tokenizer.c 2012-12-21 16:38:55 +0900 (5f06826) +++ lib/tokenizer.c 2012-12-21 16:44:23 +0900 (01e07ff) @@ -250,15 +250,7 @@ grn_tokenizer_token_push(grn_ctx *ctx, grn_tokenizer_token *token, grn_tokenizer_status status) { GRN_TEXT_SET_REF(&token->str, str_ptr, str_length); - switch (status) { - case GRN_TOKENIZER_CONTINUE : - GRN_UINT32_SET(ctx, &token->status, 0); - break; - case GRN_TOKENIZER_LAST : - default : - GRN_UINT32_SET(ctx, &token->status, GRN_TOKEN_LAST); - break; - } + GRN_UINT32_SET(ctx, &token->status, status); grn_ctx_push(ctx, &token->str); grn_ctx_push(ctx, &token->status); } -------------- next part -------------- HTML����������������������������...Download