null+****@clear*****
null+****@clear*****
2011年 2月 2日 (水) 16:20:45 JST
Daijiro MORI 2011-02-02 07:20:45 +0000 (Wed, 02 Feb 2011)
New Revision: 74721043f9161c12f88a9061fe8ffeadfd0b6237
Log:
Added a new tokenizer TokenDelimitNull.
Modified files:
lib/token.c
Modified: lib/token.c (+9 -1)
===================================================================
--- lib/token.c 2011-02-02 01:13:48 +0000 (8777070)
+++ lib/token.c 2011-02-02 07:20:45 +0000 (6bc5289)
@@ -164,6 +164,13 @@ delimit_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
return delimited_init(ctx, table, user_data, " ", 1);
}
+static grn_obj *
+delimit_null_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
+{
+ grn_obj *table = args[0];
+ return delimited_init(ctx, table, user_data, "", 1);
+}
+
/* ngram tokenizer */
typedef struct {
@@ -587,6 +594,7 @@ grn_db_init_builtin_tokenizers(grn_ctx *ctx)
bigramisa_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenBigramIgnoreBlankSplitSymbolAlphaDigit",
bigramisad_init, ngram_next, ngram_fin, vars);
-
+ DEF_TOKENIZER("TokenDelimitNull",
+ delimit_null_init, delimited_next, delimited_fin, vars);
return GRN_SUCCESS;
}