[Groonga-commit] groonga/groonga [master] Added a new tokenizer TokenDelimitNull.

Back to archive index

null+****@clear***** null+****@clear*****
2011年 2月 2日 (水) 16:20:45 JST


Daijiro MORI	2011-02-02 07:20:45 +0000 (Wed, 02 Feb 2011)

  New Revision: 74721043f9161c12f88a9061fe8ffeadfd0b6237

  Log:
    Added a new tokenizer TokenDelimitNull.

  Modified files:
    lib/token.c

  Modified: lib/token.c (+9 -1)
===================================================================
--- lib/token.c    2011-02-02 01:13:48 +0000 (8777070)
+++ lib/token.c    2011-02-02 07:20:45 +0000 (6bc5289)
@@ -164,6 +164,13 @@ delimit_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
   return delimited_init(ctx, table, user_data, " ", 1);
 }
 
+static grn_obj *
+delimit_null_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
+{
+  grn_obj *table = args[0];
+  return delimited_init(ctx, table, user_data, "", 1);
+}
+
 /* ngram tokenizer */
 
 typedef struct {
@@ -587,6 +594,7 @@ grn_db_init_builtin_tokenizers(grn_ctx *ctx)
                 bigramisa_init, ngram_next, ngram_fin, vars);
   DEF_TOKENIZER("TokenBigramIgnoreBlankSplitSymbolAlphaDigit",
                 bigramisad_init, ngram_next, ngram_fin, vars);
-
+  DEF_TOKENIZER("TokenDelimitNull",
+                delimit_null_init, delimited_next, delimited_fin, vars);
   return GRN_SUCCESS;
 }




Groonga-commit メーリングリストの案内
Back to archive index