[Groonga-commit] groonga/groonga at bd46eba [master] TokenMecab: fix a bug that loose_reading with GET returns extra token

Back to archive index

Kouhei Sutou null+****@clear*****
Wed Sep 12 18:46:44 JST 2018


Kouhei Sutou	2018-09-12 18:46:44 +0900 (Wed, 12 Sep 2018)

  Revision: bd46eba5abad571c2e739d3e6f84f41826ab9d59
  https://github.com/groonga/groonga/commit/bd46eba5abad571c2e739d3e6f84f41826ab9d59

  Message:
    TokenMecab: fix a bug that loose_reading with GET returns extra token

  Added files:
    test/command/suite/select/query/match/with_index/token_mecab/loose_reading.expected
    test/command/suite/select/query/match/with_index/token_mecab/loose_reading.test
  Modified files:
    plugins/tokenizers/mecab.c

  Modified: plugins/tokenizers/mecab.c (+2 -1)
===================================================================
--- plugins/tokenizers/mecab.c    2018-09-12 17:49:42 +0900 (d9a6f2aef)
+++ plugins/tokenizers/mecab.c    2018-09-12 18:46:44 +0900 (487358adb)
@@ -1020,7 +1020,8 @@ mecab_init(grn_ctx *ctx, grn_tokenizer_query *query)
 
   if (tokenizer->options->loose_reading &&
       grn_tokenizer_query_get_mode(ctx, tokenizer->query) == GRN_TOKEN_GET) {
-    while (mecab_next_default_format_consume_token(ctx, tokenizer, NULL) > 0) {
+    while (tokenizer->next < tokenizer->end &&
+           mecab_next_default_format_consume_token(ctx, tokenizer, NULL) > 0) {
       /* Do nothing */
     }
     tokenizer->loose.ing = GRN_TRUE;

  Added: test/command/suite/select/query/match/with_index/token_mecab/loose_reading.expected (+47 -0) 100644
===================================================================
--- /dev/null
+++ test/command/suite/select/query/match/with_index/token_mecab/loose_reading.expected    2018-09-12 18:46:44 +0900 (ba4c47454)
@@ -0,0 +1,47 @@
+table_create Menus TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Menus name COLUMN_SCALAR Text
+[[0,0.0,0.0],true]
+table_create Terms TABLE_PAT_KEY ShortText   --normalize NormalizerNFKC100   --default_tokenizer 'TokenMecab("loose_reading", true)'
+[[0,0.0,0.0],true]
+column_create Terms index COLUMN_INDEX|WITH_POSITION Menus name
+[[0,0.0,0.0],true]
+load --table Menus
+[
+{"name": "焼肉定食"},
+{"name": "やきにく定食"}
+]
+[[0,0.0,0.0],2]
+select Menus --match_columns name --query "焼き肉"
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  [
+    [
+      [
+        2
+      ],
+      [
+        [
+          "_id",
+          "UInt32"
+        ],
+        [
+          "name",
+          "Text"
+        ]
+      ],
+      [
+        1,
+        "焼肉定食"
+      ],
+      [
+        2,
+        "やきにく定食"
+      ]
+    ]
+  ]
+]

  Added: test/command/suite/select/query/match/with_index/token_mecab/loose_reading.test (+15 -0) 100644
===================================================================
--- /dev/null
+++ test/command/suite/select/query/match/with_index/token_mecab/loose_reading.test    2018-09-12 18:46:44 +0900 (46388f941)
@@ -0,0 +1,15 @@
+table_create Menus TABLE_NO_KEY
+column_create Menus name COLUMN_SCALAR Text
+
+table_create Terms TABLE_PAT_KEY ShortText \
+  --normalize NormalizerNFKC100 \
+  --default_tokenizer 'TokenMecab("loose_reading", true)'
+column_create Terms index COLUMN_INDEX|WITH_POSITION Menus name
+
+load --table Menus
+[
+{"name": "焼肉定食"},
+{"name": "やきにく定食"}
+]
+
+select Menus --match_columns name --query "焼き肉"
-------------- next part --------------
HTML����������������������������...
URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20180912/fff4a114/attachment-0001.htm 



More information about the Groonga-commit mailing list
Back to archive index