Kouhei Sutou
null+****@clear*****
Fri Mar 13 18:13:02 JST 2015
Kouhei Sutou 2015-03-13 18:13:02 +0900 (Fri, 13 Mar 2015) New Revision: cd4c80bc17d7a4b957e5843f33a51a97ba81d0ed https://github.com/groonga/groonga/commit/cd4c80bc17d7a4b957e5843f33a51a97ba81d0ed Message: doc tokenizers: add summary and figures TODO: Index column in figures is incorrect. Index column doesn't have tokens. It should be fixed. Added files: doc/source/images/reference/tokenizers/used-when-indexing.png doc/source/images/reference/tokenizers/used-when-indexing.svg doc/source/images/reference/tokenizers/used-when-searching.png doc/source/images/reference/tokenizers/used-when-searching.svg Modified files: doc/source/reference/tokenizers.rst Added: doc/source/images/reference/tokenizers/used-when-indexing.png (+0 -0) 100644 =================================================================== (Binary files differ) Added: doc/source/images/reference/tokenizers/used-when-indexing.svg (+251 -0) 100644 =================================================================== --- /dev/null +++ doc/source/images/reference/tokenizers/used-when-indexing.svg 2015-03-13 18:13:02 +0900 (78ed502) @@ -0,0 +1,251 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="823.34076" + height="561.80884" + id="svg2" + version="1.1" + inkscape:version="0.48.5 r10040" + sodipodi:docname="used-when-indexing.svg" + inkscape:export-filename="used-when-indexing.png" + inkscape:export-xdpi="72" + inkscape:export-ydpi="72"> + <defs + id="defs4"> + <marker + inkscape:stockid="Arrow1Lend" + orient="auto" + refY="0" + refX="0" + id="Arrow1Lend" + style="overflow:visible"> + <path + id="path3841" + d="M 0,0 5,-5 -12.5,0 5,5 0,0 z" + style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt" + transform="matrix(-0.8,0,0,-0.8,-10,0)" + inkscape:connector-curvature="0" /> + </marker> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="0.35" + inkscape:cx="860.7163" + inkscape:cy="202.49623" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="false" + fit-margin-top="10" + fit-margin-left="10" + fit-margin-right="10" + fit-margin-bottom="10" + inkscape:window-width="956" + inkscape:window-height="836" + inkscape:window-x="2397" + inkscape:window-y="68" + inkscape:window-maximized="0" /> + <metadata + id="metadata7"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="レイヤー 1" + inkscape:groupmode="layer" + id="layer1" + transform="translate(57.6,-98.876477)"> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="-50" + y="388.07648" + id="text2985" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan2987" + x="-50" + y="388.07648">Hello World!</tspan></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="265.71426" + y="388.07648" + id="text2989" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan2991" + x="265.71426" + y="388.07648">load</tspan></text> + <rect + style="fill:none;stroke:#204a87;stroke-width:3;stroke-linecap:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0" + id="rect2993" + width="145.71428" + height="162.85715" + x="568.14069" + y="179.50504" + rx="5" + ry="5" /> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="529.14075" + y="138.07648" + id="text2995" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan2997" + x="529.14075" + y="138.07648">data column</tspan></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="518.14075" + y="650.28534" + id="text2999" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3001" + x="518.14075" + y="650.28534">index column</tspan></text> + <rect + ry="5" + rx="5" + y="430.93359" + x="568.14069" + height="162.85715" + width="145.71428" + id="rect3003" + style="fill:none;stroke:#204a87;stroke-width:3;stroke-linecap:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0" /> + <path + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" + d="m 198.57143,376.6479 54.28571,0" + id="path3005" + inkscape:connector-curvature="0" /> + <path + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" + d="m 368.57143,370.93361 178.69037,-110" + id="path3007" + inkscape:connector-curvature="0" + sodipodi:nodetypes="cc" /> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="372.28571" + y="478.36218" + id="text3024" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3026" + x="372.28571" + y="478.36218">tokenize</tspan></text> + <path + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" + d="m 365.71429,385.21933 87.14285,52.79766" + id="path3052" + inkscape:connector-curvature="0" + sodipodi:nodetypes="cc" /> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="129.99998" + y="552.64789" + id="text3058" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3060" + x="129.99998" + y="552.64789">(tokenizer is used)</tspan></text> + <text + sodipodi:linespacing="125%" + id="text3062" + y="206.64789" + x="69.999977" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="206.64789" + x="69.999977" + id="tspan3064" + sodipodi:role="line">(tokenizer isn't used)</tspan></text> + <text + xml:space="preserve" + style="font-size:20px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="582.85803" + y="208.88481" + id="text4824" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan4826" + x="582.85803" + y="208.88481">Hello World!</tspan></text> + <text + sodipodi:linespacing="125%" + id="text4828" + y="456.88483" + x="582.85803" + style="font-size:20px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="456.88483" + x="582.85803" + id="tspan4830" + sodipodi:role="line">Hello</tspan></text> + <text + xml:space="preserve" + style="font-size:20px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="582.85803" + y="496.88483" + id="text4832" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan4834" + x="582.85803" + y="496.88483">World</tspan></text> + <text + sodipodi:linespacing="125%" + id="text4836" + y="536.88483" + x="582.85803" + style="font-size:20px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="536.88483" + x="582.85803" + id="tspan4838" + sodipodi:role="line">!</tspan></text> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path4853" + d="m 516.31983,487.41805 38.59245,27.03261" + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" /> + <text + sodipodi:linespacing="125%" + id="text3012" + y="446.93359" + x="-32.857159" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="446.93359" + x="-32.857159" + id="tspan3014" + sodipodi:role="line">(text)</tspan></text> + </g> +</svg> Added: doc/source/images/reference/tokenizers/used-when-searching.png (+0 -0) 100644 =================================================================== (Binary files differ) Added: doc/source/images/reference/tokenizers/used-when-searching.svg (+228 -0) 100644 =================================================================== --- /dev/null +++ doc/source/images/reference/tokenizers/used-when-searching.svg 2015-03-13 18:13:02 +0900 (1785d61) @@ -0,0 +1,228 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="865.50397" + height="313.52319" + id="svg2" + version="1.1" + inkscape:version="0.48.5 r10040" + sodipodi:docname="used-when-searching.svg" + inkscape:export-filename="used-when-searching.png" + inkscape:export-xdpi="72" + inkscape:export-ydpi="72"> + <defs + id="defs4"> + <marker + inkscape:stockid="Arrow1Lend" + orient="auto" + refY="0" + refX="0" + id="Arrow1Lend" + style="overflow:visible"> + <path + id="path3841" + d="M 0,0 5,-5 -12.5,0 5,5 0,0 z" + style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt" + transform="matrix(-0.8,0,0,-0.8,-10,0)" + inkscape:connector-curvature="0" /> + </marker> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="0.7" + inkscape:cx="501.07427" + inkscape:cy="247.98597" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="false" + fit-margin-top="10" + fit-margin-left="10" + fit-margin-right="10" + fit-margin-bottom="10" + inkscape:window-width="1102" + inkscape:window-height="917" + inkscape:window-x="2397" + inkscape:window-y="68" + inkscape:window-maximized="0" /> + <metadata + id="metadata7"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title></dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="レイヤー 1" + inkscape:groupmode="layer" + id="layer1" + transform="translate(54.028572,-224.70504)"> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="-30" + y="368.07648" + id="text2985" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan2987" + x="-30" + y="368.07648">world</tspan></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="165.71426" + y="368.07648" + id="text2989" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan2991" + x="165.71426" + y="368.07648">search</tspan></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="563.87537" + y="326.02426" + id="text2999" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3001" + x="563.87537" + y="326.02426">index column</tspan></text> + <rect + ry="5" + rx="5" + y="348.10107" + x="618.16101" + height="162.85715" + width="145.71428" + id="rect3003" + style="fill:none;stroke:#204a87;stroke-width:3;stroke-linecap:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0" /> + <path + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" + d="m 92.57143,356.6479 54.28571,0" + id="path3005" + inkscape:connector-curvature="0" /> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="367.20377" + y="368.45862" + id="text3024" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3026" + x="367.20377" + y="368.45862">tokenize</tspan></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="267.14282" + y="265.50504" + id="text3058" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3060" + x="267.14282" + y="265.50504">(tokenizer is used)</tspan></text> + <text + sodipodi:linespacing="125%" + id="text4828" + y="374.05231" + x="632.87836" + style="font-size:20px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="374.05231" + x="632.87836" + id="tspan4830" + sodipodi:role="line">Hello</tspan></text> + <text + xml:space="preserve" + style="font-size:20px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="632.87836" + y="414.05231" + id="text4832" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan4834" + x="632.87836" + y="414.05231">World</tspan></text> + <text + sodipodi:linespacing="125%" + id="text4836" + y="454.05231" + x="632.87836" + style="font-size:20px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="454.05231" + x="632.87836" + id="tspan4838" + sodipodi:role="line">!</tspan></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="-49.028572" + y="316.6853" + id="text3012" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3014" + x="-49.028572" + y="316.6853">(query)</tspan></text> + <path + inkscape:connector-curvature="0" + id="path3046" + d="m 302.57143,356.6479 54.28571,0" + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" /> + <path + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" + d="m 542.57143,356.6479 54.28571,0" + id="path3074" + inkscape:connector-curvature="0" /> + <path + inkscape:connector-curvature="0" + id="path3076" + d="m 596.85714,496.6479 -485.71428,0" + style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" + sodipodi:nodetypes="cc" /> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="35.014275" + y="477.82822" + id="text3078" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3080" + x="35.014275" + y="477.82822">result</tspan><tspan + sodipodi:role="line" + x="35.014275" + y="527.82825" + id="tspan3082">set</tspan></text> + <path + style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:1, 1;stroke-dashoffset:0;marker-end:url(#Arrow1Lend)" + d="m 505.71429,181.80884 0,54.28571" + id="path3010" + inkscape:connector-curvature="0" + transform="translate(-57.6,98.876477)" /> + </g> +</svg> Modified: doc/source/reference/tokenizers.rst (+114 -15) =================================================================== --- doc/source/reference/tokenizers.rst 2015-03-13 16:42:53 +0900 (7ef767f) +++ doc/source/reference/tokenizers.rst 2015-03-13 18:13:02 +0900 (9283376) @@ -2,24 +2,123 @@ .. highlightlang:: none +.. groonga-command +.. database: tokenizers + Tokenizers ========== -TODO: Write me. +Summary +------- + +Groonga has tokenizer module that tokenizes text. It is used when +indexing text and searching by query. + +.. figure:: /images/reference/tokenizers/used-when-indexing.png + :align: center + :width: 80% + + Tokenizer is used when indexing text. + +.. figure:: /images/reference/tokenizers/used-when-searching.png + :align: center + :width: 80% + + Tokenizer is used when searching by query. + + +Built-in tokenizsers +-------------------- + +Here is a list of built-in tokenizers: + + * ``TokenBigram`` + * ``TokenBigramSplitSymbol`` + * ``TokenBigramSplitSymbolAlpha`` + * ``TokenBigramSplitSymbolAlphaDigit`` + * ``TokenBigramIgnoreBlank`` + * ``TokenBigramIgnoreBlankSplitSymbol`` + * ``TokenBigramIgnoreBlankSplitAlpha`` + * ``TokenBigramIgnoreBlankSplitAlphaDigit`` + * ``TokenDelimit`` + * ``TokenDelimitNull`` + * ``TokenTrigram`` + * ``TokenUnigram`` + * ``TokenMecab`` + * ``TokenRegexp`` + +.. _token-bigram + +``TokenBigram`` +^^^^^^^^^^^^^^^ + +.. _token-bigram-split-symbol + +``TokenBigramSplitSymbol`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _token-bigram-split-symbol-alpha + +``TokenBigramSplitSymbolAlpha`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _token-bigram-split-symbol-alpha-digit + +``TokenBigramSplitSymbolAlphaDigit`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _token-bigramIgnoreBlank + +``TokenBigramIgnoreBlank`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _token-bigramIgnoreBlank-split-symbol + +``TokenBigramIgnoreBlankSplitSymbol`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _token-bigramIgnoreBlank-split-alpha + +``TokenBigramIgnoreBlankSplitAlpha`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _token-bigramIgnoreBlank-split-alpha-digit + +``TokenBigramIgnoreBlankSplitAlphaDigit`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _token-delimit + +``TokenDelimit`` +^^^^^^^^^^^^^^^^ + +.. _token-delimit-null + +``TokenDelimitNull`` +^^^^^^^^^^^^^^^^^^^^ + +.. _token-trigram + +``TokenTrigram`` +^^^^^^^^^^^^^^^^ + +.. _token-unigram + +``TokenUnigram`` +^^^^^^^^^^^^^^^^ + +.. _token-mecab + +``TokenMecab`` +^^^^^^^^^^^^^^ + +.. _token-regexp + +``TokenRegexp`` +^^^^^^^^^^^^^^^ -Here are the list of built-in tokenizers: +.. versionadded:: 5.0.1 -* TokenBigram -* TokenBigramSplitSymbol -* TokenBigramSplitSymbolAlpha -* TokenBigramSplitSymbolAlphaDigit -* TokenBigramIgnoreBlank -* TokenBigramIgnoreBlankSplitSymbol -* TokenBigramIgnoreBlankSplitAlpha -* TokenBigramIgnoreBlankSplitAlphaDigit -* TokenDelimit -* TokenDelimitNull -* TokenTrigram -* TokenUnigram -* TokenMecab +.. caution:: + This tokenizer is experimental. Specification may be changed. -------------- next part -------------- HTML����������������������������...Download