Kouhei Sutou
null+****@clear*****
Fri Nov 11 12:13:10 JST 2016
Kouhei Sutou 2016-11-11 12:13:10 +0900 (Fri, 11 Nov 2016) New Revision: a0e75696c7bf90583a0fab6a661c884bf71931cf https://github.com/groonga/groonga/commit/a0e75696c7bf90583a0fab6a661c884bf71931cf Message: Support Zstandard You can use it by adding COMPRESS_ZSTD flag to column_create. Added files: test/command/suite/column_list/compress/lz4.expected test/command/suite/column_list/compress/lz4.test test/command/suite/column_list/compress/zlib.expected test/command/suite/column_list/compress/zlib.test test/command/suite/column_list/compress/zstd.expected test/command/suite/column_list/compress/zstd.test test/command/suite/dump/schema/column/scalar/lz4.expected test/command/suite/dump/schema/column/scalar/lz4.test test/command/suite/dump/schema/column/scalar/zlib.expected test/command/suite/dump/schema/column/scalar/zlib.test test/command/suite/dump/schema/column/scalar/zstd.expected test/command/suite/dump/schema/column/scalar/zstd.test test/command/suite/schema/tables/columns/compress/zstd.expected test/command/suite/schema/tables/columns/compress/zstd.test test/command/suite/select/output/zstd/scalar/compressed.expected test/command/suite/select/output/zstd/scalar/compressed.test test/command/suite/select/output/zstd/scalar/packed.expected test/command/suite/select/output/zstd/scalar/packed.test Modified files: config.h.cmake configure.ac include/groonga/groonga.h lib/CMakeLists.txt lib/Makefile.am lib/db.c lib/dump.c lib/error.c lib/mrb/mrb_ctx.c lib/mrb/mrb_error.c lib/mrb/mrb_object_flags.c lib/mrb/scripts/context/rc.rb lib/proc/proc_column.c lib/proc/proc_schema.c lib/store.c lib/util.c src/groonga.c Modified: config.h.cmake (+1 -0) =================================================================== --- config.h.cmake 2016-11-10 12:00:16 +0900 (cc7d0a4) +++ config.h.cmake 2016-11-11 12:13:10 +0900 (d0a385f) @@ -85,6 +85,7 @@ #cmakedefine GRN_WITH_CUTTER #cmakedefine GRN_WITH_KYTEA #cmakedefine GRN_WITH_LZ4 +#cmakedefine GRN_WITH_ZSTD #cmakedefine GRN_WITH_MECAB #cmakedefine GRN_WITH_MESSAGE_PACK #cmakedefine GRN_WITH_MRUBY Modified: configure.ac (+24 -0) =================================================================== --- configure.ac 2016-11-10 12:00:16 +0900 (e314eb6) +++ configure.ac 2016-11-11 12:13:10 +0900 (61490f2) @@ -920,6 +920,30 @@ if test "x$with_lz4" != "xno"; then fi fi +# Zstandard +AC_ARG_WITH(zstd, + [AS_HELP_STRING([--with-zstd], + [Support data compression by Zstandard. [default=auto]])], + [with_zstd="$withval"], + [with_zstd="auto"]) +if test "x$with_zstd" != "xno"; then + m4_ifdef([PKG_CHECK_MODULES], [ + PKG_CHECK_MODULES([LIBZSTD], + [libzstd], + [GRN_WITH_ZSTD=yes], + [GRN_WITH_ZSTD=no]) + ], + [GRN_WITH_ZSTD=no]) + if test "$GRN_WITH_ZSTD" = "yes"; then + AC_DEFINE(GRN_WITH_ZSTD, [1], + [Support data compression by Zstandard.]) + else + if test "x$with_zstd" != "xauto"; then + AC_MSG_ERROR("No libzstd found") + fi + fi +fi + # jemalloc AC_ARG_WITH(jemalloc, [AS_HELP_STRING([--with-jemalloc], Modified: include/groonga/groonga.h (+5 -2) =================================================================== --- include/groonga/groonga.h 2016-11-10 12:00:16 +0900 (57265f9) +++ include/groonga/groonga.h 2016-11-11 12:13:10 +0900 (d08343d) @@ -126,7 +126,8 @@ typedef enum { GRN_PLUGIN_ERROR = -75, GRN_SCORER_ERROR = -76, GRN_CANCEL = -77, - GRN_WINDOW_FUNCTION_ERROR = -78 + GRN_WINDOW_FUNCTION_ERROR = -78, + GRN_ZSTD_ERROR = -79 } grn_rc; GRN_API grn_rc grn_init(void); @@ -331,6 +332,7 @@ typedef uint32_t grn_column_flags; #define GRN_OBJ_COMPRESS_LZ4 (0x02<<4) /* Just for backward compatibility. We'll remove it****@5*****. */ #define GRN_OBJ_COMPRESS_LZO GRN_OBJ_COMPRESS_LZ4 +#define GRN_OBJ_COMPRESS_ZSTD (0x03<<4) #define GRN_OBJ_WITH_SECTION (0x01<<7) #define GRN_OBJ_WITH_WEIGHT (0x01<<8) @@ -705,7 +707,8 @@ typedef enum { /* Just for backward compatibility. We'll remove it****@5*****. */ #define GRN_INFO_SUPPORT_LZO GRN_INFO_SUPPORT_LZ4 GRN_INFO_NORMALIZER, - GRN_INFO_TOKEN_FILTERS + GRN_INFO_TOKEN_FILTERS, + GRN_INFO_SUPPORT_ZSTD } grn_info_type; GRN_API grn_obj *grn_obj_get_info(grn_ctx *ctx, grn_obj *obj, grn_info_type type, grn_obj *valuebuf); Modified: lib/CMakeLists.txt (+3 -0) =================================================================== --- lib/CMakeLists.txt 2016-11-10 12:00:16 +0900 (fecdcd5) +++ lib/CMakeLists.txt 2016-11-11 12:13:10 +0900 (69b55a1) @@ -22,9 +22,11 @@ include_directories( ${ONIGMO_INCLUDE_DIRS} ${MRUBY_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS} + ${LIBZSTD_INCLUDE_DIRS} ${MESSAGE_PACK_INCLUDE_DIRS}) link_directories( ${LIBLZ4_LIBRARY_DIRS} + ${LIBZSTD_LIBRARY_DIRS} ${MESSAGE_PACK_LIBRARY_DIRS}) read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/sources.am LIBGROONGA_SOURCES) @@ -93,6 +95,7 @@ set(GRN_ALL_LIBRARIES ${PTHREAD_LIBS} ${Z_LIBS} ${LIBLZ4_LIBRARIES} + ${LIBZSTD_LIBRARIES} ${MESSAGE_PACK_LIBRARIES} ${DL_LIBS} ${M_LIBS} Modified: lib/Makefile.am (+3 -1) =================================================================== --- lib/Makefile.am 2016-11-10 12:00:16 +0900 (ee89224) +++ lib/Makefile.am 2016-11-11 12:13:10 +0900 (4cf2ac9) @@ -16,7 +16,8 @@ AM_CFLAGS = \ $(COVERAGE_CFLAGS) \ $(GRN_CFLAGS) \ $(MESSAGE_PACK_CFLAGS) \ - $(LIBLZ4_CFLAGS) + $(LIBLZ4_CFLAGS) \ + $(LIBZSTD_CFLAGS) BUNDLED_LIBRARIES_CFLAGS = \ $(MRUBY_CFLAGS) \ @@ -54,6 +55,7 @@ endif libgroonga_la_LIBADD += \ $(ONIGMO_LIBS) \ $(LIBLZ4_LIBS) \ + $(LIBZSTD_LIBS) \ $(ATOMIC_LIBS) if WITH_LEMON Modified: lib/db.c (+20 -1) =================================================================== --- lib/db.c 2016-11-10 12:00:16 +0900 (892e7cb) +++ lib/db.c 2016-11-11 12:13:10 +0900 (b3bd01a) @@ -8041,6 +8041,18 @@ grn_obj_get_info(grn_ctx *ctx, grn_obj *obj, grn_info_type type, grn_obj *valueb GRN_BOOL_PUT(ctx, valuebuf, GRN_FALSE); #endif /* GRN_WITH_LZ4 */ break; + case GRN_INFO_SUPPORT_ZSTD : + if (!valuebuf && !(valuebuf = grn_obj_open(ctx, GRN_BULK, 0, GRN_DB_BOOL))) { + ERR(GRN_INVALID_ARGUMENT, + "failed to open value buffer for GRN_INFO_ZSTD_SUPPORT"); + goto exit; + } +#ifdef GRN_WITH_ZSTD + GRN_BOOL_PUT(ctx, valuebuf, GRN_TRUE); +#else /* GRN_WITH_ZSTD */ + GRN_BOOL_PUT(ctx, valuebuf, GRN_FALSE); +#endif /* GRN_WITH_ZSTD */ + break; default : if (!obj) { ERR(GRN_INVALID_ARGUMENT, "grn_obj_get_info failed"); @@ -12007,7 +12019,14 @@ is_compressed_column(grn_ctx *ctx, grn_obj *obj) return GRN_FALSE; } - return (obj->header.flags & (GRN_OBJ_COMPRESS_ZLIB | GRN_OBJ_COMPRESS_LZ4)); + switch (obj->header.flags & GRN_OBJ_COMPRESS_MASK) { + case GRN_OBJ_COMPRESS_ZLIB : + case GRN_OBJ_COMPRESS_LZ4 : + case GRN_OBJ_COMPRESS_ZSTD : + return GRN_TRUE; + default : + return GRN_FALSE; + } } static grn_bool Modified: lib/dump.c (+3 -0) =================================================================== --- lib/dump.c 2016-11-10 12:00:16 +0900 (b69081e) +++ lib/dump.c 2016-11-11 12:13:10 +0900 (0c6eb68) @@ -100,6 +100,9 @@ grn_dump_column_create_flags(grn_ctx *ctx, case GRN_OBJ_COMPRESS_LZ4: GRN_TEXT_PUTS(ctx, buffer, "|COMPRESS_LZ4"); break; + case GRN_OBJ_COMPRESS_ZSTD: + GRN_TEXT_PUTS(ctx, buffer, "|COMPRESS_ZSTD"); + break; } if (flags & GRN_OBJ_PERSISTENT) { GRN_TEXT_PUTS(ctx, buffer, "|PERSISTENT"); Modified: lib/error.c (+3 -0) =================================================================== --- lib/error.c 2016-11-10 12:00:16 +0900 (6c8db7a) +++ lib/error.c 2016-11-11 12:13:10 +0900 (d6d14ff) @@ -441,6 +441,9 @@ grn_rc_to_string(grn_rc rc) case GRN_WINDOW_FUNCTION_ERROR : message = "window function error"; break; + case GRN_ZSTD_ERROR : + message = "Zstandard error"; + break; } return message; Modified: lib/mrb/mrb_ctx.c (+6 -0) =================================================================== --- lib/mrb/mrb_ctx.c 2016-11-10 12:00:16 +0900 (1edab7f) +++ lib/mrb/mrb_ctx.c 2016-11-11 12:13:10 +0900 (e4d9eed) @@ -762,6 +762,12 @@ grn_mrb_ctx_check(mrb_state *mrb) "window function error: <%s>(%d)", ctx->errbuf, ctx->rc); break; + case GRN_ZSTD_ERROR: + error_class = mrb_class_get_under(mrb, module, "ZstdError"); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "Zstandard error: <%s>(%d)", + ctx->errbuf, ctx->rc); + break; } if (!error_class) { Modified: lib/mrb/mrb_error.c (+2 -0) =================================================================== --- lib/mrb/mrb_error.c 2016-11-10 12:00:16 +0900 (661910f) +++ lib/mrb/mrb_error.c 2016-11-11 12:13:10 +0900 (5ba8634) @@ -196,5 +196,7 @@ grn_mrb_error_init(grn_ctx *ctx) groonga_error_class); mrb_define_class_under(mrb, module, "WindowFunctionError", groonga_error_class); + mrb_define_class_under(mrb, module, "ZstdError", + groonga_error_class); } #endif Modified: lib/mrb/mrb_object_flags.c (+1 -0) =================================================================== --- lib/mrb/mrb_object_flags.c 2016-11-10 12:00:16 +0900 (c8accad) +++ lib/mrb/mrb_object_flags.c 2016-11-11 12:13:10 +0900 (21d5b31) @@ -67,6 +67,7 @@ grn_mrb_object_flags_init(grn_ctx *ctx) MRB_DEFINE_FLAG(COMPRESS_NONE); MRB_DEFINE_FLAG(COMPRESS_ZLIB); MRB_DEFINE_FLAG(COMPRESS_LZ4); + MRB_DEFINE_FLAG(COMPRESS_ZSTD); MRB_DEFINE_FLAG(WITH_SECTION); MRB_DEFINE_FLAG(WITH_WEIGHT); Modified: lib/mrb/scripts/context/rc.rb (+2 -0) =================================================================== --- lib/mrb/scripts/context/rc.rb 2016-11-10 12:00:16 +0900 (ba327f7) +++ lib/mrb/scripts/context/rc.rb 2016-11-11 12:13:10 +0900 (45187c2) @@ -196,6 +196,8 @@ module Groonga register(:cancel, -77, Cancel) WINDOW_FUNCTION_ERROR = register(:window_function_error, -78, WindowFunctionError) + ZSTD_ERROR = + register(:zstd_error, -79, ZstdError) GroongaError.rc = UNKNOWN_ERROR end Modified: lib/proc/proc_column.c (+1 -0) =================================================================== --- lib/proc/proc_column.c 2016-11-10 12:00:16 +0900 (ee33f61) +++ lib/proc/proc_column.c 2016-11-11 12:13:10 +0900 (fa6dacf) @@ -53,6 +53,7 @@ grn_proc_column_parse_flags(grn_ctx *ctx, CHECK_FLAG(COLUMN_INDEX); CHECK_FLAG(COMPRESS_ZLIB); CHECK_FLAG(COMPRESS_LZ4); + CHECK_FLAG(COMPRESS_ZSTD); CHECK_FLAG(WITH_SECTION); CHECK_FLAG(WITH_WEIGHT); CHECK_FLAG(WITH_POSITION); Modified: lib/proc/proc_schema.c (+10 -2) =================================================================== --- lib/proc/proc_schema.c 2016-11-10 12:00:16 +0900 (e577262) +++ lib/proc/proc_schema.c 2016-11-11 12:13:10 +0900 (2ead05e) @@ -741,10 +741,18 @@ command_schema_column_output_compress(grn_ctx *ctx, grn_obj *column) const char *compress = NULL; if (column->header.type != GRN_COLUMN_INDEX) { - if (column->header.flags & GRN_OBJ_COMPRESS_ZLIB) { + switch (column->header.flags & GRN_OBJ_COMPRESS_MASK) { + case GRN_OBJ_COMPRESS_ZLIB : compress = "zlib"; - } else if (column->header.flags & GRN_OBJ_COMPRESS_LZ4) { + break; + case GRN_OBJ_COMPRESS_LZ4 : compress = "lz4"; + break; + case GRN_OBJ_COMPRESS_ZSTD : + compress = "zstd"; + break; + default : + break; } } Modified: lib/store.c (+279 -29) =================================================================== --- lib/store.c 2016-11-10 12:00:16 +0900 (154c33d) +++ lib/store.c 2016-11-11 12:13:10 +0900 (c59e35c) @@ -1209,7 +1209,11 @@ grn_ja_element_info(grn_ctx *ctx, grn_ja *ja, grn_id id, #define COMPRESS_PACKED_VALUE_SIZE_MAX 257 /* COMPRESS_THRESHOLD_BYTE - 1 + sizeof(uint64_t) = 257 */ -#if defined(GRN_WITH_ZLIB) || defined(GRN_WITH_LZ4) +#if defined(GRN_WITH_ZLIB) || defined(GRN_WITH_LZ4) || defined(GRN_WITH_ZSTD) +# define GRN_WITH_COMPRESSED +#endif + +#ifdef GRN_WITH_COMPRESSED static void * grn_ja_ref_packed(grn_ctx *ctx, grn_io_win *iw, @@ -1265,7 +1269,37 @@ grn_ja_put_packed(grn_ctx *ctx, flags, cas); } -#endif /* defined(GRN_WITH_ZLIB) || defined(GRN_WITH_LZ4) */ + +static void +grn_ja_compress_error(grn_ctx *ctx, + grn_ja *ja, + grn_id id, + grn_rc rc, + const char *message, + const char *detail) +{ + char name[GRN_TABLE_MAX_KEY_SIZE]; + int name_len; + + if (ja->obj.id == GRN_ID_NIL) { + name[0] = '\0'; + name_len = 0; + } else { + name_len = grn_obj_name(ctx, (grn_obj *)ja, name, GRN_TABLE_MAX_KEY_SIZE); + } + ERR(GRN_ZSTD_ERROR, + "[ja]%s: %s%.*s%s<%u>%s%s%s", + message, + name_len == 0 ? "" : "<", + name_len, + name, + name_len == 0 ? "" : ">: ", + id, + detail ? " :<" : "", + detail ? detail : "", + detail ? ">" : ""); +} +#endif /* GRN_WITH_COMPRESSED */ #ifdef GRN_WITH_ZLIB #include <zlib.h> @@ -1378,20 +1412,85 @@ grn_ja_ref_lz4(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *va } #endif /* GRN_WITH_LZ4 */ +#ifdef GRN_WITH_ZSTD +#include <zstd.h> + +static void * +grn_ja_ref_zstd(grn_ctx *ctx, + grn_ja *ja, + grn_id id, + grn_io_win *iw, + uint32_t *value_len) +{ + void *raw_value; + uint32_t raw_value_len; + void *zstd_value; + uint32_t zstd_value_len; + void *unpacked_value; + uint32_t uncompressed_value_len; + size_t written_len; + + if (!(raw_value = grn_ja_ref_raw(ctx, ja, id, iw, &raw_value_len))) { + iw->uncompressed_value = NULL; + *value_len = 0; + return NULL; + } + + unpacked_value = grn_ja_ref_packed(ctx, + iw, value_len, + raw_value, raw_value_len, + &zstd_value, &zstd_value_len, + &uncompressed_value_len); + if (unpacked_value) { + return unpacked_value; + } + + if (!(iw->uncompressed_value = GRN_MALLOC(uncompressed_value_len))) { + iw->uncompressed_value = NULL; + *value_len = 0; + return NULL; + } + + written_len = ZSTD_decompress((char *)(iw->uncompressed_value), + uncompressed_value_len, + zstd_value, + zstd_value_len); + if (ZSTD_isError(written_len)) { + GRN_FREE(iw->uncompressed_value); + iw->uncompressed_value = NULL; + *value_len = 0; + grn_ja_compress_error(ctx, + ja, + id, + GRN_ZSTD_ERROR, + "[zstd] failed to decompress", + ZSTD_getErrorName(written_len)); + return NULL; + } + *value_len = uncompressed_value_len; + return iw->uncompressed_value; +} +#endif /* GRN_WITH_ZSTD */ + void * grn_ja_ref(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len) { + switch (ja->header->flags & GRN_OBJ_COMPRESS_MASK) { #ifdef GRN_WITH_ZLIB - if (ja->header->flags & GRN_OBJ_COMPRESS_ZLIB) { + case GRN_OBJ_COMPRESS_ZLIB : return grn_ja_ref_zlib(ctx, ja, id, iw, value_len); - } #endif /* GRN_WITH_ZLIB */ #ifdef GRN_WITH_LZ4 - if (ja->header->flags & GRN_OBJ_COMPRESS_LZ4) { + case GRN_OBJ_COMPRESS_LZ4 : return grn_ja_ref_lz4(ctx, ja, id, iw, value_len); - } #endif /* GRN_WITH_LZ4 */ - return grn_ja_ref_raw(ctx, ja, id, iw, value_len); +#ifdef GRN_WITH_ZSTD + case GRN_OBJ_COMPRESS_ZSTD : + return grn_ja_ref_zstd(ctx, ja, id, iw, value_len); +#endif /* GRN_WITH_ZSTD */ + default : + return grn_ja_ref_raw(ctx, ja, id, iw, value_len); + } } grn_obj * @@ -1523,21 +1622,85 @@ grn_ja_put_lz4(grn_ctx *ctx, grn_ja *ja, grn_id id, } #endif /* GRN_WITH_LZ4 */ +#ifdef GRN_WITH_ZSTD +inline static grn_rc +grn_ja_put_zstd(grn_ctx *ctx, + grn_ja *ja, + grn_id id, + void *value, + uint32_t value_len, + int flags, + uint64_t *cas) +{ + grn_rc rc; + void *packed_value; + int packed_value_len_max; + int packed_value_len_real; + void *zstd_value; + int zstd_value_len_max; + int zstd_value_len_real; + int zstd_compression_level = 3; + + if (value_len == 0) { + return grn_ja_put_raw(ctx, ja, id, value, value_len, flags, cas); + } + + if (value_len < COMPRESS_THRESHOLD_BYTE) { + return grn_ja_put_packed(ctx, ja, id, value, value_len, flags, cas); + } + + zstd_value_len_max = ZSTD_compressBound(value_len); + packed_value_len_max = zstd_value_len_max + sizeof(uint64_t); + if (!(packed_value = GRN_MALLOC(packed_value_len_max))) { + return GRN_NO_MEMORY_AVAILABLE; + } + zstd_value = ((uint64_t *)packed_value) + 1; + zstd_value_len_real = ZSTD_compress(zstd_value, zstd_value_len_max, + value, value_len, + zstd_compression_level); + if (ZSTD_isError(zstd_value_len_real)) { + grn_ja_compress_error(ctx, + ja, + id, + GRN_ZSTD_ERROR, + "[zstd] failed to compress", + ZSTD_getErrorName(zstd_value_len_real)); + return ctx->rc; + } + *(uint64_t *)packed_value = value_len; + packed_value_len_real = zstd_value_len_real + sizeof(uint64_t); + rc = grn_ja_put_raw(ctx, + ja, + id, + packed_value, + packed_value_len_real, + flags, + cas); + GRN_FREE(packed_value); + return rc; +} +#endif /* GRN_WITH_ZSTD */ + grn_rc grn_ja_put(grn_ctx *ctx, grn_ja *ja, grn_id id, void *value, uint32_t value_len, int flags, uint64_t *cas) { + switch (ja->header->flags & GRN_OBJ_COMPRESS_MASK) { #ifdef GRN_WITH_ZLIB - if (ja->header->flags & GRN_OBJ_COMPRESS_ZLIB) { + case GRN_OBJ_COMPRESS_ZLIB : return grn_ja_put_zlib(ctx, ja, id, value, value_len, flags, cas); - } #endif /* GRN_WITH_ZLIB */ #ifdef GRN_WITH_LZ4 - if (ja->header->flags & GRN_OBJ_COMPRESS_LZ4) { + case GRN_OBJ_COMPRESS_LZ4 : return grn_ja_put_lz4(ctx, ja, id, value, value_len, flags, cas); - } #endif /* GRN_WITH_LZ4 */ - return grn_ja_put_raw(ctx, ja, id, value, value_len, flags, cas); +#ifdef GRN_WITH_ZSTD + case GRN_OBJ_COMPRESS_ZSTD : + return grn_ja_put_zstd(ctx, ja, id, value, value_len, flags, cas); +#endif /* GRN_WITH_ZSTD */ + default : + return grn_ja_put_raw(ctx, ja, id, value, value_len, flags, cas); + } } static grn_rc @@ -1783,7 +1946,7 @@ grn_ja_reader_close(grn_ctx *ctx, grn_ja_reader *reader) return rc; } -#if defined(GRN_WITH_ZLIB) || defined(GRN_WITH_LZ4) +#ifdef GRN_WITH_COMPRESSED /* grn_ja_reader_seek_compressed() prepares to access a compressed value. */ static grn_rc grn_ja_reader_seek_compressed(grn_ctx *ctx, grn_ja_reader *reader, grn_id id) @@ -1830,7 +1993,7 @@ grn_ja_reader_seek_compressed(grn_ctx *ctx, grn_ja_reader *reader, grn_id id) reader->value_size = (uint32_t)*(uint64_t *)seg_addr; return GRN_SUCCESS; } -#endif /* defined(GRN_WITH_ZLIB) || defined(GRN_WITH_LZ4) */ +#endif /* GRN_WITH_COMPRESSED */ /* grn_ja_reader_seek_raw() prepares to access a value. */ static grn_rc @@ -1881,17 +2044,22 @@ grn_ja_reader_seek_raw(grn_ctx *ctx, grn_ja_reader *reader, grn_id id) grn_rc grn_ja_reader_seek(grn_ctx *ctx, grn_ja_reader *reader, grn_id id) { + switch (reader->ja->header->flags & GRN_OBJ_COMPRESS_MASK) { #ifdef GRN_WITH_ZLIB - if (reader->ja->header->flags & GRN_OBJ_COMPRESS_ZLIB) { + case GRN_OBJ_COMPRESS_ZLIB : return grn_ja_reader_seek_compressed(ctx, reader, id); - } #endif /* GRN_WITH_ZLIB */ #ifdef GRN_WITH_LZ4 - if (reader->ja->header->flags & GRN_OBJ_COMPRESS_LZ4) { + case GRN_OBJ_COMPRESS_LZ4 : return grn_ja_reader_seek_compressed(ctx, reader, id); - } #endif /* GRN_WITH_LZ4 */ - return grn_ja_reader_seek_raw(ctx, reader, id); +#ifdef GRN_WITH_ZSTD + case GRN_OBJ_COMPRESS_ZSTD : + return grn_ja_reader_seek_compressed(ctx, reader, id); +#endif /* GRN_WITH_ZSTD */ + default : + return grn_ja_reader_seek_raw(ctx, reader, id); + } } grn_rc @@ -2074,6 +2242,67 @@ grn_ja_reader_read_lz4(grn_ctx *ctx, grn_ja_reader *reader, void *buf) } #endif /* GRN_WITH_LZ4 */ +#ifdef GRN_WITH_ZSTD +/* grn_ja_reader_read_zstd() reads a value compressed with Zstandard. */ +static grn_rc +grn_ja_reader_read_zstd(grn_ctx *ctx, grn_ja_reader *reader, void *buf) +{ + int src_size, dest_size; + grn_ja_einfo *einfo = (grn_ja_einfo *)reader->einfo; + if (EHUGE_P(einfo)) { + grn_io *io = reader->ja->io; + void *seg_addr; + char *packed_ptr; + uint32_t size, seg_id; + if (reader->packed_size > reader->packed_buf_size) { + void *new_buf = GRN_REALLOC(reader->packed_buf, reader->packed_size); + if (!new_buf) { + return GRN_NO_MEMORY_AVAILABLE; + } + reader->packed_buf = new_buf; + reader->packed_buf_size = reader->packed_size; + } + packed_ptr = (char *)reader->packed_buf; + grn_memcpy(packed_ptr, (char *)reader->body_seg_addr + sizeof(uint64_t), + io->header->segment_size - sizeof(uint64_t)); + packed_ptr += io->header->segment_size - sizeof(uint64_t); + size = reader->packed_size - (io->header->segment_size - sizeof(uint64_t)); + seg_id = reader->body_seg_id + 1; + while (size > io->header->segment_size) { + GRN_IO_SEG_REF(io, seg_id, seg_addr); + if (!seg_addr) { + return GRN_UNKNOWN_ERROR; + } + grn_memcpy(packed_ptr, seg_addr, io->header->segment_size); + GRN_IO_SEG_UNREF(io, seg_id); + seg_id++; + size -= io->header->segment_size; + packed_ptr += io->header->segment_size; + } + GRN_IO_SEG_REF(io, seg_id, seg_addr); + if (!seg_addr) { + return GRN_UNKNOWN_ERROR; + } + grn_memcpy(packed_ptr, seg_addr, size); + GRN_IO_SEG_UNREF(io, seg_id); + seg_id++; + src_size = (int)(reader->packed_size - sizeof(uint64_t)); + dest_size = ZSTD_decompress(reader->packed_buf, reader->value_size, + buf, src_size); + } else { + char *packed_addr = (char *)reader->body_seg_addr; + packed_addr += reader->body_seg_offset + sizeof(uint64_t); + src_size = (int)(reader->packed_size - sizeof(uint64_t)); + dest_size = ZSTD_decompress(packed_addr, reader->value_size, + buf, src_size); + } + if ((uint32_t)dest_size != reader->value_size) { + return GRN_ZSTD_ERROR; + } + return GRN_SUCCESS; +} +#endif /* GRN_WITH_ZSTD */ + /* grn_ja_reader_read_raw() reads a value. */ static grn_rc grn_ja_reader_read_raw(grn_ctx *ctx, grn_ja_reader *reader, void *buf) @@ -2121,17 +2350,22 @@ grn_ja_reader_read_raw(grn_ctx *ctx, grn_ja_reader *reader, void *buf) grn_rc grn_ja_reader_read(grn_ctx *ctx, grn_ja_reader *reader, void *buf) { + switch (reader->ja->header->flags & GRN_OBJ_COMPRESS_MASK) { #ifdef GRN_WITH_ZLIB - if (reader->ja->header->flags & GRN_OBJ_COMPRESS_ZLIB) { + case GRN_OBJ_COMPRESS_ZLIB : return grn_ja_reader_read_zlib(ctx, reader, buf); - } #endif /* GRN_WITH_ZLIB */ #ifdef GRN_WITH_LZ4 - if (reader->ja->header->flags & GRN_OBJ_COMPRESS_LZ4) { + case GRN_OBJ_COMPRESS_LZ4 : return grn_ja_reader_read_lz4(ctx, reader, buf); - } #endif /* GRN_WITH_LZ4 */ - return grn_ja_reader_read_raw(ctx, reader, buf); +#ifdef GRN_WITH_ZSTD + case GRN_OBJ_COMPRESS_ZSTD : + return grn_ja_reader_read_zstd(ctx, reader, buf); +#endif /* GRN_WITH_ZSTD */ + default : + return grn_ja_reader_read_raw(ctx, reader, buf); + } } #ifdef GRN_WITH_ZLIB @@ -2156,6 +2390,17 @@ grn_ja_reader_pread_lz4(grn_ctx *ctx, grn_ja_reader *reader, } #endif /* GRN_WITH_LZ4 */ +#ifdef GRN_WITH_ZSTD +/* grn_ja_reader_pread_zstd() reads a part of a value compressed with ZSTD. */ +static grn_rc +grn_ja_reader_pread_zstd(grn_ctx *ctx, grn_ja_reader *reader, + size_t offset, size_t size, void *buf) +{ + /* TODO: To be supported? */ + return GRN_FUNCTION_NOT_IMPLEMENTED; +} +#endif /* GRN_WITH_ZSTD */ + /* grn_ja_reader_pread_raw() reads a part of a value. */ static grn_rc grn_ja_reader_pread_raw(grn_ctx *ctx, grn_ja_reader *reader, @@ -2223,17 +2468,22 @@ grn_rc grn_ja_reader_pread(grn_ctx *ctx, grn_ja_reader *reader, size_t offset, size_t size, void *buf) { + switch (reader->ja-header->flags & GRN_OBJ_COMPRESS_MASK) { #ifdef GRN_WITH_ZLIB - if (reader->ja->header->flags & GRN_OBJ_COMPRESS_ZLIB) { + case GRN_OBJ_COMPRESS_ZLIB : return grn_ja_reader_pread_zlib(ctx, reader, offset, size, buf); - } #endif /* GRN_WITH_ZLIB */ #ifdef GRN_WITH_LZ4 - if (reader->ja->header->flags & GRN_OBJ_COMPRESS_LZ4) { + case GRN_OBJ_COMPRESS_LZ4 : return grn_ja_reader_pread_lz4(ctx, reader, offset, size, buf); - } #endif /* GRN_WITH_LZ4 */ - return grn_ja_reader_pread_raw(ctx, reader, offset, size, buf); +#ifdef GRN_WITH_ZSTD + case GRN_OBJ_COMPRESS_ZSTD : + return grn_ja_reader_pread_zstd(ctx, reader, offset, size, buf); +#endif /* GRN_WITH_ZSTD */ + default : + return grn_ja_reader_pread_raw(ctx, reader, offset, size, buf); + } } /**** vgram ****/ Modified: lib/util.c (+3 -0) =================================================================== --- lib/util.c 2016-11-10 12:00:16 +0900 (9483474) +++ lib/util.c 2016-11-11 12:13:10 +0900 (b976d65) @@ -542,6 +542,9 @@ grn_store_inspect_body(grn_ctx *ctx, grn_obj *buf, grn_obj *obj) case GRN_OBJ_COMPRESS_LZ4 : GRN_TEXT_PUTS(ctx, buf, "lz4"); break; + case GRN_OBJ_COMPRESS_ZSTD : + GRN_TEXT_PUTS(ctx, buf, "zstd"); + break; default: break; } Modified: src/groonga.c (+3 -0) =================================================================== --- src/groonga.c 2016-11-10 12:00:16 +0900 (3bb90a9) +++ src/groonga.c 2016-11-11 12:13:10 +0900 (fea7f91) @@ -3054,6 +3054,9 @@ show_version(void) #ifdef GRN_WITH_LZ4 printf(",lz4"); #endif +#ifdef GRN_WITH_ZSTD + printf(",zstd"); +#endif #ifdef USE_KQUEUE printf(",kqueue"); #endif Added: test/command/suite/column_list/compress/lz4.expected (+60 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/column_list/compress/lz4.expected 2016-11-11 12:13:10 +0900 (d63ba99) @@ -0,0 +1,60 @@ +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR|COMPRESS_LZ4 Text +[[0,0.0,0.0],true] +column_list Memos +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "id", + "UInt32" + ], + [ + "name", + "ShortText" + ], + [ + "path", + "ShortText" + ], + [ + "type", + "ShortText" + ], + [ + "flags", + "ShortText" + ], + [ + "domain", + "ShortText" + ], + [ + "range", + "ShortText" + ], + [ + "source", + "ShortText" + ] + ], + [ + 257, + "content", + "db/db.0000101", + "var", + "COLUMN_SCALAR|COMPRESS_LZ4|PERSISTENT", + "Memos", + "Text", + [ + + ] + ] + ] +] Added: test/command/suite/column_list/compress/lz4.test (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/column_list/compress/lz4.test 2016-11-11 12:13:10 +0900 (1831981) @@ -0,0 +1,4 @@ +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_LZ4 Text + +column_list Memos Added: test/command/suite/column_list/compress/zlib.expected (+60 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/column_list/compress/zlib.expected 2016-11-11 12:13:10 +0900 (b871094) @@ -0,0 +1,60 @@ +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR|COMPRESS_ZLIB Text +[[0,0.0,0.0],true] +column_list Memos +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "id", + "UInt32" + ], + [ + "name", + "ShortText" + ], + [ + "path", + "ShortText" + ], + [ + "type", + "ShortText" + ], + [ + "flags", + "ShortText" + ], + [ + "domain", + "ShortText" + ], + [ + "range", + "ShortText" + ], + [ + "source", + "ShortText" + ] + ], + [ + 257, + "content", + "db/db.0000101", + "var", + "COLUMN_SCALAR|COMPRESS_ZLIB|PERSISTENT", + "Memos", + "Text", + [ + + ] + ] + ] +] Added: test/command/suite/column_list/compress/zlib.test (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/column_list/compress/zlib.test 2016-11-11 12:13:10 +0900 (170832f) @@ -0,0 +1,4 @@ +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_ZLIB Text + +column_list Memos Added: test/command/suite/column_list/compress/zstd.expected (+60 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/column_list/compress/zstd.expected 2016-11-11 12:13:10 +0900 (546c44c) @@ -0,0 +1,60 @@ +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR|COMPRESS_ZSTD Text +[[0,0.0,0.0],true] +column_list Memos +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "id", + "UInt32" + ], + [ + "name", + "ShortText" + ], + [ + "path", + "ShortText" + ], + [ + "type", + "ShortText" + ], + [ + "flags", + "ShortText" + ], + [ + "domain", + "ShortText" + ], + [ + "range", + "ShortText" + ], + [ + "source", + "ShortText" + ] + ], + [ + 257, + "content", + "db/db.0000101", + "var", + "COLUMN_SCALAR|COMPRESS_ZSTD|PERSISTENT", + "Memos", + "Text", + [ + + ] + ] + ] +] Added: test/command/suite/column_list/compress/zstd.test (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/column_list/compress/zstd.test 2016-11-11 12:13:10 +0900 (612e445) @@ -0,0 +1,4 @@ +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_ZSTD Text + +column_list Memos Added: test/command/suite/dump/schema/column/scalar/lz4.expected (+7 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/column/scalar/lz4.expected 2016-11-11 12:13:10 +0900 (fbb6f92) @@ -0,0 +1,7 @@ +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR|COMPRESS_LZ4 Text +[[0,0.0,0.0],true] +dump +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_LZ4 Text Added: test/command/suite/dump/schema/column/scalar/lz4.test (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/column/scalar/lz4.test 2016-11-11 12:13:10 +0900 (f9008e2) @@ -0,0 +1,4 @@ +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_LZ4 Text + +dump Added: test/command/suite/dump/schema/column/scalar/zlib.expected (+7 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/column/scalar/zlib.expected 2016-11-11 12:13:10 +0900 (9524d00) @@ -0,0 +1,7 @@ +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR|COMPRESS_ZLIB Text +[[0,0.0,0.0],true] +dump +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_ZLIB Text Added: test/command/suite/dump/schema/column/scalar/zlib.test (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/column/scalar/zlib.test 2016-11-11 12:13:10 +0900 (6a560f6) @@ -0,0 +1,4 @@ +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_ZLIB Text + +dump Added: test/command/suite/dump/schema/column/scalar/zstd.expected (+7 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/column/scalar/zstd.expected 2016-11-11 12:13:10 +0900 (10cf43e) @@ -0,0 +1,7 @@ +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR|COMPRESS_ZSTD Text +[[0,0.0,0.0],true] +dump +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_ZSTD Text Added: test/command/suite/dump/schema/column/scalar/zstd.test (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/column/scalar/zstd.test 2016-11-11 12:13:10 +0900 (648bd9c) @@ -0,0 +1,4 @@ +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR|COMPRESS_ZSTD Text + +dump Added: test/command/suite/schema/tables/columns/compress/zstd.expected (+266 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/schema/tables/columns/compress/zstd.expected 2016-11-11 12:13:10 +0900 (b8bbc04) @@ -0,0 +1,266 @@ +table_create Logs TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs message COLUMN_SCALAR|COMPRESS_ZSTD Text +[[0,0.0,0.0],true] +schema +[ + [ + 0, + 0.0, + 0.0 + ], + { + "plugins": { + }, + "types": { + "Bool": { + "id": 3, + "name": "Bool", + "size": 1, + "can_be_key_type": true, + "can_be_value_type": true + }, + "Float": { + "id": 12, + "name": "Float", + "size": 8, + "can_be_key_type": true, + "can_be_value_type": true + }, + "Int16": { + "id": 6, + "name": "Int16", + "size": 2, + "can_be_key_type": true, + "can_be_value_type": true + }, + "Int32": { + "id": 8, + "name": "Int32", + "size": 4, + "can_be_key_type": true, + "can_be_value_type": true + }, + "Int64": { + "id": 10, + "name": "Int64", + "size": 8, + "can_be_key_type": true, + "can_be_value_type": true + }, + "Int8": { + "id": 4, + "name": "Int8", + "size": 1, + "can_be_key_type": true, + "can_be_value_type": true + }, + "LongText": { + "id": 16, + "name": "LongText", + "size": 2147483648, + "can_be_key_type": false, + "can_be_value_type": false + }, + "Object": { + "id": 2, + "name": "Object", + "size": 8, + "can_be_key_type": true, + "can_be_value_type": true + }, + "ShortText": { + "id": 14, + "name": "ShortText", + "size": 4096, + "can_be_key_type": true, + "can_be_value_type": false + }, + "Text": { + "id": 15, + "name": "Text", + "size": 65536, + "can_be_key_type": false, + "can_be_value_type": false + }, + "Time": { + "id": 13, + "name": "Time", + "size": 8, + "can_be_key_type": true, + "can_be_value_type": true + }, + "TokyoGeoPoint": { + "id": 17, + "name": "TokyoGeoPoint", + "size": 8, + "can_be_key_type": true, + "can_be_value_type": true + }, + "UInt16": { + "id": 7, + "name": "UInt16", + "size": 2, + "can_be_key_type": true, + "can_be_value_type": true + }, + "UInt32": { + "id": 9, + "name": "UInt32", + "size": 4, + "can_be_key_type": true, + "can_be_value_type": true + }, + "UInt64": { + "id": 11, + "name": "UInt64", + "size": 8, + "can_be_key_type": true, + "can_be_value_type": true + }, + "UInt8": { + "id": 5, + "name": "UInt8", + "size": 1, + "can_be_key_type": true, + "can_be_value_type": true + }, + "WGS84GeoPoint": { + "id": 18, + "name": "WGS84GeoPoint", + "size": 8, + "can_be_key_type": true, + "can_be_value_type": true + } + }, + "tokenizers": { + "TokenBigram": { + "id": 67, + "name": "TokenBigram" + }, + "TokenBigramIgnoreBlank": { + "id": 72, + "name": "TokenBigramIgnoreBlank" + }, + "TokenBigramIgnoreBlankSplitSymbol": { + "id": 73, + "name": "TokenBigramIgnoreBlankSplitSymbol" + }, + "TokenBigramIgnoreBlankSplitSymbolAlpha": { + "id": 74, + "name": "TokenBigramIgnoreBlankSplitSymbolAlpha" + }, + "TokenBigramIgnoreBlankSplitSymbolAlphaDigit": { + "id": 75, + "name": "TokenBigramIgnoreBlankSplitSymbolAlphaDigit" + }, + "TokenBigramSplitSymbol": { + "id": 69, + "name": "TokenBigramSplitSymbol" + }, + "TokenBigramSplitSymbolAlpha": { + "id": 70, + "name": "TokenBigramSplitSymbolAlpha" + }, + "TokenBigramSplitSymbolAlphaDigit": { + "id": 71, + "name": "TokenBigramSplitSymbolAlphaDigit" + }, + "TokenDelimit": { + "id": 65, + "name": "TokenDelimit" + }, + "TokenDelimitNull": { + "id": 76, + "name": "TokenDelimitNull" + }, + "TokenMecab": { + "id": 64, + "name": "TokenMecab" + }, + "TokenRegexp": { + "id": 77, + "name": "TokenRegexp" + }, + "TokenTrigram": { + "id": 68, + "name": "TokenTrigram" + }, + "TokenUnigram": { + "id": 66, + "name": "TokenUnigram" + } + }, + "normalizers": { + "NormalizerAuto": { + "id": 78, + "name": "NormalizerAuto" + }, + "NormalizerNFKC51": { + "id": 79, + "name": "NormalizerNFKC51" + } + }, + "token_filters": { + }, + "tables": { + "Logs": { + "id": 256, + "name": "Logs", + "type": "array", + "key_type": null, + "value_type": null, + "tokenizer": null, + "normalizer": null, + "token_filters": [ + + ], + "indexes": [ + + ], + "command": { + "name": "table_create", + "arguments": { + "name": "Logs", + "flags": "TABLE_NO_KEY" + }, + "command_line": "table_create --name Logs --flags TABLE_NO_KEY" + }, + "columns": { + "message": { + "id": 257, + "name": "message", + "table": "Logs", + "full_name": "Logs.message", + "type": "scalar", + "value_type": { + "id": 15, + "name": "Text", + "type": "type" + }, + "compress": "zstd", + "section": false, + "weight": false, + "position": false, + "sources": [ + + ], + "indexes": [ + + ], + "command": { + "name": "column_create", + "arguments": { + "table": "Logs", + "name": "message", + "flags": "COLUMN_SCALAR|COMPRESS_ZSTD", + "type": "Text" + }, + "command_line": "column_create --table Logs --name message --flags COLUMN_SCALAR|COMPRESS_ZSTD --type Text" + } + } + } + } + } + } +] Added: test/command/suite/schema/tables/columns/compress/zstd.test (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/schema/tables/columns/compress/zstd.test 2016-11-11 12:13:10 +0900 (932ce17) @@ -0,0 +1,4 @@ +table_create Logs TABLE_NO_KEY +column_create Logs message COLUMN_SCALAR|COMPRESS_ZSTD Text + +schema Added: test/command/suite/select/output/zstd/scalar/compressed.expected (+55 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/zstd/scalar/compressed.expected 2016-11-11 12:13:10 +0900 (73fbb32) @@ -0,0 +1,55 @@ +table_create Entries TABLE_PAT_KEY ShortText +[[0,0.0,0.0],true] +column_create Entries content COLUMN_SCALAR|COMPRESS_ZSTD Text +[[0,0.0,0.0],true] +load --table Entries +[ + { + "_key": "Groonga", + "content": "(256 bytes or more required for compression.) Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." + }, + { + "_key": "Mroonga", + "content": "(256 bytes or more required for compression.) Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." + } +] +[[0,0.0,0.0],2] +select Entries +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 2 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "_key", + "ShortText" + ], + [ + "content", + "Text" + ] + ], + [ + 1, + "Groonga", + "(256 bytes or more required for compression.) Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." + ], + [ + 2, + "Mroonga", + "(256 bytes or more required for compression.) Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." + ] + ] + ] +] Added: test/command/suite/select/output/zstd/scalar/compressed.test (+16 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/zstd/scalar/compressed.test 2016-11-11 12:13:10 +0900 (058d2cd) @@ -0,0 +1,16 @@ +table_create Entries TABLE_PAT_KEY ShortText +column_create Entries content COLUMN_SCALAR|COMPRESS_ZSTD Text + +load --table Entries +[ + { + "_key": "Groonga", + "content": "(256 bytes or more required for compression.) Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." + }, + { + "_key": "Mroonga", + "content": "(256 bytes or more required for compression.) Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." + } +] + +select Entries Added: test/command/suite/select/output/zstd/scalar/packed.expected (+55 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/zstd/scalar/packed.expected 2016-11-11 12:13:10 +0900 (454538f) @@ -0,0 +1,55 @@ +table_create Entries TABLE_PAT_KEY ShortText +[[0,0.0,0.0],true] +column_create Entries content COLUMN_SCALAR|COMPRESS_ZSTD Text +[[0,0.0,0.0],true] +load --table Entries +[ + { + "_key": "Groonga", + "content": "I found Groonga that is a fast fulltext search engine!" + }, + { + "_key": "Mroonga", + "content": "I found Mroonga that is a MySQL storage engine to use Groonga!" + } +] +[[0,0.0,0.0],2] +select Entries +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 2 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "_key", + "ShortText" + ], + [ + "content", + "Text" + ] + ], + [ + 1, + "Groonga", + "I found Groonga that is a fast fulltext search engine!" + ], + [ + 2, + "Mroonga", + "I found Mroonga that is a MySQL storage engine to use Groonga!" + ] + ] + ] +] Added: test/command/suite/select/output/zstd/scalar/packed.test (+16 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/zstd/scalar/packed.test 2016-11-11 12:13:10 +0900 (929ef35) @@ -0,0 +1,16 @@ +table_create Entries TABLE_PAT_KEY ShortText +column_create Entries content COLUMN_SCALAR|COMPRESS_ZSTD Text + +load --table Entries +[ + { + "_key": "Groonga", + "content": "I found Groonga that is a fast fulltext search engine!" + }, + { + "_key": "Mroonga", + "content": "I found Mroonga that is a MySQL storage engine to use Groonga!" + } +] + +select Entries -------------- next part -------------- HTML����������������������������...Download