naoa
null+****@clear*****
Tue Oct 21 20:50:41 JST 2014
naoa 2014-10-21 20:50:41 +0900 (Tue, 21 Oct 2014) New Revision: ebb6fa7104e4b0e6a396c7e687a9dcdad033dff9 https://github.com/groonga/groonga/commit/ebb6fa7104e4b0e6a396c7e687a9dcdad033dff9 Merged c22fb23: Merge pull request #223 from naoa/add-lz4-compression Message: Add LZ4 compression instead of LZO compression Modified files: configure.ac include/groonga.h lib/CMakeLists.txt lib/db.c lib/mrb/mrb_ctx.c lib/mrb/mrb_error.c lib/mrb/scripts/context/rc.rb lib/proc.c lib/store.c lib/util.c test/unit/core/test-context.c test/unit/lib/grn-test-utils.c Renamed files: test/command/suite/select/output/lz4/scalar.expected (from test/command/suite/select/output/lzo/scalar.expected) test/command/suite/select/output/lz4/scalar.test (from test/command/suite/select/output/lzo/scalar.test) Modified: configure.ac (+9 -9) =================================================================== --- configure.ac 2014-10-21 12:30:50 +0900 (2be82a1) +++ configure.ac 2014-10-21 20:50:41 +0900 (4abd18e) @@ -880,15 +880,15 @@ else fi AC_SUBST(GRN_WITH_ZLIB) -# LZO -AC_ARG_WITH(lzo, - [AS_HELP_STRING([--with-lzo], - [use LZO for data compression. [default=no]])], - [with_lzo="$withval"], - [with_lzo="no"]) -if test "x$with_lzo" = "xyes"; then - AC_DEFINE(GRN_WITH_LZO, [1], [with lzo]) - AC_SEARCH_LIBS(lzo1_compress, lzo2, [], [AC_MSG_ERROR("No liblzo2 found")]) +# LZ4 +AC_ARG_WITH(lz4, + [AS_HELP_STRING([--with-lz4], + [use LZ4 for data compression. [default=no]])], + [with_lz4="$withval"], + [with_lz4="no"]) +if test "x$with_lz4" = "xyes"; then + AC_DEFINE(GRN_WITH_LZ4, [1], [with lz4]) + AC_SEARCH_LIBS(LZ4_compress, lz4, [], [AC_MSG_ERROR("No liblz4 found")]) fi # MeCab Modified: include/groonga.h (+3 -3) =================================================================== --- include/groonga.h 2014-10-21 12:30:50 +0900 (d1cf1e2) +++ include/groonga.h 2014-10-21 20:50:41 +0900 (31a95e2) @@ -104,7 +104,7 @@ typedef enum { GRN_OPERATION_NOT_SUPPORTED = -58, GRN_ADDRESS_IS_IN_USE = -59, GRN_ZLIB_ERROR = -60, - GRN_LZO_ERROR = -61, + GRN_LZ4_ERROR = -61, GRN_STACK_OVER_FLOW = -62, GRN_SYNTAX_ERROR = -63, GRN_RETRY_MAX = -64, @@ -300,7 +300,7 @@ typedef unsigned short int grn_obj_flags; #define GRN_OBJ_COMPRESS_MASK (0x07<<4) #define GRN_OBJ_COMPRESS_NONE (0x00<<4) #define GRN_OBJ_COMPRESS_ZLIB (0x01<<4) -#define GRN_OBJ_COMPRESS_LZO (0x02<<4) +#define GRN_OBJ_COMPRESS_LZ4 (0x02<<4) #define GRN_OBJ_WITH_SECTION (0x01<<7) #define GRN_OBJ_WITH_WEIGHT (0x01<<8) @@ -791,7 +791,7 @@ typedef enum { GRN_INFO_PARTIAL_MATCH_THRESHOLD, GRN_INFO_II_SPLIT_THRESHOLD, GRN_INFO_SUPPORT_ZLIB, - GRN_INFO_SUPPORT_LZO, + GRN_INFO_SUPPORT_LZ4, GRN_INFO_NORMALIZER, GRN_INFO_TOKEN_FILTERS } grn_info_type; Modified: lib/CMakeLists.txt (+1 -1) =================================================================== --- lib/CMakeLists.txt 2014-10-21 12:30:50 +0900 (b18052a) +++ lib/CMakeLists.txt 2014-10-21 20:50:41 +0900 (abb7903) @@ -47,7 +47,7 @@ target_link_libraries(libgroonga ${RT_LIBS} ${PTHREAD_LIBS} ${Z_LIBS} - ${LZO2_LIBS} + ${LZ4_LIBS} ${DL_LIBS} ${WS2_32_LIBS}) Modified: lib/db.c (+5 -5) =================================================================== --- lib/db.c 2014-10-21 12:30:50 +0900 (f948dc7) +++ lib/db.c 2014-10-21 20:50:41 +0900 (47bca9b) @@ -6195,17 +6195,17 @@ grn_obj_get_info(grn_ctx *ctx, grn_obj *obj, grn_info_type type, grn_obj *valueb GRN_BOOL_PUT(ctx, valuebuf, GRN_FALSE); #endif break; - case GRN_INFO_SUPPORT_LZO : + case GRN_INFO_SUPPORT_LZ4 : if (!valuebuf && !(valuebuf = grn_obj_open(ctx, GRN_BULK, 0, GRN_DB_BOOL))) { ERR(GRN_INVALID_ARGUMENT, - "failed to open value buffer for GRN_INFO_LZO_SUPPORT"); + "failed to open value buffer for GRN_INFO_LZ4_SUPPORT"); goto exit; } -#ifdef GRN_WITH_LZO +#ifdef GRN_WITH_LZ4 GRN_BOOL_PUT(ctx, valuebuf, GRN_TRUE); -#else /* GRN_WITH_LZO */ +#else /* GRN_WITH_LZ4 */ GRN_BOOL_PUT(ctx, valuebuf, GRN_FALSE); -#endif /* GRN_WITH_LZO */ +#endif /* GRN_WITH_LZ4 */ break; default : if (!obj) { Modified: lib/mrb/mrb_ctx.c (+3 -3) =================================================================== --- lib/mrb/mrb_ctx.c 2014-10-21 12:30:50 +0900 (3f7bc36) +++ lib/mrb/mrb_ctx.c 2014-10-21 20:50:41 +0900 (64d6c18) @@ -568,10 +568,10 @@ grn_mrb_ctx_check(mrb_state *mrb) "zlib error: <%s>(%d)", ctx->errbuf, ctx->rc); break; - case GRN_LZO_ERROR: - error_class = mrb_class_get_under(mrb, module, "LzoError"); + case GRN_LZ4_ERROR: + error_class = mrb_class_get_under(mrb, module, "Lz4Error"); snprintf(message, MESSAGE_SIZE, - "LZO error: <%s>(%d)", + "LZ4 error: <%s>(%d)", ctx->errbuf, ctx->rc); break; case GRN_STACK_OVER_FLOW: Modified: lib/mrb/mrb_error.c (+1 -1) =================================================================== --- lib/mrb/mrb_error.c 2014-10-21 12:30:50 +0900 (2f45cfc) +++ lib/mrb/mrb_error.c 2014-10-21 20:50:41 +0900 (39ce63a) @@ -157,7 +157,7 @@ grn_mrb_error_init(grn_ctx *ctx) error_class); mrb_define_class_under(mrb, module, "ZlibError", error_class); - mrb_define_class_under(mrb, module, "LzoError", + mrb_define_class_under(mrb, module, "Lz4Error", error_class); mrb_define_class_under(mrb, module, "StackOverFlow", error_class); Modified: lib/mrb/scripts/context/rc.rb (+1 -1) =================================================================== --- lib/mrb/scripts/context/rc.rb 2014-10-21 12:30:50 +0900 (2863c7f) +++ lib/mrb/scripts/context/rc.rb 2014-10-21 20:50:41 +0900 (0f87a16) @@ -82,7 +82,7 @@ module Groonga OPERATION_NOT_SUPPORTED = new(:operation_not_supported, -58) ADDRESS_IS_IN_USE = new(:address_is_in_use, -59) ZLIB_ERROR = new(:zlib_error, -60) - LZO_ERROR = new(:lzo_error, -61) + LZ4_ERROR = new(:lz4_error, -61) STACK_OVER_FLOW = new(:stack_over_flow, -62) SYNTAX_ERROR = new(:syntax_error, -63) RETRY_MAX = new(:retry_max, -64) Modified: lib/proc.c (+4 -4) =================================================================== --- lib/proc.c 2014-10-21 12:30:50 +0900 (37b7ae6) +++ lib/proc.c 2014-10-21 20:50:41 +0900 (374b64b) @@ -1105,8 +1105,8 @@ grn_parse_column_create_flags(grn_ctx *ctx, const char *nptr, const char *end) } else if (!memcmp(nptr, "COMPRESS_ZLIB", 13)) { flags |= GRN_OBJ_COMPRESS_ZLIB; nptr += 13; - } else if (!memcmp(nptr, "COMPRESS_LZO", 12)) { - flags |= GRN_OBJ_COMPRESS_LZO; + } else if (!memcmp(nptr, "COMPRESS_LZ4", 12)) { + flags |= GRN_OBJ_COMPRESS_LZ4; nptr += 12; } else if (!memcmp(nptr, "WITH_SECTION", 12)) { flags |= GRN_OBJ_WITH_SECTION; @@ -1191,8 +1191,8 @@ grn_column_create_flags_to_text(grn_ctx *ctx, grn_obj *buf, grn_obj_flags flags) case GRN_OBJ_COMPRESS_ZLIB: GRN_TEXT_PUTS(ctx, buf, "|COMPRESS_ZLIB"); break; - case GRN_OBJ_COMPRESS_LZO: - GRN_TEXT_PUTS(ctx, buf, "|COMPRESS_LZO"); + case GRN_OBJ_COMPRESS_LZ4: + GRN_TEXT_PUTS(ctx, buf, "|COMPRESS_LZ4"); break; } if (flags & GRN_OBJ_PERSISTENT) { Modified: lib/store.c (+29 -34) =================================================================== --- lib/store.c 2014-10-21 12:30:50 +0900 (3649611) +++ lib/store.c 2014-10-21 20:50:41 +0900 (4e521b1) @@ -1226,15 +1226,16 @@ grn_ja_ref_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *v } #endif /* GRN_WITH_ZLIB */ -#ifdef GRN_WITH_LZO -#include <lzo/lzo1x.h> +#ifdef GRN_WITH_LZ4 +#include <lz4.h> static void * -grn_ja_ref_lzo(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len) +grn_ja_ref_lz4(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len) { void *lvalue; - uint32_t lvalue_len; - lzo_uint lout_len; + int lvalue_len; + int lout_len; + if (!(lvalue = grn_ja_ref_raw(ctx, ja, id, iw, &lvalue_len))) { iw->value = NULL; *value_len = 0; @@ -1246,15 +1247,10 @@ grn_ja_ref_lzo(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *va return NULL; } lout_len = *((uint64_t *)lvalue); - switch (lzo1x_decompress((lzo_bytep)(((uint64_t *)lvalue) + 1), - lvalue_len, - (lzo_bytep)(iw->value), - &lout_len, - NULL)) { - case LZO_E_OK : - case LZO_E_INPUT_NOT_CONSUMED : - break; - default : + if (LZ4_decompress_safe((const char *)((uint64_t *)lvalue + 1), + (char *)(iw->value), + lvalue_len - sizeof(uint64_t), + lout_len) < 0) { GRN_FREE(iw->value); iw->value = NULL; *value_len = 0; @@ -1263,7 +1259,7 @@ grn_ja_ref_lzo(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *va *value_len = lout_len; return iw->value; } -#endif /* GRN_WITH_LZO */ +#endif /* GRN_WITH_LZ4 */ void * grn_ja_ref(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len) @@ -1273,11 +1269,11 @@ grn_ja_ref(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_ return grn_ja_ref_zlib(ctx, ja, id, iw, value_len); } #endif /* GRN_WITH_ZLIB */ -#ifdef GRN_WITH_LZO - if (ja->header->flags & GRN_OBJ_COMPRESS_LZO) { - return grn_ja_ref_lzo(ctx, ja, id, iw, value_len); +#ifdef GRN_WITH_LZ4 + if (ja->header->flags & GRN_OBJ_COMPRESS_LZ4) { + return grn_ja_ref_lz4(ctx, ja, id, iw, value_len); } -#endif /* GRN_WITH_LZO */ +#endif /* GRN_WITH_LZ4 */ return grn_ja_ref_raw(ctx, ja, id, iw, value_len); } @@ -1359,34 +1355,33 @@ grn_ja_put_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id, } #endif /* GRN_WITH_ZLIB */ -#ifdef GRN_WITH_LZO +#ifdef GRN_WITH_LZ4 inline static grn_rc -grn_ja_put_lzo(grn_ctx *ctx, grn_ja *ja, grn_id id, +grn_ja_put_lz4(grn_ctx *ctx, grn_ja *ja, grn_id id, void *value, uint32_t value_len, int flags, uint64_t *cas) { grn_rc rc; - void *lvalue, *lwork; - lzo_uint lvalue_len = value_len + value_len / 16 + 64 + 3; - + void *lvalue; + int lvalue_len; if (value_len == 0) { return grn_ja_put_raw(ctx, ja, id, value, value_len, flags, cas); } + lvalue_len = LZ4_compressBound(value_len); if (!(lvalue = GRN_MALLOC(lvalue_len + sizeof(uint64_t)))) { return GRN_NO_MEMORY_AVAILABLE; } - if (!(lwork = GRN_MALLOC(LZO1X_1_MEM_COMPRESS))) { GRN_FREE(lvalue); return GRN_NO_MEMORY_AVAILABLE; } - if (lzo1x_1_compress(value, value_len, (lzo_bytep)((uint64_t *)lvalue + 1), &lvalue_len, lwork) != LZO_E_OK) { - GRN_FREE(lwork); + lvalue_len = LZ4_compress((const char*)value, (char *)((uint64_t *)lvalue + 1), value_len); + + if (lvalue_len <= 0) { GRN_FREE(lvalue); - ERR(GRN_LZO_ERROR, "lzo1x_1_compress"); + ERR(GRN_LZ4_ERROR, "lz4_compress"); return ctx->rc; } - GRN_FREE(lwork); *(uint64_t *)lvalue = value_len; rc = grn_ja_put_raw(ctx, ja, id, lvalue, lvalue_len + sizeof(uint64_t), flags, cas); GRN_FREE(lvalue); return rc; } -#endif /* GRN_WITH_LZO */ +#endif /* GRN_WITH_LZ4 */ grn_rc grn_ja_put(grn_ctx *ctx, grn_ja *ja, grn_id id, void *value, uint32_t value_len, @@ -1397,11 +1392,11 @@ grn_ja_put(grn_ctx *ctx, grn_ja *ja, grn_id id, void *value, uint32_t value_len, return grn_ja_put_zlib(ctx, ja, id, value, value_len, flags, cas); } #endif /* GRN_WITH_ZLIB */ -#ifdef GRN_WITH_LZO - if (ja->header->flags & GRN_OBJ_COMPRESS_LZO) { - return grn_ja_put_lzo(ctx, ja, id, value, value_len, flags, cas); +#ifdef GRN_WITH_LZ4 + if (ja->header->flags & GRN_OBJ_COMPRESS_LZ4) { + return grn_ja_put_lz4(ctx, ja, id, value, value_len, flags, cas); } -#endif /* GRN_WITH_LZO */ +#endif /* GRN_WITH_LZ4 */ return grn_ja_put_raw(ctx, ja, id, value, value_len, flags, cas); } Modified: lib/util.c (+2 -2) =================================================================== --- lib/util.c 2014-10-21 12:30:50 +0900 (67f0ff3) +++ lib/util.c 2014-10-21 20:50:41 +0900 (7455e10) @@ -400,8 +400,8 @@ grn_store_inspect_body(grn_ctx *ctx, grn_obj *buf, grn_obj *obj) case GRN_OBJ_COMPRESS_ZLIB : GRN_TEXT_PUTS(ctx, buf, "zlib"); break; - case GRN_OBJ_COMPRESS_LZO : - GRN_TEXT_PUTS(ctx, buf, "lzo"); + case GRN_OBJ_COMPRESS_LZ4 : + GRN_TEXT_PUTS(ctx, buf, "lz4"); break; default: break; Renamed: test/command/suite/select/output/lz4/scalar.expected (+1 -1) 93% =================================================================== --- test/command/suite/select/output/lzo/scalar.expected 2014-10-21 12:30:50 +0900 (cc7a8da) +++ test/command/suite/select/output/lz4/scalar.expected 2014-10-21 20:50:41 +0900 (6831a94) @@ -1,6 +1,6 @@ table_create Entries TABLE_PAT_KEY ShortText [[0,0.0,0.0],true] -column_create Entries content COLUMN_SCALAR|COMPRESS_LZO Text +column_create Entries content COLUMN_SCALAR|COMPRESS_LZ4 Text [[0,0.0,0.0],true] load --table Entries [ Renamed: test/command/suite/select/output/lz4/scalar.test (+1 -1) 82% =================================================================== --- test/command/suite/select/output/lzo/scalar.test 2014-10-21 12:30:50 +0900 (992e0fc) +++ test/command/suite/select/output/lz4/scalar.test 2014-10-21 20:50:41 +0900 (c2ed9eb) @@ -1,5 +1,5 @@ table_create Entries TABLE_PAT_KEY ShortText -column_create Entries content COLUMN_SCALAR|COMPRESS_LZO Text +column_create Entries content COLUMN_SCALAR|COMPRESS_LZ4 Text load --table Entries [ Modified: test/unit/core/test-context.c (+2 -2) =================================================================== --- test/unit/core/test-context.c 2014-10-21 12:30:50 +0900 (cc3be10) +++ test/unit/core/test-context.c 2014-10-21 20:50:41 +0900 (39b8fc0) @@ -149,11 +149,11 @@ test_support_lzo(void) cut_assert_ensure_context(); GRN_BOOL_INIT(&grn_support_p, 0); - grn_obj_get_info(context, NULL, GRN_INFO_SUPPORT_LZO, &grn_support_p); + grn_obj_get_info(context, NULL, GRN_INFO_SUPPORT_LZ4, &grn_support_p); support_p = GRN_BOOL_VALUE(&grn_support_p); GRN_OBJ_FIN(context, &grn_support_p); -#ifdef GRN_WITH_LZO +#ifdef GRN_WITH_LZ4 cut_assert_true(support_p); #else cut_assert_false(support_p); Modified: test/unit/lib/grn-test-utils.c (+2 -2) =================================================================== --- test/unit/lib/grn-test-utils.c 2014-10-21 12:30:50 +0900 (8210dfd) +++ test/unit/lib/grn-test-utils.c 2014-10-21 20:50:41 +0900 (7162768) @@ -149,8 +149,8 @@ grn_rc_to_string(grn_rc rc) return "GRN_ADDRESS_IS_IN_USE"; case GRN_ZLIB_ERROR: return "GRN_ZLIB_ERROR"; - case GRN_LZO_ERROR: - return "GRN_LZO_ERROR"; + case GRN_LZ4_ERROR: + return "GRN_LZ4_ERROR"; case GRN_STACK_OVER_FLOW: return "GRN_STACK_OVER_FLOW"; case GRN_SYNTAX_ERROR: -------------- next part -------------- HTML����������������������������...Download