naoa
null+****@clear*****
Sat Oct 18 17:18:37 JST 2014
naoa 2014-10-18 17:18:37 +0900 (Sat, 18 Oct 2014) New Revision: e407fb6a3d7240a0c99158eb432f8e9452bf2bfe https://github.com/groonga/groonga/commit/e407fb6a3d7240a0c99158eb432f8e9452bf2bfe Merged 31cd730: Merge pull request #221 from naoa/fix-memory-leak-in-zlib/lzo Message: Fix memory leak in compress zlib/lzo column Added files: test/command/suite/select/output/lzo/scalar.expected test/command/suite/select/output/lzo/scalar.test test/command/suite/select/output/zlib/scalar.expected test/command/suite/select/output/zlib/scalar.test Modified files: lib/db.c lib/io.h lib/proc.c lib/store.c Modified: lib/db.c (+0 -28) =================================================================== --- lib/db.c 2014-10-17 12:06:03 +0900 (39c40b2) +++ lib/db.c 2014-10-18 17:18:37 +0900 (f948dc7) @@ -3534,7 +3534,6 @@ grn_column_create(grn_ctx *ctx, grn_obj *table, grn_id domain = GRN_ID_NIL; char fullname[GRN_TABLE_MAX_KEY_SIZE]; char buffer[PATH_MAX]; - grn_bool ja_p = GRN_FALSE; GRN_API_ENTER; if (!table) { ERR(GRN_INVALID_ARGUMENT, "[column][create] table is missing"); @@ -3654,14 +3653,12 @@ grn_column_create(grn_ctx *ctx, grn_obj *table, case GRN_OBJ_COLUMN_SCALAR : if ((flags & GRN_OBJ_KEY_VAR_SIZE) || value_size > sizeof(int64_t)) { res = (grn_obj *)grn_ja_create(ctx, path, value_size, flags); - ja_p = GRN_TRUE; } else { res = (grn_obj *)grn_ra_create(ctx, path, value_size); } break; case GRN_OBJ_COLUMN_VECTOR : res = (grn_obj *)grn_ja_create(ctx, path, value_size * 30/*todo*/, flags); - ja_p = GRN_TRUE; //todo : zlib support break; case GRN_OBJ_COLUMN_INDEX : @@ -3674,31 +3671,6 @@ grn_column_create(grn_ctx *ctx, grn_obj *table, DB_OBJ(res)->range = range; DB_OBJ(res)->header.flags = flags; res->header.flags = flags; - if (ja_p) { - grn_bool zlib_p = GRN_FALSE; - grn_bool lzo_p = GRN_FALSE; -#ifdef GRN_WITH_ZLIB - if (flags & GRN_OBJ_COMPRESS_ZLIB) { - zlib_p = GRN_TRUE; - } -#endif /* GRN_WITH_ZLIB */ -#ifdef GRN_WITH_LZO - if (flags & GRN_OBJ_COMPRESS_LZO) { - lzo_p = GRN_TRUE; - } -#endif /* GRN_WITH_LZO */ - if (zlib_p || lzo_p) { - int table_name_len; - char table_name[GRN_TABLE_MAX_KEY_SIZE]; - table_name_len = grn_obj_name(ctx, table, table_name, - GRN_TABLE_MAX_KEY_SIZE); - GRN_LOG(ctx, GRN_LOG_WARNING, - "[column][create] " - "%s compressed column will leaks memories: <%.*s>.<%.*s>", - zlib_p ? "zlib" : "lzo", - table_name_len, table_name, name_size, name); - } - } if (grn_db_obj_init(ctx, db, id, DB_OBJ(res))) { _grn_obj_remove(ctx, res); res = NULL; Modified: lib/io.h (+1 -0) =================================================================== --- lib/io.h 2014-10-17 12:06:03 +0900 (27538b3) +++ lib/io.h 2014-10-18 17:18:37 +0900 (04c7591) @@ -75,6 +75,7 @@ typedef struct { #if defined(WIN32) && defined(WIN32_FMO_EACH) HANDLE fmo; #endif /* defined(WIN32) && defined(WIN32_FMO_EACH) */ + void *value; } grn_io_win; typedef struct { Modified: lib/proc.c (+6 -0) =================================================================== --- lib/proc.c 2014-10-17 12:06:03 +0900 (84fac0c) +++ lib/proc.c 2014-10-18 17:18:37 +0900 (37b7ae6) @@ -1102,6 +1102,12 @@ grn_parse_column_create_flags(grn_ctx *ctx, const char *nptr, const char *end) } else if (!memcmp(nptr, "COLUMN_INDEX", 12)) { flags |= GRN_OBJ_COLUMN_INDEX; nptr += 12; + } else if (!memcmp(nptr, "COMPRESS_ZLIB", 13)) { + flags |= GRN_OBJ_COMPRESS_ZLIB; + nptr += 13; + } else if (!memcmp(nptr, "COMPRESS_LZO", 12)) { + flags |= GRN_OBJ_COMPRESS_LZO; + nptr += 12; } else if (!memcmp(nptr, "WITH_SECTION", 12)) { flags |= GRN_OBJ_WITH_SECTION; nptr += 12; Modified: lib/store.c (+29 -18) =================================================================== --- lib/store.c 2014-10-17 12:06:03 +0900 (041c01e) +++ lib/store.c 2014-10-18 17:18:37 +0900 (3649611) @@ -524,6 +524,7 @@ grn_ja_ref_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *va iw->size = 0; iw->addr = NULL; iw->pseg = pseg; + iw->value = NULL; if (pseg != JA_ESEG_VOID) { grn_ja_einfo *einfo = NULL; GRN_IO_SEG_REF(ja->io, pseg, einfo); @@ -556,9 +557,14 @@ grn_ja_ref_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *va grn_rc grn_ja_unref(grn_ctx *ctx, grn_io_win *iw) { - if (!iw->addr) { return GRN_INVALID_ARGUMENT; } - GRN_IO_SEG_UNREF(iw->io, iw->pseg); - if (!iw->tiny_p) { grn_io_win_unmap2(iw); } + if (iw->value) { + GRN_FREE(iw->value); + iw->value = NULL; + } else { + if (!iw->addr) { return GRN_INVALID_ARGUMENT; } + GRN_IO_SEG_UNREF(iw->io, iw->pseg); + if (!iw->tiny_p) { grn_io_win_unmap2(iw); } + } return GRN_SUCCESS; } @@ -858,6 +864,7 @@ grn_ja_alloc(grn_ctx *ctx, grn_ja *ja, grn_id id, vp->seg = 0; vp->pos = 0; } + iw->value = NULL; grn_io_unlock(ja->io); return GRN_SUCCESS; } @@ -1176,12 +1183,11 @@ grn_ja_element_info(grn_ctx *ctx, grn_ja *ja, grn_id id, static void * grn_ja_ref_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len) { - /* TODO: This function leaks a memory. The return value - * must be freed. */ z_stream zstream; - void *value, *zvalue; + void *zvalue; uint32_t zvalue_len; if (!(zvalue = grn_ja_ref_raw(ctx, ja, id, iw, &zvalue_len))) { + iw->value = NULL; *value_len = 0; return NULL; } @@ -1190,29 +1196,33 @@ grn_ja_ref_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *v zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; if (inflateInit2(&zstream, 15 /* windowBits */) != Z_OK) { + iw->value = NULL; *value_len = 0; return NULL; } - if (!(value = GRN_MALLOC(*((uint64_t *)zvalue)))) { + if (!(iw->value = GRN_MALLOC(*((uint64_t *)zvalue)))) { inflateEnd(&zstream); + iw->value = NULL; *value_len = 0; return NULL; } - zstream.next_out = (Bytef *)value; + zstream.next_out = (Bytef *)iw->value; zstream.avail_out = *(uint64_t *)zvalue; if (inflate(&zstream, Z_FINISH) != Z_STREAM_END) { inflateEnd(&zstream); - GRN_FREE(value); + GRN_FREE(iw->value); + iw->value = NULL; *value_len = 0; return NULL; } *value_len = zstream.total_out; if (inflateEnd(&zstream) != Z_OK) { - GRN_FREE(value); + GRN_FREE(iw->value); + iw->value = NULL; *value_len = 0; return NULL; } - return value; + return iw->value; } #endif /* GRN_WITH_ZLIB */ @@ -1222,35 +1232,36 @@ grn_ja_ref_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *v static void * grn_ja_ref_lzo(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len) { - /* TODO: This function leaks a memory. The return value - * must be freed. */ - void *value, *lvalue; + void *lvalue; uint32_t lvalue_len; lzo_uint lout_len; if (!(lvalue = grn_ja_ref_raw(ctx, ja, id, iw, &lvalue_len))) { + iw->value = NULL; *value_len = 0; return NULL; } - if (!(value = GRN_MALLOC(*((uint64_t *)lvalue)))) { + if (!(iw->value = GRN_MALLOC(*((uint64_t *)lvalue)))) { + iw->value = NULL; *value_len = 0; return NULL; } lout_len = *((uint64_t *)lvalue); switch (lzo1x_decompress((lzo_bytep)(((uint64_t *)lvalue) + 1), lvalue_len, - (lzo_bytep)(value), + (lzo_bytep)(iw->value), &lout_len, NULL)) { case LZO_E_OK : case LZO_E_INPUT_NOT_CONSUMED : break; default : - GRN_FREE(value); + GRN_FREE(iw->value); + iw->value = NULL; *value_len = 0; return NULL; } *value_len = lout_len; - return value; + return iw->value; } #endif /* GRN_WITH_LZO */ Added: test/command/suite/select/output/lzo/scalar.expected (+55 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/lzo/scalar.expected 2014-10-18 17:18:37 +0900 (cc7a8da) @@ -0,0 +1,55 @@ +table_create Entries TABLE_PAT_KEY ShortText +[[0,0.0,0.0],true] +column_create Entries content COLUMN_SCALAR|COMPRESS_LZO Text +[[0,0.0,0.0],true] +load --table Entries +[ + { + "_key": "Groonga", + "content": "I found Groonga that is a fast fulltext search engine!" + }, + { + "_key": "Mroonga", + "content": "I found Mroonga that is a MySQL storage engine to use Groonga!" + } +] +[[0,0.0,0.0],2] +select Entries +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 2 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "_key", + "ShortText" + ], + [ + "content", + "Text" + ] + ], + [ + 1, + "Groonga", + "I found Groonga that is a fast fulltext search engine!" + ], + [ + 2, + "Mroonga", + "I found Mroonga that is a MySQL storage engine to use Groonga!" + ] + ] + ] +] Added: test/command/suite/select/output/lzo/scalar.test (+16 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/lzo/scalar.test 2014-10-18 17:18:37 +0900 (992e0fc) @@ -0,0 +1,16 @@ +table_create Entries TABLE_PAT_KEY ShortText +column_create Entries content COLUMN_SCALAR|COMPRESS_LZO Text + +load --table Entries +[ + { + "_key": "Groonga", + "content": "I found Groonga that is a fast fulltext search engine!" + }, + { + "_key": "Mroonga", + "content": "I found Mroonga that is a MySQL storage engine to use Groonga!" + } +] + +select Entries Added: test/command/suite/select/output/zlib/scalar.expected (+55 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/zlib/scalar.expected 2014-10-18 17:18:37 +0900 (238e266) @@ -0,0 +1,55 @@ +table_create Entries TABLE_PAT_KEY ShortText +[[0,0.0,0.0],true] +column_create Entries content COLUMN_SCALAR|COMPRESS_ZLIB Text +[[0,0.0,0.0],true] +load --table Entries +[ + { + "_key": "Groonga", + "content": "I found Groonga that is a fast fulltext search engine!" + }, + { + "_key": "Mroonga", + "content": "I found Mroonga that is a MySQL storage engine to use Groonga!" + } +] +[[0,0.0,0.0],2] +select Entries +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 2 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "_key", + "ShortText" + ], + [ + "content", + "Text" + ] + ], + [ + 1, + "Groonga", + "I found Groonga that is a fast fulltext search engine!" + ], + [ + 2, + "Mroonga", + "I found Mroonga that is a MySQL storage engine to use Groonga!" + ] + ] + ] +] Added: test/command/suite/select/output/zlib/scalar.test (+16 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/select/output/zlib/scalar.test 2014-10-18 17:18:37 +0900 (e0bb4cf) @@ -0,0 +1,16 @@ +table_create Entries TABLE_PAT_KEY ShortText +column_create Entries content COLUMN_SCALAR|COMPRESS_ZLIB Text + +load --table Entries +[ + { + "_key": "Groonga", + "content": "I found Groonga that is a fast fulltext search engine!" + }, + { + "_key": "Mroonga", + "content": "I found Mroonga that is a MySQL storage engine to use Groonga!" + } +] + +select Entries -------------- next part -------------- HTML����������������������������...Download