null+****@clear*****
null+****@clear*****
2011年 11月 4日 (金) 15:20:06 JST
Kouhei Sutou 2011-11-04 06:20:06 +0000 (Fri, 04 Nov 2011)
New Revision: d518618edf78a2a8a1118b340980c40b9d24c3d6
Log:
[zlib][lzo] support getting value from zlib/lzo compressed column. GitHub#6
But the current implementation leaks memories. We will fix
it but it will not be soon... Sorry.
Reported by Takayuki Yamaguchi. Thanks!!!
Modified files:
lib/db.c
lib/store.c
Modified: lib/db.c (+28 -0)
===================================================================
--- lib/db.c 2011-11-04 02:27:37 +0000 (7892c46)
+++ lib/db.c 2011-11-04 06:20:06 +0000 (0ce5d1c)
@@ -2980,6 +2980,7 @@ grn_column_create(grn_ctx *ctx, grn_obj *table,
grn_id domain = GRN_ID_NIL;
char fullname[GRN_PAT_MAX_KEY_SIZE];
char buffer[PATH_MAX];
+ grn_bool ja_p = GRN_FALSE;
GRN_API_ENTER;
if (!table) {
ERR(GRN_INVALID_ARGUMENT, "[column][create]: table is missing");
@@ -3100,12 +3101,14 @@ grn_column_create(grn_ctx *ctx, grn_obj *table,
case GRN_OBJ_COLUMN_SCALAR :
if ((flags & GRN_OBJ_KEY_VAR_SIZE) || value_size > sizeof(int64_t)) {
res = (grn_obj *)grn_ja_create(ctx, path, value_size, flags);
+ ja_p = GRN_TRUE;
} else {
res = (grn_obj *)grn_ra_create(ctx, path, value_size);
}
break;
case GRN_OBJ_COLUMN_VECTOR :
res = (grn_obj *)grn_ja_create(ctx, path, value_size * 30/*todo*/, flags);
+ ja_p = GRN_TRUE;
//todo : zlib support
break;
case GRN_OBJ_COLUMN_INDEX :
@@ -3118,6 +3121,31 @@ grn_column_create(grn_ctx *ctx, grn_obj *table,
DB_OBJ(res)->range = range;
DB_OBJ(res)->header.flags = flags;
res->header.flags = flags;
+ if (ja_p) {
+ grn_bool zlib_p = GRN_FALSE;
+ grn_bool lzo_p = GRN_FALSE;
+#ifndef NO_ZLIB
+ if (flags & GRN_OBJ_COMPRESS_ZLIB) {
+ zlib_p = GRN_TRUE;
+ }
+#endif /* NO_ZLIB */
+#ifndef NO_LZO
+ if (flags & GRN_OBJ_COMPRESS_LZO) {
+ lzo_p = GRN_TRUE;
+ }
+#endif /* NO_LZO */
+ if (zlib_p || lzo_p) {
+ int table_name_len;
+ char table_name[GRN_PAT_MAX_KEY_SIZE];
+ table_name_len = grn_obj_name(ctx, table, table_name,
+ GRN_PAT_MAX_KEY_SIZE);
+ GRN_LOG(ctx, GRN_LOG_WARNING,
+ "[column][create]: "
+ "%s compressed column will leaks memories: <%.*s>.<%.*s>",
+ zlib_p ? "zlib" : "lzo",
+ table_name_len, table_name, name_size, name);
+ }
+ }
if (grn_db_obj_init(ctx, db, id, DB_OBJ(res))) {
grn_obj_remove(ctx, res);
res = NULL;
Modified: lib/store.c (+26 -27)
===================================================================
--- lib/store.c 2011-11-04 02:27:37 +0000 (6a5d438)
+++ lib/store.c 2011-11-04 06:20:06 +0000 (3c044ff)
@@ -1048,46 +1048,43 @@ grn_ja_element_info(grn_ctx *ctx, grn_ja *ja, grn_id id,
static void *
grn_ja_ref_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len)
{
+ /* TODO: This function leaks a memory. The return value
+ * must be freed. */
z_stream zstream;
void *value, *zvalue;
- uint32_t zvalue_len = *value_len + sizeof (uint64_t);
+ uint32_t zvalue_len;
if (!(zvalue = grn_ja_ref_raw(ctx, ja, id, iw, &zvalue_len))) {
- *value_len = 0; return NULL;
+ *value_len = 0;
+ return NULL;
}
- zstream.next_in = (Bytef *)((uint64_t *)zvalue + 1);
- zstream.avail_in = zvalue_len - sizeof (uint64_t);
+ zstream.next_in = (Bytef *)(((uint64_t *)zvalue) + 1);
+ zstream.avail_in = zvalue_len + sizeof(uint64_t);
zstream.zalloc = Z_NULL;
zstream.zfree = Z_NULL;
if (inflateInit2(&zstream, 15 /* windowBits */) != Z_OK) {
- GRN_FREE((grn_io_ja_ehead *)zvalue - 1);
*value_len = 0;
return NULL;
}
- if (!(value = GRN_MALLOC(*(uint64_t *)zvalue + sizeof (grn_io_ja_ehead)))) {
+ if (!(value = GRN_MALLOC(*((uint64_t *)zvalue)))) {
inflateEnd(&zstream);
- GRN_FREE((grn_io_ja_ehead *)zvalue - 1);
*value_len = 0;
return NULL;
}
- zstream.next_out = (Bytef *)((grn_io_ja_ehead *)value + 1);
+ zstream.next_out = (Bytef *)value;
zstream.avail_out = *(uint64_t *)zvalue;
if (inflate(&zstream, Z_FINISH) != Z_STREAM_END) {
inflateEnd(&zstream);
GRN_FREE(value);
- GRN_FREE((grn_io_ja_ehead *)zvalue - 1);
*value_len = 0;
return NULL;
}
*value_len = zstream.total_out;
if (inflateEnd(&zstream) != Z_OK) {
GRN_FREE(value);
- GRN_FREE((grn_io_ja_ehead *)zvalue - 1);
*value_len = 0;
return NULL;
}
- *(grn_io_ja_ehead *)value = ((grn_io_ja_ehead *)zvalue)[-1];
- GRN_FREE((grn_io_ja_ehead *)zvalue - 1);
- return (grn_io_ja_ehead *)value + 1;
+ return value;
}
#endif /* NO_ZLIB */
@@ -1097,33 +1094,35 @@ grn_ja_ref_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *v
static void *
grn_ja_ref_lzo(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win *iw, uint32_t *value_len)
{
+ /* TODO: This function leaks a memory. The return value
+ * must be freed. */
void *value, *lvalue;
- uint32_t lvalue_len = *value_len + sizeof (uint64_t);
- lzo_uint loutlen;
+ uint32_t lvalue_len;
+ lzo_uint lout_len;
if (!(lvalue = grn_ja_ref_raw(ctx, ja, id, iw, &lvalue_len))) {
- *value_len = 0; return NULL;
+ *value_len = 0;
+ return NULL;
}
- if (!(value = GRN_MALLOC(*(uint64_t *)lvalue + sizeof (grn_io_ja_ehead)))) {
- GRN_FREE((grn_io_ja_ehead *)lvalue - 1);
+ if (!(value = GRN_MALLOC(*((uint64_t *)lvalue)))) {
*value_len = 0;
return NULL;
}
- loutlen = *(uint64_t *)lvalue;
- switch (lzo1x_decompress((lzo_bytep)((uint64_t *)lvalue + 1), lvalue_len - sizeof (uint64_t),
- (lzo_bytep)((grn_io_ja_ehead *)value + 1), &loutlen, NULL)) {
+ lout_len = *((uint64_t *)lvalue);
+ switch (lzo1x_decompress((lzo_bytep)(((uint64_t *)lvalue) + 1),
+ lvalue_len,
+ (lzo_bytep)(value),
+ &lout_len,
+ NULL)) {
case LZO_E_OK :
case LZO_E_INPUT_NOT_CONSUMED :
break;
default :
GRN_FREE(value);
- GRN_FREE((grn_io_ja_ehead *)lvalue - 1);
*value_len = 0;
return NULL;
}
- *value_len = loutlen;
- *(grn_io_ja_ehead *)value = ((grn_io_ja_ehead *)lvalue)[-1];
- GRN_FREE((grn_io_ja_ehead *)lvalue - 1);
- return (grn_io_ja_ehead *)value + 1;
+ *value_len = lout_len;
+ return value;
}
#endif /* NO_LZO */
@@ -1200,7 +1199,7 @@ grn_ja_put_zlib(grn_ctx *ctx, grn_ja *ja, grn_id id,
}
zvalue_len = deflateBound(&zstream, value_len);
if (!(zvalue = GRN_MALLOC(zvalue_len + sizeof (uint64_t)))) { deflateEnd(&zstream); return GRN_NO_MEMORY_AVAILABLE; }
- zstream.next_out = (Bytef *)((uint64_t *)zvalue + 1);
+ zstream.next_out = (Bytef *)(((uint64_t *)zvalue) + 1);
zstream.avail_out = zvalue_len;
if (deflate(&zstream, Z_FINISH) != Z_STREAM_END) {
deflateEnd(&zstream);