Groonga - fulltext search engine.

[Groonga-commit] groonga/groonga at 97bc3ed [master] Support having N caches in a process

Kouhei Sutou null+****@clear*****
Tue Sep 24 17:06:02 JST 2013

• Previous message: [Groonga-commit] groonga/groonga at ae2f614 [master] groonga-httpd: indent
• Next message: [Groonga-commit] groonga/groonga at 03e9958 [master] groonga-httpd: fix cache is shared by diffferent database
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Kouhei Sutou	2013-09-24 17:06:02 +0900 (Tue, 24 Sep 2013)

  New Revision: 97bc3ed8682eeead4c0e1cfccb9e749469f28b95
  https://github.com/groonga/groonga/commit/97bc3ed8682eeead4c0e1cfccb9e749469f28b95

  Message:
    Support having N caches in a process
    
    Some grn_cache_* API are exported but they are experimental.

  Modified files:
    include/groonga.h
    lib/ctx.c
    lib/ctx.h
    lib/db.c
    lib/proc.c
    src/groonga.c

  Modified: include/groonga.h (+13 -1)
===================================================================
--- include/groonga.h    2013-09-24 17:03:24 +0900 (e7ce421)
+++ include/groonga.h    2013-09-24 17:06:02 +0900 (a3cfef7)
@@ -246,7 +246,19 @@ GRN_API grn_rc grn_set_default_match_escalation_threshold(long long int threshol
 
 /* cache */
 #define GRN_CACHE_DEFAULT_MAX_N_ENTRIES 100
-GRN_API unsigned int *grn_cache_max_nentries(void);
+typedef struct _grn_cache grn_cache;
+
+GRN_API grn_cache *grn_cache_open(grn_ctx *ctx);
+GRN_API grn_rc grn_cache_close(grn_ctx *ctx, grn_cache *cache);
+
+GRN_API grn_rc grn_cache_current_set(grn_ctx *ctx, grn_cache *cache);
+GRN_API grn_cache *grn_cache_current_get(grn_ctx *ctx);
+
+GRN_API grn_rc grn_cache_set_max_n_entries(grn_ctx *ctx,
+                                           grn_cache *cache,
+                                           unsigned int n);
+GRN_API unsigned int grn_cache_get_max_n_entries(grn_ctx *ctx,
+                                                 grn_cache *cache);
 
 /* grn_encoding */
 

  Modified: lib/ctx.c (+120 -53)
===================================================================
--- lib/ctx.c    2013-09-24 17:03:24 +0900 (9fa2a3d)
+++ lib/ctx.c    2013-09-24 17:06:02 +0900 (4db42ac)
@@ -1971,7 +1971,7 @@ grn_ctx_info_get(grn_ctx *ctx, grn_ctx_info *info)
 
 typedef struct _grn_cache_entry grn_cache_entry;
 
-typedef struct {
+typedef struct _grn_cache {
   grn_cache_entry *next;
   grn_cache_entry *prev;
   grn_hash *hash;
@@ -1990,58 +1990,126 @@ struct _grn_cache_entry {
   uint32_t nref;
 };
 
-static grn_cache grn_gcache;
+static grn_cache *grn_cache_current = NULL;
+static grn_cache *grn_cache_default = NULL;
+
+grn_cache *
+grn_cache_open(grn_ctx *ctx)
+{
+  grn_cache *cache = NULL;
+
+  GRN_API_ENTER;
+  cache = GRN_MALLOC(sizeof(grn_cache));
+  if (!cache) {
+    ERR(GRN_NO_MEMORY_AVAILABLE, "[cache] failed to allocate grn_cache");
+    goto exit;
+  }
+
+  cache->next = (grn_cache_entry*)cache;
+  cache->prev = (grn_cache_entry *)cache;
+  cache->hash = grn_hash_create(&grn_gctx, NULL, GRN_TABLE_MAX_KEY_SIZE,
+                                sizeof(grn_cache_entry), GRN_OBJ_KEY_VAR_SIZE);
+  MUTEX_INIT(cache->mutex);
+  cache->max_nentries = GRN_CACHE_DEFAULT_MAX_N_ENTRIES;
+
+exit :
+  GRN_API_RETURN(cache);
+}
+
+grn_rc
+grn_cache_close(grn_ctx *ctx, grn_cache *cache)
+{
+  grn_ctx *ctx_original = ctx;
+  grn_cache_entry *vp;
+
+  GRN_API_ENTER;
+
+  ctx = &grn_gctx;
+  GRN_HASH_EACH(ctx, cache->hash, id, NULL, NULL, &vp, {
+    grn_obj_close(ctx, vp->value);
+  });
+  grn_hash_close(ctx, cache->hash);
+  MUTEX_FIN(cache->mutex);
+  ctx = ctx_original;
+  GRN_FREE(cache);
+
+  GRN_API_RETURN(ctx->rc);
+}
+
+grn_rc
+grn_cache_current_set(grn_ctx *ctx, grn_cache *cache)
+{
+  grn_cache_current = cache;
+  return GRN_SUCCESS;
+}
+
+grn_cache *
+grn_cache_current_get(grn_ctx *ctx)
+{
+  return grn_cache_current;
+}
 
 void
 grn_cache_init(void)
 {
-  grn_gcache.next = (grn_cache_entry *) &grn_gcache;
-  grn_gcache.prev = (grn_cache_entry *) &grn_gcache;
-  grn_gcache.hash = grn_hash_create(&grn_gctx, NULL, GRN_TABLE_MAX_KEY_SIZE,
-                                    sizeof(grn_cache_entry), GRN_OBJ_KEY_VAR_SIZE);
-  MUTEX_INIT(grn_gcache.mutex);
-  grn_gcache.max_nentries = GRN_CACHE_DEFAULT_MAX_N_ENTRIES;
+  grn_cache_default = grn_cache_open(&grn_gctx);
+  grn_cache_current_set(&grn_gctx, grn_cache_default);
 }
 
-uint32_t *
-grn_cache_max_nentries(void)
+grn_rc
+grn_cache_set_max_n_entries(grn_ctx *ctx, grn_cache *cache, unsigned int n)
 {
-  return &grn_gcache.max_nentries;
+  if (!cache) {
+    return GRN_INVALID_ARGUMENT;
+  }
+  cache->max_nentries = n;
+  return GRN_SUCCESS;
+}
+
+uint32_t
+grn_cache_get_max_n_entries(grn_ctx *ctx, grn_cache *cache)
+{
+  if (!cache) {
+    return 0;
+  }
+  return cache->max_nentries;
 }
 
 void
-grn_cache_get_statistics(grn_ctx *ctx, grn_cache_statistics *statistics)
+grn_cache_get_statistics(grn_ctx *ctx, grn_cache *cache,
+                         grn_cache_statistics *statistics)
 {
-  MUTEX_LOCK(grn_gcache.mutex);
-  statistics->nentries = GRN_HASH_SIZE(grn_gcache.hash);
-  statistics->max_nentries = grn_gcache.max_nentries;
-  statistics->nfetches = grn_gcache.nfetches;
-  statistics->nhits = grn_gcache.nhits;
-  MUTEX_UNLOCK(grn_gcache.mutex);
+  MUTEX_LOCK(cache->mutex);
+  statistics->nentries = GRN_HASH_SIZE(cache->hash);
+  statistics->max_nentries = cache->max_nentries;
+  statistics->nfetches = cache->nfetches;
+  statistics->nhits = cache->nhits;
+  MUTEX_UNLOCK(cache->mutex);
 }
 
 static void
-grn_cache_expire_entry(grn_cache_entry *ce)
+grn_cache_expire_entry(grn_cache *cache, grn_cache_entry *ce)
 {
   if (!ce->nref) {
     ce->prev->next = ce->next;
     ce->next->prev = ce->prev;
     grn_obj_close(&grn_gctx, ce->value);
-    grn_hash_delete_by_id(&grn_gctx, grn_gcache.hash, ce->id, NULL);
+    grn_hash_delete_by_id(&grn_gctx, cache->hash, ce->id, NULL);
   }
 }
 
 grn_obj *
-grn_cache_fetch(grn_ctx *ctx, const char *str, uint32_t str_len)
+grn_cache_fetch(grn_ctx *ctx, grn_cache *cache,
+                const char *str, uint32_t str_len)
 {
   grn_cache_entry *ce;
   grn_obj *obj = NULL;
   if (!ctx->impl || !ctx->impl->db) { return obj; }
-  MUTEX_LOCK(grn_gcache.mutex);
-  grn_gcache.nfetches++;
-  if (grn_hash_get(&grn_gctx, grn_gcache.hash, str, str_len, (void **)&ce)) {
+  MUTEX_LOCK(cache->mutex);
+  cache->nfetches++;
+  if (grn_hash_get(&grn_gctx, cache->hash, str, str_len, (void **)&ce)) {
     if (ce->tv.tv_sec <= grn_db_lastmod(ctx->impl->db)) {
-      grn_cache_expire_entry(ce);
+      grn_cache_expire_entry(cache, ce);
       goto exit;
     }
     ce->nref++;
@@ -2049,44 +2117,46 @@ grn_cache_fetch(grn_ctx *ctx, const char *str, uint32_t str_len)
     ce->prev->next = ce->next;
     ce->next->prev = ce->prev;
     {
-      grn_cache_entry *ce0 = (grn_cache_entry *)&grn_gcache;
+      grn_cache_entry *ce0 = (grn_cache_entry *)cache;
       ce->next = ce0->next;
       ce->prev = ce0;
       ce0->next->prev = ce;
       ce0->next = ce;
     }
-    grn_gcache.nhits++;
+    cache->nhits++;
   }
 exit :
-  MUTEX_UNLOCK(grn_gcache.mutex);
+  MUTEX_UNLOCK(cache->mutex);
   return obj;
 }
 
 void
-grn_cache_unref(const char *str, uint32_t str_len)
+grn_cache_unref(grn_ctx *ctx, grn_cache *cache,
+                const char *str, uint32_t str_len)
 {
   grn_cache_entry *ce;
-  grn_ctx *ctx = &grn_gctx;
-  MUTEX_LOCK(grn_gcache.mutex);
-  if (grn_hash_get(ctx, grn_gcache.hash, str, str_len, (void **)&ce)) {
+  ctx = &grn_gctx;
+  MUTEX_LOCK(cache->mutex);
+  if (grn_hash_get(ctx, cache->hash, str, str_len, (void **)&ce)) {
     if (ce->nref) { ce->nref--; }
   }
-  MUTEX_UNLOCK(grn_gcache.mutex);
+  MUTEX_UNLOCK(cache->mutex);
 }
 
 void
-grn_cache_update(grn_ctx *ctx, const char *str, uint32_t str_len, grn_obj *value)
+grn_cache_update(grn_ctx *ctx, grn_cache *cache,
+                 const char *str, uint32_t str_len, grn_obj *value)
 {
   grn_id id;
   int added = 0;
   grn_cache_entry *ce;
   grn_rc rc = GRN_SUCCESS;
   grn_obj *old = NULL, *obj;
-  if (!ctx->impl || !grn_gcache.max_nentries) { return; }
+  if (!ctx->impl || !cache->max_nentries) { return; }
   if (!(obj = grn_obj_open(&grn_gctx, GRN_BULK, 0, GRN_DB_TEXT))) { return; }
   GRN_TEXT_PUT(&grn_gctx, obj, GRN_TEXT_VALUE(value), GRN_TEXT_LEN(value));
-  MUTEX_LOCK(grn_gcache.mutex);
-  if ((id = grn_hash_add(&grn_gctx, grn_gcache.hash, str, str_len, (void **)&ce, &added))) {
+  MUTEX_LOCK(cache->mutex);
+  if ((id = grn_hash_add(&grn_gctx, cache->hash, str, str_len, (void **)&ce, &added))) {
     if (!added) {
       if (ce->nref) {
         rc = GRN_RESOURCE_BUSY;
@@ -2101,43 +2171,40 @@ grn_cache_update(grn_ctx *ctx, const char *str, uint32_t str_len, grn_obj *value
     ce->tv = ctx->impl->tv;
     ce->nref = 0;
     {
-      grn_cache_entry *ce0 = (grn_cache_entry *)&grn_gcache;
+      grn_cache_entry *ce0 = (grn_cache_entry *)cache;
       ce->next = ce0->next;
       ce->prev = ce0;
       ce0->next->prev = ce;
       ce0->next = ce;
     }
-    if (GRN_HASH_SIZE(grn_gcache.hash) > grn_gcache.max_nentries) {
-      grn_cache_expire_entry(grn_gcache.prev);
+    if (GRN_HASH_SIZE(cache->hash) > cache->max_nentries) {
+      grn_cache_expire_entry(cache, cache->prev);
     }
   } else {
     rc = GRN_NO_MEMORY_AVAILABLE;
   }
 exit :
-  MUTEX_UNLOCK(grn_gcache.mutex);
+  MUTEX_UNLOCK(cache->mutex);
   if (rc) { grn_obj_close(&grn_gctx, obj); }
   if (old) { grn_obj_close(&grn_gctx, old); }
 }
 
 void
-grn_cache_expire(int32_t size)
+grn_cache_expire(grn_cache *cache, int32_t size)
 {
-  grn_cache_entry *ce0 = (grn_cache_entry *)&grn_gcache;
-  MUTEX_LOCK(grn_gcache.mutex);
-  while (ce0 != ce0->prev && size--) { grn_cache_expire_entry(ce0->prev); }
-  MUTEX_UNLOCK(grn_gcache.mutex);
+  grn_cache_entry *ce0 = (grn_cache_entry *)cache;
+  MUTEX_LOCK(cache->mutex);
+  while (ce0 != ce0->prev && size--) {
+    grn_cache_expire_entry(cache, ce0->prev);
+  }
+  MUTEX_UNLOCK(cache->mutex);
 }
 
 void
 grn_cache_fin(void)
 {
-  grn_ctx *ctx = &grn_gctx;
-  grn_cache_entry *vp;
-  GRN_HASH_EACH(ctx, grn_gcache.hash, id, NULL, NULL, &vp, {
-    grn_obj_close(ctx, vp->value);
-  });
-  grn_hash_close(ctx, grn_gcache.hash);
-  MUTEX_FIN(grn_gcache.mutex);
+  grn_cache_current_set(&grn_gctx, NULL);
+  grn_cache_close(&grn_gctx, grn_cache_default);
 }
 
 /**** memory allocation ****/

  Modified: lib/ctx.h (+9 -5)
===================================================================
--- lib/ctx.h    2013-09-24 17:03:24 +0900 (9e3de91)
+++ lib/ctx.h    2013-09-24 17:06:02 +0900 (50df921)
@@ -477,12 +477,16 @@ typedef struct {
 } grn_cache_statistics;
 
 void grn_cache_init(void);
-grn_obj *grn_cache_fetch(grn_ctx *ctx, const char *str, uint32_t str_size);
-void grn_cache_unref(const char *str, uint32_t str_size);
-void grn_cache_update(grn_ctx *ctx, const char *str, uint32_t str_size, grn_obj *value);
-void grn_cache_expire(int32_t size);
+grn_obj *grn_cache_fetch(grn_ctx *ctx, grn_cache *cache,
+                         const char *str, uint32_t str_size);
+void grn_cache_unref(grn_ctx *ctx, grn_cache *cache,
+                     const char *str, uint32_t str_size);
+void grn_cache_update(grn_ctx *ctx, grn_cache *cache,
+                      const char *str, uint32_t str_size, grn_obj *value);
+void grn_cache_expire(grn_cache *cache, int32_t size);
 void grn_cache_fin(void);
-void grn_cache_get_statistics(grn_ctx *ctx, grn_cache_statistics *statistics);
+void grn_cache_get_statistics(grn_ctx *ctx, grn_cache *cache,
+                              grn_cache_statistics *statistics);
 
 /**** receive handler ****/
 

  Modified: lib/db.c (+5 -1)
===================================================================
--- lib/db.c    2013-09-24 17:03:24 +0900 (e6ce743)
+++ lib/db.c    2013-09-24 17:06:02 +0900 (f3754d1)
@@ -352,7 +352,11 @@ grn_db_close(grn_ctx *ctx, grn_obj *db)
   if (s->specs) { grn_ja_close(ctx, s->specs); }
   GRN_FREE(s);
   if (ctx_used_db) {
-    grn_cache_expire(-1);
+    grn_cache *cache;
+    cache = grn_cache_current_get(ctx);
+    if (cache) {
+      grn_cache_expire(cache, -1);
+    }
     ctx->impl->db = NULL;
   }
   GRN_API_RETURN(GRN_SUCCESS);

  Modified: lib/proc.c (+18 -10)
===================================================================
--- lib/proc.c    2013-09-24 17:03:24 +0900 (181dc1a)
+++ lib/proc.c    2013-09-24 17:06:02 +0900 (3cef265)
@@ -475,8 +475,9 @@ grn_select(grn_ctx *ctx, const char *table, unsigned int table_len,
     query_expander_len + 1 + query_flags_len + 1 +
     sizeof(grn_content_type) + sizeof(int) * 4;
   long long int threshold, original_threshold = 0;
+  grn_cache *cache_obj = grn_cache_current_get(ctx);
   if (cache_key_size <= GRN_TABLE_MAX_KEY_SIZE) {
-    grn_obj *cache;
+    grn_obj *cache_value;
     char *cp = cache_key;
     memcpy(cp, table, table_len);
     cp += table_len; *cp++ = '\0';
@@ -509,12 +510,15 @@ grn_select(grn_ctx *ctx, const char *table, unsigned int table_len,
     memcpy(cp, &limit, sizeof(int)); cp += sizeof(int);
     memcpy(cp, &drilldown_offset, sizeof(int)); cp += sizeof(int);
     memcpy(cp, &drilldown_limit, sizeof(int)); cp += sizeof(int);
-    if ((cache = grn_cache_fetch(ctx, cache_key, cache_key_size))) {
-      GRN_TEXT_PUT(ctx, outbuf, GRN_TEXT_VALUE(cache), GRN_TEXT_LEN(cache));
-      grn_cache_unref(cache_key, cache_key_size);
+    cache_value = grn_cache_fetch(ctx, cache_obj, cache_key, cache_key_size);
+    if (cache_value) {
+      GRN_TEXT_PUT(ctx, outbuf,
+                   GRN_TEXT_VALUE(cache_value),
+                   GRN_TEXT_LEN(cache_value));
+      grn_cache_unref(ctx, cache_obj, cache_key, cache_key_size);
       GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_CACHE,
                     ":", "cache(%" GRN_FMT_LLD ")",
-                    (long long int)GRN_TEXT_LEN(cache));
+                    (long long int)GRN_TEXT_LEN(cache_value));
       return ctx->rc;
     }
   }
@@ -770,7 +774,7 @@ grn_select(grn_ctx *ctx, const char *table, unsigned int table_len,
     GRN_OUTPUT_ARRAY_CLOSE();
     if (!ctx->rc && cacheable && cache_key_size <= GRN_TABLE_MAX_KEY_SIZE
         && (!cache || cache_len != 2 || *cache != 'n' || *(cache + 1) != 'o')) {
-      grn_cache_update(ctx, cache_key, cache_key_size, outbuf);
+      grn_cache_update(ctx, cache_obj, cache_key, cache_key_size, outbuf);
     }
     if (taintable) { grn_db_touch(ctx, DB_OBJ(table_)->db); }
     grn_obj_unlink(ctx, table_);
@@ -885,9 +889,12 @@ static grn_obj *
 proc_status(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
 {
   grn_timeval now;
+  grn_cache *cache;
   grn_cache_statistics statistics;
+
   grn_timeval_now(ctx, &now);
-  grn_cache_get_statistics(ctx, &statistics);
+  cache = grn_cache_current_get(ctx);
+  grn_cache_get_statistics(ctx, cache, &statistics);
   GRN_OUTPUT_MAP_OPEN("RESULT", 18);
   GRN_OUTPUT_CSTR("alloc_count");
   GRN_OUTPUT_INT32(grn_alloc_count());
@@ -2569,14 +2576,15 @@ proc_dump(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
 static grn_obj *
 proc_cache_limit(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
 {
-  uint32_t *mp = grn_cache_max_nentries();
-  GRN_OUTPUT_INT64(*mp);
+  grn_cache *cache;
+  cache = grn_cache_current_get(ctx);
+  GRN_OUTPUT_INT64(grn_cache_get_max_n_entries(ctx, cache));
   if (GRN_TEXT_LEN(VAR(0))) {
     const char *rest;
     uint32_t max = grn_atoui(GRN_TEXT_VALUE(VAR(0)),
                              GRN_BULK_CURR(VAR(0)), &rest);
     if (GRN_BULK_CURR(VAR(0)) == rest) {
-      *mp = max;
+      grn_cache_set_max_n_entries(ctx, cache, max);
     } else {
       ERR(GRN_INVALID_ARGUMENT,
           "max value is invalid unsigned integer format: <%.*s>",

  Modified: src/groonga.c (+3 -1)
===================================================================
--- src/groonga.c    2013-09-24 17:03:24 +0900 (b804a94)
+++ src/groonga.c    2013-09-24 17:06:02 +0900 (7e8497a)
@@ -2495,7 +2495,9 @@ main(int argc, char **argv)
   grn_set_term_handler();
 
   if (cache_limit_arg) {
-    *grn_cache_max_nentries() = cache_limit;
+    grn_cache *cache;
+    cache = grn_cache_current_get(&grn_gctx);
+    grn_cache_set_max_n_entries(&grn_gctx, cache, cache_limit);
   }
 
   newdb = (mode & MODE_NEW_DB);
-------------- next part --------------
HTML����������������������������...
Download 

• Previous message: [Groonga-commit] groonga/groonga at ae2f614 [master] groonga-httpd: indent
• Next message: [Groonga-commit] groonga/groonga at 03e9958 [master] groonga-httpd: fix cache is shared by diffferent database
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]