null+****@clear*****
null+****@clear*****
2012年 2月 13日 (月) 10:32:03 JST
Susumu Yata 2012-02-13 10:32:03 +0900 (Mon, 13 Feb 2012)
New Revision: cfe49a8ea5aea4de33edbee8ef0b196d1d45954b
Log:
removed fuctions and macros supported by the plugin API. removed grn_str (deprecated).
Modified files:
include/groonga/tokenizer.h
lib/tokenizer.c
Modified: include/groonga/tokenizer.h (+2 -102)
===================================================================
--- include/groonga/tokenizer.h 2012-02-10 19:16:20 +0900 (e00d151)
+++ include/groonga/tokenizer.h 2012-02-13 10:32:03 +0900 (e089cce)
@@ -27,107 +27,6 @@ extern "C" {
#endif /* __cplusplus */
/*
- Don't call these functions directly. Use GRN_TOKENIZER_MALLOC() and
- GRN_TOKENIZER_FREE() instead.
- */
-void *grn_tokenizer_malloc(grn_ctx *ctx, size_t size, const char *file,
- int line, const char *func);
-void grn_tokenizer_free(grn_ctx *ctx, void *ptr, const char *file,
- int line, const char *func);
-
-/*
- GRN_TOKENIZER_MALLOC() allocates `size' bytes and returns a pointer to the
- allocated memory space. Note that the memory space is associated with `ctx'.
- */
-#define GRN_TOKENIZER_MALLOC(ctx, size) \
- grn_tokenizer_malloc((ctx), (size), __FILE__, __LINE__, __FUNCTION__)
-/*
- GRN_TOKENIZER_FREE() frees a memory space allocated by
- GRN_TOKENIZER_MALLOC(). This means that `ptr' must be a pointer returned by
- GRN_TOKENIZER_MALLOC().
- */
-#define GRN_TOKENIZER_FREE(ctx, ptr) \
- grn_tokenizer_free((ctx), (ptr), __FILE__, __LINE__, __FUNCTION__)
-
-/*
- GRN_TOKENIZER_LOG() reports a log of `level'. Its error message is generated
- from the varying number of arguments, in which the first one is the format
- string and the rest are its arguments. See grn_log_level in "groonga.h" for
- more details of `level'.
- */
-#define GRN_TOKENIZER_LOG(ctx, level, ...) \
- GRN_LOG((ctx), (level), __VA_ARGS__)
-
-/*
- Don't call grn_tokenizer_set_error() directly. This function is used in
- GRN_TOKENIZER_SET_ERROR().
- */
-void grn_tokenizer_set_error(grn_ctx *ctx, grn_log_level level,
- grn_rc error_code,
- const char *file, int line, const char *func,
- const char *format, ...);
-
-/*
- Don't call these functions directly. grn_tokenizer_backtrace() and
- grn_tokenizer_logtrace() are used in GRN_TOKENIZER_SET_ERROR().
- */
-void grn_tokenizer_backtrace(grn_ctx *ctx);
-void grn_tokenizer_logtrace(grn_ctx *ctx, grn_log_level level);
-
-/*
- Don't use GRN_TOKENIZER_SET_ERROR() directly. This macro is used in
- GRN_TOKENIZER_ERROR().
- */
-#define GRN_TOKENIZER_SET_ERROR(ctx, level, error_code, ...) do { \
- grn_tokenizer_set_error(ctx, level, error_code, \
- __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__); \
- GRN_LOG(ctx, level, __VA_ARGS__); \
- grn_tokenizer_backtrace(ctx); \
- grn_tokenizer_logtrace(ctx, level); \
-} while (0)
-
-/*
- GRN_TOKENIZER_ERROR() reports an error of `error_code'. Its error message is
- generated from the varying number of arguments, in which the first one is the
- format string and the rest are its arguments. See grn_rc in "groonga.h" for
- more details of `error_code'.
- */
-#define GRN_TOKENIZER_ERROR(ctx, error_code, ...) \
- GRN_TOKENIZER_SET_ERROR(ctx, GRN_LOG_ERROR, error_code, __VA_ARGS__)
-
-/*
- grn_tokenizer_mutex is available to make a critical section. See the
- following functions.
- */
-typedef struct _grn_tokenizer_mutex grn_tokenizer_mutex;
-
-/*
- grn_tokenizer_mutex_create() returns a pointer to a new object of
- grn_tokenizer_mutex. Memory for the new object is obtained with
- GRN_TOKENIZER_MALLOC(). grn_tokenizer_mutex_create() returns NULL if
- sufficient memory is not available.
- */
-grn_tokenizer_mutex *grn_tokenizer_mutex_create(grn_ctx *ctx);
-
-/*
- grn_tokenizer_mutex_destroy() finalizes an object of grn_tokenizer_mutex
- and then frees memory allocated for that object.
- */
-void grn_tokenizer_mutex_destroy(grn_ctx *ctx, grn_tokenizer_mutex *mutex);
-
-/*
- grn_tokenizer_mutex_lock() locks a mutex object. If the object is already
- locked, the calling thread waits until the object will be unlocked.
- */
-void grn_tokenizer_mutex_lock(grn_ctx *ctx, grn_tokenizer_mutex *mutex);
-
-/*
- grn_tokenizer_mutex_unlock() unlocks a mutex object.
- grn_tokenizer_mutex_unlock() should not be called for an unlocked object.
- */
-void grn_tokenizer_mutex_unlock(grn_ctx *ctx, grn_tokenizer_mutex *mutex);
-
-/*
grn_tokenizer_charlen() returns the length (#bytes) of the first character
in the string specified by `str_ptr' and `str_length'. If the starting bytes
are invalid as a character, grn_tokenizer_charlen() returns 0. See
@@ -151,7 +50,8 @@ int grn_tokenizer_isspace(grn_ctx *ctx, const char *str_ptr,
typedef struct _grn_tokenizer_query grn_tokenizer_query;
struct _grn_tokenizer_query {
- grn_str *str;
+ grn_obj *normalized_query;
+ char *query_buf;
const char *ptr;
unsigned int length;
grn_encoding encoding;
Modified: lib/tokenizer.c (+40 -104)
===================================================================
--- lib/tokenizer.c 2012-02-10 19:16:20 +0900 (7d4415f)
+++ lib/tokenizer.c 2012-02-13 10:32:03 +0900 (b47f1db)
@@ -26,89 +26,6 @@
#include "str.h"
#include "token.h"
-void *grn_tokenizer_malloc(grn_ctx *ctx, size_t size, const char *file,
- int line, const char *func) {
- return grn_malloc(ctx, size, file, line, func);
-}
-
-void grn_tokenizer_free(grn_ctx *ctx, void *ptr, const char *file,
- int line, const char *func) {
- return grn_free(ctx, ptr, file, line, func);
-}
-
-/*
- grn_tokenizer_ctx_log() is a clone of grn_ctx_log() in ctx.c. The only
- difference is that grn_tokenizer_ctx_log() uses va_list instead of `...'.
- */
-static void grn_tokenizer_ctx_log(grn_ctx *ctx, const char *format,
- va_list ap) {
- va_list aq;
- va_copy(aq, ap);
- vsnprintf(ctx->errbuf, GRN_CTX_MSGSIZE, format, aq);
- va_end(aq);
-}
-
-void grn_tokenizer_set_error(grn_ctx *ctx, grn_log_level level,
- grn_rc error_code,
- const char *file, int line, const char *func,
- const char *format, ...) {
- ctx->errlvl = level;
- ctx->rc = error_code;
- ctx->errfile = file;
- ctx->errline = line;
- ctx->errfunc = func;
- grn_ctx_impl_err(ctx);
-
- {
- va_list ap;
- va_start(ap, format);
- grn_tokenizer_ctx_log(ctx, format, ap);
- va_end(ap);
- }
-}
-
-void grn_tokenizer_backtrace(grn_ctx *ctx) {
- BACKTRACE(ctx);
-}
-
-void grn_tokenizer_logtrace(grn_ctx *ctx, grn_log_level level) {
- if (level <= GRN_LOG_ERROR) {
- LOGTRACE(ctx, level);
- }
-}
-
-struct _grn_tokenizer_mutex {
- grn_critical_section critical_section;
-};
-
-grn_tokenizer_mutex *grn_tokenizer_mutex_create(grn_ctx *ctx) {
- grn_tokenizer_mutex * const mutex =
- GRN_TOKENIZER_MALLOC(ctx, sizeof(grn_tokenizer_mutex));
- if (mutex != NULL) {
- CRITICAL_SECTION_INIT(mutex->critical_section);
- }
- return mutex;
-}
-
-void grn_tokenizer_mutex_destroy(grn_ctx *ctx, grn_tokenizer_mutex *mutex) {
- if (mutex != NULL) {
- CRITICAL_SECTION_FIN(mutex->critical_section);
- GRN_TOKENIZER_FREE(ctx, mutex);
- }
-}
-
-void grn_tokenizer_mutex_lock(grn_ctx *ctx, grn_tokenizer_mutex *mutex) {
- if (mutex != NULL) {
- CRITICAL_SECTION_ENTER(mutex->critical_section);
- }
-}
-
-void grn_tokenizer_mutex_unlock(grn_ctx *ctx, grn_tokenizer_mutex *mutex) {
- if (mutex != NULL) {
- CRITICAL_SECTION_LEAVE(mutex->critical_section);
- }
-}
-
/*
grn_tokenizer_charlen() takes the length of a string, unlike grn_charlen_().
*/
@@ -166,40 +83,57 @@ grn_tokenizer_query *grn_tokenizer_query_create(grn_ctx *ctx,
int num_args, grn_obj **args) {
grn_obj *query_str = grn_ctx_pop(ctx);
if (query_str == NULL) {
- GRN_TOKENIZER_ERROR(ctx, GRN_INVALID_ARGUMENT, "missing argument");
+ GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "missing argument");
return NULL;
}
if ((num_args < 1) || (args == NULL) || (args[0] == NULL)) {
- GRN_TOKENIZER_ERROR(ctx, GRN_INVALID_ARGUMENT, "invalid NULL pointer");
+ GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "invalid NULL pointer");
return NULL;
}
{
grn_tokenizer_query * const query =
- GRN_TOKENIZER_MALLOC(ctx, sizeof(grn_tokenizer_query));
+ GRN_PLUGIN_MALLOC(ctx, sizeof(grn_tokenizer_query));
if (query == NULL) {
return NULL;
}
+ query->normalized_query = NULL;
+ query->query_buf = NULL;
{
grn_obj * const table = args[0];
- grn_encoding table_encoding;
- int flags = 0;
- grn_table_get_info(ctx, table, NULL, &table_encoding, NULL, NULL);
- {
- grn_str * const str = grn_str_open_(ctx, GRN_TEXT_VALUE(query_str),
- GRN_TEXT_LEN(query_str),
- flags | GRN_OBJ_KEY_NORMALIZE,
- table_encoding);
- if (str == NULL) {
- GRN_TOKENIZER_FREE(ctx, query);
+ grn_encoding table_encoding = GRN_ENC_NONE;
+ grn_obj *normalizer = NULL;
+ grn_table_get_info(ctx, table, NULL, &table_encoding, NULL, &normalizer);
+ if (normalizer != NULL) {
+ grn_obj * const normalized_query = grn_normalized_text_open(
+ ctx, normalizer, GRN_TEXT_VALUE(query_str),
+ GRN_TEXT_LEN(query_str), table_encoding, 0);
+ if (query->normalized_query == NULL) {
+ GRN_PLUGIN_FREE(ctx, query);
+ GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR,
+ "[tokenizer] failed to open normalized text");
+ return NULL;
+ }
+ query->normalized_query = normalized_query;
+ grn_normalized_text_get_value(ctx, query->normalized_query,
+ &query->ptr, NULL, &query->length);
+ } else {
+ unsigned int query_length = GRN_TEXT_LEN(query_str);
+ char *query_buf = (char *)GRN_PLUGIN_MALLOC(ctx, query_length + 1);
+ if (query_buf == NULL) {
+ GRN_PLUGIN_FREE(ctx, query);
+ GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR,
+ "[tokenizer] failed to duplicate query");
return NULL;
}
- query->str = str;
+ memcpy(query_buf, GRN_TEXT_VALUE(query_str), query_length);
+ query_buf[query_length] = '\0';
+ query->query_buf = query_buf;
+ query->ptr = query_buf;
+ query->length = query_length;
}
- query->ptr = query->str->norm;
- query->length = query->str->norm_blen;
query->encoding = table_encoding;
}
return query;
@@ -208,10 +142,13 @@ grn_tokenizer_query *grn_tokenizer_query_create(grn_ctx *ctx,
void grn_tokenizer_query_destroy(grn_ctx *ctx, grn_tokenizer_query *query) {
if (query != NULL) {
- if (query->str != NULL) {
- grn_str_close(ctx, query->str);
+ if (query->normalized_query != NULL) {
+ grn_obj_unlink(ctx, query->normalized_query);
+ }
+ if (query->query_buf != NULL) {
+ GRN_PLUGIN_FREE(ctx, query->query_buf);
}
- GRN_TOKENIZER_FREE(ctx, query);
+ GRN_PLUGIN_FREE(ctx, query);
}
}
@@ -265,8 +202,7 @@ grn_rc grn_tokenizer_register(grn_ctx *ctx, const char *plugin_name_ptr,
GRN_PROC_TOKENIZER,
init, next, fin, 3, vars);
if (obj == NULL) {
- GRN_TOKENIZER_ERROR(ctx, GRN_TOKENIZER_ERROR,
- "grn_proc_create() failed");
+ GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR, "grn_proc_create() failed");
return ctx->rc;
}
}