Kouhei Sutou
null+****@clear*****
Fri Oct 28 00:30:26 JST 2016
Kouhei Sutou 2016-10-28 00:30:26 +0900 (Fri, 28 Oct 2016) New Revision: d2930f71668a02e3525c63b78a4c4cdd49fa7b1b https://github.com/pgroonga/pgroonga/commit/d2930f71668a02e3525c63b78a4c4cdd49fa7b1b Message: wal: support logging create table, create column and set source Modified files: src/pgrn_create.c src/pgrn_wal.c src/pgrn_wal.h src/pgroonga.c Modified: src/pgrn_create.c (+47 -9) =================================================================== --- src/pgrn_create.c 2016-10-28 00:13:51 +0900 (2fd2f49) +++ src/pgrn_create.c 2016-10-28 00:30:26 +0900 (b5a0ea9) @@ -6,28 +6,45 @@ #include "pgrn_groonga.h" #include "pgrn_options.h" #include "pgrn_value.h" +#include "pgrn_wal.h" static grn_ctx *ctx = &PGrnContext; void PGrnCreateSourcesCtidColumn(PGrnCreateData *data) { + grn_column_flags flags = GRN_OBJ_COLUMN_SCALAR; + grn_obj *type; + + type = grn_ctx_at(ctx, GRN_DB_UINT64); data->sourcesCtidColumn = PGrnCreateColumn(data->sourcesTable, PGrnSourcesCtidColumnName, GRN_OBJ_COLUMN_SCALAR, - grn_ctx_at(ctx, GRN_DB_UINT64)); + type); + PGrnWALCreateColumn(data->index, + data->sourcesTable, + PGrnSourcesCtidColumnName, + flags, + type); } void PGrnCreateSourcesTable(PGrnCreateData *data) { char sourcesTableName[GRN_TABLE_MAX_KEY_SIZE]; + grn_table_flags flags = GRN_OBJ_TABLE_NO_KEY; snprintf(sourcesTableName, sizeof(sourcesTableName), PGrnSourcesTableNameFormat, data->relNode); data->sourcesTable = PGrnCreateTable(sourcesTableName, GRN_OBJ_TABLE_NO_KEY, NULL); + PGrnWALCreateTable(data->index, + sourcesTableName, + flags, + NULL, + NULL, + NULL); PGrnCreateSourcesCtidColumn(data); } @@ -35,7 +52,7 @@ PGrnCreateSourcesTable(PGrnCreateData *data) void PGrnCreateDataColumn(PGrnCreateData *data) { - grn_obj_flags flags = 0; + grn_column_flags flags = 0; grn_obj *range; grn_id rangeID; @@ -80,6 +97,11 @@ PGrnCreateDataColumn(PGrnCreateData *data) columnName, flags, range); + PGrnWALCreateColumn(data->index, + data->sourcesTable, + columnName, + flags, + range); } } @@ -88,7 +110,11 @@ PGrnCreateLexicon(PGrnCreateData *data) { grn_id typeID = GRN_ID_NIL; char lexiconName[GRN_TABLE_MAX_KEY_SIZE]; + grn_table_flags flags = GRN_OBJ_TABLE_PAT_KEY; + grn_obj *type; grn_obj *lexicon; + grn_obj *tokenizer = NULL; + grn_obj *normalizer = NULL; switch (data->attributeTypeID) { @@ -103,9 +129,8 @@ PGrnCreateLexicon(PGrnCreateData *data) snprintf(lexiconName, sizeof(lexiconName), PGrnLexiconNameFormat, data->relNode, data->i); - lexicon = PGrnCreateTable(lexiconName, - GRN_OBJ_TABLE_PAT_KEY, - grn_ctx_at(ctx, typeID)); + type = grn_ctx_at(ctx, typeID); + lexicon = PGrnCreateTable(lexiconName, flags, type); GRN_PTR_PUT(ctx, data->lexicons, lexicon); if (data->forFullTextSearch || @@ -127,17 +152,25 @@ PGrnCreateLexicon(PGrnCreateData *data) { if (!PGrnIsNoneValue(tokenizerName)) { + tokenizer = PGrnLookup(tokenizerName, ERROR); grn_obj_set_info(ctx, lexicon, GRN_INFO_DEFAULT_TOKENIZER, - PGrnLookup(tokenizerName, ERROR)); + tokenizer); } } if (!PGrnIsNoneValue(normalizerName)) { - grn_obj_set_info(ctx, lexicon, GRN_INFO_NORMALIZER, - PGrnLookup(normalizerName, ERROR)); + normalizer = PGrnLookup(normalizerName, ERROR); + grn_obj_set_info(ctx, lexicon, GRN_INFO_NORMALIZER, normalizer); } } + + PGrnWALCreateTable(data->index, + lexiconName, + flags, + type, + tokenizer, + normalizer); } void @@ -145,7 +178,7 @@ PGrnCreateIndexColumn(PGrnCreateData *data) { char lexiconName[GRN_TABLE_MAX_KEY_SIZE]; grn_obj *lexicon; - grn_obj_flags flags = GRN_OBJ_COLUMN_INDEX; + grn_column_flags flags = GRN_OBJ_COLUMN_INDEX; snprintf(lexiconName, sizeof(lexiconName), PGrnLexiconNameFormat, data->relNode, data->i); @@ -157,4 +190,9 @@ PGrnCreateIndexColumn(PGrnCreateData *data) PGrnIndexColumnName, flags, data->sourcesTable); + PGrnWALCreateColumn(data->index, + lexicon, + PGrnIndexColumnName, + flags, + data->sourcesTable); } Modified: src/pgrn_wal.c (+207 -7) =================================================================== --- src/pgrn_wal.c 2016-10-28 00:13:51 +0900 (1ec52d8) +++ src/pgrn_wal.c 2016-10-28 00:30:26 +0900 (ff398a2) @@ -45,7 +45,8 @@ static struct PGrnBuffers *buffers = &PGrnBuffers; typedef enum { PGRN_WAL_ACTION_INSERT, PGRN_WAL_ACTION_CREATE_TABLE, - PGRN_WAL_ACTION_CREATE_COLUMN + PGRN_WAL_ACTION_CREATE_COLUMN, + PGRN_WAL_ACTION_SET_SOURCES } PGrnWALAction; typedef struct { @@ -94,10 +95,37 @@ struct PGrnWALData_ #define PGRN_WAL_STATUES_TABLE_NAME_SIZE strlen(PGRN_WAL_STATUES_TABLE_NAME) #define PGRN_WAL_STATUES_CURRENT_COLUMN_NAME "current" +#ifdef PGRN_SUPPORT_WAL +static void +msgpack_pack_cstr(msgpack_packer *packer, const char *string) +{ + size_t size; + + size = strlen(string); + msgpack_pack_str(packer, size); + msgpack_pack_str_body(packer, string, size); +} + +static void +msgpack_pack_grn_obj(msgpack_packer *packer, grn_obj *object) +{ + if (object) + { + char name[GRN_TABLE_MAX_KEY_SIZE]; + int nameSize; + nameSize = grn_obj_name(ctx, object, name, GRN_TABLE_MAX_KEY_SIZE); + msgpack_pack_str(packer, nameSize); + msgpack_pack_str_body(packer, name, nameSize); + } + else + { + msgpack_pack_nil(packer); + } +} + static void PGrnWALEnsureStatusesTable(void) { -#ifdef PGRN_SUPPORT_WAL grn_obj *walStatuses; walStatuses = grn_ctx_get(ctx, @@ -113,10 +141,8 @@ PGrnWALEnsureStatusesTable(void) PGRN_WAL_STATUES_CURRENT_COLUMN_NAME, GRN_OBJ_COLUMN_SCALAR, grn_ctx_at(ctx, GRN_DB_UINT64)); -#endif } -#ifdef PGRN_SUPPORT_WAL static uint64_t PGrnWALPackPosition(BlockNumber block, OffsetNumber offset) { @@ -507,6 +533,162 @@ PGrnWALInsertColumn(PGrnWALData *data, #endif } +void +PGrnWALCreateTable(Relation index, + const char *name, + grn_obj_flags flags, + grn_obj *type, + grn_obj *tokenizer, + grn_obj *normalizer) +{ +#ifdef PGRN_SUPPORT_WAL + PGrnWALData *data; + msgpack_packer *packer; + size_t nElements = 6; + + if (!PGrnWALEnabled) + return; + + data = PGrnWALStart(index); + + packer = &(data->packer); + msgpack_pack_map(packer, nElements); + + msgpack_pack_cstr(packer, "_action"); + msgpack_pack_uint32(packer, PGRN_WAL_ACTION_CREATE_TABLE); + + msgpack_pack_cstr(packer, "name"); + msgpack_pack_cstr(packer, name); + + msgpack_pack_cstr(packer, "flags"); + msgpack_pack_uint32(packer, flags); + + msgpack_pack_cstr(packer, "type"); + msgpack_pack_grn_obj(packer, type); + + msgpack_pack_cstr(packer, "tokenizer"); + msgpack_pack_grn_obj(packer, tokenizer); + + msgpack_pack_cstr(packer, "normalizer"); + msgpack_pack_grn_obj(packer, normalizer); + + PGrnWALFinish(data); +#endif +} + +void +PGrnWALCreateColumn(Relation index, + grn_obj *table, + const char *name, + grn_obj_flags flags, + grn_obj *type) +{ +#ifdef PGRN_SUPPORT_WAL + PGrnWALData *data; + msgpack_packer *packer; + size_t nElements = 5; + + if (!PGrnWALEnabled) + return; + + data = PGrnWALStart(index); + + packer = &(data->packer); + msgpack_pack_map(packer, nElements); + + msgpack_pack_cstr(packer, "_action"); + msgpack_pack_uint32(packer, PGRN_WAL_ACTION_CREATE_COLUMN); + + msgpack_pack_cstr(packer, "table"); + msgpack_pack_grn_obj(packer, table); + + msgpack_pack_cstr(packer, "name"); + msgpack_pack_cstr(packer, name); + + msgpack_pack_cstr(packer, "flags"); + msgpack_pack_uint32(packer, flags); + + msgpack_pack_cstr(packer, "type"); + msgpack_pack_grn_obj(packer, type); + + PGrnWALFinish(data); +#endif +} + +void +PGrnWALSetSource(Relation index, + grn_obj *column, + grn_obj *source) +{ +#ifdef PGRN_SUPPORT_WAL + PGrnWALData *data; + msgpack_packer *packer; + size_t nElements = 3; + + if (!PGrnWALEnabled) + return; + + data = PGrnWALStart(index); + + packer = &(data->packer); + msgpack_pack_map(packer, nElements); + + msgpack_pack_cstr(packer, "_action"); + msgpack_pack_uint32(packer, PGRN_WAL_ACTION_SET_SOURCES); + + msgpack_pack_cstr(packer, "column"); + msgpack_pack_grn_obj(packer, column); + + msgpack_pack_cstr(packer, "sources"); + msgpack_pack_array(packer, 1); + msgpack_pack_grn_obj(packer, source); + + PGrnWALFinish(data); +#endif +} + +void +PGrnWALSetSources(Relation index, + grn_obj *column, + grn_obj *sources) +{ +#ifdef PGRN_SUPPORT_WAL + PGrnWALData *data; + msgpack_packer *packer; + size_t nElements = 3; + + if (!PGrnWALEnabled) + return; + + data = PGrnWALStart(index); + + packer = &(data->packer); + msgpack_pack_map(packer, nElements); + + msgpack_pack_cstr(packer, "_action"); + msgpack_pack_uint32(packer, PGRN_WAL_ACTION_SET_SOURCES); + + msgpack_pack_cstr(packer, "column"); + msgpack_pack_grn_obj(packer, column); + + msgpack_pack_cstr(packer, "sources"); + { + unsigned int i, nElements; + + nElements = GRN_BULK_VSIZE(sources) / sizeof(grn_id); + msgpack_pack_array(packer, nElements); + for (i = 0; i < nElements; i++) + { + grn_obj *source; + source = grn_ctx_at(ctx, GRN_RECORD_VALUE_AT(sources, i)); + msgpack_pack_grn_obj(packer, source); + } + } + + PGrnWALFinish(data); +#endif +} + #ifdef PGRN_SUPPORT_WAL typedef struct { Relation index; @@ -587,7 +769,7 @@ PGrnWALApplyInsert(PGrnWALApplyData *data, msgpack_object_map *map, uint32_t currentElement) { - grn_obj *table = data->sources; + grn_obj *table = NULL; const char *key = NULL; size_t keySize = 0; grn_id id; @@ -615,6 +797,12 @@ PGrnWALApplyInsert(PGrnWALApplyData *data, currentElement++; } } + if (!table) + { + if (!data->sources) + data->sources = PGrnLookupSourcesTable(data->index, ERROR); + table = data->sources; + } if (currentElement < map->size) { @@ -724,6 +912,14 @@ PGrnWALApplyCreateColumn(PGrnWALApplyData *data, } static void +PGrnWALApplySetSources(PGrnWALApplyData *data, + msgpack_object_map *map, + uint32_t currentElement) +{ + /* TODO */ +} + +static void PGrnWALApplyObject(PGrnWALApplyData *data, msgpack_object *object) { msgpack_object_map *map; @@ -775,6 +971,9 @@ PGrnWALApplyObject(PGrnWALApplyData *data, msgpack_object *object) case PGRN_WAL_ACTION_CREATE_COLUMN: PGrnWALApplyCreateColumn(data, map, currentElement); break; + case PGRN_WAL_ACTION_SET_SOURCES: + PGrnWALApplySetSources(data, map, currentElement); + break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -793,8 +992,6 @@ PGrnWALApplyConsume(PGrnWALApplyData *data) BlockNumber lastBlock = data->current.block; OffsetNumber lastOffset = data->current.offset; - data->sources = PGrnLookupSourcesTable(data->index, ERROR); - msgpack_unpacker_init(&unpacker, PGRN_PAGE_DATA_SIZE); msgpack_unpacked_init(&unpacked); nBlocks = RelationGetNumberOfBlocks(data->index); @@ -805,6 +1002,9 @@ PGrnWALApplyConsume(PGrnWALApplyData *data) PGrnPageSpecial *pageSpecial; size_t dataSize; + if (i == PGRN_WAL_META_PAGE_BLOCK_NUMBER) + continue; + buffer = ReadBuffer(data->index, i); LockBuffer(buffer, BUFFER_LOCK_SHARE); page = BufferGetPage(buffer); Modified: src/pgrn_wal.h (+20 -0) =================================================================== --- src/pgrn_wal.h 2016-10-28 00:13:51 +0900 (79e44e2) +++ src/pgrn_wal.h 2016-10-28 00:30:26 +0900 (1e559c1) @@ -23,4 +23,24 @@ void PGrnWALInsertColumn(PGrnWALData *data, const char *name, grn_obj *value); +void PGrnWALCreateTable(Relation index, + const char *name, + grn_obj_flags flags, + grn_obj *type, + grn_obj *tokenizer, + grn_obj *normalizer); + +void PGrnWALCreateColumn(Relation index, + grn_obj *table, + const char *name, + grn_obj_flags flags, + grn_obj *type); + +void PGrnWALSetSource(Relation index, + grn_obj *column, + grn_obj *source); +void PGrnWALSetSources(Relation index, + grn_obj *column, + grn_obj *sources); + void PGrnWALApply(Relation index); Modified: src/pgroonga.c (+1 -0) =================================================================== --- src/pgroonga.c 2016-10-28 00:13:51 +0900 (6723c24) +++ src/pgroonga.c 2016-10-28 00:30:26 +0900 (dcee142) @@ -877,6 +877,7 @@ PGrnSetSources(Relation index, grn_obj *sourcesTable) source = PGrnLookupColumn(sourcesTable, name->data, ERROR); PGrnIndexColumnSetSource(indexColumn, source); + PGrnWALSetSource(index, indexColumn, source); grn_obj_unlink(ctx, source); grn_obj_unlink(ctx, indexColumn); } -------------- next part -------------- HTML����������������������������...Download