Kouhei Sutou
null+****@clear*****
Thu Nov 3 00:06:50 JST 2016
Kouhei Sutou 2016-11-03 00:06:50 +0900 (Thu, 03 Nov 2016) New Revision: d17d6ca88a2e728a7c48237102c92ae2d524ec6d https://github.com/pgroonga/pgroonga/commit/d17d6ca88a2e728a7c48237102c92ae2d524ec6d Message: Support disabling index only scan when there is one or more long record Added files: src/pgrn_index_status.c src/pgrn_index_status.h Modified files: sources.am src/pgrn_global.c src/pgrn_global.h src/pgrn_wal.c src/pgroonga.c Modified: sources.am (+2 -1) =================================================================== --- sources.am 2016-11-02 22:49:23 +0900 (175b8a3) +++ sources.am 2016-11-03 00:06:50 +0900 (763741c) @@ -7,8 +7,9 @@ SRCS = \ src/pgrn_global.c \ src/pgrn_groonga.c \ src/pgrn_highlight_html.c \ - src/pgrn_keywords.c \ + src/pgrn_index_status.c \ src/pgrn_jsonb.c \ + src/pgrn_keywords.c \ src/pgrn_match_positions_byte.c \ src/pgrn_match_positions_character.c \ src/pgrn_options.c \ Modified: src/pgrn_global.c (+4 -0) =================================================================== --- src/pgrn_global.c 2016-11-02 22:49:23 +0900 (6d420cc) +++ src/pgrn_global.c 2016-11-03 00:06:50 +0900 (b83b7f8) @@ -20,6 +20,8 @@ PGrnInitializeBuffers(void) GRN_UINT64_INIT(&(PGrnBuffers.jsonbValueKeys), GRN_OBJ_VECTOR); GRN_UINT64_INIT(&(PGrnBuffers.walPosition), 0); GRN_VOID_INIT(&(PGrnBuffers.walValue)); + GRN_UINT32_INIT(&(PGrnBuffers.maxRecordSize), 0); + GRN_UINT64_INIT(&(PGrnBuffers.walAppliedPosition), 0); GRN_TEXT_INIT(&(PGrnBuffers.head), 0); GRN_TEXT_INIT(&(PGrnBuffers.body), 0); GRN_TEXT_INIT(&(PGrnBuffers.foot), 0); @@ -39,6 +41,8 @@ PGrnFinalizeBuffers(void) GRN_OBJ_FIN(ctx, &(PGrnBuffers.jsonbValueKeys)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.walPosition)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.walValue)); + GRN_OBJ_FIN(ctx, &(PGrnBuffers.maxRecordSize)); + GRN_OBJ_FIN(ctx, &(PGrnBuffers.walAppliedPosition)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.head)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.body)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.foot)); Modified: src/pgrn_global.h (+2 -0) =================================================================== --- src/pgrn_global.h 2016-11-02 22:49:23 +0900 (a4bf3c1) +++ src/pgrn_global.h 2016-11-03 00:06:50 +0900 (96078cd) @@ -14,6 +14,8 @@ struct PGrnBuffers grn_obj jsonbValueKeys; grn_obj walPosition; grn_obj walValue; + grn_obj maxRecordSize; + grn_obj walAppliedPosition; grn_obj head; grn_obj body; grn_obj foot; Added: src/pgrn_index_status.c (+133 -0) 100644 =================================================================== --- /dev/null +++ src/pgrn_index_status.c 2016-11-03 00:06:50 +0900 (63d5478) @@ -0,0 +1,133 @@ +#include "pgroonga.h" + +#include "pgrn_global.h" +#include "pgrn_groonga.h" +#include "pgrn_index_status.h" + +#include <miscadmin.h> + +static grn_ctx *ctx = &PGrnContext; +static struct PGrnBuffers *buffers = &PGrnBuffers; + +#define TABLE_NAME "IndexStatuses" +#define TABLE_NAME_SIZE (sizeof(TABLE_NAME) - 1) +#define MAX_RECORD_SIZE_COLUMN_NAME "max_record_size" +#define WAL_APPLIED_POSITION_COLUMN_NAME "wal_applied_position" + +void +PGrnInitializeIndexStatus(void) +{ + grn_obj *table; + + table = grn_ctx_get(ctx, + TABLE_NAME, + TABLE_NAME_SIZE); + if (!table) + { + table = PGrnCreateTableWithSize(NULL, + TABLE_NAME, + TABLE_NAME_SIZE, + GRN_OBJ_TABLE_HASH_KEY, + grn_ctx_at(ctx, GRN_DB_UINT32), + NULL, + NULL); + } + + if (!grn_ctx_get(ctx, TABLE_NAME "." MAX_RECORD_SIZE_COLUMN_NAME, -1)) + { + PGrnCreateColumn(NULL, + table, + MAX_RECORD_SIZE_COLUMN_NAME, + GRN_OBJ_COLUMN_SCALAR, + grn_ctx_at(ctx, GRN_DB_UINT32)); + } + + if (!grn_ctx_get(ctx, TABLE_NAME "." WAL_APPLIED_POSITION_COLUMN_NAME, -1)) + { + PGrnCreateColumn(NULL, + table, + WAL_APPLIED_POSITION_COLUMN_NAME, + GRN_OBJ_COLUMN_SCALAR, + grn_ctx_at(ctx, GRN_DB_UINT64)); + } +} + +static grn_id +PGrnIndexStatusGetRecordID(Relation index) +{ + grn_obj *table; + + table = PGrnLookupWithSize(TABLE_NAME, TABLE_NAME_SIZE, ERROR); + return grn_table_add(ctx, + table, + &(index->rd_node.relNode), + sizeof(uint32_t), + NULL); +} + +uint32_t +PGrnIndexStatusGetMaxRecordSize(Relation index) +{ + grn_id id; + grn_obj *column; + grn_obj *maxRecordSize = &(buffers->maxRecordSize); + + id = PGrnIndexStatusGetRecordID(index); + column = PGrnLookup(TABLE_NAME "." MAX_RECORD_SIZE_COLUMN_NAME, + ERROR); + GRN_BULK_REWIND(maxRecordSize); + grn_obj_get_value(ctx, column, id, maxRecordSize); + return GRN_UINT32_VALUE(maxRecordSize); +} + +void +PGrnIndexStatusSetMaxRecordSize(Relation index, uint32_t size) +{ + grn_id id; + grn_obj *column; + grn_obj *maxRecordSize = &(buffers->maxRecordSize); + + id = PGrnIndexStatusGetRecordID(index); + column = PGrnLookup(TABLE_NAME "." MAX_RECORD_SIZE_COLUMN_NAME, + ERROR); + GRN_UINT32_SET(ctx, maxRecordSize, size); + grn_obj_set_value(ctx, column, id, maxRecordSize, GRN_OBJ_SET); +} + +void +PGrnIndexStatusGetWALAppliedPosition(Relation index, + BlockNumber *block, + OffsetNumber *offset) +{ + grn_id id; + grn_obj *column; + grn_obj *position = &(buffers->walAppliedPosition); + uint64_t positionRaw; + + id = PGrnIndexStatusGetRecordID(index); + column = PGrnLookup(TABLE_NAME "." WAL_APPLIED_POSITION_COLUMN_NAME, + ERROR); + GRN_BULK_REWIND(position); + grn_obj_get_value(ctx, column, id, position); + positionRaw = GRN_UINT64_VALUE(position); + *block = (BlockNumber)(positionRaw >> 32); + *offset = (OffsetNumber)(positionRaw & ((1 << 16) - 1)); +} + +void +PGrnIndexStatusSetWALAppliedPosition(Relation index, + BlockNumber block, + OffsetNumber offset) +{ + grn_id id; + grn_obj *column; + grn_obj *position = &(buffers->walAppliedPosition); + uint64_t positionRaw; + + id = PGrnIndexStatusGetRecordID(index); + column = PGrnLookup(TABLE_NAME "." WAL_APPLIED_POSITION_COLUMN_NAME, + ERROR); + positionRaw = (((uint64_t)block) << 32) + (uint64_t)offset; + GRN_UINT64_SET(ctx, position, positionRaw); + grn_obj_set_value(ctx, column, id, position, GRN_OBJ_SET); +} Added: src/pgrn_index_status.h (+19 -0) 100644 =================================================================== --- /dev/null +++ src/pgrn_index_status.h 2016-11-03 00:06:50 +0900 (274a55d) @@ -0,0 +1,19 @@ +#pragma once + +#include <postgres.h> +#include <storage/off.h> +#include <utils/rel.h> + +#include <groonga.h> + +void PGrnInitializeIndexStatus(void); + +uint32_t PGrnIndexStatusGetMaxRecordSize(Relation index); +void PGrnIndexStatusSetMaxRecordSize(Relation index, uint32_t size); + +void PGrnIndexStatusGetWALAppliedPosition(Relation index, + BlockNumber *block, + OffsetNumber *offset); +void PGrnIndexStatusSetWALAppliedPosition(Relation index, + BlockNumber block, + OffsetNumber offset); Modified: src/pgrn_wal.c (+21 -111) =================================================================== --- src/pgrn_wal.c 2016-11-02 22:49:23 +0900 (3f38f8a) +++ src/pgrn_wal.c 2016-11-03 00:06:50 +0900 (dbeb33f) @@ -4,6 +4,7 @@ #include "pgrn_global.h" #include "pgrn_groonga.h" +#include "pgrn_index_status.h" #include "pgrn_wal.h" static bool PGrnWALEnabled = false; @@ -94,10 +95,6 @@ struct PGrnWALData_ #endif }; -#define PGRN_WAL_STATUES_TABLE_NAME "WALStatuses" -#define PGRN_WAL_STATUES_TABLE_NAME_SIZE strlen(PGRN_WAL_STATUES_TABLE_NAME) -#define PGRN_WAL_STATUES_CURRENT_COLUMN_NAME "current" - #ifdef PGRN_SUPPORT_WAL static void msgpack_pack_cstr(msgpack_packer *packer, const char *string) @@ -125,71 +122,6 @@ msgpack_pack_grn_obj(msgpack_packer *packer, grn_obj *object) msgpack_pack_nil(packer); } } - -static void -PGrnWALEnsureStatusesTable(void) -{ - grn_obj *walStatuses; - - walStatuses = grn_ctx_get(ctx, - PGRN_WAL_STATUES_TABLE_NAME, - PGRN_WAL_STATUES_TABLE_NAME_SIZE); - if (walStatuses) - return; - - walStatuses = PGrnCreateTable(NULL, - PGRN_WAL_STATUES_TABLE_NAME, - GRN_OBJ_TABLE_HASH_KEY, - grn_ctx_at(ctx, GRN_DB_UINT32), - NULL, - NULL); - PGrnCreateColumn(NULL, - walStatuses, - PGRN_WAL_STATUES_CURRENT_COLUMN_NAME, - GRN_OBJ_COLUMN_SCALAR, - grn_ctx_at(ctx, GRN_DB_UINT64)); -} - -static uint64_t -PGrnWALPackPosition(BlockNumber block, OffsetNumber offset) -{ - return (((uint64_t)block) << 32) + (uint64_t)offset; -} - -static void -PGrnWALUnpackPosition(uint64_t position, - BlockNumber *block, - OffsetNumber *offset) -{ - *block = (BlockNumber)(position >> 32); - *offset = (OffsetNumber)(position & ((1 << 16) - 1)); -} - -static void -PGrnWALUpdateStatus(Relation index, - BlockNumber block, - OffsetNumber offset) -{ - grn_obj *statusesTable; - grn_obj *currentColumn; - uint32_t oid; - grn_id id; - uint64_t positionRaw; - grn_obj *position = &(buffers->walPosition); - - PGrnWALEnsureStatusesTable(); - - statusesTable = PGrnLookup(PGRN_WAL_STATUES_TABLE_NAME, ERROR); - currentColumn = PGrnLookupColumn(statusesTable, - PGRN_WAL_STATUES_CURRENT_COLUMN_NAME, - ERROR); - oid = RelationGetRelid(index); - id = grn_table_add(ctx, statusesTable, &oid, sizeof(uint32_t), NULL); - positionRaw = PGrnWALPackPosition(block, offset); - GRN_BULK_REWIND(position); - GRN_UINT64_SET(ctx, position, positionRaw); - grn_obj_set_value(ctx, currentColumn, id, position, GRN_OBJ_SET); -} #endif #define PGRN_WAL_META_PAGE_BLOCK_NUMBER 0 @@ -297,9 +229,10 @@ PGrnWALPageWriter(void *userData, buffer, length); data->current.pageSpecial->current += length; - PGrnWALUpdateStatus(data->index, - BufferGetBlockNumber(data->current.buffer), - data->current.pageSpecial->current); + PGrnIndexStatusSetWALAppliedPosition( + data->index, + BufferGetBlockNumber(data->current.buffer), + data->current.pageSpecial->current); written += length; } else @@ -313,9 +246,10 @@ PGrnWALPageWriter(void *userData, buffer, writableSize); data->current.pageSpecial->current += writableSize; - PGrnWALUpdateStatus(data->index, - BufferGetBlockNumber(data->current.buffer), - data->current.pageSpecial->current); + PGrnIndexStatusSetWALAppliedPosition( + data->index, + BufferGetBlockNumber(data->current.buffer), + data->current.pageSpecial->current); written += writableSize; length -= writableSize; buffer += writableSize; @@ -818,9 +752,6 @@ PGrnWALSetSourceIDs(Relation index, #ifdef PGRN_SUPPORT_WAL typedef struct { Relation index; - grn_obj *statusesTable; - grn_obj *currentColumn; - grn_id statusID; struct { BlockNumber block; OffsetNumber offset; @@ -835,14 +766,9 @@ PGrnWALApplyNeeded(PGrnWALApplyData *data) OffsetNumber currentOffset; BlockNumber nBlocks; - { - grn_obj *position = &(buffers->walPosition); - GRN_BULK_REWIND(position); - grn_obj_get_value(ctx, data->currentColumn, data->statusID, position); - PGrnWALUnpackPosition(GRN_UINT64_VALUE(position), - ¤tBlock, - ¤tOffset); - } + PGrnIndexStatusGetWALAppliedPosition(data->index, + ¤tBlock, + ¤tOffset); nBlocks = RelationGetNumberOfBlocks(data->index); if (currentBlock >= nBlocks) @@ -1445,10 +1371,13 @@ PGrnWALApplyConsume(PGrnWALApplyData *data) while (msgpack_unpacker_next(&unpacker, &unpacked) == MSGPACK_UNPACK_SUCCESS) { + OffsetNumber appliedOffset; + PGrnWALApplyObject(data, &unpacked.data); - PGrnWALUpdateStatus(data->index, - i, - unpacker.off - unpackerBaseOffset); + appliedOffset = unpacker.off - unpackerBaseOffset; + PGrnIndexStatusSetWALAppliedPosition(data->index, + i, + appliedOffset); } } msgpack_unpacked_destroy(&unpacked); @@ -1461,34 +1390,15 @@ PGrnWALApply(Relation index) { #ifdef PGRN_SUPPORT_WAL PGrnWALApplyData data; - uint32_t oid; - - PGrnWALEnsureStatusesTable(); data.index = index; - data.statusesTable = PGrnLookup(PGRN_WAL_STATUES_TABLE_NAME, ERROR); - data.currentColumn = PGrnLookupColumn(data.statusesTable, - PGRN_WAL_STATUES_CURRENT_COLUMN_NAME, - ERROR); - oid = RelationGetRelid(index); - data.statusID = grn_table_add(ctx, - data.statusesTable, - &oid, - sizeof(uint32_t), - NULL); if (!PGrnWALApplyNeeded(&data)) return; LockRelation(index, RowExclusiveLock); - { - grn_obj *position = &(buffers->walPosition); - - GRN_BULK_REWIND(position); - grn_obj_get_value(ctx, data.currentColumn, data.statusID, position); - PGrnWALUnpackPosition(GRN_UINT64_VALUE(position), - &(data.current.block), - &(data.current.offset)); - } + PGrnIndexStatusGetWALAppliedPosition(data.index, + &(data.current.block), + &(data.current.offset)); data.sources = NULL; PGrnWALApplyConsume(&data); UnlockRelation(index, RowExclusiveLock); Modified: src/pgroonga.c (+23 -1) =================================================================== --- src/pgroonga.c 2016-11-02 22:49:23 +0900 (d8e201a) +++ src/pgroonga.c 2016-11-03 00:06:50 +0900 (caaa358) @@ -7,8 +7,9 @@ #include "pgrn_global.h" #include "pgrn_groonga.h" #include "pgrn_highlight_html.h" -#include "pgrn_keywords.h" +#include "pgrn_index_status.h" #include "pgrn_jsonb.h" +#include "pgrn_keywords.h" #include "pgrn_match_positions_byte.h" #include "pgrn_match_positions_character.h" #include "pgrn_options.h" @@ -399,6 +400,8 @@ _PG_init(void) PGrnInitializeOptions(); + PGrnInitializeIndexStatus(); + PGrnInitializeMatchSequentialSearchData(); PGrnInitializePrefixRKSequentialSearchData(); @@ -1933,6 +1936,22 @@ PGrnInsert(Relation index, PGrnWALData *walData; unsigned int i; + { + uint32_t currentMaxRecordSize; + + currentMaxRecordSize = PGrnIndexStatusGetMaxRecordSize(index); + if (currentMaxRecordSize < INDEX_SIZE_MASK) + { + Size recordSize; + + recordSize = heap_compute_data_size(desc, values, isnull); + if (recordSize >= INDEX_SIZE_MASK) + { + PGrnIndexStatusSetMaxRecordSize(index, recordSize); + } + } + } + if (desc->natts == 1 && PGrnAttributeIsJSONB(desc->attrs[0]->atttypid)) { PGrnJSONBInsert(index, @@ -3973,6 +3992,9 @@ pgroonga_canreturn_raw(Relation index, TupleDesc desc; unsigned int i; + if (PGrnIndexStatusGetMaxRecordSize(index) >= INDEX_SIZE_MASK) + return false; + desc = RelationGetDescr(index); for (i = 0; i < desc->natts; i++) { -------------- next part -------------- HTML����������������������������...Download