susumu.yata
null+****@clear*****
Thu Apr 25 14:46:36 JST 2013
susumu.yata 2013-04-25 14:46:36 +0900 (Thu, 25 Apr 2013) New Revision: 2d41b7045e9835ed8ea8e029f8e11c96961824e7 https://github.com/groonga/grnxx/commit/2d41b7045e9835ed8ea8e029f8e11c96961824e7 Message: Implement a part of grnxx::Storage. Modified files: lib/grnxx/storage.cpp lib/grnxx/storage.hpp lib/grnxx/storage/chunk_index.cpp lib/grnxx/storage/chunk_index.hpp lib/grnxx/storage/header.cpp lib/grnxx/storage/header.hpp lib/grnxx/storage/node_header.cpp lib/grnxx/storage/node_header.hpp lib/grnxx/storage/storage_impl.cpp lib/grnxx/storage/storage_impl.hpp Modified: lib/grnxx/storage.cpp (+1 -1) =================================================================== --- lib/grnxx/storage.cpp 2013-04-25 11:13:26 +0900 (8d68dc7) +++ lib/grnxx/storage.cpp 2013-04-25 14:46:36 +0900 (cfcff89) @@ -111,7 +111,7 @@ StorageNodeStatus StorageNode::status() const { } uint64_t StorageNode::size() const { - return header_->size << header_->bits; + return header_->size; } Time StorageNode::modified_time() const { Modified: lib/grnxx/storage.hpp (+1 -1) =================================================================== --- lib/grnxx/storage.hpp 2013-04-25 11:13:26 +0900 (7de8ab2) +++ lib/grnxx/storage.hpp 2013-04-25 14:46:36 +0900 (e5302e7) @@ -75,7 +75,7 @@ constexpr uint32_t STORAGE_INVALID_NODE_ID = storage::INVALID_NODE_ID; struct StorageOptions { // The maximum number of files. - uint32_t max_num_files; + uint16_t max_num_files; // The maximum size of each file. uint64_t max_file_size; // The size of the root node. Modified: lib/grnxx/storage/chunk_index.cpp (+2 -1) =================================================================== --- lib/grnxx/storage/chunk_index.cpp 2013-04-25 11:13:26 +0900 (627428b) +++ lib/grnxx/storage/chunk_index.cpp 2013-04-25 14:46:36 +0900 (80aaae7) @@ -23,9 +23,10 @@ namespace storage { ChunkIndex::ChunkIndex(uint32_t id) : id_(id), file_id_(0), + reserved_0_(0), offset_(0), size_(0), - reserved_(0) {} + reserved_1_(0) {} } // namespace storage } // namespace grnxx Modified: lib/grnxx/storage/chunk_index.hpp (+8 -2) =================================================================== --- lib/grnxx/storage/chunk_index.hpp 2013-04-25 11:13:26 +0900 (f21dab3) +++ lib/grnxx/storage/chunk_index.hpp 2013-04-25 14:46:36 +0900 (ba71d1f) @@ -23,16 +23,22 @@ namespace grnxx { namespace storage { +constexpr size_t CHUNK_INDEX_SIZE = 32; + struct ChunkIndex { uint32_t id_; - uint32_t file_id_; + uint16_t file_id_; + uint16_t reserved_0_; uint64_t offset_; uint64_t size_; - uint64_t reserved_; + uint64_t reserved_1_; explicit ChunkIndex(uint32_t id); }; +static_assert(sizeof(ChunkIndex) == CHUNK_INDEX_SIZE, + "sizeof(ChunkIndex) != CHUNK_INDEX_SIZE"); + } // namespace storage } // namespace grnxx Modified: lib/grnxx/storage/header.cpp (+9 -5) =================================================================== --- lib/grnxx/storage/header.cpp 2013-04-25 11:13:26 +0900 (6752f69) +++ lib/grnxx/storage/header.cpp 2013-04-25 14:46:36 +0900 (26d7dc0) @@ -33,20 +33,20 @@ constexpr char HEADER_FORMAT[HEADER_FORMAT_SIZE] = GRNXX_STORAGE_HEADER_FORMAT; } // namespace Header::Header() - : format{ GRNXX_STORAGE_HEADER_FORMAT }, + : format{}, version{ GRNXX_STORAGE_HEADER_VERSION }, - max_num_files(0), max_file_size(0), - total_size(0), - num_files(0), + max_num_files(0), num_node_chunks(0), num_nodes(0), + total_size(0), max_num_nodes(0), latest_phantom_node_id(INVALID_NODE_ID), latest_unlinked_node_id(INVALID_NODE_ID), oldest_idle_node_ids(), inter_process_data_mutex(MUTEX_UNLOCKED), - inter_process_file_mutex(MUTEX_UNLOCKED) { + inter_process_file_mutex(MUTEX_UNLOCKED), + reserved_() { static constexpr size_t NUM_LISTS = sizeof(oldest_idle_node_ids) / sizeof(*oldest_idle_node_ids); for (size_t i = 0; i < NUM_LISTS; ++i) { @@ -58,5 +58,9 @@ bool Header::is_valid() const { return std::memcmp(format, HEADER_FORMAT, HEADER_FORMAT_SIZE) == 0; } +void Header::validate() { + std::memcpy(format, HEADER_FORMAT, sizeof(HEADER_FORMAT_SIZE)); +} + } // namespace storage } // namespace grnxx Modified: lib/grnxx/storage/header.hpp (+11 -9) =================================================================== --- lib/grnxx/storage/header.hpp 2013-04-25 11:13:26 +0900 (f3763c3) +++ lib/grnxx/storage/header.hpp 2013-04-25 14:46:36 +0900 (8667b64) @@ -38,18 +38,16 @@ struct Header { char format[HEADER_FORMAT_SIZE]; // The grnxx version. char version[HEADER_VERSION_SIZE]; - // The maximum number of files. - uint32_t max_num_files; // The maximum size of each file. uint64_t max_file_size; - // The total size including headers. - uint64_t total_size; - // The number of files. - uint32_t num_files; + // The maximum number of files. + uint16_t max_num_files; // The number of node chunks. - uint32_t num_node_chunks; + uint16_t num_node_chunks; // The number of nodes. uint32_t num_nodes; + // The total size including headers. + uint64_t total_size; // The upper limit of the number of nodes. // This value is extended when a node header chunk is added. uint32_t max_num_nodes; @@ -63,15 +61,19 @@ struct Header { Mutex inter_process_data_mutex; // A mutex object for exclusively update files. Mutex inter_process_file_mutex; + uint8_t reserved_[148]; - // Initialize the members. + // Initialize the members except "format". Header(); // Return true if the header seems to be correct. bool is_valid() const; + + // Initialize "format". + void validate(); }; -static_assert(sizeof(Header) <= HEADER_SIZE, "sizeof(Header) > HEADER_SIZE"); +static_assert(sizeof(Header) == HEADER_SIZE, "sizeof(Header) != HEADER_SIZE"); } // namespace storage } // namespace grnxx Modified: lib/grnxx/storage/node_header.cpp (+1 -2) =================================================================== --- lib/grnxx/storage/node_header.cpp 2013-04-25 11:13:26 +0900 (0436eb2) +++ lib/grnxx/storage/node_header.cpp 2013-04-25 14:46:36 +0900 (c6474ea) @@ -23,7 +23,7 @@ namespace storage { NodeHeader::NodeHeader() : id(0), status(STORAGE_NODE_PHANTOM), - bits(0), + reserved_(0), chunk_id(0), offset(0), size(0), @@ -32,7 +32,6 @@ NodeHeader::NodeHeader() next_phantom_node_id(0), sibling_node_id(0), modified_time(0), - reserved{}, user_data{} {} } // namespace storage Modified: lib/grnxx/storage/node_header.hpp (+19 -18) =================================================================== --- lib/grnxx/storage/node_header.hpp 2013-04-25 11:13:26 +0900 (49d4781) +++ lib/grnxx/storage/node_header.hpp 2013-04-25 14:46:36 +0900 (7fcdfc4) @@ -25,31 +25,30 @@ namespace storage { using NodeStatus = StorageNodeStatus; +constexpr size_t NODE_HEADER_SIZE = 64; + struct NodeHeader { - // The ID of this node. + // The node ID. uint32_t id; - // The status of this node. + // The node status. NodeStatus status; + uint8_t reserved_; // (Non-phantom) - // For calculating the actual offset and size, see also "offset" and "size". - uint8_t bits; - // (Non-phantom) - // The ID of the chunk to which this node belongs. + // The ID of the chunk to which the node belongs. uint16_t chunk_id; // (Non-phantom) - // The offset of this node in chunk. - // The actual offset is "offset" << "bits". - uint32_t offset; + // The offset in chunk. + uint64_t offset; // (Non-phantom) - // The size of this node. The actual size is "size" << "bits". - uint32_t size; + // The body size. + uint64_t size; // (Non-phantom) // The ID of the next node in chunk. - // INVALID_ID indicates that this node is the last node in chunk. + // STORAGE_INVALID_NODE_ID indicates that the node is the last node in chunk. uint32_t next_node_id; // (Non-phantom) // The ID of the previous node in chunk. - // INVALID_ID indicates that this node is the first node in chunk. + // STORAGE_INVALID_NODE_ID indicates that the node is the first node in chunk. uint32_t prev_node_id; union { // (Phantom) @@ -57,7 +56,7 @@ struct NodeHeader { uint32_t next_phantom_node_id; // (Active, marked, or unlinked) // The ID of the latest child node. - // INVALID_ID indicates that this node has no children. + // STORAGE_INVALID_NODE_ID indicates that the node has no children. uint32_t child_id; // (Idle) // The ID of the next idle node. @@ -66,11 +65,12 @@ struct NodeHeader { union { // (Active or marked) // The ID of the next sibling node. - // INVALID_ID indicates that this node has no elder siblings. + // STORAGE_INVALID_NODE_ID indicates that the node has no elder siblings. uint32_t sibling_node_id; // (Unlinked) // The ID of the next unlinked node. - // INVALID_ID indicates that this node is the last unlinked node. + // STORAGE_INVALID_NODE_ID indicates that the node is the last unlinked + // node. uint32_t next_unlinked_node_id; // (Idle) // The ID of the previous idle node. @@ -78,8 +78,6 @@ struct NodeHeader { }; // The last modified time. Time modified_time; - // Reserved for future use. - uint8_t reserved[8]; // User data. uint8_t user_data[16]; @@ -87,6 +85,9 @@ struct NodeHeader { NodeHeader(); }; +static_assert(sizeof(NodeHeader) == NODE_HEADER_SIZE, + "sizeof(NodeHeader) != NODE_HEADER_SIZE"); + } // namespace storage } // namespace grnxx Modified: lib/grnxx/storage/storage_impl.cpp (+130 -38) =================================================================== --- lib/grnxx/storage/storage_impl.cpp 2013-04-25 11:13:26 +0900 (c1edb55) +++ lib/grnxx/storage/storage_impl.cpp 2013-04-25 14:46:36 +0900 (044a2e0) @@ -30,8 +30,17 @@ namespace { constexpr uint32_t MAX_NODE_ID = INVALID_NODE_ID - 1; -constexpr uint32_t MAX_NUM_NODE_HEADER_CHUNKS = 32; -constexpr uint32_t MAX_NUM_NODE_BODY_CHUNKS = 2048; +constexpr uint16_t MAX_NUM_NODE_HEADER_CHUNKS = 32; +constexpr uint16_t MAX_NUM_NODE_BODY_CHUNKS = 2048; + +constexpr uint64_t CHUNK_UNIT_SIZE = 1 << 16; +constexpr size_t CHUNK_INDEX_TOTAL_SIZE = + CHUNK_INDEX_SIZE * (MAX_NUM_NODE_HEADER_CHUNKS + MAX_NUM_NODE_BODY_CHUNKS); + +constexpr uint64_t HEADER_CHUNK_SIZE = CHUNK_UNIT_SIZE; + +static_assert(HEADER_CHUNK_SIZE <= (HEADER_SIZE + CHUNK_INDEX_TOTAL_SIZE), + "HEADER_CHUNK_SIZE > (HEADER_SIZE + CHUNK_INDEX_TOTAL_SIZE)"); // TODO: Define constant values. @@ -63,12 +72,8 @@ StorageImpl *StorageImpl::create(const char *path, GRNXX_ERROR() << "new grnxx::storage::StorageImpl failed"; return nullptr; } - if (path && (~flags & STORAGE_TEMPORARY)) { - if (!storage->create_persistent_storage(path, flags, options)) { - return nullptr; - } - } else if (flags & STORAGE_TEMPORARY) { - if (!storage->create_temporary_storage(path, flags, options)) { + if (path || (flags & STORAGE_TEMPORARY)) { + if (!storage->create_file_backed_storage(path, flags, options)) { return nullptr; } } else { @@ -160,7 +165,7 @@ StorageNode StorageImpl::open_node(uint32_t node_id) { } bool StorageImpl::unlink_node(uint32_t node_id) { - if (node_id >= header_->num_nodes) { + if ((node_id == STORAGE_ROOT_NODE_ID) || (node_id >= header_->num_nodes)) { GRNXX_ERROR() << "invalid argument: node_id = " << node_id << ", num_nodes = " << header_->num_nodes; return false; @@ -182,34 +187,44 @@ StorageFlags StorageImpl::flags() const { return flags_; } -bool StorageImpl::create_persistent_storage(const char *path, - StorageFlags flags, - const StorageOptions &options) { - path_.reset(Path::clone_path(path)); - if (!path_) { - return false; - } - if (flags & STORAGE_HUGE_TLB) { - flags_ |= STORAGE_HUGE_TLB; - } - // TODO - return false; -} - -bool StorageImpl::create_temporary_storage(const char *path, - StorageFlags flags, - const StorageOptions &options) { +bool StorageImpl::create_file_backed_storage(const char *path, + StorageFlags flags, + const StorageOptions &options) { if (path) { path_.reset(Path::clone_path(path)); if (!path_) { return false; } } - flags_ |= STORAGE_TEMPORARY; + if (flags & STORAGE_TEMPORARY) { + flags_ |= STORAGE_TEMPORARY; + } if (flags & STORAGE_HUGE_TLB) { flags_ |= STORAGE_HUGE_TLB; } - // TODO + FileFlags file_flags = FILE_DEFAULT; + if (flags_ & STORAGE_TEMPORARY) { + file_flags |= FILE_TEMPORARY; + } + std::unique_ptr<File> header_file(File::create(path, file_flags)); + if (!header_file) { + return false; + } + if (!header_file->resize(HEADER_CHUNK_SIZE)) { + return false; + } + std::unique_ptr<Chunk> header_chunk( + create_chunk(header_file.get(), 0, HEADER_CHUNK_SIZE)); + header_ = static_cast<Header *>(header_chunk->address()); + *header_ = Header(); + // TODO: Create the root node. + // TODO: Initialize the header. + header_->validate(); + if (!prepare_pointers()) { + return false; + } + files_[0] = std::move(header_file); + header_chunk_ = std::move(header_chunk); return false; } @@ -219,8 +234,18 @@ bool StorageImpl::create_anonymous_storage(StorageFlags flags, if (flags & STORAGE_HUGE_TLB) { flags_ |= STORAGE_HUGE_TLB; } - // TODO - return false; + std::unique_ptr<Chunk> header_chunk( + create_chunk(nullptr, 0, HEADER_CHUNK_SIZE)); + header_ = static_cast<Header *>(header_chunk->address()); + *header_ = Header(); + // TODO: Create the root node. + // TODO: Initialize the header. + header_->validate(); + if (!prepare_pointers()) { + return false; + } + header_chunk_ = std::move(header_chunk); + return true; } bool StorageImpl::open_storage(const char *path, StorageFlags flags) { @@ -234,8 +259,26 @@ bool StorageImpl::open_storage(const char *path, StorageFlags flags) { if (flags & STORAGE_HUGE_TLB) { flags_ |= STORAGE_HUGE_TLB; } - // TODO - return false; + FileFlags file_flags = FILE_DEFAULT; + if (flags_ & STORAGE_READ_ONLY) { + file_flags |= FILE_READ_ONLY; + } + std::unique_ptr<File> header_file(File::open(path, file_flags)); + if (!header_file) { + return false; + } + std::unique_ptr<Chunk> header_chunk( + create_chunk(header_file.get(), 0, HEADER_CHUNK_SIZE)); + header_ = static_cast<Header *>(header_chunk->address()); + if (!header_->is_valid()) { + return false; + } + if (!prepare_pointers()) { + return false; + } + files_[0] = std::move(header_file); + header_chunk_ = std::move(header_chunk); + return true; } bool StorageImpl::open_or_create_storage(const char *path, @@ -249,17 +292,58 @@ bool StorageImpl::open_or_create_storage(const char *path, flags_ |= STORAGE_HUGE_TLB; } // TODO + if (!prepare_pointers()) { + return false; + } + // TODO return false; } -bool StorageImpl::prepare_files_and_chunks(const StorageOptions &options) { - files_.reset( - new (std::nothrow) std::unique_ptr<File>[options.max_num_files]); - if (!files_) { - GRNXX_ERROR() << "new std::unique_ptr<grnxx::storage::File>[] failed: " - << "size = " << options.max_num_files; +bool StorageImpl::prepare_pointers() { + node_header_chunk_indexes_ = reinterpret_cast<ChunkIndex *>(header_ + 1); + node_body_chunk_indexes_ = + node_header_chunk_indexes_ + MAX_NUM_NODE_HEADER_CHUNKS; + if (~flags_ & STORAGE_ANONYMOUS) { + files_.reset( + new (std::nothrow) std::unique_ptr<File>[header_->max_num_files]); + if (!files_) { + GRNXX_ERROR() << "new std::unique_ptr<grnxx::storage::File>[] failed: " + << "size = " << header_->max_num_files; + return false; + } + } + node_header_chunks_.reset( + new (std::nothrow) std::unique_ptr<Chunk>[MAX_NUM_NODE_HEADER_CHUNKS]); + if (!node_header_chunks_) { + GRNXX_ERROR() << "new std::unique_ptr<grnxx::storage::Chunk>[] failed: " + << "size = " << MAX_NUM_NODE_HEADER_CHUNKS; return false; } + node_body_chunks_.reset( + new (std::nothrow) std::unique_ptr<Chunk>[MAX_NUM_NODE_BODY_CHUNKS]); + if (!node_header_chunks_) { + GRNXX_ERROR() << "new std::unique_ptr<grnxx::storage::Chunk>[] failed: " + << "size = " << MAX_NUM_NODE_BODY_CHUNKS; + return false; + } + return true; +} + +void StorageImpl::prepare_indexes() { + node_header_chunk_indexes_ = reinterpret_cast<ChunkIndex *>(header_ + 1); + node_body_chunk_indexes_ = + node_header_chunk_indexes_ + MAX_NUM_NODE_HEADER_CHUNKS; +} + +bool StorageImpl::prepare_files_and_chunks(uint16_t max_num_files) { + if (~flags_ & STORAGE_ANONYMOUS) { + files_.reset(new (std::nothrow) std::unique_ptr<File>[max_num_files]); + if (!files_) { + GRNXX_ERROR() << "new std::unique_ptr<grnxx::storage::File>[] failed: " + << "size = " << max_num_files; + return false; + } + } node_header_chunks_.reset( new (std::nothrow) std::unique_ptr<Chunk>[MAX_NUM_NODE_HEADER_CHUNKS]); if (!node_header_chunks_) { @@ -277,5 +361,13 @@ bool StorageImpl::prepare_files_and_chunks(const StorageOptions &options) { return true; } +Chunk *StorageImpl::create_chunk(File *file, int64_t offset, int64_t size) { + ChunkFlags chunk_flags = CHUNK_DEFAULT; + if (flags_ & STORAGE_HUGE_TLB) { + chunk_flags |= CHUNK_HUGE_TLB; + } + return Chunk::create(file, offset, size, chunk_flags); +} + } // namespace storage } // namespace grnxx Modified: lib/grnxx/storage/storage_impl.hpp (+7 -5) =================================================================== --- lib/grnxx/storage/storage_impl.hpp 2013-04-25 11:13:26 +0900 (ca14564) +++ lib/grnxx/storage/storage_impl.hpp 2013-04-25 14:46:36 +0900 (7ff0e0e) @@ -68,17 +68,19 @@ class StorageImpl : public Storage { std::unique_ptr<std::unique_ptr<Chunk>[]> node_header_chunks_; std::unique_ptr<std::unique_ptr<Chunk>[]> node_body_chunks_; - bool create_persistent_storage(const char *path, StorageFlags flags, - const StorageOptions &options); - bool create_temporary_storage(const char *path, StorageFlags flags, - const StorageOptions &options); + bool create_file_backed_storage(const char *path, StorageFlags flags, + const StorageOptions &options); bool create_anonymous_storage(StorageFlags flags, const StorageOptions &options); bool open_storage(const char *path, StorageFlags flags); bool open_or_create_storage(const char *path, StorageFlags flags, const StorageOptions &options); - bool prepare_files_and_chunks(const StorageOptions &options); + bool prepare_pointers(); + void prepare_indexes(); + bool prepare_files_and_chunks(uint16_t max_num_files); + + Chunk *create_chunk(File *file, int64_t offset, int64_t size); }; } // namespace storage -------------- next part -------------- HTML����������������������������...Download