YUKI Hiroshi
null+****@clear*****
Tue Jan 6 11:43:27 JST 2015
YUKI Hiroshi 2015-01-06 11:43:27 +0900 (Tue, 06 Jan 2015) New Revision: a5a133848ba08e81c869d25d7b1a9044802d7c9a https://github.com/droonga/droonga-engine/commit/a5a133848ba08e81c869d25d7b1a9044802d7c9a Message: Merge changes in the master correctly Modified files: .travis.yml bin/droonga-engine-absorb-data bin/droonga-engine-join bin/droonga-engine-set-role bin/droonga-engine-unjoin droonga-engine.gemspec lib/droonga/command/droonga_engine.rb lib/droonga/command/remote.rb lib/droonga/dispatcher.rb lib/droonga/farm.rb lib/droonga/serf.rb sample/cluster/Rakefile Modified: .travis.yml (+1 -0) =================================================================== --- .travis.yml 2015-01-06 11:03:49 +0900 (d6db562) +++ .travis.yml 2015-01-06 11:43:27 +0900 (2b4b377) @@ -9,6 +9,7 @@ rvm: - 1.9.3 - 2.0.0 - 2.1 + - 2.2 # - ruby-head before_install: # - GROONGA_MASTER=yes curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh Modified: bin/droonga-engine-absorb-data (+2 -5) =================================================================== --- bin/droonga-engine-absorb-data 2015-01-06 11:03:49 +0900 (49e4783) +++ bin/droonga-engine-absorb-data 2015-01-06 11:43:27 +0900 (23b2ced) @@ -132,11 +132,8 @@ class AbsorbDataCommand end def run_remote_command(target, command, options) - serf = Droonga::Serf.new(nil, target) - result = serf.send_query(command, options) - #puts result[:result] - puts result[:error] unless result[:error].empty? - result[:response] + serf = Droonga::Serf.new(target) + serf.send_query(command, options) end def absorber Modified: bin/droonga-engine-join (+2 -5) =================================================================== --- bin/droonga-engine-join 2015-01-06 11:03:49 +0900 (4547634) +++ bin/droonga-engine-join 2015-01-06 11:43:27 +0900 (55192cb) @@ -125,11 +125,8 @@ class JoinCommand end def run_remote_command(target, command, options) - serf = Droonga::Serf.new(nil, target) - result = serf.send_query(command, options) - #puts(result[:result]) - puts(result[:error]) unless result[:error].empty? - result[:response] + serf = Droonga::Serf.new(target) + serf.send_query(command, options) end def absorber Modified: bin/droonga-engine-set-role (+2 -5) =================================================================== --- bin/droonga-engine-set-role 2015-01-06 11:03:49 +0900 (39c5fed) +++ bin/droonga-engine-set-role 2015-01-06 11:43:27 +0900 (6a92b68) @@ -67,11 +67,8 @@ class SetRoleCommand end def run_remote_command(target, command, options) - serf = Droonga::Serf.new(nil, target) - result = serf.send_query(command, options) - #puts(result[:result]) - puts(result[:error]) unless result[:error].empty? - result[:response] + serf = Droonga::Serf.new(target) + serf.send_query(command, options) end def set_node_role Modified: bin/droonga-engine-unjoin (+2 -5) =================================================================== --- bin/droonga-engine-unjoin 2015-01-06 11:03:49 +0900 (634f609) +++ bin/droonga-engine-unjoin 2015-01-06 11:43:27 +0900 (6ae7df2) @@ -133,11 +133,8 @@ class UnjoinCommand end def run_remote_command(target, command, options) - serf = Droonga::Serf.new(nil, target) - result = serf.send_query(command, options) - puts(result[:result]) - puts(result[:error]) unless result[:error].empty? - result[:response] + serf = Droonga::Serf.new(target) + serf.send_query(command, options) end def do_unjoin Modified: droonga-engine.gemspec (+1 -1) =================================================================== --- droonga-engine.gemspec 2015-01-06 11:03:49 +0900 (e9c1928) +++ droonga-engine.gemspec 2015-01-06 11:43:27 +0900 (da3a2d9) @@ -46,7 +46,7 @@ Gem::Specification.new do |gem| gem.add_dependency "json" gem.add_dependency "rroonga", ">= 4.0.4" gem.add_dependency "sigdump" - gem.add_dependency "slop" + gem.add_dependency "slop", "<= 3.6.0" gem.add_dependency "sys-proctable" gem.add_development_dependency "kramdown" gem.add_development_dependency "bundler" Modified: lib/droonga/command/droonga_engine.rb (+49 -66) =================================================================== --- lib/droonga/command/droonga_engine.rb 2015-01-06 11:03:49 +0900 (6cc6451) +++ lib/droonga/command/droonga_engine.rb 2015-01-06 11:43:27 +0900 (1aa3134) @@ -23,6 +23,7 @@ require "coolio" require "sigdump/setup" require "droonga/engine/version" +require "droonga/loggable" require "droonga/path" require "droonga/address" require "droonga/serf" @@ -48,16 +49,13 @@ module Droonga parse_command_line_arguments!(command_line_arguments) setup_path + setup_log if****@confi*****? Process.daemon end - open_log_file do - write_pid_file do - run_main_loop - end - end + run_main_loop end private @@ -80,38 +78,15 @@ module Droonga end end + def setup_log + ENV["DROONGA_LOG_LEVEL"] =****@confi*****_level + end + def run_main_loop main_loop = MainLoop.new(@configuration) main_loop.run end - def open_log_file - if****@confi*****_file - File.open(@configuration.log_file, "a") do |file| - $stdout.reopen(file) - $stderr.reopen(file) - yield - end - else - yield - end - end - - def write_pid_file - if****@confi*****_file_path - @configuration.pid_file_path.open("w") do |file| - file.puts(Process.pid) - end - begin - yield - ensure - FileUtils.rm_f(@configuration.pid_file_path.to_s) - end - else - yield - end - end - class Configuration attr_reader :ready_notify_fd def initialize @@ -121,6 +96,7 @@ module Droonga @port = nil @tag = nil + @log_level = nil @log_file = nil @daemon = nil @pid_file_path = nil @@ -149,34 +125,19 @@ module Droonga end def tag - @port || config["tag"] || default_tag + @tag || config["tag"] || default_tag end def log_level - ENV["DROONGA_LOG_LEVEL"] || config["log_level"] || default_log_level + @log_level || config["log_level"] || default_log_level end - def log_level=(level) - ENV["DROONGA_LOG_LEVEL"] = level - end - - def log_file - file = @log_file || config["log_file"] || default_log_file - File.expand_path(file) - end - - def log_file=(file) - @log_file = File.expand_path(file) + def log_file_path + @log_file_path || config["log_file"] || default_log_file_path end def pid_file_path - path = @pid_file_path || config["pid_file"] || default_pid_file_path - return nil if path.nil? - Pathname.new(path.to_s).expand_path - end - - def pid_file_path=(path) - @pid_file_path = Pathname.new(path).expand_path + @pid_file_path || config["pid_file"] || default_pid_file_path end def daemon? @@ -252,25 +213,43 @@ module Droonga ENV["DROONGA_LOG_LEVEL"] || Logger::Level.default end - def default_log_file - Path.default_log_file + def default_log_file_path + nil end def default_pid_file_path nil end + def normalize_path(path) + if path == "-" + nil + else + Pathname.new(path).expand_path + end + end + def config @config ||= load_config end def load_config - config = Path.config - if config.exist? - YAML.load_file(config) - else - {} + config_path = Path.config + return {} unless config_path.exist? + + config = YAML.load_file(config_path) + path_keys = ["log_file", "pid_file"] + path_keys.each do |path_key| + path = config[path_key] + next if path.nil? + + path = Pathname.new(path) + unless path.absolute? + path = (config_path.dirname + path).expand_path + end + config[path_key] = path end + config end def add_connection_options(parser) @@ -301,12 +280,13 @@ module Droonga parser.on("--log-level=LEVEL", levels, "The log level of the Droonga engine", "[#{levels_label}]", - "(#{log_level})") do |level| - self.log_level = level + "(#{default_log_level})") do |level| + @log_level = level end parser.on("--log-file=FILE", - "Output logs to FILE") do |file| - self.log_file = file + "Output logs to FILE", + "(#{default_log_file_path})") do |path| + @log_file_path = normalize_path(path) end end @@ -323,7 +303,7 @@ module Droonga end parser.on("--pid-file=PATH", "Put PID to PATH") do |path| - self.pid_file_path = path + @pid_file_path = normalize_path(path) end end @@ -386,6 +366,8 @@ module Droonga end class MainLoop + include Loggable + def initialize(configuration) @configuration = configuration @loop = Coolio::Loop.default @@ -438,7 +420,6 @@ module Droonga @service_runner.success? end - private def setup_initial_on_ready return if****@confi*****_notify_fd.nil? @service_runner.on_ready = lambda do @@ -490,6 +471,7 @@ module Droonga def restart_graceful old_service_runner = @service_runner + reopen_log_file @service_runner = run_service @service_runner.on_ready = lambda do @service_runner.on_failure = nil @@ -503,6 +485,7 @@ module Droonga def restart_immediately old_service_runner = @service_runner + reopen_log_file @service_runner = run_service old_service_runner.stop_immediately end @@ -550,7 +533,7 @@ module Droonga catalog_observer = FileObserver.new(@loop, Path.catalog) catalog_observer.on_change = lambda do restart_graceful - @serf.update_cluster_id if @serf and****@serf*****? + @serf.update_cluster_id end catalog_observer.start catalog_observer Modified: lib/droonga/command/remote.rb (+1 -1) =================================================================== --- lib/droonga/command/remote.rb 2015-01-06 11:03:49 +0900 (2ffd9ec) +++ lib/droonga/command/remote.rb 2015-01-06 11:43:27 +0900 (e12b93d) @@ -37,7 +37,7 @@ module Droonga @response = { "log" => [] } - @serf = Serf.new(nil, @serf_name) + @serf = Serf.new(@serf_name) @service_installation = ServiceInstallation.new @service_installation.ensure_using_service_base_directory Modified: lib/droonga/dispatcher.rb (+1 -1) =================================================================== --- lib/droonga/dispatcher.rb 2015-01-06 11:03:49 +0900 (4e11ffe) +++ lib/droonga/dispatcher.rb 2015-01-06 11:43:27 +0900 (b3e1daa) @@ -89,7 +89,7 @@ module Droonga @adapter_runners.each_value do |adapter_runner| adapter_runner.shutdown end - @farm.shutdown + @farm.stop_immediately logger.trace("stop_immediately: done") end Modified: lib/droonga/farm.rb (+19 -3) =================================================================== --- lib/droonga/farm.rb 2015-01-06 11:03:49 +0900 (31de0c5) +++ lib/droonga/farm.rb 2015-01-06 11:43:27 +0900 (792b318) @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2013 Droonga Project +# Copyright (C) 2013-2015 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -37,12 +37,18 @@ module Droonga end def start + n_slices =****@slice***** + if n_slices.zero? + on_ready + return + end + n_ready_slices = 0 @slices.each_value do |slice| slice.on_ready = lambda do n_ready_slices += 1 - if n_ready_slices ==****@slice***** - @on_ready.call if @on_ready + if n_ready_slices == n_slices + on_ready end end slice.start @@ -51,6 +57,11 @@ module Droonga def stop_gracefully n_slices =****@slice***** + if n_slices.zero? + yield if block_given? + return + end + n_done_slices = 0 @slices.each_value do |slice| slice.stop_gracefully do @@ -71,5 +82,10 @@ module Droonga def process(slice_name, message) @slices[slice_name].process(message) end + + private + def on_ready + @on_ready.call if @on_ready + end end end Modified: lib/droonga/serf.rb (+54 -234) =================================================================== --- lib/droonga/serf.rb 2015-01-06 11:03:49 +0900 (d470f03) +++ lib/droonga/serf.rb 2015-01-06 11:43:27 +0900 (f21e6cc) @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Droonga Project +# Copyright (C) 2014-2015 Droonga Project # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,11 +13,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -require "English" - require "json" -require "coolio" -require "open3" require "droonga/path" require "droonga/loggable" @@ -32,9 +28,6 @@ require "droonga/service_installation" module Droonga class Serf - # the port must be different from droonga-http-server's agent! - AGENT_PORT = 7946 - class << self def path Droonga::Path.base + "serf" @@ -43,87 +36,68 @@ module Droonga include Loggable - def initialize(loop, name) - # TODO: Don't allow nil for loop. It reduces nil checks and - # simplifies source code. - @loop = loop + def initialize(name) + @serf = nil @name = name - @agent = nil @service_installation = ServiceInstallation.new end - def start - logger.trace("start: start") + def run_agent(loop) + logger.trace("run_agent: start") ensure_serf - ENV["SERF"] = @serf - ENV["SERF_RPC_ADDRESS"] = rpc_address retry_joins = [] detect_other_hosts.each do |other_host| retry_joins.push("-retry-join", other_host) end - @agent = run("agent", - "-node", @name, - "-bind", "#{extract_host(@name)}:#{port}", - "-event-handler", "droonga-engine-serf-event-handler", - "-log-level", log_level, - "-tag", "type=engine", - "-tag", "role=#{role}", - "-tag", "cluster_id=#{cluster_id}", - *retry_joins) - Thread.new do - sleep 1 # wait until the serf agent becomes running - update_cluster_state if****@agent*****? + agent = Agent.new(loop, @serf, + extract_host(@name), agent_port, rpc_port, + "-node", @name, + "-event-handler", "droonga-engine-serf-event-handler", + "-tag", "type=engine", + "-tag", "role=#{role}", + "-tag", "cluster_id=#{cluster_id}", + *retry_joins) + agent.on_ready = lambda do + update_cluster_state end - logger.trace("start: done") - end - - def running? - @agent and****@agent*****? + agent.start + logger.trace("run_agent: done") + agent end - def stop - logger.trace("stop: start") - run("leave").stop - @agent.stop - @agent = nil - logger.trace("stop: done") - end - - def restart - logger.trace("restart: start") - stop - start - logger.trace("restart: done") + def leave + run_command("leave") end def join(*hosts) - ensure_serf nodes = hosts.collect do |host| - "#{host}:#{port}" + "#{host}:#{agent_port}" end - run_once("join", *nodes) + run_command("join", *nodes) end def send_query(query, payload) - ensure_serf options = ["-format", "json"] + additional_options_from_payload(payload) options += [query, JSON.generate(payload)] - result = run_once("query", *options) - result[:result] = JSON.parse(result[:result]) - if payload["node"] - responses = result[:result]["Responses"] - response = responses[payload["node"]] + raw_serf_response = run_command("query", *options) + serf_response = JSON.parse(raw_serf_response) + + node = payload["node"] + if node + responses = serf_response["Responses"] + response = responses[node] if response.is_a?(String) begin - result[:response] = JSON.parse(response) + JSON.parse(response) rescue JSON::ParserError - result[:response] = response + response end else - result[:response] = response + response end + else + response end - result end def update_cluster_state @@ -137,13 +111,12 @@ module Droonga end def current_cluster_state - ensure_serf - nodes = {} - result = run_once("members", "-format", "json") - result[:result] = JSON.parse(result[:result]) - members = result[:result] + raw_response = run_command("members", "-format", "json") + response = JSON.parse(raw_response) + current_cluster_id = cluster_id - members["members"].each do |member| + nodes = {} + response["members"].each do |member| foreign = member["tags"]["cluster_id"] != current_cluster_id next if foreign @@ -157,13 +130,11 @@ module Droonga end def set_tag(name, value) - ensure_serf - run_once("tags", "-set", "#{name}=#{value}") + run_command("tags", "-set", "#{name}=#{value}") end def delete_tag(name) - ensure_serf - run_once("tags", "-delete", name) + run_command("tags", "-delete", name) end def update_cluster_id @@ -188,7 +159,7 @@ module Droonga private def ensure_serf - @serf = find_system_serf + @serf ||= find_system_serf return if @serf serf_path = self.class.path @@ -207,19 +178,12 @@ module Droonga nil end - def run(command, *options) - process = SerfProcess.new(@loop, @serf, command, - "-rpc-addr", rpc_address, - *options) - process.start - process - end - - def run_once(command, *options) - process = SerfProcess.new(@loop, @serf, command, - "-rpc-addr", rpc_address, - *options) - process.run_once + def run_command(command, *options) + ensure_serf + command = Command.new(@serf, command, + "-rpc-addr", rpc_address, + *options) + command.run end def additional_options_from_payload(payload) @@ -234,28 +198,20 @@ module Droonga node_name.split(":").first end - def log_level - level = Logger::Level.default - case level - when "trace", "debug", "info", "warn" - level - when "error", "fatal" - "err" - else - level # Or error? - end + def rpc_address + "#{extract_host(@name)}:#{rpc_port}" end - def rpc_address - "#{extract_host(@name)}:7373" + def rpc_port + 7373 end def node_metadata @node_metadata ||= NodeMetadata.new end - def port - AGENT_PORT + def agent_port + Agent::PORT end def detect_other_hosts @@ -272,141 +228,5 @@ module Droonga def log_tag "serf" end - - class SerfProcess - include Loggable - - def initialize(loop, serf, command, *options) - @loop = loop - @serf = serf - @command = command - @options = options - @pid = nil - end - - def start - capture_output do |output_write, error_write| - env = {} - spawn_options = { - :out => output_write, - :err => error_write, - } - @pid = spawn(env, @serf, @command, *@options, spawn_options) - end - end - - def stop - return if****@pid*****? - Process.waitpid(@pid) - @output_io.close - @error_io.close - @pid = nil - end - - def running? - not****@pid*****? - end - - def run_once - stdout, stderror, status = Open3.capture3(@serf, @command, *@options, :pgroup => true) - { - :result => stdout, - :error => stderror, - :status => status, - } - end - - private - def capture_output - result = nil - output_read, output_write = IO.pipe - error_read, error_write = IO.pipe - - begin - result = yield(output_write, error_write) - rescue - output_read.close unless output_read.closed? - output_write.close unless output_write.closed? - error_read.close unless error_read.closed? - error_write.close unless error_write.closed? - raise - end - - output_line_buffer = LineBuffer.new - on_read_output = lambda do |data| - on_standard_output(output_line_buffer, data) - end - @output_io = Coolio::IO.new(output_read) - @output_io.on_read do |data| - on_read_output.call(data) - end - # TODO: Don't allow nil for loop. It reduces nil checks and - # simplifies source code. - @loop.attach(@output_io) if @loop - - error_line_buffer = LineBuffer.new - on_read_error = lambda do |data| - on_error_output(error_line_buffer, data) - end - @error_io = Coolio::IO.new(error_read) - @error_io.on_read do |data| - on_read_error.call(data) - end - # TODO: Don't allow nil for loop. It reduces nil checks and - # simplifies source code. - @loop.attach(@error_io) if @loop - - result - end - - def on_standard_output(line_buffer, data) - line_buffer.feed(data) do |line| - line = line.chomp - case line - when /\A==> / - content = $POSTMATCH - logger.info(content) - when /\A / - content = $POSTMATCH - case content - when /\A(\d{4})\/(\d{2})\/(\d{2}) (\d{2}):(\d{2}):(\d{2}) \[(\w+)\] / - year, month, day = $1, $2, $3 - hour, minute, second = $4, $5, $6 - level = $7 - content = $POSTMATCH - level = normalize_level(level) - logger.send(level, content) - else - logger.info(content) - end - else - logger.info(line) - end - end - end - - def normalize_level(level) - level = level.downcase - case level - when "err" - "error" - else - level - end - end - - def on_error_output(line_buffer, data) - line_buffer.feed(data) do |line| - line = line.chomp - logger.error(line.gsub(/\A==> /, "")) - end - end - - def log_tag - tag = "serf" - tag << "[#{@pid}]" if @pid - tag - end - end end end Modified: sample/cluster/Rakefile (+5 -6) =================================================================== --- sample/cluster/Rakefile 2015-01-06 11:03:49 +0900 (b99ed1d) +++ sample/cluster/Rakefile 2015-01-06 11:43:27 +0900 (744ad87) @@ -17,6 +17,7 @@ require "pathname" require "json" +require "yaml" base_dir_path = Pathname.new(__FILE__).dirname top_dir_path = base_dir_path.parent.parent @@ -48,14 +49,12 @@ class DroongaEngine rm_rf(working_dir_path.to_s) mkdir_p(working_dir_path.to_s) generate_catalog(node_ids) + generate_config end def start @pid = spawn("droonga-engine", - "--base-dir", working_dir_path.to_s, - "--host", host, - "--port", self.class.port.to_s, - "--pid-file", pid_file_path.to_s) + "--base-dir", working_dir_path.to_s) end def stop @@ -81,8 +80,8 @@ class DroongaEngine working_dir_path + "catalog.json" end - def pid_file_path - working_dir_path + "droonga-engine.pid" + def config_path + working_dir_path + "droonga-engine.yaml" end def pid_path_base_name -------------- next part -------------- HTML����������������������������...Download