Kouhei Sutou 2019-03-28 15:45:11 +0900 (Thu, 28 Mar 2019) Revision: a1de97f87143bcd1e5cb5b1ae47aadd9086d4f42 https://github.com/ranguba/chupa-text/commit/a1de97f87143bcd1e5cb5b1ae47aadd9086d4f42 Message: http-server: add error handling Modified files: lib/chupa-text/decomposers/http-server.rb test/decomposers/test-http-server.rb Modified: lib/chupa-text/decomposers/http-server.rb (+79 -33) =================================================================== --- lib/chupa-text/decomposers/http-server.rb 2019-03-28 14:56:21 +0900 (4f698cc) +++ lib/chupa-text/decomposers/http-server.rb 2019-03-28 15:45:11 +0900 (b2f277c) @@ -58,40 +58,49 @@ module ChupaText end end - def decompose(data) + def decompose(data, &block) http = Net::HTTP.new(@url.host, @url.port) http.use_ssl = true if****@url*****_a?(URI::HTTPS) - http.start do - request = Net::HTTP::Post.new(@url) - request["transfer-encoding"] = "chunked" - data.open do |input| - request.set_form([ - [ - "data", - input, - { - filename: data.path.to_s, - content_type: data.mime_type, - }, - ], - ], - "multipart/form-data") - response = http.request(request) - unless response.is_a?(Net::HTTPOK) - error do - message = "#{log_tag} Failed to process data in server: " - message << "#{@url}: " - message << "#{response.code}: #{response.message.strip}\n" - case response.content_type - when "application/json" - PP.pp(JSON.parse(response.body), message) - else - message << response.body - end - message - end - break - end + if data.timeout.is_a?(Numeric) + http.open_timeout = data.timeout * 1.5 + http.read_timeout = data.timeout * 1.5 + http.write_timeout = data.timeout * 1.5 + end + begin + http.start do + process_request(http, data, &block) + end + rescue SystemCallError => error + error do + message = "#{log_tag}[connection] " + message << "Failed to process data in server: " + message << "#{@url}: " + message << "#{error.class}: #{error.message}\n" + message << error.backtrace.join("\n") + message + end + rescue Net::ReadTimeout => error + error do + message = "#{log_tag}[timeout] " + message << "Failed to process data in server: " + message << "#{@url}: " + message << "#{error.class}: #{error.message}\n" + message << error.backtrace.join("\n") + message + end + end + end + + private + def process_request(http, data) + request = Net::HTTP::Post.new(@url) + request["transfer-encoding"] = "chunked" + data.open do |input| + request.set_form(build_parameters(data, input), + "multipart/form-data") + response = http.request(request) + case response + when Net::HTTPOK extracted = JSON.parse(response.body) (extracted["texts"] || []).each do |text| text_data = TextData.new(text["body"], source_data: data) @@ -101,11 +110,48 @@ module ChupaText end yield(text_data) end + else + error do + message = "#{log_tag} Failed to process data in server: " + message << "#{@url}: " + message << "#{response.code}: #{response.message.strip}\n" + case response.content_type + when "application/json" + PP.pp(JSON.parse(response.body), message) + else + message << response.body + end + message + end end end end - private + def build_parameters(data, input) + parameters = [] + [ + ["timeout", + data.timeout || ChupaText::ExternalCommand.default_timeout], + ["limit_cpu", + data.limit_cpu || ChupaText::ExternalCommand.default_limit_cpu], + ["limit_as", + data.limit_as || ChupaText::ExternalCommand.default_limit_as], + ["max_body_size", data.max_body_size], + ].each do |key, value| + next if value.nil? + parameters << [key, StringIO.new(value.to_s)] + end + parameters << [ + "data", + input, + { + filename: data.path.to_s, + content_type: data.mime_type, + }, + ] + parameters + end + def log_tag "[decomposer][http-server]" end Modified: test/decomposers/test-http-server.rb (+44 -3) =================================================================== --- test/decomposers/test-http-server.rb 2019-03-28 14:56:21 +0900 (6696971) +++ test/decomposers/test-http-server.rb 2019-03-28 15:45:11 +0900 (da1725f) @@ -35,6 +35,7 @@ class TestDecomposersHTTPServer < Test::Unit::TestCase AccessLog: []) @response_status = 200 @server.mount_proc(@path) do |request, response| + sleep(@timeout * 2) if @timeout response.status = @response_status response.content_type = "application/json" response.body = JSON.generate(@response) @@ -51,6 +52,7 @@ Ruby,ChupaText CSV @input_mime_type = "text/csv" @input_path = "/tmp/hello.csv" + @timeout = nil @extracted_text = @input_data.gsub(/,/, "\t") @extracted_path = @input_path.gsub(/\.csv\z/, ".txt") @response = { @@ -71,7 +73,6 @@ Ruby,ChupaText }, ], } - end def setup_decomposer @@ -88,12 +89,12 @@ Ruby,ChupaText end sub_test_case("decompose") do - def test_valid + def test_success assert_equal([@extracted_text], decompose.collect(&:body)) end - def test_invalid + def test_not_ok @response_status = 404 messages = capture_log do assert_equal([], decompose.collect(&:body)) @@ -110,6 +111,45 @@ Ruby,ChupaText messages) end + def test_no_server + no_server_url = "http://127.0.0.1:2929/extraction.json" + @decomposer = ChupaText::Decomposers::HTTPServer.new(:url => no_server_url) + messages = capture_log do + assert_equal([], decompose.collect(&:body)) + end + messages = messages.collect do |level, message| + [level, message.gsub(/Errno::.*\z/, "")] + end + assert_equal([ + [ + :error, + "[decomposer][http-server][connection] " + + "Failed to process data in server: " + + "#{no_server_url}: ", + ], + ], + messages) + end + + def test_read_timeout + @timeout = 0.1 + messages = capture_log do + assert_equal([], decompose.collect(&:body)) + end + messages = messages.collect do |level, message| + [level, message.gsub(/Net::.*\z/, "")] + end + assert_equal([ + [ + :error, + "[decomposer][http-server][timeout] " + + "Failed to process data in server: " + + "#{@server_url}: ", + ], + ], + messages) + end + def test_default_url ChupaText::Decomposers::HTTPServer.default_url = @server_url @decomposer = ChupaText::Decomposers::HTTPServer.new({}) @@ -123,6 +163,7 @@ Ruby,ChupaText data.path = @input_path data.mime_type = @input_mime_type data.body = @input_data + data.timeout = @timeout decomposed = [] @decomposer.decompose(data) do |decomposed_data| -------------- next part -------------- An HTML attachment was scrubbed... URL: <https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20190328/a278e2c2/attachment-0001.html>
