[Groonga-commit] ranguba/chupa-text at a1de97f [master] http-server: add error handling

Back to archive index
Kouhei Sutou null+****@clear*****
Thu Mar 28 15:45:11 JST 2019


Kouhei Sutou	2019-03-28 15:45:11 +0900 (Thu, 28 Mar 2019)

  Revision: a1de97f87143bcd1e5cb5b1ae47aadd9086d4f42
  https://github.com/ranguba/chupa-text/commit/a1de97f87143bcd1e5cb5b1ae47aadd9086d4f42

  Message:
    http-server: add error handling

  Modified files:
    lib/chupa-text/decomposers/http-server.rb
    test/decomposers/test-http-server.rb

  Modified: lib/chupa-text/decomposers/http-server.rb (+79 -33)
===================================================================
--- lib/chupa-text/decomposers/http-server.rb    2019-03-28 14:56:21 +0900 (4f698cc)
+++ lib/chupa-text/decomposers/http-server.rb    2019-03-28 15:45:11 +0900 (b2f277c)
@@ -58,40 +58,49 @@ module ChupaText
         end
       end
 
-      def decompose(data)
+      def decompose(data, &block)
         http = Net::HTTP.new(@url.host, @url.port)
         http.use_ssl = true if****@url*****_a?(URI::HTTPS)
-        http.start do
-          request = Net::HTTP::Post.new(@url)
-          request["transfer-encoding"] = "chunked"
-          data.open do |input|
-            request.set_form([
-                               [
-                                 "data",
-                                 input,
-                                 {
-                                   filename: data.path.to_s,
-                                   content_type: data.mime_type,
-                                 },
-                               ],
-                             ],
-                             "multipart/form-data")
-            response = http.request(request)
-            unless response.is_a?(Net::HTTPOK)
-              error do
-                message = "#{log_tag} Failed to process data in server: "
-                message << "#{@url}: "
-                message << "#{response.code}: #{response.message.strip}\n"
-                case response.content_type
-                when "application/json"
-                  PP.pp(JSON.parse(response.body), message)
-                else
-                  message << response.body
-                end
-                message
-              end
-              break
-            end
+        if data.timeout.is_a?(Numeric)
+          http.open_timeout = data.timeout * 1.5
+          http.read_timeout = data.timeout * 1.5
+          http.write_timeout = data.timeout * 1.5
+        end
+        begin
+          http.start do
+            process_request(http, data, &block)
+          end
+        rescue SystemCallError => error
+          error do
+            message = "#{log_tag}[connection] "
+            message << "Failed to process data in server: "
+            message << "#{@url}: "
+            message << "#{error.class}: #{error.message}\n"
+            message << error.backtrace.join("\n")
+            message
+          end
+        rescue Net::ReadTimeout => error
+          error do
+            message = "#{log_tag}[timeout] "
+            message << "Failed to process data in server: "
+            message << "#{@url}: "
+            message << "#{error.class}: #{error.message}\n"
+            message << error.backtrace.join("\n")
+            message
+          end
+        end
+      end
+
+      private
+      def process_request(http, data)
+        request = Net::HTTP::Post.new(@url)
+        request["transfer-encoding"] = "chunked"
+        data.open do |input|
+          request.set_form(build_parameters(data, input),
+                           "multipart/form-data")
+          response = http.request(request)
+          case response
+          when Net::HTTPOK
             extracted = JSON.parse(response.body)
             (extracted["texts"] || []).each do |text|
               text_data = TextData.new(text["body"], source_data: data)
@@ -101,11 +110,48 @@ module ChupaText
               end
               yield(text_data)
             end
+          else
+            error do
+              message = "#{log_tag} Failed to process data in server: "
+              message << "#{@url}: "
+              message << "#{response.code}: #{response.message.strip}\n"
+              case response.content_type
+              when "application/json"
+                PP.pp(JSON.parse(response.body), message)
+              else
+                message << response.body
+              end
+              message
+            end
           end
         end
       end
 
-      private
+      def build_parameters(data, input)
+        parameters = []
+        [
+          ["timeout",
+           data.timeout || ChupaText::ExternalCommand.default_timeout],
+          ["limit_cpu",
+           data.limit_cpu || ChupaText::ExternalCommand.default_limit_cpu],
+          ["limit_as",
+           data.limit_as || ChupaText::ExternalCommand.default_limit_as],
+          ["max_body_size", data.max_body_size],
+        ].each do |key, value|
+          next if value.nil?
+          parameters << [key, StringIO.new(value.to_s)]
+        end
+        parameters << [
+          "data",
+          input,
+          {
+            filename: data.path.to_s,
+            content_type: data.mime_type,
+          },
+        ]
+        parameters
+      end
+
       def log_tag
         "[decomposer][http-server]"
       end

  Modified: test/decomposers/test-http-server.rb (+44 -3)
===================================================================
--- test/decomposers/test-http-server.rb    2019-03-28 14:56:21 +0900 (6696971)
+++ test/decomposers/test-http-server.rb    2019-03-28 15:45:11 +0900 (da1725f)
@@ -35,6 +35,7 @@ class TestDecomposersHTTPServer < Test::Unit::TestCase
                                       AccessLog: [])
     @response_status = 200
     @server.mount_proc(@path) do |request, response|
+      sleep(@timeout * 2) if @timeout
       response.status = @response_status
       response.content_type = "application/json"
       response.body = JSON.generate(@response)
@@ -51,6 +52,7 @@ Ruby,ChupaText
     CSV
     @input_mime_type = "text/csv"
     @input_path = "/tmp/hello.csv"
+    @timeout = nil
     @extracted_text = @input_data.gsub(/,/, "\t")
     @extracted_path = @input_path.gsub(/\.csv\z/, ".txt")
     @response = {
@@ -71,7 +73,6 @@ Ruby,ChupaText
         },
       ],
     }
-
   end
 
   def setup_decomposer
@@ -88,12 +89,12 @@ Ruby,ChupaText
   end
 
   sub_test_case("decompose") do
-    def test_valid
+    def test_success
       assert_equal([@extracted_text],
                    decompose.collect(&:body))
     end
 
-    def test_invalid
+    def test_not_ok
       @response_status = 404
       messages = capture_log do
         assert_equal([], decompose.collect(&:body))
@@ -110,6 +111,45 @@ Ruby,ChupaText
                    messages)
     end
 
+    def test_no_server
+      no_server_url = "http://127.0.0.1:2929/extraction.json"
+      @decomposer = ChupaText::Decomposers::HTTPServer.new(:url => no_server_url)
+      messages = capture_log do
+        assert_equal([], decompose.collect(&:body))
+      end
+      messages = messages.collect do |level, message|
+        [level, message.gsub(/Errno::.*\z/, "")]
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][http-server][connection] " +
+                       "Failed to process data in server: " +
+                       "#{no_server_url}: ",
+                     ],
+                   ],
+                   messages)
+    end
+
+    def test_read_timeout
+      @timeout = 0.1
+      messages = capture_log do
+        assert_equal([], decompose.collect(&:body))
+      end
+      messages = messages.collect do |level, message|
+        [level, message.gsub(/Net::.*\z/, "")]
+      end
+      assert_equal([
+                     [
+                       :error,
+                       "[decomposer][http-server][timeout] " +
+                       "Failed to process data in server: " +
+                       "#{@server_url}: ",
+                     ],
+                   ],
+                   messages)
+    end
+
     def test_default_url
       ChupaText::Decomposers::HTTPServer.default_url = @server_url
       @decomposer = ChupaText::Decomposers::HTTPServer.new({})
@@ -123,6 +163,7 @@ Ruby,ChupaText
       data.path = @input_path
       data.mime_type = @input_mime_type
       data.body = @input_data
+      data.timeout = @timeout
 
       decomposed = []
       @decomposer.decompose(data) do |decomposed_data|
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20190328/a278e2c2/attachment-0001.html>


More information about the Groonga-commit mailing list
Back to archive index