[Groonga-commit] ranguba/chupa-text at 9ef3cde [master] Data has timeout and limits

Back to archive index
Kouhei Sutou null+****@clear*****
Thu Mar 28 14:56:21 JST 2019


Kouhei Sutou	2019-03-28 14:56:21 +0900 (Thu, 28 Mar 2019)

  Revision: 9ef3cdef9b7cad539fe4b43b09ebc1dcb3f52f5c
  https://github.com/ranguba/chupa-text/commit/9ef3cdef9b7cad539fe4b43b09ebc1dcb3f52f5c

  Message:
    Data has timeout and limits

  Modified files:
    lib/chupa-text/data.rb
    lib/chupa-text/external-command.rb
    test/test-external-command.rb

  Modified: lib/chupa-text/data.rb (+17 -0)
===================================================================
--- lib/chupa-text/data.rb    2019-03-28 14:11:27 +0900 (f10c31b)
+++ lib/chupa-text/data.rb    2019-03-28 14:56:21 +0900 (ff2f8db)
@@ -68,6 +68,17 @@ module ChupaText
     # @return [Integer, nil] the max body size in bytes.
     attr_accessor :max_body_size
 
+    # @return [Numeric, String, nil] the timeout on extraction.
+    attr_accessor :timeout
+
+    # @return [Numeric, String, nil] the max CPU time on extraction by
+    #   external command.
+    attr_accessor :limit_cpu
+
+    # @return [Numeric, String, nil] the max memory on extraction by
+    #   external command.
+    attr_accessor :limit_as
+
     def initialize(options={})
       @uri = nil
       @body = nil
@@ -80,6 +91,9 @@ module ChupaText
       @need_screenshot = true
       @expected_screenshot_size = [200, 200]
       @max_body_size = nil
+      @timeout = nil
+      @limit_cpu = nil
+      @limit_as = nil
       @options = options || {}
       source_data = @options[:source_data]
       if source_data
@@ -112,6 +126,9 @@ module ChupaText
       self.need_screenshot = data.need_screenshot?
       self.expected_screenshot_size = data.expected_screenshot_size
       self.max_body_size = data.max_body_size
+      self.timeout = data.timeout
+      self.limit_cpu = data.limit_cpu
+      self.limit_as = data.limit_as
     end
 
     # @param [String, URI, nil] uri The URI for the data. If `uri` is

  Modified: lib/chupa-text/external-command.rb (+21 -38)
===================================================================
--- lib/chupa-text/external-command.rb    2019-03-28 14:11:27 +0900 (2a2d524)
+++ lib/chupa-text/external-command.rb    2019-03-28 14:56:21 +0900 (bcef54b)
@@ -23,11 +23,8 @@ module ChupaText
     include Loggable
 
     @default_timeout = nil
-    @default_soft_timeout = nil
     @default_limit_cpu = nil
-    @default_soft_limit_cpu = nil
     @default_limit_as = nil
-    @default_soft_limit_as = nil
     class << self
       def default_timeout
         @default_timeout || ENV["CHUPA_TEXT_EXTERNAL_COMMAND_TIMEOUT"]
@@ -37,14 +34,6 @@ module ChupaText
         @default_timeout = timeout
       end
 
-      def default_soft_timeout
-        @default_soft_timeout
-      end
-
-      def default_soft_timeout=(timeout)
-        @default_soft_timeout = timeout
-      end
-
       def default_limit_cpu
         @default_limit_cpu || limit_env("CPU")
       end
@@ -53,14 +42,6 @@ module ChupaText
         @default_limit_cpu = cpu
       end
 
-      def default_soft_limit_cpu
-        @default_soft_limit_cpu
-      end
-
-      def default_soft_limit_cpu=(cpu)
-        @default_soft_limit_cpu = cpu
-      end
-
       def default_limit_as
         @default_limit_as || limit_env("AS")
       end
@@ -69,14 +50,6 @@ module ChupaText
         @default_limit_as = as
       end
 
-      def default_soft_limit_as
-        @default_soft_limit_as
-      end
-
-      def default_soft_limit_as=(as)
-        @default_soft_limit_as = as
-      end
-
       private
       def limit_env(name)
         ENV["CHUPA_TEXT_EXTERNAL_COMMAND_LIMIT_#{name}"] ||
@@ -96,13 +69,19 @@ module ChupaText
       else
         options = {}
       end
+      data = options[:data]
       pid = spawn(options[:env] || {},
                   @path.to_s,
                   *arguments,
-                  spawn_options(options[:spawn_options]))
+                  spawn_options(options[:spawn_options], data))
+      if data
+        soft_timeout = data.timeout
+      else
+        soft_timeout = nil
+      end
       status = nil
       begin
-        status = wait_process(pid, options[:timeout], options[:soft_timeout])
+        status = wait_process(pid, options[:timeout], soft_timeout)
       ensure
         unless status
           begin
@@ -126,14 +105,21 @@ module ChupaText
     end
 
     private
-    def spawn_options(user_options)
+    def spawn_options(user_options, data)
       options = (user_options || {}).dup
-      apply_default_spawn_limit(options, :cpu, :time)
-      apply_default_spawn_limit(options, :as, :size)
+      if data
+        soft_limit_cpu = data.limit_cpu
+        soft_limit_as = data.limit_as
+      else
+        soft_limit_cpu = nil
+        soft_limit_as = nil
+      end
+      apply_default_spawn_limit(options, soft_limit_cpu, :cpu, :time)
+      apply_default_spawn_limit(options, soft_limit_as, :as, :size)
       options
     end
 
-    def apply_default_spawn_limit(options, key, type)
+    def apply_default_spawn_limit(options, soft_value, key, type)
       # TODO: Workaround for Ruby 2.3.3p222
       case key
       when :cpu
@@ -151,7 +137,6 @@ module ChupaText
       tag = "[limit][#{key}]"
       value = self.class.__send__("default_limit_#{key}")
       value = __send__("parse_#{type}", tag, value)
-      soft_value = self.class.__send__("default_soft_limit_#{key}")
       soft_value = __send__("parse_#{type}", tag, soft_value)
       if value
         value = soft_value if soft_value and soft_value < value
@@ -275,10 +260,8 @@ module ChupaText
 
     def wait_process(pid, timeout, soft_timeout)
       tag = "[timeout]"
-      timeout = parse_time(tag,
-                           timeout || self.class.default_timeout)
-      soft_timeout = parse_time(tag,
-                                soft_timeout || self.class.default_soft_timeout)
+      timeout = parse_time(tag, timeout || self.class.default_timeout)
+      soft_timeout = parse_time(tag, soft_timeout)
       if timeout
         timeout = soft_timeout if soft_timeout and soft_timeout < timeout
       else

  Modified: test/test-external-command.rb (+32 -32)
===================================================================
--- test/test-external-command.rb    2019-03-28 14:11:27 +0900 (54d69a7)
+++ test/test-external-command.rb    2019-03-28 14:56:21 +0900 (53a0842)
@@ -75,13 +75,12 @@ class TestExternalCommand < Test::Unit::TestCase
 
   class TestTimeout < self
     def setup
+      @data = ChupaText::TextData.new("Hello")
       timeout = ChupaText::ExternalCommand.default_timeout
-      soft_timeout = ChupaText::ExternalCommand.default_soft_timeout
       begin
         yield
       ensure
         ChupaText::ExternalCommand.default_timeout = timeout
-        ChupaText::ExternalCommand.default_soft_timeout = soft_timeout
       end
     end
 
@@ -89,7 +88,8 @@ class TestExternalCommand < Test::Unit::TestCase
       IO.pipe do |input, output|
         command = create_command(ruby)
         command.run("-e", "puts(Process.pid)",
-                    options.merge(spawn_options: {out: output}))
+                    options.merge(data: @data,
+                                  spawn_options: {out: output}))
         input.gets.chomp
       end
     end
@@ -108,11 +108,11 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_option_soft_not_use
+    def test_data_not_use
+      @data.timeout = "90s"
       pid = nil
       messages = capture_log do
-        pid = run_command(timeout: "60s",
-                          soft_timeout: "90s")
+        pid = run_command(timeout: "60s")
       end
       assert_equal([
                      [
@@ -123,11 +123,11 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_option_soft_use
+    def test_data_use
+      @data.timeout = "30s"
       pid = nil
       messages = capture_log do
-        pid = run_command(timeout: "60s",
-                          soft_timeout: "30s")
+        pid = run_command(timeout: "60s")
       end
       assert_equal([
                      [
@@ -138,10 +138,11 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_option_soft_only
+    def test_data_only
+      @data.timeout = "30s"
       pid = nil
       messages = capture_log do
-        pid = run_command(soft_timeout: "30s")
+        pid = run_command
       end
       assert_equal([
                      [
@@ -167,9 +168,9 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_default_soft_not_use
+    def test_default_data_not_use
       ChupaText::ExternalCommand.default_timeout = "60s"
-      ChupaText::ExternalCommand.default_soft_timeout = "90s"
+      @data.timeout = "90s"
       pid = nil
       messages = capture_log do
         pid = run_command
@@ -183,9 +184,9 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_default_soft_use
+    def test_default_data_use
       ChupaText::ExternalCommand.default_timeout = "60s"
-      ChupaText::ExternalCommand.default_soft_timeout = "30s"
+      @data.timeout = "30s"
       pid = nil
       messages = capture_log do
         pid = run_command
@@ -199,9 +200,8 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_default_soft_only
-      ChupaText::ExternalCommand.default_timeout = nil
-      ChupaText::ExternalCommand.default_soft_timeout = "30s"
+    def test_default_data_only
+      @data.timeout = "30s"
       pid = nil
       messages = capture_log do
         pid = run_command
@@ -218,19 +218,19 @@ class TestExternalCommand < Test::Unit::TestCase
 
   class TestLimitCPU < self
     def setup
+      @data = ChupaText::TextData.new("Hello")
       limit_cpu = ChupaText::ExternalCommand.default_limit_cpu
-      soft_limit_cpu = ChupaText::ExternalCommand.default_soft_limit_cpu
       begin
         yield
       ensure
         ChupaText::ExternalCommand.default_limit_cpu = limit_cpu
-        ChupaText::ExternalCommand.default_soft_limit_cpu = soft_limit_cpu
       end
     end
 
     def run_command(spawn_options={})
       command = create_command(ruby)
       command.run("-e", "true",
+                  data: @data,
                   spawn_options: spawn_options)
     end
 
@@ -250,9 +250,9 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_default_soft_not_use
+    def test_default_data_not_use
       ChupaText::ExternalCommand.default_limit_cpu = "60s"
-      ChupaText::ExternalCommand.default_soft_limit_cpu = "90s"
+      @data.limit_cpu = "90s"
       messages = capture_log do
         run_command
       end
@@ -267,9 +267,9 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_default_soft_use
+    def test_default_data_use
       ChupaText::ExternalCommand.default_limit_cpu = "60s"
-      ChupaText::ExternalCommand.default_soft_limit_cpu = "30s"
+      @data.limit_cpu = "30s"
       messages = capture_log do
         run_command
       end
@@ -284,8 +284,8 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_default_soft_only
-      ChupaText::ExternalCommand.default_soft_limit_cpu = "30s"
+    def test_default_data_only
+      @data.limit_cpu = "30s"
       messages = capture_log do
         run_command
       end
@@ -303,19 +303,19 @@ class TestExternalCommand < Test::Unit::TestCase
 
   class TestLimitAS < self
     def setup
+      @data = ChupaText::TextData.new("Hello")
       limit_as = ChupaText::ExternalCommand.default_limit_as
-      soft_limit_as = ChupaText::ExternalCommand.default_soft_limit_as
       begin
         yield
       ensure
         ChupaText::ExternalCommand.default_limit_as = limit_as
-        ChupaText::ExternalCommand.default_soft_limit_as = soft_limit_as
       end
     end
 
     def run_command(spawn_options={})
       command = create_command(ruby)
       command.run("-e", "true",
+                  data: @data,
                   spawn_options: spawn_options)
     end
 
@@ -336,9 +336,9 @@ class TestExternalCommand < Test::Unit::TestCase
                    messages)
     end
 
-    def test_default_soft_not_use
+    def test_default_data_not_use
       ChupaText::ExternalCommand.default_limit_as = "100MiB"
-      ChupaText::ExternalCommand.default_soft_limit_as = "150MiB"
+      @data.limit_as = "150MiB"
       messages = capture_log do
         run_command
       end
@@ -356,7 +356,7 @@ class TestExternalCommand < Test::Unit::TestCase
 
     def test_default_soft_use
       ChupaText::ExternalCommand.default_limit_as = "100MiB"
-      ChupaText::ExternalCommand.default_soft_limit_as = "50MiB"
+      @data.limit_as = "50MiB"
       messages = capture_log do
         run_command
       end
@@ -373,7 +373,7 @@ class TestExternalCommand < Test::Unit::TestCase
     end
 
     def test_default_soft_only
-      ChupaText::ExternalCommand.default_soft_limit_as = "50MiB"
+      @data.limit_as = "50MiB"
       messages = capture_log do
         run_command
       end
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20190328/b9821605/attachment-0001.html>


More information about the Groonga-commit mailing list
Back to archive index