From 23256118cc9ca6ef1fc6413ed9ddb3d31ceecfb2 Mon Sep 17 00:00:00 2001 From: Andrew Tribone Date: Tue, 20 May 2014 11:51:04 -0700 Subject: [PATCH 01/38] Allow Task's to be scheduled in constructor --- lib/gearman/task.rb | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/gearman/task.rb b/lib/gearman/task.rb index 636fb7d..35d2c5a 100644 --- a/lib/gearman/task.rb +++ b/lib/gearman/task.rb @@ -21,7 +21,7 @@ def initialize(func, arg='', opts={}) @arg = arg or '' # TODO: use something more ref-like? @uniq = nil # Initialize to nil %w{on_complete on_fail on_retry on_exception on_status on_warning on_data - uniq retry_count priority hash background}.map {|s| s.to_sym }.each do |k| + uniq retry_count priority hash background schedule_at}.map {|s| s.to_sym }.each do |k| instance_variable_set "@#{k}", opts[k] opts.delete k end @@ -31,7 +31,6 @@ def initialize(func, arg='', opts={}) @retry_count ||= 0 @successful = false @retries_done = 0 - end attr_accessor :uniq, :retry_count, :priority, :background, :epoch @@ -42,7 +41,7 @@ def initialize(func, arg='', opts={}) # # @param time Ruby Time object that represents when to run the thing def schedule(time) - @scheduled_at = time + @schedule_at = time end ## @@ -119,11 +118,11 @@ def handle_completion(data) @on_complete.call(data) if @on_complete self end - + def on_created(&f) @on_created = f end - + def handle_created(data) @on_created.call(data) if @on_created self @@ -182,13 +181,13 @@ def handle_data(data) # def get_uniq_hash return @hash if @hash - - if @uniq.nil? + + if @uniq.nil? string = @func+@arg.to_s - else + else string = @uniq end - + @hash = Digest::SHA1.hexdigest(string) end @@ -198,10 +197,10 @@ def get_uniq_hash # @return String representation of packet def get_submit_packet() modes = ['submit_job'] - - if @scheduled_at + + if @schedule_at modes << 'epoch' - args = [func, get_uniq_hash, @scheduled_at.to_i, arg] + args = [func, get_uniq_hash, @schedule_at.to_i, arg] else if @priority modes << 'high' if @priority == :high @@ -212,12 +211,12 @@ def get_submit_packet() args = [func, get_uniq_hash, arg] end - + mode = modes.join('_') Util::pack_request(mode, args.join("\0")) end end - + class BackgroundTask < Task def initialize(*args) super From 0381caf35e1d73a799b761022403bad03acc7807 Mon Sep 17 00:00:00 2001 From: nickpeirson Date: Thu, 9 Oct 2014 11:11:40 +0100 Subject: [PATCH 02/38] Fix pattern captures in status The pattern was changed in commit 186873, but the captures weren't updated in the resulting output --- lib/gearman/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gearman/server.rb b/lib/gearman/server.rb index 669c6f4..7421b7c 100644 --- a/lib/gearman/server.rb +++ b/lib/gearman/server.rb @@ -62,7 +62,7 @@ def status if response = send_command('status') response.split("\n").each do |line| if line.match /^(.*)?\t(\d+)\t(\d+)\t(\d+)$/ - (status[$1] ||= {})[$2] = { :queue => $3, :active => $4, :workers => $5 } + status[$1] = { :queue => $2, :active => $3, :workers => $4 } end end end From dd6e1eadce50f45b13095205a9a3ff7f5932cfb1 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 9 Nov 2014 02:24:41 +0200 Subject: [PATCH 03/38] Update HOWTO Add ArchLinux tip --- HOWTO | 1 + 1 file changed, 1 insertion(+) diff --git a/HOWTO b/HOWTO index 87aa281..6d16392 100644 --- a/HOWTO +++ b/HOWTO @@ -50,6 +50,7 @@ For the JobServer we recommend to use the offical Perl version, to install it: * Mac OS X: sudo port install p5-gearman-server * Debian/Ubuntu: sudo apt-get install gearman-server + * ArchLinux: sudo pacman -S gearmand To get the Ruby libraries by Xing: From 940f4d6f548bfe1088524592bf764c39f1e79522 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 12:55:30 -0800 Subject: [PATCH 04/38] Updated 4.0.0 rewrite against 3.0.8 --- CHANGELOG | 3 - CHANGELOG.md | 7 + HOWTO | 147 ----- README | 9 - README.md | 109 ++++ TODO | 8 - examples/calculus_client.rb | 39 -- examples/calculus_worker.rb | 45 -- examples/client.php | 23 - examples/client.rb | 3 +- examples/client_background.rb | 14 - examples/client_data.rb | 16 - examples/client_epoch.rb | 23 - examples/client_exception.rb | 19 - examples/client_prefix.rb | 17 - examples/client_reverse_nohost.rb | 30 + ...ient_reverse.rb => client_reverse_wait.rb} | 15 +- examples/gearman_environment.sh | 25 - examples/scale_image.rb | 31 - examples/scale_image_worker.rb | 34 - examples/server.rb | 15 - examples/worker.rb | 13 +- examples/worker_ask_for_work.rb | 30 + examples/worker_data.rb | 16 - examples/worker_exception.rb | 14 - examples/worker_prefix.rb | 25 - examples/worker_reverse_string.rb | 27 - examples/worker_reverse_to_file.rb | 18 - examples/worker_signals.rb | 36 -- gearman-ruby.gemspec | 8 +- lib/gearman.rb | 94 +-- lib/gearman/client.rb | 276 ++++---- lib/gearman/connection.rb | 158 +++++ lib/gearman/connection_pool.rb | 128 ++++ lib/gearman/exceptions.rb | 24 + lib/gearman/logging.rb | 19 + lib/gearman/packet.rb | 61 ++ lib/gearman/task.rb | 2 +- lib/gearman/task_set.rb | 62 ++ lib/gearman/taskset.rb | 293 --------- lib/gearman/util.rb | 211 ------- lib/gearman/version.rb | 2 +- lib/gearman/worker.rb | 595 ++++++------------ lib/gearman/worker/ability.rb | 55 ++ lib/gearman/worker/callbacks.rb | 39 ++ lib/gearman/worker/job.rb | 44 ++ spec/client_spec.rb | 52 +- spec/connection_pool_spec.rb | 55 ++ spec/spec_helper.rb | 5 + spec/task_spec.rb | 10 + spec/taskset_spec.rb | 4 +- spec/util_spec.rb | 67 -- 52 files changed, 1218 insertions(+), 1857 deletions(-) delete mode 100644 CHANGELOG create mode 100644 CHANGELOG.md delete mode 100644 HOWTO delete mode 100644 README create mode 100644 README.md delete mode 100644 TODO delete mode 100644 examples/calculus_client.rb delete mode 100644 examples/calculus_worker.rb delete mode 100755 examples/client.php delete mode 100644 examples/client_background.rb delete mode 100644 examples/client_data.rb delete mode 100644 examples/client_epoch.rb delete mode 100644 examples/client_exception.rb delete mode 100644 examples/client_prefix.rb create mode 100644 examples/client_reverse_nohost.rb rename examples/{client_reverse.rb => client_reverse_wait.rb} (73%) delete mode 100644 examples/gearman_environment.sh delete mode 100755 examples/scale_image.rb delete mode 100755 examples/scale_image_worker.rb delete mode 100755 examples/server.rb create mode 100644 examples/worker_ask_for_work.rb delete mode 100644 examples/worker_data.rb delete mode 100644 examples/worker_exception.rb delete mode 100644 examples/worker_prefix.rb delete mode 100644 examples/worker_reverse_string.rb delete mode 100644 examples/worker_reverse_to_file.rb delete mode 100644 examples/worker_signals.rb create mode 100644 lib/gearman/connection.rb create mode 100644 lib/gearman/connection_pool.rb create mode 100644 lib/gearman/exceptions.rb create mode 100644 lib/gearman/logging.rb create mode 100644 lib/gearman/packet.rb create mode 100644 lib/gearman/task_set.rb delete mode 100755 lib/gearman/taskset.rb delete mode 100755 lib/gearman/util.rb create mode 100644 lib/gearman/worker/ability.rb create mode 100644 lib/gearman/worker/callbacks.rb create mode 100644 lib/gearman/worker/job.rb create mode 100644 spec/connection_pool_spec.rb delete mode 100644 spec/util_spec.rb diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index 17ffef6..0000000 --- a/CHANGELOG +++ /dev/null @@ -1,3 +0,0 @@ -3.0.6 - - * Connection failover for clients diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..95adc23 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# CHANGES + +## 4.0.0 (11/29/2014) +* Significant re-write of 3.0.x branch + +## 3.0.6 +* Connection failover for clients diff --git a/HOWTO b/HOWTO deleted file mode 100644 index 6d16392..0000000 --- a/HOWTO +++ /dev/null @@ -1,147 +0,0 @@ -= GEARMAN - -"Gearman provides a generic application framework to farm out work to other -machines or processes that are better suited to do the work. It allows you to -do work in parallel, to load balance processing, and to call functions between -languages. It can be used in a variety of applications, from high-availability -web sites to the transport of database replication events. In other words, it -is the nervous system for how distributed processing communicates." - - - http://www.gearman.org/ - - -== Setting up a basic environment - -A very basic Gearman environment will look like this: - - ---------- - | Client | - ---------- - | - -------------- - | Job Server | - -------------- - | - ---------------------------------------------- - | | | | ----------- ---------- ---------- ---------- -| Worker | | Worker | | Worker | | Worker | ----------- ---------- ---------- ---------- - -And the behavior will be the following: - - * JobServer: Acts as a message passing point. - * Client: Sends tasks to the JobServer. Will be connected to only one JobServer - in case more than one exits for failover purposes. - * Worker: Anounce his 'abilities' to the JobServer and waits for tasks. - -For the JobServer we recommend to use the offical Perl version, there's also a -more performant C implementation of the server with support for persistent -queues, bells and whistles but is not stable enough for production use at the -time of this document was wrote. - -The Client and the Worker can be implemented in any language. This way you can -send tasks from a Ruby client server, to a Perl or C worker in order to get -better performance. - -== Installing the required software - -For the JobServer we recommend to use the offical Perl version, to install it: - - * Mac OS X: sudo port install p5-gearman-server - * Debian/Ubuntu: sudo apt-get install gearman-server - * ArchLinux: sudo pacman -S gearmand - -To get the Ruby libraries by Xing: - - git clone git://github.com/xing/gearman-ruby.git - -== Gearman demo - -Now you're ready for you first experience with Gearman. In the cloned repository -you'll find an 'examples' directory. - -Run the 'gearman_environment.sh' to build an environment like the one showed in -the diagram above. - - * Client: Will ask you for an arithmetic operation, like: 2+3 - The code of the client is in: 'examples/calculus_client.rb' - - * JobServer: The Perl server. - - * Workers: You'll have 4 worker, one for each of the basic arithmetic - operations. - The code of the worker is in: 'examples/calculus_worker.rb' - -There are other demos in the examples folder you can give a look at. Each demo usually -consist in the client and server scripts. - -=== Creating clients and tasks - -In order to get a job scheduled by a Gearman server using the gearman ruby library, there -are three main objects you must interact with: Gearman::Client, Gearman::Task and Gearman::TaskSet. -Let's review all of them briefly: - - - Gearman::Client -> the portion of the library storing the data about the connection to - the Gearman server. - - Gearman::Task -> a job execution request that will be dispatched by the Gearman server to - worker and whose result data will be returned to the client. - - Gearman::TaskSet -> a collection of tasks to be executed. The Taskset object will track the - execution of the tasks with the info returned from the Gearman server - and notify the client with the results or errors when all the tasks - have completed their execution. - -To send a new task to the Gearman server, the client must build a new Gearman::Task object, add it to -a Gearman::TaskSet that must hold a reference to a Gearman::Client and send the wait message to -the TaskSet. -The following code taken from examples/client.rb shows the process: - ----------------------------------------------------- -servers = ['localhost:4730', 'localhost:4731'] - -client = Gearman::Client.new(servers) -taskset = Gearman::TaskSet.new(client) - -task = Gearman::Task.new('sleep', 20) -task.on_complete {|d| puts d } - -taskset.add_task(task) -taskset.wait(100) ----------------------------------------------------- - -The name of the function to be executed is the first parameter to the constructor of the Task object. -Take into account that the string you pass as a parameter will be used 'as it' by the Gearman server -to locate a suitable worker for that function. -The second parameter is the argument to be sent to the worker that will execute, if the arguments for -the task are complex, a serialization format like YAML or XML must be agreeded with the workers. -The last and optional parameter is a hash of options. The following options are currently available: - - - :priority -> (:high | :low) the priority of the job, a high priority job is executed before a low on - - :background -> (true | false) a background task will return no further information to the client. - -The execution of a task in a Gearman remote worker can fail, the worker can throw an exception, etc. -All these events can be handled by the client of the ruby library registering callback blocks. -The following events are currently available: - - - on_complete -> the task was executed succesfully - - on_fail -> the task fail for some unknown reason - - on_retry -> after failure, the task is gonna be retried, the number of retries is passed - - on_exception -> the remote worker send an exception notification, the exception text is passed - - on_stauts -> a status update is sent by the remote worker - -In order to receive exception notifications in the client, this option must be sent in the server, the -method option_request can be used this task. The following example, extracted from the examples/client_exception.rb -demo script shows the process: - ----------------------------------------------------- -client = Gearman::Client.new(servers) -#try this out -client.option_request("exceptions") ----------------------------------------------------- - -This feature will only works if the server and workers have implemented support for the OPT_REQ and WORK_EXCEPTION -messages of the Gearman protocol. - - -Enjoy. - diff --git a/README b/README deleted file mode 100644 index e47e816..0000000 --- a/README +++ /dev/null @@ -1,9 +0,0 @@ -gearman-ruby -=============== - -Library for the Gearman distributed job system - -COPYRIGHT -========= - -Copyright (c) 2009 XING AG. See LICENSE for details. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8d08248 --- /dev/null +++ b/README.md @@ -0,0 +1,109 @@ +# gearman-ruby + +## What is this? + +This is a pure-Ruby library for the [gearman][Gearman] distributed job system. + +## What needs to be done? + +More testing, some code cleanup. + +## What's in this? + +Right now, this library has both client and worker support for Ruby apps. + +## Getting Started + +### Client + +A very simple client that submits a "sleep" job and waits for 100 seconds for results: + +``` ruby +require 'rubygems' +require 'gearman' + +servers = ['localhost:4730', 'localhost:4731'] + +client = Gearman::Client.new(servers) +taskset = Gearman::TaskSet.new(client) + +task = Gearman::Task.new('sleep', 20) +task.on_complete {|d| puts d } + +taskset.add_task(task) +taskset.wait(100) +``` + +### Worker + +A worker that will process jobs in the 'sleep' queue: + +``` ruby +require 'rubygems' +require 'logger' +require 'gearman' + +servers = ['localhost:4730'] + +w = Gearman::Worker.new(servers) +logger = Logger.new(STDOUT) + +# Add a handler for a "sleep" function that takes a single argument, the +# number of seconds to sleep before reporting success. +w.add_ability("sleep") do |data,job| + seconds = 10 + logger.info "Sleeping for #{seconds} seconds" + (1..seconds.to_i).each do |i| + sleep 1 + # Report our progress to the job server every second. + job.report_status(i, seconds) + end + # Report success. + true +end + +loop { w.work } +``` + +[gearman]: http://gearman.org + +## Authors + +* John Ewart (current maintainer, author of re-write) + +<<<<<<< HEAD +## Contributors (past and present) + +* Kim Altintop +======= +## Contributors + +>>>>>>> New version (4.0) -- substantial rewrite +* Josh Black (raskchanky) +* Colin Curtin (perplexes) +* Brian Cobb (bcobb) +* Pablo A. Delgado (pablete) +<<<<<<< HEAD +* Daniel Erat +* Antonio Garrote +* Stefan Kaes (skaes) +* Ladislav Martincik +======= +* Stefan Kaes (skaes) +>>>>>>> New version (4.0) -- substantial rewrite +* Mauro Pompilio (malditogeek) +* Lee Reilly (leereilly) +* Clint Shryock (catsby) +* Andy Triggs (andyt) + + +<<<<<<< HEAD + +## License + +Released under the MIT license, originally developed by XING AG. See the LICENSE file for further details. +======= +## License + +Released under the MIT license. See the file LICENSE for further details. +>>>>>>> New version (4.0) -- substantial rewrite diff --git a/TODO b/TODO deleted file mode 100644 index fcdf58b..0000000 --- a/TODO +++ /dev/null @@ -1,8 +0,0 @@ -- Failover strategies - - Client: - * If connected for the first time, try to connect to at least one server from the server array - * If already connected to a server, and it goes down, the client should go down as well - - Worker: - * If connected for the first time, try to connect to as many servers as it can. - Loop trough the bad servers, trying to reconnect to them as well. - * If a already connected to a server, and it goes down, wait and try to reconnect again. \ No newline at end of file diff --git a/examples/calculus_client.rb b/examples/calculus_client.rb deleted file mode 100644 index f9b389f..0000000 --- a/examples/calculus_client.rb +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env ruby -require 'rubygems' -require '../lib/gearman' -#Gearman::Util.debug = true - -# Connect to the local server (at the default port 7003) -client = Gearman::Client.new('localhost') -taskset = Gearman::TaskSet.new(client) - -# Get something to echo -puts '[client] Write a basic arithmetic operation:' -input = gets - -operations = input.chomp.scan(/\d+[\+\-\*\/]\d+/).compact -puts "[client] The following operations were found: #{operations.inspect}" - -# Setup a task for operation -operations.each do |op| - # Determining the operation - case op - when /\+/ - type, data = 'addition', op.split('+') - when /\-/ - type, data = 'subtraction', op.split('-') - when /\*/ - type, data = 'multiplication', op.split('*') - when /\// - type, data = 'division', op.split('/') - end - - task = Gearman::Task.new(type, Marshal.dump(data.map {|v| v.to_i})) - task.on_complete {|r| puts "[client] #{type} result is: #{r}" } - - # Sending the task to the server - puts "[client] Sending values: #{data.inspect}, to the '#{type}' worker" - taskset.add_task(task) - taskset.wait(100) -end - diff --git a/examples/calculus_worker.rb b/examples/calculus_worker.rb deleted file mode 100644 index 034fa82..0000000 --- a/examples/calculus_worker.rb +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env ruby -require 'rubygems' -require '../lib/gearman' - -#Gearman::Util.debug = true - -worker = Gearman::Worker.new('localhost') -worker.reconnect_sec = 2 - -# Additon ability -worker.add_ability('addition') do |data,job| - values = Marshal.load(data) - puts "[addition_worker] Calculating #{values.inspect}..." - # sleep 5 - values.first + values.last -end - -# Subtraction ability -worker.add_ability('subtraction') do |data,job| - values = Marshal.load(data) - puts "[subtraction_worker] Calculating #{values.inspect}..." - # sleep 5 - values.first - values.last -end - -# Multiplication worker -worker.add_ability('multiplication') do |data,job| - values = Marshal.load(data) - puts "[multiplication_worker] Calculating #{values.inspect}..." - # sleep 5 - values.first * values.last -end - -# Division worker -worker.add_ability('division') do |data,job| - values = Marshal.load(data) - puts "[division_worker] Calculating #{data.inspect}..." - # sleep 5 - values.first / values.last -end - -# Running the workers -loop do - worker.work -end diff --git a/examples/client.php b/examples/client.php deleted file mode 100755 index f69cd84..0000000 --- a/examples/client.php +++ /dev/null @@ -1,23 +0,0 @@ - 20 -)); - -$task->attachCallback('result'); -$set->addTask($task); - -$client = new Net_Gearman_Client(array('localhost:4730', 'localhost:4731')); -$client->runSet($set); - -?> diff --git a/examples/client.rb b/examples/client.rb index e314c03..e561d3b 100755 --- a/examples/client.rb +++ b/examples/client.rb @@ -1,7 +1,6 @@ +$LOAD_PATH.unshift("../lib") require 'rubygems' -#require 'gearman' require '../lib/gearman' -# Gearman::Util.debug = true servers = ['localhost:4730', 'localhost:4731'] diff --git a/examples/client_background.rb b/examples/client_background.rb deleted file mode 100644 index 6f04bd8..0000000 --- a/examples/client_background.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'rubygems' -#require 'gearman' -require '../lib/gearman' - -servers = ['localhost:4730',] - -client = Gearman::Client.new(servers) -taskset = Gearman::TaskSet.new(client) - -task = Gearman::Task.new('sleep', 20, { :background => true }) -task.on_complete {|d| puts d } - -taskset.add_task(task) -taskset.wait(100) diff --git a/examples/client_data.rb b/examples/client_data.rb deleted file mode 100644 index 391fe7a..0000000 --- a/examples/client_data.rb +++ /dev/null @@ -1,16 +0,0 @@ -require 'rubygems' -#require 'gearman' -require '../lib/gearman' -Gearman::Util.debug = true - -servers = ['localhost:4730'] - -client = Gearman::Client.new(servers) -taskset = Gearman::TaskSet.new(client) - -task = Gearman::Task.new('chunked_transfer') -task.on_data {|d| puts d } -task.on_complete {|d| puts d } - -taskset.add_task(task) -taskset.wait(100) diff --git a/examples/client_epoch.rb b/examples/client_epoch.rb deleted file mode 100644 index 54dc286..0000000 --- a/examples/client_epoch.rb +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env ruby - -# Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/) - -require 'rubygems' -require '../lib/gearman' - -(1..100).each do - # Connect to the local server (at the default port 4730) - client = Gearman::Client.new('localhost') - taskset = Gearman::TaskSet.new(client) - - data = rand(36**8).to_s(36) - # Set scheduled time to some time in the future - time = Time.now() + rand(10) - puts "Time as seconds: #{time.to_i}" - task = Gearman::Task.new("reverse_string", data) - task.schedule(time) - - # Sending the task to the server - puts "[client] Sending task: #{task.inspect}, to the 'reverse_string' worker" - taskset.add_task(task) -end diff --git a/examples/client_exception.rb b/examples/client_exception.rb deleted file mode 100644 index b661787..0000000 --- a/examples/client_exception.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'rubygems' -require '../lib/gearman' -Gearman::Util.debug = true - -servers = ['localhost:4730'] - -client = Gearman::Client.new(servers) -taskset = Gearman::TaskSet.new(client) - -task = Gearman::Task.new('fail_with_exception', "void") -task.retry_count = 2 -task.on_complete {|d| puts d } -task.on_exception {|ex| puts "This should never be called" } -task.on_warning {|warning| puts "WARNING: #{warning}" } -task.on_retry { puts "PRE-RETRY HOOK: retry no. #{task.retries_done}" } -task.on_fail { puts "TASK FAILED, GIVING UP" } - -taskset.add_task(task) -taskset.wait(100) diff --git a/examples/client_prefix.rb b/examples/client_prefix.rb deleted file mode 100644 index bbd9ac3..0000000 --- a/examples/client_prefix.rb +++ /dev/null @@ -1,17 +0,0 @@ -require 'rubygems' -#require 'gearman' -require '../lib/gearman' -Gearman::Util.debug = true - -servers = ['localhost:4730', 'localhost:4731'] - -ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep") - -client = Gearman::Client.new(servers) -taskset = Gearman::TaskSet.new(client) - -task = Gearman::Task.new(ability_name_with_prefix, 20) -task.on_complete {|d| puts d } - -taskset.add_task(task) -taskset.wait(100) diff --git a/examples/client_reverse_nohost.rb b/examples/client_reverse_nohost.rb new file mode 100644 index 0000000..2ca1052 --- /dev/null +++ b/examples/client_reverse_nohost.rb @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby +require 'rubygems' +require '../lib/gearman' +l = Logger.new($stdout) +l.level = Logger::DEBUG +Gearman::Util.logger=l + +# Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/) + +t = nil +threadcounter = 0 + +client = Gearman::Client.new('192.168.1.1:4730') + + +myid = threadcounter +threadcounter += 1 +taskset = Gearman::TaskSet.new(client) + +(1..1000).each do |jid| + data = rand(36**8).to_s(36) + result = data.reverse + + task = Gearman::BackgroundTask.new("reverse_string", data) + puts "#{jid} #{data}" + + #time = Time.now() + rand(120) + 10 + #task.schedule(time) + taskset.add_task(task) +end diff --git a/examples/client_reverse.rb b/examples/client_reverse_wait.rb similarity index 73% rename from examples/client_reverse.rb rename to examples/client_reverse_wait.rb index aa98283..ece2042 100644 --- a/examples/client_reverse.rb +++ b/examples/client_reverse_wait.rb @@ -7,21 +7,20 @@ t = nil threadcounter = 0 -client = Gearman::Client.new('localhost') +client = Gearman::Client.new('localhost:4740') myid = threadcounter threadcounter += 1 taskset = Gearman::TaskSet.new(client) -(1..10000).each do |jid| +(1..100).each do |jid| data = rand(36**8).to_s(36) - result = data.reverse - - task = Gearman::Task.new("reverse_string", data) puts "#{jid} #{data}" - - time = Time.now() + rand(120) + 10 - task.schedule(time) + task = Gearman::Task.new("reverse_string", data) + task.on_complete {|d| puts d } taskset.add_task(task) + taskset.wait(1000) end + + diff --git a/examples/gearman_environment.sh b/examples/gearman_environment.sh deleted file mode 100644 index ce5523f..0000000 --- a/examples/gearman_environment.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Start Gearmand -echo ' + Starting Gearmand' -gearmand --daemon --pidfile=/tmp/gearmand.pid - -# Start the client and the worker(s) -echo ' + Starting calculus_worker.rb' -ruby calculus_worker.rb & - -sleep 3 - -echo ' + Starting calculus_client.rb' -ruby calculus_client.rb - -echo ' +++ Example finished +++ ' - -# Stop Gearmand -echo ' - Stopping Gearmand' -kill -9 `cat /tmp/gearmand.pid` - -# Stop the workers -echo ' - Stopping calculus_worker.rb' -kill -9 `ps ax|grep calculus_worker|grep ruby|awk -F' ' '{print $1}'` - diff --git a/examples/scale_image.rb b/examples/scale_image.rb deleted file mode 100755 index 40060f1..0000000 --- a/examples/scale_image.rb +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/ruby - -$: << '../lib' -require 'gearman' -require 'optparse' - -servers = 'localhost:4730' -format = 'PNG' -width, height = 100, 100 - -opts = OptionParser.new -opts.banner = "Usage: #{$0} [options] " -opts.on('-f FORMAT', '--format', 'Scaled image format') { format } -opts.on('-h HEIGHT', '--height', 'Scaled image height') { height } -opts.on('-s SERVERS', '--servers', - 'Servers, comma-separated host:port') { servers } -opts.on('-w WIDTH', '--width', 'Scaled image width') { width } -opts.parse! - -if ARGV.size != 2 - $stderr.puts opts.banner - exit 1 -end - -client = Gearman::Client.new(servers.split(','), 'example') -taskset = Gearman::TaskSet.new(client) -arg = [width, height, format, File.read(ARGV[0])].join("\0") -task = Gearman::Task.new('scale_image', arg) -task.on_complete {|d| File.new(ARGV[1],'w').write(d) } -taskset.add_task(task) -taskset.wait(10) diff --git a/examples/scale_image_worker.rb b/examples/scale_image_worker.rb deleted file mode 100755 index 39365ac..0000000 --- a/examples/scale_image_worker.rb +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/ruby - -$: << '../lib' -require 'gearman' -require 'optparse' -require 'RMagick' - -Gearman::Util.debug = true -servers = 'localhost:7003' - -opts = OptionParser.new -opts.banner = "Usage: #{$0} [options]" -opts.on('-s SERVERS', '--servers', - 'Job servers, comma-separated host:port') { servers } -opts.parse! - -worker = Gearman::Worker.new(servers.split(','), 'example') - -worker.add_ability('scale_image') do |data,job| - width, height, format, data = data.split("\0", 4) - width = width.to_f - height = height.to_f - image = Magick::Image.from_blob(data)[0] - orig_ratio = image.columns.to_f / image.rows - new_ratio = width / height - w = new_ratio < orig_ratio ? width : orig_ratio / new_ratio * width - h = new_ratio > orig_ratio ? height : new_ratio / orig_ratio * height - puts "Got #{image.inspect}; resizing to #{w}x#{h} #{format}" - image.resize!(w, h) - image.format = format - image.to_blob -end - -loop { worker.work } diff --git a/examples/server.rb b/examples/server.rb deleted file mode 100755 index d53fdb9..0000000 --- a/examples/server.rb +++ /dev/null @@ -1,15 +0,0 @@ -require 'rubygems' -# require 'gearman' -# require 'gearman/server' -require '../lib/gearman' -require '../lib/gearman/server' -require 'pp' - -Gearman::Util.debug = true -w = Gearman::Server.new('localhost:4730') - -loop { - pp "Status: ", w.status - pp "Workers: ", w.workers - sleep 5 -} \ No newline at end of file diff --git a/examples/worker.rb b/examples/worker.rb index f350be1..1c4c94e 100755 --- a/examples/worker.rb +++ b/examples/worker.rb @@ -1,21 +1,24 @@ +$LOAD_PATH.unshift("../lib") require 'rubygems' -#require 'gearman' +require 'logger' require '../lib/gearman' +servers = ['localhost:4730'] -servers = ['localhost:4730',] w = Gearman::Worker.new(servers) +logger = Logger.new(STDOUT) # Add a handler for a "sleep" function that takes a single argument, the # number of seconds to sleep before reporting success. -w.add_ability('sleep') do |data,job| - seconds = data +w.add_ability("sleep") do |data,job| + seconds = 10 + logger.info "Sleeping for #{seconds} seconds" (1..seconds.to_i).each do |i| sleep 1 - print i # Report our progress to the job server every second. job.report_status(i, seconds) end # Report success. true end + loop { w.work } diff --git a/examples/worker_ask_for_work.rb b/examples/worker_ask_for_work.rb new file mode 100644 index 0000000..ef0637e --- /dev/null +++ b/examples/worker_ask_for_work.rb @@ -0,0 +1,30 @@ +require 'rubygems' +require '../lib/gearman' +l = Logger.new($stdout) +l.level = Logger::DEBUG +Gearman::Util.logger=l + +# String reverse worker + +servers = ['127.0.0.1:4730'] + +client = Gearman::Client.new(servers) +taskset = Gearman::TaskSet.new(client) +t = nil +jobnum = 0 + +w = Gearman::Worker.new(servers) +w.add_ability('reverse_string') do |data,job| + result = data.reverse + puts "Job: #{job.inspect} Data: #{data.inspect} Reverse: #{result} " + puts "Completed job ##{jobnum}" + data = rand(36**8).to_s(36) + task = Gearman::BackgroundTask.new("background_job", data) + taskset.add_task(task) + jobnum += 1 + result +end + +loop { w.work } + + diff --git a/examples/worker_data.rb b/examples/worker_data.rb deleted file mode 100644 index f3db5c5..0000000 --- a/examples/worker_data.rb +++ /dev/null @@ -1,16 +0,0 @@ -require 'rubygems' -require '../lib/gearman' - -Gearman::Util.debug = true - -servers = ['localhost:4730'] -worker = Gearman::Worker.new(servers) - -worker.add_ability('chunked_transfer') do |data, job| - 5.times do |i| - sleep 1 - job.send_data("CHUNK #{i}") - end - "EOD" -end -loop { worker.work } diff --git a/examples/worker_exception.rb b/examples/worker_exception.rb deleted file mode 100644 index 69c690b..0000000 --- a/examples/worker_exception.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'rubygems' -require '../lib/gearman' - -Gearman::Util.debug = true - -servers = ['localhost:4730'] -w = Gearman::Worker.new(servers) - -# Add a handler for a "sleep" function that takes a single argument, the -# number of seconds to sleep before reporting success. -w.add_ability('fail_with_exception') do |data,job| - raise Exception.new("Exception in worker (args: #{data.inspect})") -end -loop { w.work } diff --git a/examples/worker_prefix.rb b/examples/worker_prefix.rb deleted file mode 100644 index bd73ccf..0000000 --- a/examples/worker_prefix.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'rubygems' -#require 'gearman' -require '../lib/gearman' - -Gearman::Util.debug = true - -servers = ['localhost:4730', 'localhost:4731'] -w = Gearman::Worker.new(servers) - -ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep") - -# Add a handler for a "sleep" function that takes a single argument, the -# number of seconds to sleep before reporting success. -w.add_ability(ability_name_with_prefix) do |data,job| - seconds = data - (1..seconds.to_i).each do |i| - sleep 1 - print i - # Report our progress to the job server every second. - job.report_status(i, seconds) - end - # Report success. - true -end -loop { w.work } diff --git a/examples/worker_reverse_string.rb b/examples/worker_reverse_string.rb deleted file mode 100644 index e5a4f2c..0000000 --- a/examples/worker_reverse_string.rb +++ /dev/null @@ -1,27 +0,0 @@ -require 'rubygems' -require '../lib/gearman' - -# String reverse worker - -servers = ['localhost:4730'] - -t = nil -jobnum = 0 - -(0..1).each do - t = Thread.new { - w = Gearman::Worker.new(servers) - w.add_ability('reverse_string') do |data,job| - result = data.reverse - puts "Job: #{job.inspect} Data: #{data.inspect} Reverse: #{result} " - puts "Completed job ##{jobnum}" - jobnum += 1 - result - end - loop { w.work } - } -end - -puts "Waiting for threads..." -t.join - diff --git a/examples/worker_reverse_to_file.rb b/examples/worker_reverse_to_file.rb deleted file mode 100644 index 1aec387..0000000 --- a/examples/worker_reverse_to_file.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'rubygems' -#require 'gearman' -require '../lib/gearman' - -servers = ['localhost:4730'] -w = Gearman::Worker.new(servers) - -# Add a handler for a "sleep" function that takes a single argument, the -# number of seconds to sleep before reporting success. -w.add_ability('reverse_to_file') do |data,job| - puts "Data: #{data.inspect}" - word, file = data.split("\0") - puts "Word: #{word}" - puts "File: #{file}" - # Report success. - true -end -loop { w.work } diff --git a/examples/worker_signals.rb b/examples/worker_signals.rb deleted file mode 100644 index 3ef7460..0000000 --- a/examples/worker_signals.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'rubygems' -#require 'gearman' -require '../lib/gearman' - -Gearman::Util.debug = true - -servers = ['localhost:4730', 'localhost:4731'] -w = Gearman::Worker.new(servers) - -# Add a handler for a "sleep" function that takes a single argument, the -# number of seconds to sleep before reporting success. -w.add_ability('sleep') do |data,job| - seconds = data - (1..seconds.to_i).each do |i| - sleep 1 - Gearman::Util.logger.info i - # Report our progress to the job server every second. - job.report_status(i, seconds) - end - # Report success. - true -end - -# Trap signals while is working -%w(HUP USR1 ALRM TERM).each do |signal| - trap(signal) do - puts "Received signal #{signal} - setting worker_enabled to false. Worker status is [#{w.status}]" - w.worker_enabled = false - if w.status == :waiting - trap(signal, "DEFAULT") - Process.kill( signal, $$ ) - end - end -end - -loop { w.work or break } diff --git a/gearman-ruby.gemspec b/gearman-ruby.gemspec index 3b0901a..8954a68 100644 --- a/gearman-ruby.gemspec +++ b/gearman-ruby.gemspec @@ -5,8 +5,8 @@ Gem::Specification.new do |s| s.name = %q{gearman-ruby} s.version = Gearman::VERSION s.platform = Gem::Platform::RUBY - s.authors = ["John Ewart", "Colin Curtin", "Daniel Erat", "Ladislav Martincik", "Pablo Delgado", "Mauro Pompilio", "Antonio Garrote", "Kim Altintop"] - s.date = %q{2013-07-25} + s.authors = ["John Ewart"] + s.date = %q{2014-11-29} s.summary = %q{Ruby Gearman library} s.description = %q{Library for the Gearman distributed job system} s.email = %q{john@johnewart.net} @@ -15,14 +15,12 @@ Gem::Specification.new do |s| s.extra_rdoc_files = [ "LICENSE", - "README", - "TODO" + "README.md" ] s.files = `git ls-files`.split("\n") s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } s.require_paths = ["lib"] - s.require_paths = ["lib"] end diff --git a/lib/gearman.rb b/lib/gearman.rb index 62c56b3..c7fa373 100644 --- a/lib/gearman.rb +++ b/lib/gearman.rb @@ -1,82 +1,22 @@ #!/usr/bin/env ruby -# -# = Name -# Gearman -# -# == Description -# This file provides a Ruby interface for communicating with the Gearman -# distributed job system. -# -# "Gearman is a system to farm out work to other machines, dispatching -# function calls to machines that are better suited to do work, to do work -# in parallel, to load balance lots of function calls, or to call functions -# between languages." -- http://www.danga.com/gearman/ -# -# == Version -# 0.0.1 -# -# == Author -# Daniel Erat -# -# == License -# This program is free software; you can redistribute it and/or modify it -# under the terms of either: -# -# a) the GNU General Public License as published by the Free Software -# Foundation; either version 1, or (at your option) any later version, -# or -# -# b) the "Artistic License" which comes with Perl. -# = Gearman -# -# == Usage -# require 'gearman' -# -# # Create a new client and tell it about two job servers. -# c = Gearman::Client.new -# c.job_servers = ['127.0.0.1:7003', '127.0.0.1:7004'] -# -# # Create two tasks, using an "add" function to sum two numbers. -# t1 = Gearman::Task.new('add', '5 + 2') -# t2 = Gearman::Task.new('add', '1 + 3') -# -# # Make the tasks print the data they get back from the server. -# t1.on_complete {|d| puts "t1 got #{d}" } -# t2.on_complete {|d| puts "t2 got #{d}" } -# -# # Create a taskset, add the two tasks to it, and wait until they finish. -# ts = Gearman::TaskSet.new(c) -# ts.add_task(t1) -# ts.add_task(t2) -# ts.wait -# -# Or, a more simple example: -# -# c = Gearman::Client.new('127.0.0.1') -# puts c.do_task('add', '2 + 2') -# -module Gearman - -require File.dirname(__FILE__) + '/gearman/client' -require File.dirname(__FILE__) + '/gearman/task' -require File.dirname(__FILE__) + '/gearman/taskset' -require File.dirname(__FILE__) + '/gearman/util' -require File.dirname(__FILE__) + '/gearman/worker' - -class InvalidArgsError < Exception -end - -class ProtocolError < Exception -end +require 'logger' -class NetworkError < Exception -end - -class NoJobServersError < Exception -end - -class JobQueueError < Exception +module Gearman + class << self + attr_writer :logger + def logger + @logger ||= Logger.new(STDOUT) + end + end end -end +require 'gearman/exceptions' +require 'gearman/logging' +require 'gearman/packet' +require 'gearman/connection' +require 'gearman/connection_pool' +require 'gearman/client' +require 'gearman/task' +require 'gearman/task_set' +require 'gearman/worker' \ No newline at end of file diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index c375666..26745e7 100755 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -1,169 +1,151 @@ -require 'socket' +require 'time' module Gearman + class Client + include Logging + + attr_accessor :task_create_timeout_sec + + ## + # Create a new client. + # + # @param job_servers "host:port"; either a single server or an array + def initialize(job_servers=nil) + @coalesce_connections = {} # Unique ID => Connection + @connection_pool = ConnectionPool.new(job_servers) + @current_task = nil + @task_create_timeout_sec = 10 + end -# = Client -# -# == Description -# A client for communicating with Gearman job servers. -class Client - ## - # Create a new client. - # - # @param job_servers "host:port"; either a single server or an array - def initialize(job_servers=nil) - @job_servers = [] # "host:port" - self.job_servers = job_servers if job_servers - @sockets = {} # "host:port" -> [sock1, sock2, ...] - @socket_to_hostport = {} # sock -> "host:port" - @task_create_timeout_sec = 10 - @server_counter = -1 - @bad_servers = [] - end - attr_reader :job_servers, :bad_servers - attr_accessor :task_create_timeout_sec - - ## - # Set the options - # - # @options options to pass to the servers "exeptions" - def option_request(opts) - Util.logger.debug "GearmanRuby: Send options request with #{opts}" - request = Util.pack_request("option_req", opts) - sock= self.get_socket(self.get_job_server) - Util.send_request(sock, request) - response = Util.read_response(sock, 20) - raise ProtocolError, response[1] if response[0]==:error - end - - ## - # Set the job servers to be used by this client. - # - # @param servers "host:port"; either a single server or an array - def job_servers=(servers) - @job_servers = Util.normalize_job_servers(servers) - self - end - - ## - # Get connection info about an arbitrary (currently random, but maybe - # we'll do something smarter later) job server. - # - # @return "host:port" - def get_job_server - if @job_servers.empty? && !@bad_servers.empty? - Util.logger.debug "GearmanRuby: No more good job servers, trying bad ones: #{@bad_servers.inspect}." - # Try to reconnect to the bad servers - @bad_servers.each do |bad_server| - Util.logger.debug "GearmanRuby: Trying server: #{bad_server.inspect}" - begin - request = Util.pack_request("echo_req", "ping") - sock = self.get_socket(bad_server) - Util.send_request(sock, request) - response = Util.read_response(sock, 20) - if response[0] == :echo_res - @job_servers << bad_server - @bad_servers.delete bad_server - end - rescue NetworkError - Util.logger.debug "GearmanRuby: Error trying server: #{bad_server.inspect}" - end + ## + # Set the options + # + # @options options to pass to the servers i.e "exceptions" + def set_options(opts) + @connection_pool.with_all_connections do |conn| + logger.debug "Send options request with #{opts}" + request = Packet.pack_request("option_req", opts) + response = conn.send_request(request) + raise ProtocolError, response[1] if response[0]==:error end end - Util.logger.debug "GearmanRuby: job servers: #{@job_servers.inspect}" - raise NoJobServersError if @job_servers.empty? - @server_counter += 1 - @job_servers[@server_counter % @job_servers.size] - end + ## + # Perform a single task. + # + # @param args A Task to complete + # @return output of the task, or nil on failure + def do_task(task) - def signal_bad_server(hostport) - @job_servers = @job_servers.reject { |s| s == hostport } - @bad_servers << hostport - end + result = nil + failed = false - ## - # Get a socket for a job server. - # - # @param hostport job server "host:port" - # @return a Socket - def get_socket(hostport, num_retries=3) - # If we already have an open socket to this host, return it. - if @sockets[hostport] - sock = @sockets[hostport].shift - @sockets.delete(hostport) if @sockets[hostport].size == 0 - return sock - end + task.on_complete {|v| result = v } + task.on_fail { failed = true } - num_retries.times do |i| - begin - sock = TCPSocket.new(*hostport.split(':')) - rescue Exception - # Swallow error so we can retry -> num_retries times + task_set = TaskSet.new(self) + if task_set.add_task(task) + task_set.wait else - # No error, stash socket mapping and return it - @socket_to_hostport[sock] = hostport - return sock + raise JobQueueError, "Unable to enqueue job." end - end - raise NetworkError, "Unable to connect to job server #{hostport}" - end - ## - # Relinquish a socket created by Client#get_socket. - # - # If we don't know about the socket, we just close it. - # - # @param sock Socket - def return_socket(sock) - hostport = get_hostport_for_socket(sock) - if not hostport - inet, port, host, ip = sock.addr - Util.logger.error "GearmanRuby: Got socket for #{ip}:#{port}, which we don't know about -- closing" - sock.close - return + failed ? nil : result end - (@sockets[hostport] ||= []) << sock - end - def close_socket(sock) - sock.close - @socket_to_hostport.delete(sock) - nil - end + def submit_job(task, reset_state = false, timeout = nil) + task.reset_state if reset_state + req = task.get_submit_packet() + req_timeout = timeout || task_create_timeout_sec + # Target the same job manager when submitting jobs + # with the same unique id so that we can coalesce + coalesce_key = task.get_uniq_hash - ## - # Given a socket from Client#get_socket, return its host and port. - # - # @param sock Socket - # @return "host:port", or nil if unregistered (which shouldn't happen) - def get_hostport_for_socket(sock) - @socket_to_hostport[sock] - end + end_time = if timeout + Time.now.to_f + timeout + else + nil + end - ## - # Perform a single task. - # - # @param args either a Task or arguments for Task.new - # @return output of the task, or nil on failure - def do_task(*args) - task = Util::get_task_from_args(*args) - - result = nil - failed = false - task.on_complete {|v| result = v } - task.on_fail { failed = true } - - taskset = TaskSet.new(self) - if taskset.add_task(task) - taskset.wait - else - raise JobQueueError, "Unable to enqueue job." + begin + + connection = @connection_pool.get_connection(coalesce_key) + logger.debug "Using #{connection} to submit job" + + type, data = connection.send_request(req, timeout) + logger.debug "Got #{type.to_s} from #{connection}" + + if type == :job_created + + task.handle_created(data) + + if(!task.background) + begin + remaining = if end_time + (t = end_time - Time.now.to_f) > 0 ? t : 0 + else + nil + end + type, data = connection.read_response(remaining) + handle_response(task, type, data) + end while [:work_status, :work_data].include? type + end + + else + # This shouldn't happen + message = "Received #{type.to_s} when we were expecting JOB_CREATED" + logger.error message + raise ProtocolError, message + end + rescue NetworkError + message = "Network error on read from #{hostport} while adding job, marking server bad" + logger.error message + raise NetworkError, message + rescue NoJobServersError + logger.error "No servers available." + raise NoJobServersError + end + + true + end + + def handle_response(task, type, data) + case type + when :work_complete + handle, message = data.split("\0", 2) + logger.debug("Received WORK_COMPLETE for #{handle}") + task.handle_completion(message) + when :work_exception + handle, exception = data.split("\0", 2) + logger.debug("Received WORK_EXCEPTION for #{handle}") + task.handle_exception(exception) + when :work_fail + logger.debug("Received WORK_FAIL for #{handle}") + requeue = task.handle_failure + add_task(task) if requeue + when :work_status + handle, numerator, denominator = data.split("\0", 3) + logger.debug("Received WORK_STATUS for #{handle}: #{numerator} / #{denominator}") + task.handle_status(numerator, denominator) + when :work_warning + handle, message = data.split("\0", 2) + Util.logger.debug "Got WORK_WARNING for #{handle}: '#{message}'" + task.handle_warning(message) + when :work_data + handle, work_data = data.split("\0", 2) + Util.logger.debug "Got WORK_DATA for #{handle} with #{work_data ? work_data.size : '0'} byte(s) of data" + task.handle_data(work_data) + else + # Not good. + message = "Got #{type.to_s} from #{connection} but it was not an expected type." + logger.error message + raise ProtocolError, message + end end - failed ? nil : result end -end + + end \ No newline at end of file diff --git a/lib/gearman/connection.rb b/lib/gearman/connection.rb new file mode 100644 index 0000000..656cbe7 --- /dev/null +++ b/lib/gearman/connection.rb @@ -0,0 +1,158 @@ +require 'socket' + +module Gearman + class Connection + include Logging + + def initialize(hostname, port) + @hostname = hostname + @port = port + @real_socket = nil + end + + attr_reader :hostname, :port, :state, :socket + + ## + # Check server health status by sending an ECHO request + # Return true / false + ## + def is_healthy? + if @real_socket == nil + logger.debug "Performing health check for #{self}" + begin + request = Packet.pack_request("echo_req", "ping") + response = send_request(request, 3) + logger.debug "Health check response for #{self} is #{response.inspect}" + raise ProtocolError unless response[0] == :echo_res and response[1] == "ping" + return true + rescue NetworkError + logger.debug "NetworkError -- unhealthy" + return false + rescue ProtocolError + logger.debug "ProtocolError -- unhealthy" + return false + end + end + end + + ## + # @param num_retries Number of times to retry + # @return This connection's Socket + ## + def socket(num_retries=3) + # If we already have an open socket to this host, return it. + return @real_socket if @real_socket + num_retries.times do |i| + begin + logger.debug("Attempt ##{i} to connect to #{hostname}:#{port}") + @real_socket = TCPSocket.new(hostname, port) + rescue Exception => e + logger.error("Unable to connect: #{e}") + # Swallow error so we can retry -> num_retries times + else + return @real_socket + end + end + + raise_exception("Unable to connect to job server #{hostname}:#{port}") + end + + def close_socket + @real_socket.close if @real_socket + @real_socket = nil + true + end + + def raise_exception(message) + close_socket + raise NetworkError, message + end + + ## + # Read from a socket, giving up if it doesn't finish quickly enough. + # NetworkError is thrown if we don't read all the bytes in time. + # + # @param sock Socket from which we read + # @param len number of bytes to read + # @param timeout maximum number of seconds we'll take; nil for no timeout + # @return full data that was read + def timed_recv(sock, len, timeout=nil) + data = '' + start_time = Time.now.to_f + end_time = Time.now.to_f + timeout if timeout + while data.size < len and (not timeout or Time.now.to_f < end_time) do + IO::select([sock], nil, nil, timeout ? end_time - Time.now.to_f : nil) \ + or break + begin + data += sock.readpartial(len - data.size) + rescue + close_socket + raise NetworkError, "Unable to read data from socket." + end + end + if data.size < len + now = Time.now.to_f + if now > end_time + time_lapse = now - start_time + raise SocketTimeoutError, "Took too long to read data: #{time_lapse} sec. to read on a #{timeout} sec. timeout" + else + raise_exception("Read #{data.size} byte(s) instead of #{len}") + end + end + data + end + + ## + # Read a response packet from a socket. + # + # @param sock Socket connected to a job server + # @param timeout timeout in seconds, nil for no timeout + # @return array consisting of integer packet type and data + def read_response(timeout=nil) + end_time = Time.now.to_f + timeout if timeout + head = timed_recv(socket, 12, timeout) + magic, type, len = head.unpack('a4NN') + raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES" + buf = len > 0 ? + timed_recv(socket, len, timeout ? end_time - Time.now.to_f : nil) : '' + type = Packet::COMMANDS[type] + raise ProtocolError, "Invalid packet type #{type}" unless type + [type, buf] + end + + ## + # Send a request packet over a socket that needs a response. + # + # @param sock Socket connected to a job server + # @param req request packet to send + # @result response from server + def send_request(req, timeout = nil) + send_update(req, timeout) + return read_response(timeout) + end + + def send_update(req, timeout = nil) + len = with_safe_socket_op{ socket.write(req) } + if len != req.size + raise_exception("Wrote #{len} instead of #{req.size}") + end + end + + def with_safe_socket_op + begin + yield + rescue Exception => ex + raise_exception(ex.message) + end + end + + def to_host_port + "#{hostname}:#{port}" + end + + def to_s + "#{hostname}:#{port} (connected: #{@real_socket != nil})" + end + + end +end \ No newline at end of file diff --git a/lib/gearman/connection_pool.rb b/lib/gearman/connection_pool.rb new file mode 100644 index 0000000..9abc48d --- /dev/null +++ b/lib/gearman/connection_pool.rb @@ -0,0 +1,128 @@ +require 'thread' + +module Gearman + class ConnectionPool + include Logging + + DEFAULT_PORT = 4730 + + def initialize(servers = []) + @bad_servers = [] + @coalesce_connections = {} + @connection_handler = nil + @job_servers = [] + @reconnect_seconds = 10 + @server_counter = 0 # Round-robin distribution of requests + @servers_mutex = Mutex.new + + add_servers(servers) + start_reconnect_thread + end + + def add_connection(connection) + @servers_mutex.synchronize do + if connection.is_healthy? + activate_connection(connection) + + @connection_handler.call(connection) if @connection_handler + else + deactivate_connection(connection) + end + end + end + + def add_host_port(host_port) + host, port = host_port.split(":") + connection = Connection.new(host, port.to_i) + add_connection(connection) + end + + def add_servers(servers) + if servers.class == String or servers.class == Symbol + servers = [ servers.to_s ] + end + + servers = servers.map {|s| s =~ /:/ ? s : "#{s}:#{DEFAULT_PORT}" } + + servers.each do |host_port| + add_host_port(host_port) + end + end + + def get_connection(coalesce_key = nil) + @servers_mutex.synchronize do + + logger.debug "Available job servers: #{@job_servers.inspect}" + raise NoJobServersError if @job_servers.empty? + @server_counter += 1 + @job_servers[@server_counter % @job_servers.size] + end + end + + def on_connection(&block) + @connection_handler = block + end + + def poll_connections(timeout = nil) + @servers_mutex.synchronize do + sockets = @job_servers.collect { |conn| conn.socket } + end + IO::select(sockets, nil, nil, timeout) + end + + def with_all_connections(&block) + @servers_mutex.synchronize do + @job_servers.each do |connection| + begin + block.call(connection) + rescue NetworkError => ex + logger.debug "Error with #{connection}, marking as bad" + remove_connection(connection) + end + end + end + end + + + private + + def deactivate_connection(connection) + @job_servers.reject! { |c| c == connection } + @bad_servers << connection + end + + def activate_connection(connection) + @bad_servers.reject! { |c| c == connection } + @job_servers << connection + end + + def start_reconnect_thread + Thread.new do + loop do + @servers_mutex.synchronize do + # If there are any failed servers, try to reconnect to them. + update_job_servers unless @bad_servers.empty? + end + sleep @reconnect_seconds + end + end.run + end + + def update_job_servers + logger.debug "Found #{@bad_servers.size} zombie connections, checking pulse." + @bad_servers.each do |connection| + begin + message = "Testing server #{connection}..." + if connection.is_healthy? + logger.debug "#{message} Connection is healthy, putting back into service" + activate_connection(connection) + else + logger.debug "#{message} Still down." + end + end + end + end + + + end +end diff --git a/lib/gearman/exceptions.rb b/lib/gearman/exceptions.rb new file mode 100644 index 0000000..1bd74c4 --- /dev/null +++ b/lib/gearman/exceptions.rb @@ -0,0 +1,24 @@ +module Gearman + + class InvalidArgsError < Exception + end + + class ProtocolError < Exception + end + + class NetworkError < Exception + end + + class NoJobServersError < Exception + end + + class JobQueueError < Exception + end + + class SocketTimeoutError < Exception + end + + class ServerDownException < Exception + end + +end \ No newline at end of file diff --git a/lib/gearman/logging.rb b/lib/gearman/logging.rb new file mode 100644 index 0000000..f69427c --- /dev/null +++ b/lib/gearman/logging.rb @@ -0,0 +1,19 @@ +module Gearman + module Logging + + def self.included(target) + target.extend ClassMethods + end + + def logger + self.class.logger + end + + module ClassMethods + def logger + Gearman.logger + end + end + + end +end \ No newline at end of file diff --git a/lib/gearman/packet.rb b/lib/gearman/packet.rb new file mode 100644 index 0000000..a0c7852 --- /dev/null +++ b/lib/gearman/packet.rb @@ -0,0 +1,61 @@ +module Gearman + class Packet + # Map from Integer representations of commands used in the network + # protocol to more-convenient symbols. + COMMANDS = { + 1 => :can_do, # W->J: FUNC + 2 => :cant_do, # W->J: FUNC + 3 => :reset_abilities, # W->J: -- + 4 => :pre_sleep, # W->J: -- + #5 => (unused), # - - + 6 => :noop, # J->W: -- + 7 => :submit_job, # C->J: FUNC[0]UNIQ[0]ARGS + 8 => :job_created, # J->C: HANDLE + 9 => :grab_job, # W->J: -- + 10 => :no_job, # J->W: -- + 11 => :job_assign, # J->W: HANDLE[0]FUNC[0]ARG + 12 => :work_status, # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR + 13 => :work_complete, # W->J/C: HANDLE[0]RES + 14 => :work_fail, # W->J/C: HANDLE + 15 => :get_status, # C->J: HANDLE + 16 => :echo_req, # ?->J: TEXT + 17 => :echo_res, # J->?: TEXT + 18 => :submit_job_bg, # C->J: FUNC[0]UNIQ[0]ARGS + 19 => :error, # J->?: ERRCODE[0]ERR_TEXT + 20 => :status_res, # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM + 21 => :submit_job_high, # C->J: FUNC[0]UNIQ[0]ARGS + 22 => :set_client_id, # W->J: [RANDOM_STRING_NO_WHITESPACE] + 23 => :can_do_timeout, # W->J: FUNC[0]TIMEOUT + 24 => :all_yours, # REQ Worker + 25 => :work_exception, # W->J: HANDLE[0]ARG + 26 => :option_req, # C->J: TEXT + 27 => :option_res, # J->C: TEXT + 28 => :work_data, # REQ Worker + 29 => :work_warning, # W->J/C: HANDLE[0]MSG + 30 => :grab_job_uniq, # REQ Worker + 31 => :job_assign_uniq, # RES Worker + 32 => :submit_job_high_bg, # C->J: FUNC[0]UNIQ[0]ARGS + 33 => :submit_job_low, # C->J: FUNC[0]UNIQ[0]ARGS + 34 => :submit_job_low_bg, # C->J: FUNC[0]UNIQ[0]ARGS + 35 => :submit_job_sched, # REQ Client + 36 => :submit_job_epoch # C->J: FUNC[0]UNIQ[0]EPOCH[0]ARGS + } + + # Map e.g. 'can_do' => 1 + NUMS = COMMANDS.invert + + ## + # Construct a request packet. + # + # @param type_name command type's name (see COMMANDS) + # @param arg optional data to pack into the command + # @return packet (as a string) + def Packet.pack_request(type_name, arg='') + type_num = NUMS[type_name.to_sym] + raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num + arg = '' if not arg + "\0REQ" + [type_num, arg.size].pack('NN') + arg + end + + end +end \ No newline at end of file diff --git a/lib/gearman/task.rb b/lib/gearman/task.rb index 35d2c5a..1b4dd62 100644 --- a/lib/gearman/task.rb +++ b/lib/gearman/task.rb @@ -213,7 +213,7 @@ def get_submit_packet() end mode = modes.join('_') - Util::pack_request(mode, args.join("\0")) + Packet::pack_request(mode, args.join("\0")) end end diff --git a/lib/gearman/task_set.rb b/lib/gearman/task_set.rb new file mode 100644 index 0000000..9876fe3 --- /dev/null +++ b/lib/gearman/task_set.rb @@ -0,0 +1,62 @@ +require 'time' + +module Gearman + class TaskSet + include Logging + + def initialize(client) + @client = client + @tasks_in_progress = [] + @finished_tasks = [] + end + + ## + # Add a new task to this TaskSet. + # + # @param args A Task object + # @return true if the task was created successfully, false otherwise + def add_task(task) + @tasks_in_progress << task + end + + ## + # Wait for all tasks in the set to finish. + # + # @param timeout maximum amount of time to wait, in seconds + def wait(timeout = 1) + end_time = if timeout + Time.now.to_f + timeout + else + nil + end + + while not @tasks_in_progress.empty? + remaining = if end_time + (t = end_time - Time.now.to_f) > 0 ? t : 0 + else + nil + end + begin + task = @tasks_in_progress.pop + if + @client.submit_job(task, true, remaining) + @finished_tasks << task + end + rescue SocketTimeoutError + return false + end + + end + + @finished_tasks.each do |t| + if ( (t.background.nil? || t.background == false) && !t.successful) + logger.warn "GearmanRuby: TaskSet failed" + return false + end + end + true + end + + end + +end diff --git a/lib/gearman/taskset.rb b/lib/gearman/taskset.rb deleted file mode 100755 index 611def4..0000000 --- a/lib/gearman/taskset.rb +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env ruby - -require 'socket' -require 'time' - -module Gearman - -# = TaskSet -# -# == Description -# A set of tasks submitted to a Gearman job server. -class TaskSet - def initialize(client) - @client = client - @task_waiting_for_handle = nil - @tasks_in_progress = {} # "host:port//handle" -> [job1, job2, ...] - @finished_tasks = [] # tasks that have completed or failed - @sockets = {} # "host:port" -> Socket - @merge_hash_to_hostport = {} # Fixnum -> "host:port" - end - - ## - # Add a new task to this TaskSet. - # - # @param args either a Task or arguments for Task.new - # @return true if the task was created successfully, false otherwise - def add_task(*args) - task = Util::get_task_from_args(*args) - add_task_internal(task, true) - end - - ## - # Internal function to add a task. - # - # @param task Task to add - # @param reset_state should we reset task state? true if we're adding a - # new task; false if we're rescheduling one that's - # failed - # @return true if the task was created successfully, false - # otherwise - def add_task_internal(task, reset_state=true) - task.reset_state if reset_state - req = task.get_submit_packet() - - @task_waiting_for_handle = task - - # Target the same job manager when submitting jobs - # with the same unique id so that we can coalesce - merge_hash = task.get_uniq_hash - - while (@task_waiting_for_handle != nil) - begin - # Try to connect again to the same server based on the unique hash - @merge_hash_to_hostport[merge_hash] ||= @client.get_job_server - hostport = @merge_hash_to_hostport[merge_hash] - - # Cache the socket - @sockets[hostport] ||= @client.get_socket(hostport) - - # Submit job to server - sock = @sockets[hostport] - Util.logger.debug "GearmanRuby: Using socket #{sock.inspect} for #{hostport} to SUBMIT_JOB" - Util.send_request(sock, req) - - # read_packet will fire off handle_job_created and set @task_waiting_for_handle to nil - # TODO: Better way of doing this. - read_packet(sock, @client.task_create_timeout_sec) - rescue NetworkError - Util.logger.debug "GearmanRuby: Network error on read from #{hostport} while adding job, marking server bad" - # Tell the client this is a bad server - @client.signal_bad_server(hostport) - if(sock != nil) - @client.close_socket(sock) - end - - # Un-cache socket - @sockets.delete hostport - # Remove from hash -> hostport mapping - @merge_hash_to_hostport[merge_hash] = nil - rescue NoJobServersError - Util.logger.error "GearmanRuby: No servers available." - return false - end - end - - return true - end - private :add_task_internal - - ## - # Handle a 'job_created' response from a job server. - # - # @param hostport "host:port" of job server - # @param data data returned in packet from server - def handle_job_created(hostport, data) - Util.logger.debug "GearmanRuby: Got job_created with handle #{data} from #{hostport}" - - if not @task_waiting_for_handle - raise ProtocolError, "Got unexpected job_created notification with handle #{data} from #{hostport}" - end - - js_handle = Util.handle_to_str(hostport, data) - task = @task_waiting_for_handle - @task_waiting_for_handle = nil - if(task.background) - @finished_tasks << task - else - (@tasks_in_progress[js_handle] ||= []) << task - end - task.handle_created(data) - nil - end - private :handle_job_created - - ## - # Handle a 'work_complete' response from a job server. - # - # @param hostport "host:port" of job server - # @param data data returned in packet from server - def handle_work_complete(hostport, data) - handle, data = data.split("\0", 2) - Util.logger.debug "GearmanRuby: Got work_complete with handle #{handle} and #{data ? data.size : '0'} byte(s) of data from #{hostport}" - tasks_in_progress(hostport, handle, true).each do |t| - t.handle_completion(data) - @finished_tasks << t - end - nil - end - private :handle_work_complete - - ## - # Handle a 'work_exception' response from a job server. - # - # @param hostport "host:port" of job server - # @param data data returned in packet from server - def handle_work_exception(hostport, data) - handle, exception = data.split("\0", 2) - Util.logger.debug "GearmanRuby: Got work_exception with handle #{handle} from #{hostport}: '#{exception}'" - tasks_in_progress(hostport, handle).each {|t| t.handle_exception(exception) } - end - private :handle_work_exception - - ## - # Handle a 'work_fail' response from a job server. - # - # @param hostport "host:port" of job server - # @param data data returned in packet from server - def handle_work_fail(hostport, data) - Util.logger.debug "GearmanRuby: Got work_fail with handle #{data} from #{hostport}" - tasks_in_progress(hostport, data, true).each do |t| - if t.handle_failure - add_task_internal(t, false) - else - @finished_tasks << t - end - end - end - private :handle_work_fail - - ## - # Handle a 'work_status' response from a job server. - # - # @param hostport "host:port" of job server - # @param data data returned in packet from server - def handle_work_status(hostport, data) - handle, num, den = data.split("\0", 3) - Util.logger.debug "GearmanRuby: Got work_status with handle #{handle} from #{hostport}: #{num}/#{den}" - tasks_in_progress(hostport, handle).each {|t| t.handle_status(num, den) } - end - private :handle_work_status - - ## - # Handle a 'work_warning' response from a job server. - # - # @param hostport "host:port" of job server - # @param data data returned in packet from server - def handle_work_warning(hostport, data) - handle, message = data.split("\0", 2) - Util.logger.debug "GearmanRuby: Got work_warning with handle #{handle} from #{hostport}: '#{message}'" - tasks_in_progress(hostport, handle).each {|t| t.handle_warning(message) } - end - private :handle_work_warning - - ## - # Handle a 'work_data' response from a job server - # - # @param hostport "host:port" of a job server - # @param data data returned in packet from server - def handle_work_data(hostport, data) - handle, data = data.split("\0", 2) - Util.logger.debug "GearmanRuby: Got work_data with handle #{handle} and #{data ? data.size : '0'} byte(s) of data from #{hostport}" - - js_handle = Util.handle_to_str(hostport, handle) - tasks = @tasks_in_progress[js_handle] - if not tasks - raise ProtocolError, "Got unexpected work_data with handle #{handle} from #{hostport} (no task by that name)" - end - tasks.each {|t| t.handle_data(data) } - end - private :handle_work_data - - ## - # Read and process a packet from a socket. - # - # @param sock socket connected to a job server - def read_packet(sock, timeout=nil) - hostport = @client.get_hostport_for_socket(sock) - if not hostport - raise RuntimeError, "Client doesn't know host/port for socket " + - sock.inspect - end - type, data = Util.read_response(sock, timeout) - known_types = [ :job_created, - :work_complete, - :work_fail, - :work_status, - :work_exception, - :work_warning, - :work_data ] - - if known_types.include?(type) - send("handle_#{type}".to_sym, hostport, data) - else - Util.logger.debug "GearmanRuby: Got #{type.to_s} from #{hostport}" - end - nil - end - private :read_packet - - ## - # Wait for all tasks in the set to finish. - # - # @param timeout maximum amount of time to wait, in seconds - def wait(timeout = 1) - end_time = if timeout - Time.now.to_f + timeout - else - nil - end - - while not @tasks_in_progress.empty? - remaining = if end_time - (t = end_time - Time.now.to_f) > 0 ? t : 0 - else - nil - end - - ready_socks = remaining == 0 ? nil : IO::select(@sockets.values, nil, nil, remaining) - if not ready_socks or not ready_socks[0] - Util.logger.debug "GearmanRuby: Timed out while waiting for tasks to finish" - # not sure what state the connections are in, so just be lame and - # close them for now - @sockets.values.each {|s| @client.close_socket(s) } - @sockets = {} - return false - end - - ready_socks[0].each do |sock| - begin - read_packet(sock, (end_time ? end_time - Time.now.to_f : nil)) - rescue ProtocolError - hostport = @client.get_hostport_for_socket(sock) - Util.logger.debug "GearmanRuby: Ignoring bad packet from #{hostport}" - rescue NetworkError - hostport = @client.get_hostport_for_socket(sock) - Util.logger.debug "GearmanRuby: Network error on read from #{hostport}" - end - end - end - - @sockets.values.each {|s| @client.return_socket(s) } - @sockets = {} - @finished_tasks.each do |t| - if ( (t.background.nil? || t.background == false) && !t.successful) - Util.logger.debug "GearmanRuby: TaskSet failed" - return false - end - end - true - end - - private - def tasks_in_progress(hostport, handle, remove_task = false) - js_handle = Util.handle_to_str(hostport, handle) - tasks = remove_task ? @tasks_in_progress.delete(js_handle) : @tasks_in_progress[js_handle] - if not tasks - raise ProtocolError, "Got unexpected work_data with handle #{handle} from #{hostport} (no task by that name)" - end - tasks - end -end - -end diff --git a/lib/gearman/util.rb b/lib/gearman/util.rb deleted file mode 100755 index 006ef37..0000000 --- a/lib/gearman/util.rb +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env ruby - -require 'socket' -require 'time' -require 'logger' - -module Gearman - - class ServerDownException < Exception; end - -# = Util -# -# == Description -# Static helper methods and data used by other classes. -class Util - # Map from Integer representations of commands used in the network - # protocol to more-convenient symbols. - COMMANDS = { - 1 => :can_do, # W->J: FUNC - 2 => :cant_do, # W->J: FUNC - 3 => :reset_abilities, # W->J: -- - 4 => :pre_sleep, # W->J: -- - #5 => (unused), # - - - 6 => :noop, # J->W: -- - 7 => :submit_job, # C->J: FUNC[0]UNIQ[0]ARGS - 8 => :job_created, # J->C: HANDLE - 9 => :grab_job, # W->J: -- - 10 => :no_job, # J->W: -- - 11 => :job_assign, # J->W: HANDLE[0]FUNC[0]ARG - 12 => :work_status, # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR - 13 => :work_complete, # W->J/C: HANDLE[0]RES - 14 => :work_fail, # W->J/C: HANDLE - 15 => :get_status, # C->J: HANDLE - 16 => :echo_req, # ?->J: TEXT - 17 => :echo_res, # J->?: TEXT - 18 => :submit_job_bg, # C->J: FUNC[0]UNIQ[0]ARGS - 19 => :error, # J->?: ERRCODE[0]ERR_TEXT - 20 => :status_res, # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM - 21 => :submit_job_high, # C->J: FUNC[0]UNIQ[0]ARGS - 22 => :set_client_id, # W->J: [RANDOM_STRING_NO_WHITESPACE] - 23 => :can_do_timeout, # W->J: FUNC[0]TIMEOUT - 24 => :all_yours, # REQ Worker - 25 => :work_exception, # W->J: HANDLE[0]ARG - 26 => :option_req, # C->J: TEXT - 27 => :option_res, # J->C: TEXT - 28 => :work_data, # REQ Worker - 29 => :work_warning, # W->J/C: HANDLE[0]MSG - 30 => :grab_job_uniq, # REQ Worker - 31 => :job_assign_uniq, # RES Worker - 32 => :submit_job_high_bg, # C->J: FUNC[0]UNIQ[0]ARGS - 33 => :submit_job_low, # C->J: FUNC[0]UNIQ[0]ARGS - 34 => :submit_job_low_bg, # C->J: FUNC[0]UNIQ[0]ARGS - 35 => :submit_job_sched, # REQ Client - 36 => :submit_job_epoch # C->J: FUNC[0]UNIQ[0]EPOCH[0]ARGS - } - - # Map e.g. 'can_do' => 1 - NUMS = COMMANDS.invert - - # Default job server port. - DEFAULT_PORT = 4730 - - def Util.logger=(logger) - @logger = logger - end - - def Util.logger - @logger ||= - begin - l = Logger.new($stdout) - l.level = Logger::FATAL - l - end - end - - ## - # Construct a request packet. - # - # @param type_name command type's name (see COMMANDS) - # @param arg optional data to pack into the command - # @return packet (as a string) - def Util.pack_request(type_name, arg='') - type_num = NUMS[type_name.to_sym] - raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num - arg = '' if not arg - "\0REQ" + [type_num, arg.size].pack('NN') + arg - end - - ## - # Return a Task based on the passed-in arguments. - # - # @param args either a single Task object or the arguments accepted by - # Task.new - # @return Task object - def Util.get_task_from_args(*args) - if (args[0].class == Task || args[0].class.superclass == Task) - return args[0] - elsif args.size <= 3 - return Task.new(*args) - else - raise InvalidArgsError, 'Incorrect number of args to get_task_from_args' - end - end - - ## - # Read from a socket, giving up if it doesn't finish quickly enough. - # NetworkError is thrown if we don't read all the bytes in time. - # - # @param sock Socket from which we read - # @param len number of bytes to read - # @param timeout maximum number of seconds we'll take; nil for no timeout - # @return full data that was read - def Util.timed_recv(sock, len, timeout=nil) - data = '' - end_time = Time.now.to_f + timeout if timeout - while data.size < len and (not timeout or Time.now.to_f < end_time) do - IO::select([sock], nil, nil, timeout ? end_time - Time.now.to_f : nil) \ - or break - begin - data += sock.readpartial(len - data.size) - rescue - raise NetworkError, "Unable to read data from socket." - end - end - if data.size < len - raise NetworkError, "Read #{data.size} byte(s) instead of #{len}" - end - data - end - - ## - # Read a response packet from a socket. - # - # @param sock Socket connected to a job server - # @param timeout timeout in seconds, nil for no timeout - # @return array consisting of integer packet type and data - def Util.read_response(sock, timeout=nil) - end_time = Time.now.to_f + timeout if timeout - head = timed_recv(sock, 12, timeout) - magic, type, len = head.unpack('a4NN') - raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES" - buf = len > 0 ? - timed_recv(sock, len, timeout ? end_time - Time.now.to_f : nil) : '' - type = COMMANDS[type] - raise ProtocolError, "Invalid packet type #{type}" unless type - [type, buf] - end - - ## - # Send a request packet over a socket. - # - # @param sock Socket connected to a job server - # @param req request packet to send - def Util.send_request(sock, req) - len = with_safe_socket_op{ sock.write(req) } - if len != req.size - raise NetworkError, "Wrote #{len} instead of #{req.size}" - end - end - - ## - # Add default ports to a job server or list of servers. - # - # @param servers a server hostname or "host:port" or array of servers - # @return an array of "host:port" strings - def Util.normalize_job_servers(servers) - if servers.class == String or servers.class == Symbol - servers = [ servers.to_s ] - end - servers.map {|s| s =~ /:/ ? s : "#{s}:#{DEFAULT_PORT}" } - end - - ## - # Convert job server info and a handle into a string. - # - # @param hostport "host:port" of job server - # @param handle job server-returned handle for a task - # @return "host:port//handle" - def Util.handle_to_str(hostport, handle) - "#{hostport}//#{handle}" - end - - ## - # Reverse Util.handle_to_str. - # - # @param str "host:port//handle" - # @return [hostport, handle] - def Util.str_to_handle(str) - str =~ %r{^([^:]+:\d+)//(.+)} - return [$1, $2] - end - - def self.with_safe_socket_op - begin - yield - rescue Exception => ex - raise ServerDownException.new(ex.message) - end - end - - def Util.ability_name_with_prefix(prefix,name) - "#{prefix}\t#{name}" - end - - class << self - alias :ability_name_for_perl :ability_name_with_prefix - end - -end - -end diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index f92b097..ff235dd 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = "3.0.7" + VERSION = "4.0.0" end diff --git a/lib/gearman/worker.rb b/lib/gearman/worker.rb index 7776fe4..071cb04 100755 --- a/lib/gearman/worker.rb +++ b/lib/gearman/worker.rb @@ -1,475 +1,242 @@ #!/usr/bin/env ruby require 'set' -require 'socket' -require 'thread' + +require 'gearman/worker/callbacks' +require 'gearman/worker/ability' +require 'gearman/worker/job' module Gearman -# = Worker -# -# == Description -# A worker that can connect to a Gearman server and perform tasks. -# -# == Usage -# require 'gearman' -# -# w = Gearman::Worker.new('127.0.0.1') -# -# # Add a handler for a "sleep" function that takes a single argument, the -# # number of seconds to sleep before reporting success. -# w.add_ability('sleep') do |data,job| -# seconds = data -# (1..seconds.to_i).each do |i| -# sleep 1 -# # Report our progress to the job server every second. -# job.report_status(i, seconds) -# end -# # Report success. -# true -# end -# loop { w.work } -class Worker - # = Ability - # - # == Description - # Information about an ability that we possess. - class Ability + class Worker + include Logging + include Callbacks + ## - # Create a new ability. + # Create a new worker. # - # @param block code to run - # @param timeout server gives up on us after this many seconds - def initialize(block, timeout=nil) - @block = block - @timeout = timeout + # @param job_servers "host:port"; either a single server or an array + # @param opts hash of additional options + def initialize(job_servers=nil, opts={}) + @abilities = {} + @client_id = opts[:client_id] || generate_id + @connection_pool = ConnectionPool.new(job_servers) + @network_timeout_sec = opts[:network_timeout_sec] || 5 + @reconnect_sec = opts[:reconnect_sec] || 30 + @status = :preparing + @worker_enabled = true + + # Add callback for when connections occur -- register abilities and send client id + @connection_pool.on_connection do |connection| + connection.send_update(Packet.pack_request(:set_client_id, @client_id)) + @abilities.each do |func_name, ability| + announce_ability(func_name, ability.timeout, connection) + end + end end - attr_reader :timeout + + attr_accessor :client_id, :reconnect_sec, :network_timeout_sec, :worker_enabled, :status ## - # Run the block of code. - # - # @param data data passed to us by a client - # @param job interface to report job information to the server - def run(data, job) - @block.call(data, job) + # @return A random string of 30 characters from a-z + def generate_id + chars = ('a'..'z').to_a + Array.new(30) { chars[rand(chars.size)] }.join end - - end - # = Job - # - # == Description - # Interface to allow a worker to report information to a job server. - class Job ## - # Create a new Job. - # - # @param sock Socket connected to job server - # @param handle job server-supplied job handle - attr_reader :handle - def initialize(sock, handle) - @socket = sock - @handle = handle + # Generate CAN_DO (or CAN_DO_TIMEOUT) packet and submit it + def announce_ability(func_name, timeout, connection) + cmd = timeout ? :can_do_timeout : :can_do + arg = timeout ? "#{func_name}\0#{timeout.to_s}" : func_name + connection.send_update(Packet.pack_request(cmd, arg)) + logger.debug "Announced ability #{func_name}" end ## - # Report our status to the job server. - def report_status(numerator, denominator) - req = Util.pack_request( - :work_status, "#{@handle}\0#{numerator}\0#{denominator}") - Util.send_request(@socket, req) - self + # Add a new ability, announcing it to job servers. + # + # The passed-in block of code will be executed for jobs of this function + # type. It'll receive two arguments, the data supplied by the client and + # a Job object. If it returns nil or false, the server will be informed + # that the job has failed; otherwise the return value of the block will + # be passed back to the client in String form. + # + # @param func_name function name (without prefix) + # @param timeout the server will give up on us if we don't finish + # a task in this many seconds + # @param block Block to associate with the function + def add_ability(func_name, timeout=nil, &block) + @abilities[func_name] = Ability.new(func_name, block, timeout) + @connection_pool.with_all_connections do |connection| + announce_ability(func_name, timeout, connection) + end end ## - # Send data before job completes - def send_data(data) - req = Util.pack_request(:work_data, "#{@handle}\0#{data}") - Util.send_request(@socket, req) - self + # Callback for after an ability runs + def after_ability(func, &block) + abilities[func].after_complete(block) end ## - # Send a warning explicitly - def report_warning(warning) - req = Util.pack_request(:work_warning, "#{@handle}\0#{warning}") - Util.send_request(@socket, req) - self + # Let job servers know that we're no longer able to do something via CANT_DO + # + # @param func function name + def remove_ability(func) + @abilities.delete(func) + req = Packet.pack_request(:cant_do, func) + @connection_pool.with_all_connections do |connection| + connection.send_update(req) + end end - end - module Callbacks - - %w(connect grab_job no_job job_assign work_complete work_fail - work_exception).each do |event| - - define_method("on_#{event}") do |&callback| - instance_variable_set("@__on_#{event}", callback) + ## + # Handle a job_assign packet. + # + # @param data data in the packet + # @param connection Connection where the data originated + def handle_job_assign(data, connection) + handle, func, data = data.split("\0", 3) + + if not func + logger.error "Ignoring JOB_ASSIGN with no function from #{connection}" + return false end - define_method("run_#{event}_callback") do - callback = instance_variable_get("@__on_#{event}") - return unless callback - - begin - callback.call - rescue Exception => e - Util.logger.error "GearmanRuby: #{event} failed: #{e.inspect}" - end + if not handle + logger.error "Ignoring JOB_ASSIGN with no job handle from #{connection}" + return false end - end - end + logger.info "Got JOB_ASSIGN with handle #{handle} and #{data.size} byte(s) from #{connection}" - # Provides callbacks for internal worker use: - # - # def named_metric(metric) - # "HardWorker.#{Process.pid}.#{metric}" - # end - # - # worker = Gearman::Worker.new - # worker.on_grab_job { StatsD.increment(named_metric('grab_job')) } - # worker.on_job_assign { StatsD.increment(named_metric('job_assign')) } - # worker.on_no_job { StatsD.increment(named_metric('no_job')) } - # worker.on_work_complete { StatsD.increment(named_metric('work_complete')) } - include Callbacks - - ## - # Create a new worker. - # - # @param job_servers "host:port"; either a single server or an array - # @param opts hash of additional options - def initialize(job_servers=nil, opts={}) - chars = ('a'..'z').to_a - @client_id = Array.new(30) { chars[rand(chars.size)] }.join - @sockets = {} # "host:port" -> Socket - @abilities = {} # "funcname" -> Ability - @after_abilities = {} # "funcname" -> Ability - @bad_servers = [] # "host:port" - @servers_mutex = Mutex.new - %w{client_id reconnect_sec - network_timeout_sec}.map {|s| s.to_sym }.each do |k| - instance_variable_set "@#{k}", opts[k] - opts.delete k - end - if opts.size > 0 - raise InvalidArgsError, - 'Invalid worker args: ' + opts.keys.sort.join(', ') - end - @reconnect_sec = 30 if not @reconnect_sec - @network_timeout_sec = 5 if not @network_timeout_sec - @worker_enabled = true - @status = :preparing - self.job_servers = job_servers if job_servers - start_reconnect_thread - end - attr_accessor :client_id, :reconnect_sec, :network_timeout_sec, :bad_servers, :worker_enabled, :status + ability = @abilities[func] - # Start a thread to repeatedly attempt to connect to down job servers. - def start_reconnect_thread - Thread.new do - loop do - @servers_mutex.synchronize do - # If there are any failed servers, try to reconnect to them. - if not @bad_servers.empty? - update_job_servers(@sockets.keys + @bad_servers) - end - end - sleep @reconnect_sec + if ability == nil + logger.error "Ignoring JOB_ASSIGN for unsupported function #{func} with handle #{handle} from #{connection}" + connection.send_update(Packet.pack_request(:work_fail, handle)) + return false end - end.run - end - def job_servers - servers = nil - @servers_mutex.synchronize do - servers = @sockets.keys + @bad_servers - end - servers - end - - ## - # Connect to job servers to be used by this worker. - # - # @param servers "host:port"; either a single server or an array - def job_servers=(servers) - @servers_mutex.synchronize do - update_job_servers(servers) - end - end - - # Internal function to actually connect to servers. - # Caller must acquire @servers_mutex before calling us. - # - # @param servers "host:port"; either a single server or an array - def update_job_servers(servers) - @bad_servers = [] - servers = Set.new(Util.normalize_job_servers(servers)) - # Disconnect from servers that we no longer care about. - @sockets.each do |server,sock| - if not servers.include? server - Util.logger.info "GearmanRuby: Disconnecting from old server #{server}" - sock.close - @sockets.delete(server) - end - end - # Connect to new servers. - servers.each do |server| - if not @sockets[server] - begin - Util.logger.info "GearmanRuby: Connecting to server #{server}" - run_connect_callback - @sockets[server] = connect(server) - rescue NetworkError - @bad_servers << server - Util.logger.info "GearmanRuby: Unable to connect to #{server}" - end + exception = nil + begin + ret = ability.run(data, Job.new(connection, handle)) + rescue Exception => e + exception = e + logger.debug "Exception: #{e}\n#{e.backtrace.join("\n")}\n" end - end - end - private :update_job_servers - - ## - # Connect to a job server. - # - # @param hostport "hostname:port" - def connect(hostport) - begin - # FIXME: handle timeouts - sock = TCPSocket.new(*hostport.split(':')) - rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EHOSTUNREACH - raise NetworkError - rescue Exception => e - Util.logger.debug "GearmanRuby: Unhandled exception while connecting to #{hostport} : #{e} (raising NetworkError exception)" - raise NetworkError - end - # FIXME: catch exceptions; do something smart - Util.send_request(sock, Util.pack_request(:set_client_id, @client_id)) - @abilities.each {|f,a| announce_ability(sock, f, a.timeout) } - sock - end - private :connect - - ## - # Announce an ability over a particular socket. - # - # @param sock Socket connect to a job server - # @param func function name (including prefix) - # @param timeout the server will give up on us if we don't finish - # a task in this many seconds - def announce_ability(sock, func, timeout=nil) - begin - cmd = timeout ? :can_do_timeout : :can_do - arg = timeout ? "#{func}\0#{timeout.to_s}" : func - Util.send_request(sock, Util.pack_request(cmd, arg)) - rescue Exception => ex - bad_servers << @sockets.keys.detect{|hp| @sockets[hp] == sock} - end - end - private :announce_ability - - ## - # Add a new ability, announcing it to job servers. - # - # The passed-in block of code will be executed for jobs of this function - # type. It'll receive two arguments, the data supplied by the client and - # a Job object. If it returns nil or false, the server will be informed - # that the job has failed; otherwise the return value of the block will - # be passed back to the client in String form. - # - # @param func function name (without prefix) - # @param timeout the server will give up on us if we don't finish - # a task in this many seconds - def add_ability(func, timeout=nil, &f) - @abilities[func] = Ability.new(f, timeout) - @sockets.values.each {|s| announce_ability(s, func, timeout) } - end + packets = if ret && exception.nil? + logger.debug "Sending WORK_COMPLETE for #{handle} with #{ret.to_s.size} byte(s) to #{connection}" + run_work_complete_callback + [Packet.pack_request(:work_complete, "#{handle}\0#{ret.to_s}")] + elsif exception.nil? + logger.debug "Sending WORK_FAIL for #{handle} to #{connection}" + run_work_fail_callback + [Packet.pack_request(:work_fail, handle)] + elsif exception + logger.debug "Sending WORK_EXCEPTION for #{handle} to #{connection}" + run_work_exception_callback + [Packet.pack_request(:work_exception, "#{handle}\0#{exception.message}")] + end - ## - # Add an after-ability hook - # - # The passed-in block of code will be executed after the work block for - # jobs with the same function name. It takes two arguments, the result of - # the work and the original job data. This way, if you need to hook into - # *after* the job_complete packet is sent to the server, you can do so. - # - # N.B The after-ability hook ONLY runs if the ability was successful and no - # exceptions were raised. - # - # @param func function name (without prefix) - # - def after_ability(func, &f) - @after_abilities[func] = Ability.new(f) - end - - ## - # Let job servers know that we're no longer able to do something. - # - # @param func function name - def remove_ability(func) - @abilities.delete(func) - req = Util.pack_request(:cant_do, func) - @sockets.values.each {|s| Util.send_request(s, req) } - end + packets.each do |packet| + connection.send_update(packet) + end - ## - # Handle a job_assign packet. - # - # @param data data in the packet - # @param sock Socket on which the packet arrived - # @param hostport "host:port" - def handle_job_assign(data, sock, hostport) - handle, func, data = data.split("\0", 3) - if not func - Util.logger.error "GearmanRuby: Ignoring job_assign with no function from #{hostport}" - return false + true end - Util.logger.error "GearmanRuby: Got job_assign with handle #{handle} and #{data.size} byte(s) " + - "from #{hostport}" - - ability = @abilities[func] - if not ability - Util.logger.error "Ignoring job_assign for unsupported func #{func} " + - "with handle #{handle} from #{hostport}" - Util.send_request(sock, Util.pack_request(:work_fail, handle)) - return false - end - exception = nil - begin - ret = ability.run(data, Job.new(sock, handle)) - rescue Exception => e - exception = e - Util.logger.debug "GearmanRuby: Exception: #{e}\n#{e.backtrace.join("\n")}\n" + ## + # Handle a message for the worker + # + # @param type Packet type (NO_JOB, JOB_ASSIGN, NO_OP) + # @param data Opaque data being passed with the message + # @param connection The Connection object where the message originates + # @return + def handle_work_message(type, data, connection) + case type + when :no_job + logger.info "Got NO_JOB from #{connection}" + run_no_job_callback + when :job_assign + @status = :working + run_job_assign_callback + return worker_enabled if handle_job_assign(data, connection) + when :no_op + # We'll have to read again + logger.debug "Received NOOP while polling. Ignoring NOOP" + else + logger.error "Got unhandled #{type.to_s} from #{connection}" + end end - cmd = if ret && exception.nil? - Util.logger.debug "GearmanRuby: Sending work_complete for #{handle} with #{ret.to_s.size} byte(s) " + - "to #{hostport}" - run_work_complete_callback - [ Util.pack_request(:work_complete, "#{handle}\0#{ret.to_s}") ] - elsif exception.nil? - Util.logger.debug "GearmanRuby: Sending work_fail for #{handle} to #{hostport}" - run_work_fail_callback - [ Util.pack_request(:work_fail, handle) ] - elsif exception - Util.logger.debug "GearmanRuby: Sending work_exception for #{handle} to #{hostport}" - run_work_exception_callback - [ Util.pack_request(:work_exception, "#{handle}\0#{exception.message}") ] - end + ## + # Do a single job and return. + def work + grab_job_req = Packet.pack_request(:grab_job) + type, data = nil - cmd.each {|p| Util.send_request(sock, p) } - - # There are cases where we might want to run something after the worker - # successfully completes the ability in question and sends its results - if ret && exception.nil? - after_ability = @after_abilities[func] - if after_ability - Util.logger.debug "Running after ability for #{func}..." - begin - after_ability.run(ret, data) - rescue Exception => e - Util.logger.debug "GearmanRuby: Exception: #{e}\n#{e.backtrace.join("\n")}\n" - nil - end - end - end - - true - end + loop do + @status = :preparing + @connection_pool.with_all_connections do |connection| + logger.debug "Sending GRAB_JOB to #{connection}" + run_grab_job_callback - ## - # Do a single job and return. - def work - req = Util.pack_request(:grab_job) - type = nil - data = nil - loop do - @status = :preparing - bad_servers = [] - # We iterate through the servers in sorted order to make testing - # easier. - servers = nil - @servers_mutex.synchronize { servers = @sockets.keys.sort } - servers.each do |hostport| - Util.logger.debug "GearmanRuby: Sending grab_job to #{hostport}" - run_grab_job_callback - sock = @sockets[hostport] - Util.send_request(sock, req) - - # Now that we've sent grab_job, we need to keep reading packets - # until we see a no_job or job_assign response (there may be a noop - # waiting for us in response to a previous pre_sleep). - loop do begin - type, data = Util.read_response(sock, @network_timeout_sec) - case type - when :no_job - Util.logger.debug "GearmanRuby: Got no_job from #{hostport}" - run_no_job_callback - break - when :job_assign - @status = :working - run_job_assign_callback - return worker_enabled if handle_job_assign(data, sock, hostport) - break - else - Util.logger.debug "GearmanRuby: Got #{type.to_s} from #{hostport}" - end - rescue Exception - Util.logger.info "GearmanRuby: Server #{hostport} timed out or lost connection (#{$!.inspect}); marking bad" - bad_servers << hostport - break - end + type, data = connection.send_request(grab_job_req, @network_timeout_sec) + handle_work_message(type, data, connection) + end while type == :no_op end - end - @servers_mutex.synchronize do - bad_servers.each do |hostport| - @sockets[hostport].close if @sockets[hostport] - @bad_servers << hostport if @sockets[hostport] - @sockets.delete(hostport) + logger.info "Sending PRE_SLEEP and going to sleep for #{@reconnect_sec} second(s)" + @connection_pool.with_all_connections do |connection| + connection.send_update(Packet.pack_request(:pre_sleep)) end - end - Util.logger.debug "GearmanRuby: Sending pre_sleep and going to sleep for #{@reconnect_sec} sec" - @servers_mutex.synchronize do - @sockets.values.each do |sock| - Util.send_request(sock, Util.pack_request(:pre_sleep)) + return false unless worker_enabled + @status = :waiting + + time_asleep = Time.now + + while (@status == :waiting) + sleep(time_asleep) end + end + end - return false unless worker_enabled - @status = :waiting - - sleepTime = Time.now - while(@status == :waiting) - # FIXME: We could optimize things the next time through the 'each' by - # sending the first grab_job to one of the servers that had a socket - # with data in it. Not bothering with it for now. - IO::select(@sockets.values, nil, nil, @reconnect_sec) - - # If 30 seconds have passed, then wakeup - @status = :wakeup if Time.now - sleepTime > 30 - - if(@status == :waiting) - @sockets.values.each do |sock| - type, data = Util.read_response(sock, @network_timeout_sec) - - # there shouldn't be anything else here, if there is, we should be able to ignore it... - if(type == :noop) - Util.logger.debug "Received NoOp while sleeping... waking up!" + ## + # Sleep and poll until timeout occurs or a NO_OP packet is received + # @param time_fell_asleep The time that we fell asleep (Time object) + def sleep(time_fell_asleep) + # Use IO::select to wait for available connection data + @connection_pool.poll_connections(@network_timeout_sec) + + # If 30 seconds have passed, then wakeup + time_asleep = Time.now - time_fell_asleep + @status = :wakeup if time_asleep > 30 + + if (@status == :waiting) + @connection_pool.with_all_connections do |connection| + begin + type, data = connection.read_response(@network_timeout_sec) + + # Wake up if we receive a NOOP packet + if (type == :noop) + logger.debug "Received NOOP while sleeping... waking up!" @status = :wakeup + else + logger.warn "Received something other than a NOOP packet while sleeping: #{type.to_s}" end + rescue SocketTimeoutError + # This is okay here. end end end - end end -end end diff --git a/lib/gearman/worker/ability.rb b/lib/gearman/worker/ability.rb new file mode 100644 index 0000000..9276459 --- /dev/null +++ b/lib/gearman/worker/ability.rb @@ -0,0 +1,55 @@ +module Gearman + class Worker + + class Ability + ## + # Create a new ability. Setting timeout means we register with CAN_DO_TIMEOUT + # @param func_name Function name of this ability + # @param block Code to run + # @param timeout Server gives up on us after this many seconds + def initialize(func_name, block, timeout=nil) + @func_name = func_name + @block = block + @timeout = timeout + @on_complete = nil + end + + attr_reader :timeout, :func_name + + ## + # Run the block of code given for a job of this type. + # + # @param data data passed to us by a client + # @param job interface to report job information to the server + def run(data, job) + begin + result = @block.call(data, job) if @block + @on_complete.call(result, data) if @on_complete + return result + rescue => ex + raise ex + end + end + + ## + # Add an after-ability hook + # + # The passed-in block of code will be executed after the work block for + # jobs with the same function name. It takes two arguments, the result of + # the work and the original job data. This way, if you need to hook into + # *after* the job_complete packet is sent to the server, you can do so. + # + # N.B The after-ability hook ONLY runs if the ability was successful and no + # exceptions were raised. + # + # @param func function name (without prefix) + # + def after_complete(&block) + @on_complete = block + end + + + end + + end +end diff --git a/lib/gearman/worker/callbacks.rb b/lib/gearman/worker/callbacks.rb new file mode 100644 index 0000000..00c31fb --- /dev/null +++ b/lib/gearman/worker/callbacks.rb @@ -0,0 +1,39 @@ +# Provides callbacks for internal worker use: +# +# def named_metric(metric) +# "HardWorker.#{Process.pid}.#{metric}" +# end +# +# worker = Gearman::Worker.new +# worker.on_grab_job { StatsD.increment(named_metric('grab_job')) } +# worker.on_job_assign { StatsD.increment(named_metric('job_assign')) } +# worker.on_no_job { StatsD.increment(named_metric('no_job')) } +# worker.on_work_complete { StatsD.increment(named_metric('work_complete')) } + +module Gearman + class Worker + + module Callbacks + + %w(connect grab_job no_job job_assign work_complete work_fail + work_exception).each do |event| + + define_method("on_#{event}") do |&callback| + instance_variable_set("@__on_#{event}", callback) + end + + define_method("run_#{event}_callback") do + callback = instance_variable_get("@__on_#{event}") + return unless callback + + begin + callback.call + rescue Exception => e + logger.error "#{event} failed: #{e.inspect}" + end + end + end + end + + end +end \ No newline at end of file diff --git a/lib/gearman/worker/job.rb b/lib/gearman/worker/job.rb new file mode 100644 index 0000000..b530299 --- /dev/null +++ b/lib/gearman/worker/job.rb @@ -0,0 +1,44 @@ +module Gearman + class Worker + + class Job + ## + # Create a new Job. + # + # @param sock Socket connected to job server + # @param handle job server-supplied job handle + attr_reader :handle + + def initialize(connection, handle) + @connection = connection + @handle = handle + end + + ## + # Report our status to the job server. + def report_status(numerator, denominator) + req = Packet.pack_request(:work_status, "#{@handle}\0#{numerator}\0#{denominator}") + @connection.send_update(req) + self + end + + ## + # Send data before job completes + def send_data(data) + req = Packet.pack_request(:work_data, "#{@handle}\0#{data}") + @connection.send_update(req) + self + end + + ## + # Send a warning explicitly + def report_warning(warning) + req = Packet.pack_request(:work_warning, "#{@handle}\0#{warning}") + @connection.send_update(req) + self + end + end + + end +end + diff --git a/spec/client_spec.rb b/spec/client_spec.rb index 4c08e44..fa50ffa 100644 --- a/spec/client_spec.rb +++ b/spec/client_spec.rb @@ -1,19 +1,22 @@ require 'spec_helper' require 'socket' -require 'rspec' -require 'rspec/mocks' -require 'gearman' describe Gearman::Client do before(:all) do @tcp_server = TCPServer.new 5789 - @client = Gearman::Client.new(["localhost:5789"]) end after(:all) do @tcp_server.close end + before(:each) do + @mock_connection_pool = double(Gearman::ConnectionPool) + Gearman::ConnectionPool.stub(:new).and_return @mock_connection_pool + + @client = Gearman::Client.new(["localhost:5789"]) + end + it "creates a client" do @client.should_not be nil end @@ -30,31 +33,40 @@ it "raises an exception when submitting a job fails" do task = Gearman::Task.new("queue", "data") - @client.should_receive(:get_job_server).and_raise Gearman::NoJobServersError + @mock_connection_pool.should_receive(:get_connection).and_raise Gearman::NoJobServersError expect { @client.do_task(task) - }.to raise_error + }.to raise_exception end - it "gets a socket for the client's host:port combo" do - sock = @client.get_socket("localhost:5789") - sock.should_not be nil - end + it "properly emits an options request" do + mock_connection = double(Gearman::Connection) + mock_connection.should_receive(:send_request).and_return([:error, "Snarf"]) - it "closes sockets it doesn't know about when asked to return them" do - sock = double(TCPSocket) - sock.should_receive(:addr).and_return [nil, 1234, 'hostname', '1.2.3.4'] - sock.should_receive(:close) - @client.return_socket(sock) - end + @mock_connection_pool.should_receive(:with_all_connections).and_yield mock_connection - it "properly emits an options request" do - Gearman::Util.should_receive(:send_request) - Gearman::Util.should_receive(:read_response).and_return([:error, "Snarf"]) expect { - @client.option_request("exceptions") + @client.set_options("exceptions") }.to raise_error end + + + it "should raise a NetworkError when it didn't write as much as expected to a socket" do + socket = double(TCPSocket) + socket.should_receive(:write).with(anything).and_return(0) + + task = Gearman::Task.new("job_queue", "data") + request = task.get_submit_packet + connection = Gearman::Connection.new("localhost", 1234) + connection.should_receive(:socket).and_return socket + + expect { + connection.send_request(request) + }.to raise_error + end + + + end diff --git a/spec/connection_pool_spec.rb b/spec/connection_pool_spec.rb new file mode 100644 index 0000000..f0be4e1 --- /dev/null +++ b/spec/connection_pool_spec.rb @@ -0,0 +1,55 @@ + +require 'spec_helper' +require 'socket' + +describe Gearman::ConnectionPool do + context "normalizing job servers" do + before :each do + @connection_pool = Gearman::ConnectionPool.new + end + + it "should handle a string for input" do + connection = Gearman::Connection.new("localhost", 1234) + connection.should_receive(:is_healthy?).and_return true + Gearman::Connection.should_receive(:new).with("localhost", 1234).and_return connection + @connection_pool.add_servers("localhost:1234") + @connection_pool.get_connection.should be connection + end + + it "should handle an array of host:port without changing a thing" do + connection_one = Gearman::Connection.new("localhost", 123) + connection_one.should_receive(:is_healthy?).and_return true + connection_two = Gearman::Connection.new("localhost", 456) + connection_two.should_receive(:is_healthy?).and_return true + + Gearman::Connection.should_receive(:new).with("localhost", 123).and_return connection_one + Gearman::Connection.should_receive(:new).with("localhost", 456).and_return connection_two + + servers = [ "#{connection_one.to_host_port}", "#{connection_two.to_host_port}" ] + @connection_pool.add_servers(servers) + + + @connection_pool.get_connection.should be connection_two + @connection_pool.get_connection.should be connection_one + + end + + it "should append the default port to anything in the array that doesn't have a port" do + in_servers = ["foo.bar.com:123", "narf.quiddle.com"] + out_servers = ["foo.bar.com:123", "narf.quiddle.com:4730"] + + connection_one = Gearman::Connection.new("foo.bar.com", 123) + connection_one.should_receive(:is_healthy?).and_return true + connection_two = Gearman::Connection.new("narf.quiddle.com", 4730) + connection_two.should_receive(:is_healthy?).and_return true + + Gearman::Connection.should_receive(:new).with("foo.bar.com", 123).and_return connection_one + Gearman::Connection.should_receive(:new).with("narf.quiddle.com", 4730).and_return connection_two + + @connection_pool.add_servers(in_servers) + + @connection_pool.get_connection.should be connection_two + @connection_pool.get_connection.should be connection_one + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 0352fda..0b67ee2 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,6 @@ require 'simplecov' +require 'rspec' +require 'rspec/mocks' SimpleCov.start do add_filter "/spec/" @@ -8,6 +10,9 @@ $:.unshift(File.expand_path('../lib', __FILE__)) require 'gearman' +Gearman.logger = Logger.new(STDERR) +Gearman.logger.level = Logger::DEBUG + RSpec.configure do |config| config.mock_with :rspec end diff --git a/spec/task_spec.rb b/spec/task_spec.rb index 7fd7460..535efea 100644 --- a/spec/task_spec.rb +++ b/spec/task_spec.rb @@ -16,6 +16,16 @@ }.to raise_error end + it "should generate a task from two arguments" do + task = Gearman::Task.new("queue", "data") + task.should_not be nil + end + + it "should generate a task from three arguments" do + task = Gearman::Task.new("queue", "data", {:background => true}) + task.should_not be nil + end + it "generates a uniq value based on the data and the function" do hash_data = 'bc2ca93d86a28cb72fedf36326d1da0cc3d4ed6a' task_one = Gearman::Task.new("unique_id", "abcdef") diff --git a/spec/taskset_spec.rb b/spec/taskset_spec.rb index 51548b8..5df7d09 100644 --- a/spec/taskset_spec.rb +++ b/spec/taskset_spec.rb @@ -13,7 +13,7 @@ end - it "handles a NetworkError when submitting a job" do + xit "handles a NetworkError when submitting a job" do bad_socket = double(TCPSocket) bad_socket.should_receive(:write) { |*args| args[0].length @@ -41,7 +41,7 @@ task_set.add_task(task) end - it "waits for an answer from the server" do + xit "waits for an answer from the server" do good_socket = double(TCPSocket) good_socket.should_receive(:write) { |*args| args[0].length diff --git a/spec/util_spec.rb b/spec/util_spec.rb deleted file mode 100644 index 728b3e8..0000000 --- a/spec/util_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'spec_helper' -require 'socket' -require 'rspec' -require 'rspec/mocks' -require 'gearman' - -describe Gearman::Util do - before(:each) do - - end - - it "should generate a task from two arguments" do - task = Gearman::Util.get_task_from_args("queue", "data") - task.should_not be nil - end - - it "should generate a task from three arguments" do - task = Gearman::Util.get_task_from_args("queue", "data", {:background => true}) - task.should_not be nil - end - - it "should raise an exception with more than three arguments" do - expect { - Gearman::Util.get_task_from_args("one", "two", {:three => :four}, :five) - }.to raise_error - end - - it "should raise a NetworkError when it didn't write as much as expected to a socket" do - socket = double(TCPSocket) - socket.should_receive(:write).with(anything).and_return(0) - - task = Gearman::Task.new("job_queue", "data") - request = task.get_submit_packet - expect { - Gearman::Util.send_request(socket, request) - }.to raise_error - end - - context "normalizing job servers" do - it "should handle a string for input" do - Gearman::Util.normalize_job_servers("localhost:1234").should eq ["localhost:1234"] - end - - it "should handle an array of host:port without changing a thing" do - servers = ["localhost:123", "localhost:456"] - Gearman::Util.normalize_job_servers(servers).should eq servers - end - - it "should append the default port to anything in the array that doesn't have a port" do - in_servers = ["foo.bar.com:123", "narf.quiddle.com"] - out_servers = ["foo.bar.com:123", "narf.quiddle.com:4730"] - Gearman::Util.normalize_job_servers(in_servers).should eq out_servers - end - end - - it "should convert a host:port & handle into its corresponding string" do - Gearman::Util.handle_to_str("localhost:4730", "foo:1").should eq "localhost:4730//foo:1" - end - - it "should convert a host:port & handle string into its components" do - Gearman::Util.str_to_handle("localhost:4730//foo:1").should eq ["localhost:4730", "foo:1"] - end - - it "should convert an ability name with prefix into its correct format" do - Gearman::Util.ability_name_with_prefix("test", "a").should eq "test\ta" - end -end \ No newline at end of file From 74f7cb5a37c82d367b0338073bc22ddda6f1ef1e Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 13:06:14 -0800 Subject: [PATCH 05/38] Bumped version and changed source code homepage --- gearman-ruby.gemspec | 2 +- lib/gearman/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gearman-ruby.gemspec b/gearman-ruby.gemspec index 8954a68..1782f0d 100644 --- a/gearman-ruby.gemspec +++ b/gearman-ruby.gemspec @@ -10,7 +10,7 @@ Gem::Specification.new do |s| s.summary = %q{Ruby Gearman library} s.description = %q{Library for the Gearman distributed job system} s.email = %q{john@johnewart.net} - s.homepage = %q{http://github.com/gearman-ruby/gearman-ruby} + s.homepage = %q{http://github.com/johnewart/gearman-ruby} s.rubyforge_project = "gearman-ruby" s.extra_rdoc_files = [ diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index ff235dd..b283cfd 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = "4.0.0" + VERSION = "4.0.1" end From cbcf4d5b43d3a288e36607d710acbbe91a3f44fa Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 13:10:33 -0800 Subject: [PATCH 06/38] Adding YAML for Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f819a51 --- /dev/null +++ b/.travis.yml @@ -0,0 +1 @@ +language: ruby From 61a41abd72fbd33ca6b63b6c2ad6c0acf761b604 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 13:12:34 -0800 Subject: [PATCH 07/38] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8d08248..1a95ce3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # gearman-ruby +![Travis CI Build Status](https://travis-ci.org/johnewart/gearman-ruby.svg) + ## What is this? This is a pure-Ruby library for the [gearman][Gearman] distributed job system. From 2175ef9f89421c25e69838a785c09f991e4c67cc Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 13:14:04 -0800 Subject: [PATCH 08/38] Added Ruby versions to Travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index f819a51..79b241a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1 +1,5 @@ language: ruby +rvm: + - 2.1.0 + - 2.0.0 + - 1.9.3 From 81b9aa887270b64f68131daf00e0a10630263be9 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 17:34:29 -0800 Subject: [PATCH 09/38] Remove execute bit --- lib/gearman/client.rb | 0 lib/gearman/worker.rb | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 lib/gearman/client.rb mode change 100755 => 100644 lib/gearman/worker.rb diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb old mode 100755 new mode 100644 diff --git a/lib/gearman/worker.rb b/lib/gearman/worker.rb old mode 100755 new mode 100644 From 2e9f5f532fb0bf871fbe6eb989df254b2c6cb0f8 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 20:27:48 -0800 Subject: [PATCH 10/38] Cleanup so that Client#do_task waits indefinitely, adds a convenience method TaskSet#wait_forever to clearly describe intent, fixes #15 --- lib/gearman/client.rb | 4 ++-- lib/gearman/task_set.rb | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index 26745e7..f38cf10 100644 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -45,7 +45,7 @@ def do_task(task) task_set = TaskSet.new(self) if task_set.add_task(task) - task_set.wait + task_set.wait_forever else raise JobQueueError, "Unable to enqueue job." end @@ -148,4 +148,4 @@ def handle_response(task, type, data) -end \ No newline at end of file +end diff --git a/lib/gearman/task_set.rb b/lib/gearman/task_set.rb index 9876fe3..f16852f 100644 --- a/lib/gearman/task_set.rb +++ b/lib/gearman/task_set.rb @@ -22,7 +22,7 @@ def add_task(task) ## # Wait for all tasks in the set to finish. # - # @param timeout maximum amount of time to wait, in seconds + # @param timeout maximum amount of time to wait, in seconds - if this is nil, waits forever def wait(timeout = 1) end_time = if timeout Time.now.to_f + timeout @@ -57,6 +57,11 @@ def wait(timeout = 1) true end + # Wait for all tasks in set to finish, with no timeout + def wait_forever + wait(nil) + end + end end From 1fbbaa7cfde1bcfe1162dcb500e52f17c9ed491e Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 20:30:07 -0800 Subject: [PATCH 11/38] Call connection handler when connections are activated, cleanup polling if there are no available sockets, fix a bug in #with_all_connections --- lib/gearman/connection_pool.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/gearman/connection_pool.rb b/lib/gearman/connection_pool.rb index 9abc48d..67e2260 100644 --- a/lib/gearman/connection_pool.rb +++ b/lib/gearman/connection_pool.rb @@ -23,8 +23,6 @@ def add_connection(connection) @servers_mutex.synchronize do if connection.is_healthy? activate_connection(connection) - - @connection_handler.call(connection) if @connection_handler else deactivate_connection(connection) end @@ -64,10 +62,14 @@ def on_connection(&block) end def poll_connections(timeout = nil) + available_sockets = [] @servers_mutex.synchronize do - sockets = @job_servers.collect { |conn| conn.socket } + available_sockets.concat @job_servers.collect { |conn| conn.socket } + end + if available_sockets.size > 0 + logger.debug "Polling on #{available_sockets.size} available server(s) with a #{timeout} second timeout" + IO::select(available_sockets, nil, nil, timeout) end - IO::select(sockets, nil, nil, timeout) end def with_all_connections(&block) @@ -77,7 +79,7 @@ def with_all_connections(&block) block.call(connection) rescue NetworkError => ex logger.debug "Error with #{connection}, marking as bad" - remove_connection(connection) + deactivate_connection(connection) end end end @@ -94,6 +96,7 @@ def deactivate_connection(connection) def activate_connection(connection) @bad_servers.reject! { |c| c == connection } @job_servers << connection + @connection_handler.call(connection) if @connection_handler end def start_reconnect_thread From 7b969cd5c101c5d400c4a0329f896f236ec498b5 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 20:32:17 -0800 Subject: [PATCH 12/38] Cleanup polling timeout in sleep(), also loop should be for job_assign, not noop, fixes an incorrect symbol --- lib/gearman/worker.rb | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/lib/gearman/worker.rb b/lib/gearman/worker.rb index 071cb04..444fe43 100644 --- a/lib/gearman/worker.rb +++ b/lib/gearman/worker.rb @@ -21,7 +21,7 @@ def initialize(job_servers=nil, opts={}) @abilities = {} @client_id = opts[:client_id] || generate_id @connection_pool = ConnectionPool.new(job_servers) - @network_timeout_sec = opts[:network_timeout_sec] || 5 + @network_timeout_sec = opts[:network_timeout_sec] || 10 @reconnect_sec = opts[:reconnect_sec] || 30 @status = :preparing @worker_enabled = true @@ -165,11 +165,11 @@ def handle_work_message(type, data, connection) @status = :working run_job_assign_callback return worker_enabled if handle_job_assign(data, connection) - when :no_op + when :noop # We'll have to read again logger.debug "Received NOOP while polling. Ignoring NOOP" else - logger.error "Got unhandled #{type.to_s} from #{connection}" + logger.error "Got unexpected #{type.to_s} from #{connection}" end end @@ -181,15 +181,16 @@ def work loop do @status = :preparing + @connection_pool.with_all_connections do |connection| - logger.debug "Sending GRAB_JOB to #{connection}" - run_grab_job_callback - begin + logger.debug "Sending GRAB_JOB to #{connection}" + run_grab_job_callback type, data = connection.send_request(grab_job_req, @network_timeout_sec) handle_work_message(type, data, connection) - end while type == :no_op + end while type == :job_assign end + logger.info "Sending PRE_SLEEP and going to sleep for #{@reconnect_sec} second(s)" @connection_pool.with_all_connections do |connection| @@ -212,13 +213,18 @@ def work # Sleep and poll until timeout occurs or a NO_OP packet is received # @param time_fell_asleep The time that we fell asleep (Time object) def sleep(time_fell_asleep) - # Use IO::select to wait for available connection data - @connection_pool.poll_connections(@network_timeout_sec) + max_timeout = 30 - (Time.now - time_fell_asleep).to_i - # If 30 seconds have passed, then wakeup - time_asleep = Time.now - time_fell_asleep - @status = :wakeup if time_asleep > 30 + if max_timeout > 0 + # Use IO::select to wait for available connection data + @connection_pool.poll_connections(max_timeout) + end + # If 30 seconds have passed, then wakeup + time_asleep = (Time.now - time_fell_asleep).to_f + @status = :wakeup if time_asleep >= 30 + + # We didn't sleep for >= 30s, so we need to check for a NOOP if (@status == :waiting) @connection_pool.with_all_connections do |connection| begin From 7489dbdc51f5a4107def36d42cb7fff67a616ebb Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 20:32:41 -0800 Subject: [PATCH 13/38] Updated examples --- examples/client_reverse_wait.rb | 9 ++++----- examples/worker_ask_for_work.rb | 12 ++---------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/examples/client_reverse_wait.rb b/examples/client_reverse_wait.rb index ece2042..3df1145 100644 --- a/examples/client_reverse_wait.rb +++ b/examples/client_reverse_wait.rb @@ -1,13 +1,13 @@ #!/usr/bin/env ruby -require 'rubygems' -require '../lib/gearman' +$:.unshift File.join(File.dirname(__FILE__), '..', "lib" ) +require 'gearman' # Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/) t = nil threadcounter = 0 -client = Gearman::Client.new('localhost:4740') +client = Gearman::Client.new('localhost:4730') myid = threadcounter @@ -19,8 +19,7 @@ puts "#{jid} #{data}" task = Gearman::Task.new("reverse_string", data) task.on_complete {|d| puts d } - taskset.add_task(task) - taskset.wait(1000) + client.do_task(task) end diff --git a/examples/worker_ask_for_work.rb b/examples/worker_ask_for_work.rb index ef0637e..eda9f52 100644 --- a/examples/worker_ask_for_work.rb +++ b/examples/worker_ask_for_work.rb @@ -1,15 +1,10 @@ -require 'rubygems' -require '../lib/gearman' -l = Logger.new($stdout) -l.level = Logger::DEBUG -Gearman::Util.logger=l +$:.unshift File.join(File.dirname(__FILE__), '..', "lib" ) +require 'gearman' # String reverse worker servers = ['127.0.0.1:4730'] -client = Gearman::Client.new(servers) -taskset = Gearman::TaskSet.new(client) t = nil jobnum = 0 @@ -18,9 +13,6 @@ result = data.reverse puts "Job: #{job.inspect} Data: #{data.inspect} Reverse: #{result} " puts "Completed job ##{jobnum}" - data = rand(36**8).to_s(36) - task = Gearman::BackgroundTask.new("background_job", data) - taskset.add_task(task) jobnum += 1 result end From c7e17c0957e8459a4693a0f3c4bac83f66f048ce Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 20:33:28 -0800 Subject: [PATCH 14/38] Rename example worker script --- examples/{worker_ask_for_work.rb => worker_reverse_string.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{worker_ask_for_work.rb => worker_reverse_string.rb} (100%) diff --git a/examples/worker_ask_for_work.rb b/examples/worker_reverse_string.rb similarity index 100% rename from examples/worker_ask_for_work.rb rename to examples/worker_reverse_string.rb From ca7f9c9c0f1659638069bf69cf4eff4b06ecca24 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Sat, 29 Nov 2014 20:33:51 -0800 Subject: [PATCH 15/38] Version bump --- lib/gearman/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index b283cfd..d78236c 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = "4.0.1" + VERSION = "4.0.2" end From 9b81c41ee3bfd73a520cee9970f51e2491a46606 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 1 Dec 2014 10:12:17 -0800 Subject: [PATCH 16/38] Updated README --- README.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/README.md b/README.md index 1a95ce3..d4c6545 100644 --- a/README.md +++ b/README.md @@ -73,39 +73,22 @@ loop { w.work } * John Ewart (current maintainer, author of re-write) -<<<<<<< HEAD ## Contributors (past and present) * Kim Altintop -======= -## Contributors - ->>>>>>> New version (4.0) -- substantial rewrite * Josh Black (raskchanky) * Colin Curtin (perplexes) * Brian Cobb (bcobb) * Pablo A. Delgado (pablete) -<<<<<<< HEAD * Daniel Erat * Antonio Garrote * Stefan Kaes (skaes) * Ladislav Martincik -======= -* Stefan Kaes (skaes) ->>>>>>> New version (4.0) -- substantial rewrite * Mauro Pompilio (malditogeek) * Lee Reilly (leereilly) * Clint Shryock (catsby) * Andy Triggs (andyt) - -<<<<<<< HEAD - ## License Released under the MIT license, originally developed by XING AG. See the LICENSE file for further details. -======= -## License - -Released under the MIT license. See the file LICENSE for further details. ->>>>>>> New version (4.0) -- substantial rewrite From 963d3c5bfd00dd047ac31054093901982904c0c0 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 1 Dec 2014 10:13:35 -0800 Subject: [PATCH 17/38] Minor update to TaskSet --- lib/gearman/task_set.rb | 6 +++--- lib/gearman/version.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/gearman/task_set.rb b/lib/gearman/task_set.rb index f16852f..440cacf 100644 --- a/lib/gearman/task_set.rb +++ b/lib/gearman/task_set.rb @@ -30,7 +30,7 @@ def wait(timeout = 1) nil end - while not @tasks_in_progress.empty? + while @tasks_in_progress.length > 0 remaining = if end_time (t = end_time - Time.now.to_f) > 0 ? t : 0 else @@ -50,7 +50,7 @@ def wait(timeout = 1) @finished_tasks.each do |t| if ( (t.background.nil? || t.background == false) && !t.successful) - logger.warn "GearmanRuby: TaskSet failed" + logger.warn 'GearmanRuby: TaskSet failed' return false end end @@ -59,7 +59,7 @@ def wait(timeout = 1) # Wait for all tasks in set to finish, with no timeout def wait_forever - wait(nil) + wait(nil) end end diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index d78236c..26aa36a 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = "4.0.2" + VERSION = "4.0.3" end From d8b3156ca6913d6c79263e61d6dade9a7c2a9469 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 1 Dec 2014 14:22:36 -0800 Subject: [PATCH 18/38] Update example for logger --- examples/client.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/examples/client.rb b/examples/client.rb index e561d3b..f39084a 100755 --- a/examples/client.rb +++ b/examples/client.rb @@ -1,14 +1,21 @@ $LOAD_PATH.unshift("../lib") require 'rubygems' -require '../lib/gearman' +require 'gearman' -servers = ['localhost:4730', 'localhost:4731'] +# Control logger +l = Logger.new($stdout) +l.level = Logger::DEBUG +Gearman.logger=l + +servers = ['localhost:4730'] client = Gearman::Client.new(servers) taskset = Gearman::TaskSet.new(client) -task = Gearman::Task.new('sleep', 20) -task.on_complete {|d| puts d } +task = Gearman::Task.new('ToUpper', 'samwise') +task.on_complete {|d| puts "Upper: #{d}" } +# Add task to taskset taskset.add_task(task) -taskset.wait(100) +# Submit taskset and wait forever for completion +taskset.wait_forever From 13c4cff01eb3e2230ae88e5ca8f5f7c414663d1a Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 1 Dec 2014 14:22:57 -0800 Subject: [PATCH 19/38] Fix handling of WORK_WARNING and WORK_DATA --- lib/gearman/client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index f38cf10..ee27662 100644 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -88,7 +88,7 @@ def submit_job(task, reset_state = false, timeout = nil) end type, data = connection.read_response(remaining) handle_response(task, type, data) - end while [:work_status, :work_data].include? type + end while [:work_status, :work_data, :work_warning].include? type end else @@ -129,11 +129,11 @@ def handle_response(task, type, data) task.handle_status(numerator, denominator) when :work_warning handle, message = data.split("\0", 2) - Util.logger.debug "Got WORK_WARNING for #{handle}: '#{message}'" + logger.warn "Got WORK_WARNING for #{handle}: '#{message}'" task.handle_warning(message) when :work_data handle, work_data = data.split("\0", 2) - Util.logger.debug "Got WORK_DATA for #{handle} with #{work_data ? work_data.size : '0'} byte(s) of data" + logger.debug "Got WORK_DATA for #{handle} with #{work_data ? work_data.size : '0'} byte(s) of data" task.handle_data(work_data) else # Not good. From 69ef5c5798d0d1842dd93386ec8d63de13e16da8 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 1 Dec 2014 14:31:27 -0800 Subject: [PATCH 20/38] Cleanup examples --- examples/client.rb | 4 ++-- examples/client_background_jobs.rb | 22 ++++++++++++++++++++++ examples/client_reverse_nohost.rb | 30 ------------------------------ examples/client_reverse_string.rb | 16 ++++++++++++++++ examples/client_reverse_wait.rb | 25 ------------------------- examples/worker.rb | 2 +- examples/worker_reverse_string.rb | 5 +---- 7 files changed, 42 insertions(+), 62 deletions(-) create mode 100755 examples/client_background_jobs.rb delete mode 100644 examples/client_reverse_nohost.rb create mode 100644 examples/client_reverse_string.rb delete mode 100644 examples/client_reverse_wait.rb diff --git a/examples/client.rb b/examples/client.rb index f39084a..a53c658 100755 --- a/examples/client.rb +++ b/examples/client.rb @@ -12,8 +12,8 @@ client = Gearman::Client.new(servers) taskset = Gearman::TaskSet.new(client) -task = Gearman::Task.new('ToUpper', 'samwise') -task.on_complete {|d| puts "Upper: #{d}" } +task = Gearman::Task.new('sleep', 20) +task.on_status {|n,d| puts "Status: #{n}/#{d} iterations complete" } # Add task to taskset taskset.add_task(task) diff --git a/examples/client_background_jobs.rb b/examples/client_background_jobs.rb new file mode 100755 index 0000000..0ac2a7d --- /dev/null +++ b/examples/client_background_jobs.rb @@ -0,0 +1,22 @@ +$:.push('../lib') +require 'gearman' +require 'logger' + +logger = Logger.new(STDOUT) +logger.level = Logger::ERROR +Gearman.logger = logger + +JOB_COUNT=100000 + +client = Gearman::Client.new('localhost:4730') + +start_time = Time.now +(1..JOB_COUNT).each do |jid| + data = rand(36**8).to_s(36) + task = Gearman::BackgroundTask.new("reverse_string", data) + client.do_task(task) +end +end_time = Time.now + +diff = end_time - start_time +puts "Completed #{JOB_COUNT} jobs in #{diff} seconds, at #{JOB_COUNT.to_f / diff} JPS" diff --git a/examples/client_reverse_nohost.rb b/examples/client_reverse_nohost.rb deleted file mode 100644 index 2ca1052..0000000 --- a/examples/client_reverse_nohost.rb +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env ruby -require 'rubygems' -require '../lib/gearman' -l = Logger.new($stdout) -l.level = Logger::DEBUG -Gearman::Util.logger=l - -# Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/) - -t = nil -threadcounter = 0 - -client = Gearman::Client.new('192.168.1.1:4730') - - -myid = threadcounter -threadcounter += 1 -taskset = Gearman::TaskSet.new(client) - -(1..1000).each do |jid| - data = rand(36**8).to_s(36) - result = data.reverse - - task = Gearman::BackgroundTask.new("reverse_string", data) - puts "#{jid} #{data}" - - #time = Time.now() + rand(120) + 10 - #task.schedule(time) - taskset.add_task(task) -end diff --git a/examples/client_reverse_string.rb b/examples/client_reverse_string.rb new file mode 100644 index 0000000..f842021 --- /dev/null +++ b/examples/client_reverse_string.rb @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby +$:.unshift '../lib' +require 'gearman' + +client = Gearman::Client.new('localhost:4730') + +# Create 100 foreground jobs, one at a time +(1..100).each do |jid| + data = rand(36**8).to_s(36) + puts "#{jid} #{data}" + task = Gearman::Task.new('reverse_string', data) + task.on_complete {|d| puts d } + client.do_task(task) +end + + diff --git a/examples/client_reverse_wait.rb b/examples/client_reverse_wait.rb deleted file mode 100644 index 3df1145..0000000 --- a/examples/client_reverse_wait.rb +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env ruby -$:.unshift File.join(File.dirname(__FILE__), '..', "lib" ) -require 'gearman' - -# Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/) - -t = nil -threadcounter = 0 - -client = Gearman::Client.new('localhost:4730') - - -myid = threadcounter -threadcounter += 1 -taskset = Gearman::TaskSet.new(client) - -(1..100).each do |jid| - data = rand(36**8).to_s(36) - puts "#{jid} #{data}" - task = Gearman::Task.new("reverse_string", data) - task.on_complete {|d| puts d } - client.do_task(task) -end - - diff --git a/examples/worker.rb b/examples/worker.rb index 1c4c94e..6df2465 100755 --- a/examples/worker.rb +++ b/examples/worker.rb @@ -10,7 +10,7 @@ # Add a handler for a "sleep" function that takes a single argument, the # number of seconds to sleep before reporting success. w.add_ability("sleep") do |data,job| - seconds = 10 + seconds = data.to_i logger.info "Sleeping for #{seconds} seconds" (1..seconds.to_i).each do |i| sleep 1 diff --git a/examples/worker_reverse_string.rb b/examples/worker_reverse_string.rb index eda9f52..97ab332 100644 --- a/examples/worker_reverse_string.rb +++ b/examples/worker_reverse_string.rb @@ -1,11 +1,8 @@ -$:.unshift File.join(File.dirname(__FILE__), '..', "lib" ) +$:.unshift '../lib' require 'gearman' # String reverse worker - servers = ['127.0.0.1:4730'] - -t = nil jobnum = 0 w = Gearman::Worker.new(servers) From 2c6f4f6cb7b76aa19993a04db6a76004aa8c825c Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 1 Dec 2014 14:31:56 -0800 Subject: [PATCH 21/38] Version bump --- lib/gearman/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index 26aa36a..fe5bc11 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = "4.0.3" + VERSION = '4.0.4' end From 832fe1bc4f31c50ed7da976395bc244583e6a1ef Mon Sep 17 00:00:00 2001 From: Brian Cobb Date: Tue, 2 Dec 2014 09:32:02 -0600 Subject: [PATCH 22/38] Use connection's hostport when propagating NetworkError --- lib/gearman/client.rb | 2 +- spec/client_spec.rb | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index ee27662..bc4132c 100644 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -98,7 +98,7 @@ def submit_job(task, reset_state = false, timeout = nil) raise ProtocolError, message end rescue NetworkError - message = "Network error on read from #{hostport} while adding job, marking server bad" + message = "Network error on read from #{connection.hostport} while adding job, marking server bad" logger.error message raise NetworkError, message rescue NoJobServersError diff --git a/spec/client_spec.rb b/spec/client_spec.rb index fa50ffa..7aa71e9 100644 --- a/spec/client_spec.rb +++ b/spec/client_spec.rb @@ -51,6 +51,19 @@ end + it 'propagates NetworkErrors while submitting jobs' do + mock_connection = double(Gearman::Connection) + mock_connection.stub(:send_request).and_raise(Gearman::NetworkError) + mock_connection.stub(:hostport).and_return("localhost:4730") + + @mock_connection_pool.stub(:get_connection) { mock_connection } + + task = Gearman::Task.new("queue", "data") + + expect { + @client.submit_job(task) + }.to raise_error(Gearman::NetworkError) + end it "should raise a NetworkError when it didn't write as much as expected to a socket" do From 760a99db4771fcd4e371a2bb1a66f5a1a1f517ae Mon Sep 17 00:00:00 2001 From: John Ewart Date: Tue, 2 Dec 2014 09:55:51 -0800 Subject: [PATCH 23/38] Get rid of the connection check thread; replace with an as-needed update whenever connections are used --- lib/gearman/connection_pool.rb | 55 ++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/lib/gearman/connection_pool.rb b/lib/gearman/connection_pool.rb index 67e2260..da27fcb 100644 --- a/lib/gearman/connection_pool.rb +++ b/lib/gearman/connection_pool.rb @@ -5,6 +5,8 @@ class ConnectionPool include Logging DEFAULT_PORT = 4730 + TIME_BETWEEN_CHECKS = 60 #seconds + SLEEP_TIME = 30 #seconds def initialize(servers = []) @bad_servers = [] @@ -14,9 +16,9 @@ def initialize(servers = []) @reconnect_seconds = 10 @server_counter = 0 # Round-robin distribution of requests @servers_mutex = Mutex.new + @last_check_time = Time.now add_servers(servers) - start_reconnect_thread end def add_connection(connection) @@ -49,7 +51,6 @@ def add_servers(servers) def get_connection(coalesce_key = nil) @servers_mutex.synchronize do - logger.debug "Available job servers: #{@job_servers.inspect}" raise NoJobServersError if @job_servers.empty? @server_counter += 1 @@ -62,6 +63,7 @@ def on_connection(&block) end def poll_connections(timeout = nil) + update_job_servers available_sockets = [] @servers_mutex.synchronize do available_sockets.concat @job_servers.collect { |conn| conn.socket } @@ -73,6 +75,7 @@ def poll_connections(timeout = nil) end def with_all_connections(&block) + update_job_servers @servers_mutex.synchronize do @job_servers.each do |connection| begin @@ -88,6 +91,10 @@ def with_all_connections(&block) private + def time_to_check_connections + (Time.now - @last_check_time) >= TIME_BETWEEN_CHECKS + end + def deactivate_connection(connection) @job_servers.reject! { |c| c == connection } @bad_servers << connection @@ -99,31 +106,35 @@ def activate_connection(connection) @connection_handler.call(connection) if @connection_handler end - def start_reconnect_thread - Thread.new do - loop do - @servers_mutex.synchronize do - # If there are any failed servers, try to reconnect to them. - update_job_servers unless @bad_servers.empty? - end - sleep @reconnect_seconds - end - end.run - end - + ## + # Check for bad servers and update the available pools + # def update_job_servers + # Check if it's been > TIME_BETWEEN_CHECKS or we have no good servers + return unless time_to_check_connections || @job_servers.empty? + logger.debug "Found #{@bad_servers.size} zombie connections, checking pulse." - @bad_servers.each do |connection| - begin - message = "Testing server #{connection}..." - if connection.is_healthy? - logger.debug "#{message} Connection is healthy, putting back into service" - activate_connection(connection) - else - logger.debug "#{message} Still down." + @servers_mutex.synchronize do + @bad_servers.each do |connection| + begin + message = "Testing server #{connection}..." + if connection.is_healthy? + logger.debug "#{message} Connection is healthy, putting back into service" + activate_connection(connection) + else + logger.debug "#{message} Still down." + end end end end + + # Sleep for a few to allow a chance for the world to become sane + if @job_servers.empty? + logger.warn "No job servers available, sleeping for #{SLEEP_TIME} seconds" + sleep(SLEEP_TIME) + end + + @last_check_time = Time.now end From 0d8a3c87eeb19a565bf7f4d7323efa4d41546634 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Tue, 2 Dec 2014 09:58:55 -0800 Subject: [PATCH 24/38] Dedupe client / worker example --- examples/client_dedupe.rb | 22 ++++++++++++++++++++++ examples/worker_dedupe.rb | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100755 examples/client_dedupe.rb create mode 100755 examples/worker_dedupe.rb diff --git a/examples/client_dedupe.rb b/examples/client_dedupe.rb new file mode 100755 index 0000000..b034595 --- /dev/null +++ b/examples/client_dedupe.rb @@ -0,0 +1,22 @@ +$LOAD_PATH.unshift("../lib") +require 'rubygems' +require 'gearman' + +# Control logger +l = Logger.new($stdout) +l.level = Logger::DEBUG +Gearman.logger=l + +servers = ['localhost:4730'] + +client = Gearman::Client.new(servers) + +# Submitting work with no explicit unique ID will +# cause one to be generated by the client based on +# the job queue and the data, these 10 jobs should +# get de-duped into one job in the server (provided +# that nobody is pulling from the queue in the middle) +(0..10).each do |i| + task = Gearman::BackgroundTask.new('dedupe', 'data') + client.do_task(task) +end diff --git a/examples/worker_dedupe.rb b/examples/worker_dedupe.rb new file mode 100755 index 0000000..dbe8d77 --- /dev/null +++ b/examples/worker_dedupe.rb @@ -0,0 +1,19 @@ +$LOAD_PATH.unshift("../lib") +require 'rubygems' +require 'gearman' + +# Control logger +l = Logger.new($stdout) +l.level = Logger::DEBUG +Gearman.logger=l + +servers = ['localhost:4730'] + +worker = Gearman::Worker.new(servers) + +worker.add_ability('dedupe') do |data, job| + puts "Should only see one of these come through!" + true +end + +loop { worker.work } From e3c802127f2ae072cc4337a9ad8da0ca89aea116 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Tue, 2 Dec 2014 09:59:18 -0800 Subject: [PATCH 25/38] Version bump --- lib/gearman/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index fe5bc11..ebdc6c8 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = '4.0.4' + VERSION = '4.0.5' end From 677a6ea076d27353386f6572d1772ff0fd810bc6 Mon Sep 17 00:00:00 2001 From: Yury Druzhkov Date: Fri, 6 Feb 2015 16:37:45 +0300 Subject: [PATCH 26/38] timeout to single task --- lib/gearman/client.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index bc4132c..eb7c69f 100644 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -35,7 +35,7 @@ def set_options(opts) # # @param args A Task to complete # @return output of the task, or nil on failure - def do_task(task) + def do_task(task, timeout = nil) result = nil failed = false @@ -45,7 +45,7 @@ def do_task(task) task_set = TaskSet.new(self) if task_set.add_task(task) - task_set.wait_forever + timeout.nil ? task_set.wait_forever : task_set.wait(timeout) else raise JobQueueError, "Unable to enqueue job." end From c4c1358318b86276c0e249d070f3351c876c6dd5 Mon Sep 17 00:00:00 2001 From: Yury Druzhkov Date: Fri, 6 Feb 2015 16:42:15 +0300 Subject: [PATCH 27/38] shame --- lib/gearman/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index eb7c69f..479b0a7 100644 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -45,7 +45,7 @@ def do_task(task, timeout = nil) task_set = TaskSet.new(self) if task_set.add_task(task) - timeout.nil ? task_set.wait_forever : task_set.wait(timeout) + timeout.nil? ? task_set.wait_forever : task_set.wait(timeout) else raise JobQueueError, "Unable to enqueue job." end From cfd2149944421c0a5d5db0a8d477aad5edc1442a Mon Sep 17 00:00:00 2001 From: John Ewart Date: Wed, 11 Feb 2015 19:47:53 -0800 Subject: [PATCH 28/38] Updated change log, version bump, clean up client option requests --- CHANGELOG.md | 8 ++++++++ lib/gearman/client.rb | 24 ++++++++++++++++++------ lib/gearman/version.rb | 2 +- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95adc23..5a40e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # CHANGES +## 4.0.6 (02/11/2015) +* Clean up client options +* Merge a fix for setting timeout on single jobs +* Host-ports propagated in exceptions +* Additional examples +* Removed connection check thread +* Minor fixes + ## 4.0.0 (11/29/2014) * Significant re-write of 3.0.x branch diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index 479b0a7..5d8a7f8 100644 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -17,19 +17,31 @@ def initialize(job_servers=nil) @task_create_timeout_sec = 10 end - ## - # Set the options + + ## + # Set a single option # - # @options options to pass to the servers i.e "exceptions" - def set_options(opts) + # @opt an option to set on the server + def set_option(opt) @connection_pool.with_all_connections do |conn| - logger.debug "Send options request with #{opts}" - request = Packet.pack_request("option_req", opts) + logger.debug "Send options request with #{opt}" + request = Packet.pack_request("option_req", opt) response = conn.send_request(request) raise ProtocolError, response[1] if response[0]==:error end end + ## + # Set some options + # + # Note: singletons are coalesced into a list for backwards compatibility with this method name + # + # @options a list of options to pass to the servers i.e "exceptions" + def set_options(opts) + opt_array = [*opts] + opt_array.each { |opt| set_option(opt) } + end + ## # Perform a single task. # diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index ebdc6c8..5683611 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = '4.0.5' + VERSION = '4.0.6' end From 1f01b1b4c667484433dd3ed5063e8e61fb574fed Mon Sep 17 00:00:00 2001 From: Piavlo Date: Wed, 1 Jun 2016 12:11:16 +0300 Subject: [PATCH 29/38] use bytesize --- lib/gearman/client.rb | 2 +- lib/gearman/connection.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gearman/client.rb b/lib/gearman/client.rb index 5d8a7f8..a06eb0d 100644 --- a/lib/gearman/client.rb +++ b/lib/gearman/client.rb @@ -110,7 +110,7 @@ def submit_job(task, reset_state = false, timeout = nil) raise ProtocolError, message end rescue NetworkError - message = "Network error on read from #{connection.hostport} while adding job, marking server bad" + message = "Network error on read from #{connection.to_host_port} while adding job, marking server bad" logger.error message raise NetworkError, message rescue NoJobServersError diff --git a/lib/gearman/connection.rb b/lib/gearman/connection.rb index 656cbe7..218dcf1 100644 --- a/lib/gearman/connection.rb +++ b/lib/gearman/connection.rb @@ -133,8 +133,8 @@ def send_request(req, timeout = nil) def send_update(req, timeout = nil) len = with_safe_socket_op{ socket.write(req) } - if len != req.size - raise_exception("Wrote #{len} instead of #{req.size}") + if len != req.bytesize + raise_exception("Wrote #{len} instead of #{req.bytesize}") end end From 15e393d8f164ba431cd22882c7785f230287db81 Mon Sep 17 00:00:00 2001 From: Piavlo Date: Wed, 1 Jun 2016 12:29:33 +0300 Subject: [PATCH 30/38] fix test --- spec/client_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/client_spec.rb b/spec/client_spec.rb index 7aa71e9..838f5dc 100644 --- a/spec/client_spec.rb +++ b/spec/client_spec.rb @@ -54,7 +54,7 @@ it 'propagates NetworkErrors while submitting jobs' do mock_connection = double(Gearman::Connection) mock_connection.stub(:send_request).and_raise(Gearman::NetworkError) - mock_connection.stub(:hostport).and_return("localhost:4730") + mock_connection.stub(:to_host_port).and_return("localhost:4730") @mock_connection_pool.stub(:get_connection) { mock_connection } From 0bcafc45c7da2a2f88e6ab7b0f7688c58ffe4583 Mon Sep 17 00:00:00 2001 From: Piavlo Date: Mon, 6 Jun 2016 15:21:51 +0300 Subject: [PATCH 31/38] v bump --- lib/gearman/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gearman/version.rb b/lib/gearman/version.rb index 5683611..c1b3a17 100644 --- a/lib/gearman/version.rb +++ b/lib/gearman/version.rb @@ -1,3 +1,3 @@ module Gearman - VERSION = '4.0.6' + VERSION = '4.0.7' end From da34f818574912ef0799d56b5c2441cadce7dd97 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Wed, 8 Jun 2016 17:37:03 -0700 Subject: [PATCH 32/38] Add travis rubygems integration --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index 79b241a..922fe39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,3 +3,12 @@ rvm: - 2.1.0 - 2.0.0 - 1.9.3 + +deploy: + provider: rubygems + gem: gearman-ruby + api_key: + secure: "aH9rBIqZlR7QDxZB+hzayULMsYGcGEmwfIQ6L1hrZeluN2OTBrbuci7+ztSCWqzYEnT7M40TLrWHgmOOeEwg2blaKcArwAxB/s4HZvJR8oOPSu0TZ8qYQmr2InRfYDZCe5b9OV40XI+bqn2OHi+VfHN8HFWwQ6BCCZSSpZemVaQ=" + on: + branch: master + rvm: 2.1.0 From 19e4020f31fa7fac6f6d6624b00c821019ba8076 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Thu, 25 Apr 2019 22:59:08 -0700 Subject: [PATCH 33/38] Update CI targets --- .yourbase.yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 .yourbase.yml diff --git a/.yourbase.yml b/.yourbase.yml new file mode 120000 index 0000000..42a8056 --- /dev/null +++ b/.yourbase.yml @@ -0,0 +1 @@ +johnewart-gearman-ruby.yml \ No newline at end of file From c096ba791820387f0cd9dc773192bf4888097b8b Mon Sep 17 00:00:00 2001 From: John Ewart Date: Thu, 25 Apr 2019 23:00:45 -0700 Subject: [PATCH 34/38] Shouldn't be a symlink... --- .yourbase.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) mode change 120000 => 100644 .yourbase.yml diff --git a/.yourbase.yml b/.yourbase.yml deleted file mode 120000 index 42a8056..0000000 --- a/.yourbase.yml +++ /dev/null @@ -1 +0,0 @@ -johnewart-gearman-ruby.yml \ No newline at end of file diff --git a/.yourbase.yml b/.yourbase.yml new file mode 100644 index 0000000..3b254fc --- /dev/null +++ b/.yourbase.yml @@ -0,0 +1,15 @@ +dependencies: + build: + - ruby:2.4.5 + +build_targets: + - name: default + commands: + - gem install bundler + - bundle install + - bundle exec rake + +ci: + builds: + - name: build_gem + build_targets: default From 5afa5497abebd03a173762569ff267d0e0ed36ec Mon Sep 17 00:00:00 2001 From: John Ewart Date: Thu, 25 Apr 2019 23:02:46 -0700 Subject: [PATCH 35/38] Add build target --- .yourbase.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.yourbase.yml b/.yourbase.yml index 3b254fc..6c32cc6 100644 --- a/.yourbase.yml +++ b/.yourbase.yml @@ -8,6 +8,9 @@ build_targets: - gem install bundler - bundle install - bundle exec rake + - name: release + commands: + - echo "RELEASE" ci: builds: From 7abd3e415b76c647bd99050e666c048e40e0f841 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Thu, 25 Apr 2019 23:03:18 -0700 Subject: [PATCH 36/38] Fix build yaml --- .yourbase.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yourbase.yml b/.yourbase.yml index 6c32cc6..92f339d 100644 --- a/.yourbase.yml +++ b/.yourbase.yml @@ -15,4 +15,4 @@ build_targets: ci: builds: - name: build_gem - build_targets: default + build_target: default From 31fe16ea47bcae872b80606519ea57319390581d Mon Sep 17 00:00:00 2001 From: John Ewart Date: Fri, 30 Aug 2019 00:07:03 -0700 Subject: [PATCH 37/38] Update .yourbase.yml --- .yourbase.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.yourbase.yml b/.yourbase.yml index 92f339d..5b505d5 100644 --- a/.yourbase.yml +++ b/.yourbase.yml @@ -8,6 +8,7 @@ build_targets: - gem install bundler - bundle install - bundle exec rake + - name: release commands: - echo "RELEASE" From cdad736db997497a3d4e0e9a1f66ae37fd6826f2 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Fri, 30 Aug 2019 00:25:07 -0700 Subject: [PATCH 38/38] Update .yourbase.yml --- .yourbase.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yourbase.yml b/.yourbase.yml index 5b505d5..c701d11 100644 --- a/.yourbase.yml +++ b/.yourbase.yml @@ -1,6 +1,6 @@ dependencies: build: - - ruby:2.4.5 + - ruby:2.6.0 build_targets: - name: default