RubyConf Brazil 2010

OMG HELLO!
Friday, November 5, 2010

THANK YOU

Aaron Patterson

AT&T, AT&T logo and all AT&T related marks are trademarks of AT&T Intellectual Property and/or AT&T affiliated companies.

@tenderlove

nokogiri

ruby core
committer

Psych

rails core
committer

ActiveRecord

anywhere...

aaron.patterson@gmail.com

WWFMD?

Preparations
the @tenderlove way

What should
I talk about?

Yehuda?

Evan?

I will talk
about
Concurrency!

Threads!

Iterators!

Generators!

Fibers!

Coroutines!

the @tenderlove
process for
producing slides

BOOKS ARE IN
STORES NOW!

ONLY $24.99!!!

BUY TODAY!

Fun with Fibers!

Concurrency

WARNING!

fork()

fork()
foo = "hello"
fork do
foo << " world"
puts foo
end
fork do
foo << " everyone!"
puts foo
end
puts foo
Process.wait

Copy on Write

Threads

Why Use Threads?

Thread API

thread = Thread.new {
... do some work ...
}
thread.join
new and join

100.times { Thread.new { ... } }
(Thread.list - [Thread.main]).each { |t|
t.join
}
list / main

Return Value
thread = Thread.new do
10
end
thread.join
thread.value # => 10

Shared Data

n = 0
def increment(v)
v + 1
end
threads = 2.times.map {
Thread.new {
1_000_000.times { n = increment(n) }
}
}
threads.each { |x| x.join }
puts n

2,000,000?

Read
Increment
Update
v = n
v += 1
n = v
n = 2
n = 2
n = 3

Thread 1 Thread 2

Read
Read
Thread 1 Thread 2
Increment
Increment
Update
Update

Read
Read
Thread 1 Thread 2
Increment
Increment
Update
Update
n = 2
n = 3

Solutions

Locking

Mutex
n = 0
mutex = Mutex.new
def increment(v)
v + 1
end
2.times do
Thread.new {
1_000_000.times {
mutex.synchronize { n = increment(n) }
}
}
end

Mutex_m
require 'mutex_m'
class Tenderlove
include Mutex_m
end
n = 0
mutex = Tenderlove.new
mutex.synchronize { ... }

Conditional Variable

mutex = Mutex.new
resource = ConditionVariable.new
a = Thread.new {
mutex.synchronize {
puts "hello"
resource.wait(mutex)
puts "world"
}
}
b = Thread.new {
mutex.synchronize {
puts "hello1"
sleep 1
resource.signal
}
}
a.join
b.join

Thread Safe
Objects

Queue
queue = Queue.new
[
Thread.new { 100.times {
queue << "omg!"
} },
Thread.new { 100.times {
queue << "omgagain!"
} },
].each { |t| t.join }

SizedQueue

Iterators

Internal

%w{ a b c d e }.each do
...
...
end

External

list = %w{ a b }
iter = Enumerator.new(list)
iter.next # => a
iter.next # => b
iter.next # raises StopIteration

list = %w{ a b }
begin
3.times do
iter.next
end
rescue StopIteration => e
p e.result
end

list = %w{ a b }
begin
3.times do
iter.next
end
rescue StopIteration => e
p e.result # => ["a", "b"]
end

External => Internal

list = %w{ a b }
iter.each do |thing|
p thing
end

Specify a Method

list = %w{ a b }
iter = Enumerator.new(list, :map)
x = iter.each do |thing|
thing + " hello!"
end
p x

list = %w{ a b }
x = iter.each do |thing|
thing + " hello!"
end
p x # => ["a hello!", "b hello!"]

list = %w{ a b }
x = iter.each_with_index do |thing, i|
thing + ":#{i}"
end
p x

list = %w{ a b }
x = iter.each_with_index do |thing, i|
thing + ":#{i}"
end
p x # => ["a:0", "b:1"]

DRY it up

list = %w{ a b }
iter = list.enum_for(:each)
3.times do
p iter.next
end

list = %w{ a b }
iter = list.each
3.times do
p itr.next
end

list = %w{ a b }
list.map.each_with_index { |obj, i|
"#{obj} => #{i}"
}

list = %w{ a b }
list.map.each_with_index { |obj, i|
"#{obj} => #{i}"
} # => ["a => 0", "b => 1"]

Fun with Iterators!

Parallel Iteration

a = %w{ a b c d }
b = %w{ a b c d }
iter = a.each
b.all? { |v| iter.next == v }

Iteration Strategy

Tree
Node
NodeNode
NodeNode

Node
class Node < Struct.new(:children)
def initialize(children = [])
super
end
end

Bottom Up
class DepthFirst < Struct.new(:tree)
def each(&block)
iterate(tree, block)
end
private
def iterate(node, block)
node.children.each { |n|
iterate(n, block)
}
block.call node
end
end

Node
NodeNode
NodeNode
Bottom Up
Node
NodeNode
NodeNode

Top Down
class TopDown < Struct.new(:tree)
def each(&block)
iterate(tree, block)
end
private
def iterate(node, block)
block.call node
node.children.each { |n|
iterate(n, block)
}
end
end

Node
NodeNode
NodeNode
Top Down
Node
NodeNode
NodeNode

Node
class Node < Struct.new(:children)
def initialize(children = [])
super
end
def each(&block)
return enum_for(:each) unless block_given?
DepthFirst.new(self).each(&block)
end
end

Parallel Iteration
iter = tree1.each
tree2.all? { |node| node == iter.next }

To Inﬁnity, and
Beyond!

Inﬁnite Series
n → ∞

class Infinity
include Enumerable
def each
return enum_for(:each) unless block_given?
i = 0
loop do
yield i
i += 1
end
end
end

infinity = Enumerator.new do |yielder|
i = 0
loop do
yielder << i
i += 1
end
end

π
Pi

Mādhavan of Sangamagrāmam

pi = Enumerator.new do |yielder|
root_12 = Math.sqrt(12)
pi = 0
infinity.each do |k|
pi += (-1 / 3.0) ** k / (2 * k + 1).to_f
yielder << (root_12 * pi)
end
end

Generators

Iterator.is_a?
Generator

Generator Client

Generator Client
Value

Fibers

Abstract Methods

method = Proc.new { ... }
method.call # => "hello"
method.call # => "world"
method.call # => "how are you?"

method = Proc.new { ... }
method.call('1') # => "1: hello"
method.call('2') # => "2: world"
method.call('3') # => "3: how are you?"

Proc => Fiber

call => resume

method = Fiber.new { ... }
method.resume('1') # => "1: hello"
method.resume('2') # => "2: world"
method.resume('3') # => "3: how are you?"

Arbitrary Exits

method = Fiber.new do
Fiber.yield "hello"
Fiber.yield "world"
end
method.resume # => "hello"
method.resume # => "world"
method.resume # => nil
method.resume # => (FiberError)

Accept Parameters

method = Fiber.new do |val|
val = Fiber.yield "#{val}: hello"
val = Fiber.yield "#{val}: world"
val
end
method.resume(1) # => "1: hello"
method.resume(2) # => "2: world"
method.resume(3) # => 3
method.resume(4) # => (FiberError)

First "resume" is
the block argument

The rest are
returned by
Fiber.yield

Cool story bro

Callbacks -> FSM

YAML
because XML is boring

YAML
yes, I wrote nokogiri. :-P

Psych
Ruby 1.9.2 Only

class Foo < Psych::Handler
def start_document(...)
...
end
def end_document(...)
...
end
end
SAX Style Handler

Event Handler
class EventHandler
def initialize(fiber)
@fiber = fiber
end
def method_missing(name, *args)
@fiber.resume(name, args)
end
end

fiber = Fiber.new do |method, args|
stack = []
loop do
case method
when :start_mapping
stack << []
when :end_mapping
h = Hash[*stack.pop]
if stack.empty?
p h
else
stack.last << h
end
when :scalar
stack.last << args.first
end
method, args = Fiber.yield
end
end

case method
when :start_mapping
stack << []
when :end_mapping
h = Hash[*stack.pop]
if stack.empty?
stdout.resume h
else
stack.last << h
end
when :scalar
stack.last << args.first
end
method, args = Fiber.yield

StdOut Fiber
stdout = Fiber.new do |obj|
loop do
p obj
obj = Fiber.yield
end
end

Tied Together
parser = Psych::Parser.new EventHandler.new
fiber
parser.parse(<<-eoyaml)
---
- foo: { fiz: buzz }
- bar: { fiz: buzz }
- baz: { fiz: buzz }
eoyaml

Filtering Results

Foo Filter
foo_filter = Fiber.new do |hash|
loop do
stdout.resume hash if hash.key? 'foo'
hash = Fiber.yield
end
end

Output
{"foo"=>{"fiz"=>"buzz"}}

Scheduling
Threads Without Threads

P1
P2
Processes

P1
P2
Processes
fopen()
fread()
fseek()
fopen()
fopen()

Simple Example

Producer
producer = Fiber.new do
loop do
5.times do |i|
queue << "hello :#{i}"
end
Fiber.yield
end
end

Consumer
hungry = Fiber.new do
loop do
while !queue.empty?
puts queue.pop
end
Fiber.yield
end
end

Scheduler
fibers = [producer, hungry]
loop do
fibers.each do |fiber|
fiber.resume
end
end

Slow Consumer
slow = Fiber.new do
loop do
while !queue.empty?
sleep 1
puts "########### slow: " + queue.pop
Fiber.yield
end
Fiber.yield
end
end

Random Scheduler
fibers = [producer, hungry, slow]
loop do
fibers.sort_by { rand }.each do |fiber|
fiber.resume
end
end

"Processes"
•Anonymous
•We control when to "give up"

Homework
•Favor fast processes
•Favor queue ﬁllers
•Favor queue drainers
•Deal with multiple queues

Generators
The Fiber Edition

infinity = Fiber.new do
i = 0
loop do
Fiber.yield i
i += 1
end
end

Our Own
Enumerator

class SimpleEnum
def initialize &block
@fiber = Fiber.new do
block.call(self)
end
end
def << value
Fiber.yield value
end
def next
@fiber.resume
end
end

infinity = SimpleEnum.new do |yielder|
i = 0
loop do
yielder << i
i += 1
end
end

•Capture return value
•Add StopIteration Exception
Homework

Coroutines

Fiber implements
Coroutine

Coroutine Client
Value

Pipelines

Coroutine CoroutineGenerator

Our Friend, Inﬁnity
infinity = Fiber.new do
i = 0
loop do
Fiber.yield i
i += 1
end
end

Even Numbers
evens = Fiber.new do
loop do
value = infinity.resume
Fiber.yield value if value % 2 == 0
end
end

Three's
threes = Fiber.new do
loop do
value = evens.resume
Fiber.yield value if value % 3 == 0
end
end

Abstraction

Common Elements
•Source
•Sink
•Runloop
•Work

class Pipe < Struct.new(:source)
def initialize(source = nil)
super
@fiber = Fiber.new { self.loop }
end
def loop
while value = read
process value
end
end
def read; source.resume end
def write(value); Fiber.yield(value) end
def process(value); write(value) end
def resume; @fiber.resume end
end

class Infinity < Pipe
def loop
i = 0
while true
write i
i += 1
end
end
end
Inﬁnity (again)

Evens
class Evens < Pipe
def process(value)
super if value % 2 == 0
end
end

StdOut
class StdOut < Pipe
def write value
puts value
end
end

Connection
infinity = Infinity.new
even = Evens.new infinity
stdout = StdOut.new even
stdout.resume

Pretty Syntax
class Pipe
def |(other)
other.source = self
other
end
end

Connection
(Infinity.new | Evens.new | StdOut.new).resume

Producer / Consumer

Compiled Template
template = ERB.new('<%= "hello" %>')
puts template.src

Compiled Template
#coding:US-ASCII
_erbout = '';
_erbout.concat(( "hello" ).to_s);
_erbout.force_encoding(__ENCODING__)

Redirect Output

Compiled Template
erb = ERB.new('<%= "hello" %>', nil, nil, 'self')
puts erb.src

Compiled Template
#coding:US-ASCII
self = '';
self.concat(( "hello" ).to_s);
self.force_encoding(__ENCODING__)

Monkey Patch!
class ERB::Compiler
def pre_cmd= x; end
end
erb = ERB.new('<%= "hello" %>', nil, nil,
'self')
puts erb.src

Compiled Source
#coding:US-ASCII
self.concat(( "hello" ).to_s);
self.force_encoding(__ENCODING__)

ERb Pipe

ERb Pipe
class PipERB < Pipe
def initialize data
super()
@erb = ERB.new(data, nil, nil, 'self')
end
alias :concat :process
def read; @erb.result(binding) end
def force_encoding encoding; end
end

Pipe to StdOut
(PipERB.new("<%= 'hello' %>") | StdOut.new).resume

ERb Pipe + Rack

Slow Template
<ul>
<% a = b = 1 %>
<% 10.times do %>
<li><%= a %></li>
<% sleep(1) # simulate work %>
<% a, b = b, a + b %>
<% end %>
</ul>

Normal Handler
class ERBHandler
def initialize
@fib_template = DATA.read
end
def call env
result = [
ERB.new(@fib_template).result(binding)
]
[ 200, {'X-Hello' => 'World'}, result ]
end
end

Rack API
Demands "each"

Each Pipe
class Each < Pipe
def each(&block)
@block = block
resume
end
def process(value)
@block.call(value)
end
end

Each Pipe
(PipERB.new(DATA.read) | Each.new).each do |chunk|
print chunk
end
__END__
<ul>
<% a = b = 1 %>
<% 10.times do %>
<li><%= a %></li>
<% sleep 1; a, b = b, a + b %>
<% end %>
</ul>

Fiber Handler
class ERBHandler
def initialize
@fib_template = DATA.read
end
def call env
[ 200, {},
(PipERB.new(@fib_template) | Each.new)
]
end
end

Cooperative
Scheduling

ERb
Rack

Caveats?

Conclusion

Two Pipelines

Real Life?

Fibers are
Interesting!

Concurrency is
Interesting

But it requires a
shift in thinking

Install Ruby 1.9.2

Play with this at
home!

Questions?

RubyConf Brazil 2010

Empfohlen

Empfohlen

Weitere ähnliche Inhalte

Kürzlich hochgeladen

Kürzlich hochgeladen (20)

Empfohlen

Empfohlen (20)

RubyConf Brazil 2010