Twisted is an event-driven networking engine written in Python. It provides tools for developing asynchronous network applications including an asynchronous I/O core, Deferreds (promises and pipelines), and implementations of many common protocols like HTTP, FTP, SMTP, POP3, IMAP, SSH, DNS, IRC, NNTP, XMPP, and more. Twisted also includes frameworks for application development, services, plugins, remote procedure calls (RPC), threads, databases, and a web server.

1. Twisted
Жлобич Андрей
Python Minsk Meetup – 03.2014

2. C10K problem
много соединений
*
поток на каждое
=
:(плохо

3. 1 2 3
a
X Y Z
1 a X
b c
1 поток, неблокирующий IO
3 потока, блокирующий IO
block
Y
2 b Y
block
3 c Z

4. 3 потока, блокирующий IO + GIL
1 a X
1 поток, неблокирующий IO
2 b Y 3 c Z

5. Event loop
started exit()False
sched[0][0]
<= time()
True
sched.pop(0)[1]()
rr, rw, _ = select(
reads.keys(), writes.keys(),
[], sched[0][0] – time())
for fd in rr: reads[fd](fd)
for fd in rw: writes[fd](fd)
Input / Output

6. Event loop
def mainLoop(self):
while self._started:
try:
while self._started:
self.runUntilCurrent()
t2 = self.timeout()
t = self.running and t2
self.doIteration(t)
except:
log.msg("unexpected error")
log.err()
else:
log.msg('main loop terminated')

7. Twisted is an event-driven
networking engine written
in Python

8. Twisted – это
● Асинхронное ядро (реактор)
select, poll, epoll, kqueue, cfrunloop, glib, gtk, qt, wx
● Deferred (promise + pipeline)
● Набор разнообразных абстракций
(protocol, transport, factory, endpoint, etc)
● Готовая реализация многих протоколов
(HTTP, FTP, SMTP, POP3, IMAP, SSH, DNS, IRC,
NNTP, XMPP, OSCAR, SOCKSv4, Telnet, NNTP, ...)
● Инфраструктурные решения
(демонизация, тестирование, etc)

9. from twisted.internet import reactor, protocol
from twisted.protocols.basic import LineReceiver
class MyChatProtocol(LineReceiver):
def connectionMade(self):
self.factory.clients.append(self)
def connectionLost(self, reason):
self.factory.clients.remove(self)
def lineReceived(self, line):
for c in self.factory.clients:
if c is not self:
c.sendLine(line)
class MyChatFactory(protocol.ServerFactory):
protocol = MyChatProtocol
def startFactory(self):
self.clients = []
reactor.listenTCP(1234, MyChatFactory())
reactor.run()

10. Transport – Protocol – Factory
Транспорт
прием-передача данных
tcp, udp, unix, ssl, process, loopback
Протокол
парсинг входящих данных
state уровня соединения
Фабрика
создание протоколов (reconnect etc)
state уровня приложения

11. from twisted.internet import reactor
from twisted.internet.protocol import Protocol, ClientCreator
class Greeter(Protocol):
def dataReceived(self, data):
print(data)
def send_message(self, msg):
self.transport.write("%srn" % msg)
def gotProtocol(g):
g.send_message("Hello!")
reactor.callLater(5, g.send_message, "How are you?")
reactor.callLater(20, g.send_message, "Bye.")
reactor.callLater(21, g.transport.loseConnection)
c = ClientCreator(reactor, Greeter)
c.connectTCP("localhost", 1234).addCallback(gotProtocol)
reactor.run()

12. Deferred
● Аналог Future/Promise + Pipeline
● В Deferred лежит:
– список пар (callback, errback)
– готовое значение / исключение
● Возможность приостановки
● Поддержка отмен (cancellation)
● Но нету таймаутов (раньше были)
● Печать errback в лог из __del__

13. import random
from twisted.internet import reactor, defer
def lowlevel_async_op(data, callback, errback):
if random.randint(0, 42):
reactor.callLater(3, callback, data)
else:
reactor.callLater(2, errback, Exception())
def do_request():
d = defer.Deferred()
lowlevel_async_op("123", d.callback, d.errback)
return d
def clean_data(data):
return int(data)
def process_data(data):
print("got %r" % data)
def log_error(f):
print("error: %s" % f)
d = do_request()
d.addCallback(clean_data)
d.addCallback(process_data)
d.addErrback(log_error)
d.addBoth(lambda _: reactor.stop())
reactor.run()

14. cb1 eb1
cb2 lambda x: x
lambda x: x eb3
log log
...
d.addCallbacks(cb1, eb1)
d.callback(...) d.errback(...)
d.addCallback(cb2)
d.addErrback(eb3)
d.addBoth(log)

15. return Deferred()
log log
sc1 se1
sc2 se2
se3sc3

16. def get_data():
return str(random.randint(0, 10))
def get_parsed_data():
data = get_data()
return int(data) * 2
def work_with_data():
x1 = get_parsed_data()
x2 = get_parsed_data()
print(x1 * x2)

17. def get_data():
d = defer.Deferred()
reactor.callLater(3,
lambda: d.callback(str(random.randint(0, 10))))
return d
def get_parsed_data():
def parse_data(data):
return int(data) * 2
return get_data().addCallback(parse_data)
def work_with_data():
def got_x1(x1):
def got_x2(x2):
print(x1 * x2)
return get_parsed_data().addCallback(got_x2)
return get_parsed_data().addCallback(got_x1)

18. def get_data():
return (
task.deferLater(reactor, 3, random.randint, 0, 10)
.addCallback(str))
def get_parsed_data():
return (
get_data()
.addCallback(int)
.addCallback(lambda x: x * 2))
def work_with_data():
return defer.gatherResults( # parallel!
[get_parsed_data(), get_parsed_data()]
).addCallback(
lambda xs: util.println(xs[0] * xs[1])
)

19. @defer.inlineCallbacks
def get_data():
yield task.deferLater(reactor, 3, int) # sleep
defer.returnValue(str(random.randint(0, 10)))
@defer.inlineCallbacks
def get_parsed_data():
data = yield get_data()
defer.returnValue(int(data) * 2)
@defer.inlineCallbacks
def work_with_data():
x1, x2 = yield defer.gatherResults(
[get_parsed_data(), get_parsed_data()])
print(x1 * x2)

20. ООП-интерфейсы?!
В моем любимом Python?

21. class IProtocol(zope.interface.Interface):
def dataReceived(data):
"""
Called whenever data is received.
"""
def connectionLost(reason):
"""
Called when the connection is shut down.
@type reason: L{twisted.python.failure.Failure}
"""
def makeConnection(transport):
"""
Make a connection to a transport and a server.
"""
def connectionMade():
"""
Called when a connection is made.
"""

22. twisted.internet
IAddress IConnector IResolverSimple IResolver IReactorTCP
IReactorSSL IReactorUNIX IReactorUNIXDatagram
IReactorWin32Events IReactorUDP IReactorMulticast
IReactorSocket IReactorProcess IReactorTime IDelayedCall
IReactorThreads IReactorCore IReactorPluggableResolver
IReactorDaemonize IReactorFDSet IListeningPort ILoggingContext
IFileDescriptor IReadDescriptor IWriteDescriptor
IReadWriteDescriptor IHalfCloseableDescriptor ISystemHandle
IConsumer IProducer IPushProducer IPullProducer IProtocol
IProcessProtocol IHalfCloseableProtocol IFileDescriptorReceiver
IProtocolFactory ITransport ITCPTransport IUNIXTransport
ITLSTransport ISSLTransport IProcessTransport
IServiceCollection IUDPTransport IUNIXDatagramTransport
IUNIXDatagramConnectedTransport IMulticastTransport
IStreamClientEndpoint IStreamServerEndpoint
IStreamServerEndpointStringParser
IStreamClientEndpointStringParser

23. twisted.cred
class ICredentials(Interface): ""
class IAnonymous(ICredentials): ""
class IUsernameDigestHash(ICredentials):
def checkHash(digestHash): ""
class IUsernamePassword(ICredentials):
def checkPassword(password): ""
class ICredentialsChecker(Interface):
credentialInterfaces = Attribute("")
def requestAvatarId(credentials): ""
class IRealm(Interface):
def requestAvatar(avatarId, mind, *interfaces): ""

24. class Portal:
def __init__(self, realm, checkers=()):
self.realm = realm
self.checkers = {}
for checker in checkers:
self.registerChecker(checker)
def listCredentialsInterfaces(self):
return self.checkers.keys()
def registerChecker(self, checker, *credentialInterfaces):
if not credentialInterfaces:
credentialInterfaces = checker.credentialInterfaces
for credentialInterface in credentialInterfaces:
self.checkers[credentialInterface] = checker
def login(self, credentials, mind, *interfaces):
for i in self.checkers:
if i.providedBy(credentials):
return maybeDeferred(
self.checkers[i].requestAvatarId, credentials
).addCallback(self.realm.requestAvatar, mind, *interfaces)
ifac = zope.interface.providedBy(credentials)
return defer.fail(failure.Failure(error.UnhandledCredentials(
"No checker for %s" % ', '.join(map(reflect.qual, ifac)))))

25. class ISiteUser(Interface):
def get_private_setting(self, pref_key): ""
def check_permission(self, perm):
def logout(self): ""
class IMailbox(Interface):
def load_incoming_mails(): ""
def send_mail(dest, message): ""
@implementer(IRealm)
class MyRealm(object):
def requestAvatar(self, avatarId, mind, *interfaces):
if ISiteUser in interfaces:
a = SiteUserImpl(avatarId, ...)
return ISiteUser, a, a.logout
if IMailbox in interfaces:
a = UserMailboxImpl(avatarId)
return ISiteUser, a, lambda: None
raise NotImplementedError

26. Plugin system
Разработчик приложения
● Пишем свой интерфейс
ISomeLogic
● Вызываем
twisted.plugin.getPlugins(ISomeLogic)
Разработчик плагина
● Пишем класс CustomLogic, реализующий
ISomeLogic + twisted.plugin.IPlugin
● Создаем файл twisted/plugins/xxx.py
● В файл xxx.py создаем экземпляр CustomLogic
yyy = CustomLogic()

27. Application framework
● Инициализация приложения
● Демонизация
● Выбор реактора
● Логирование
● Профилирование
● Упрощение конфигурации
● Не DI-фреймворк!
● Deprecated: персистентность приложения

28. Application framework
class IService(Interface):
def setServiceParent(parent):
""" Set the parent of the service. """
def disownServiceParent():
"""Remove L{IService} from L{IServiceCollection}."""
def privilegedStartService():
""" Do preparation work for starting the service."""
def startService():
""" Start the service. """
def stopService():
""" Stop the service. """

29. Application framework
● TCPServer, TCPClient
● SSLServer, SSLClient
● UDPServer, UDPClient
● UNIXServer, UNIXClient
● UNIXDatagramServer, UNIXDatagramClient
● StreamServerEndpointService
● MulticastServer
● TimerService
● CooperatorService

30. Application framework
Application
MultiService
MultiService
TCPClient
TCPServer
TimerServiceCustomService2
CustomService1
factory
factory

31. # file 'simple_pinger.py'
from twisted.internet import protocol
class SimplePinger(protocol.Protocol):
def connectionMade(self):
print("connection made")
self.factory.resetDelay()
self.factory.client = self
def connectionLost(self, reason):
print("connection lost")
self.factory.client = None
def ping(self, token):
self.transport.write("ping {0}rn".format(token))
class PingerFactory(protocol.ReconnectingClientFactory):
client = None
protocol = SimplePinger
counter = 0
def ping(self):
if self.client:
self.counter += 1
return self.client.ping(self.counter)

32. Application framework
# file 'pinger_1234.tac'
from twisted.application import internet, service
from simple_pinger import PingerFactory
client = PingerFactory()
timer = internet.TimerService(1, client.ping)
clcon = internet.TCPClient("localhost", 1234, client)
application = service.Application("demo")
timer.setServiceParent(application)
clcon.setServiceParent(application)
~> twistd -y pinger_1234.tac

33. twistd plugins
class IServiceMaker(Interface):
tapname = Attribute("A short string naming this Twisted plugin")
description = Attribute("A brief summary of the features")
options = Attribute("A C{twisted.python.usage.Options} subclass")
def makeService(options):
"""
Create and return an object providing
L{twisted.application.service.IService}.
"""
~> twistd [opts] <<tapname>> … [plugin-opts]
~> twistd -l bs.log procmon buggy-script.py
Например

34. Spread – perspective broker
● Удаленный вызов методов (RPC)
● Асинхронный
● Симметричный
● Своя сериализация (jelly / banana)
● Передача объектов как по ссылке
(Referencable) так и значению (Copyable)
● Передача больших объектов (Cacheable)

35. from twisted.spread import pb
from twisted.application import service, internet
class Formatter(pb.Referenceable):
def __init__(self, format_spec):
self.format_spec = format_spec
def remote_formatIt(self, value):
return format(value, self.format_spec)
class ServerObject(pb.Root):
def remote_getFormatter(self, format_spec):
return Formatter(format_spec)
application = service.Application("pb-server")
sobj = ServerObject()
bs = internet.TCPServer(8800, pb.PBServerFactory(sobj))
bs.setServiceParent(application)

36. from twisted.internet import defer, task
from twisted.spread import pb
@defer.inlineCallbacks
def main(reactor):
cf = pb.PBClientFactory()
reactor.connectTCP("localhost", 8800, cf)
root = yield cf.getRootObject()
fmt = yield root.callRemote('getFormatter', ".2f")
res = yield fmt.callRemote('formatIt', 1.2345)
print(res)
if __name__ == '__main__':
task.react(main)

37. Библиотеки
websocket, memcache, redis, riak,
couchdb, cassandra, postgresql,
amqp, stomp, solr, xmpp, oscar, msn,
snmp, smpp, ldap, webdav
…
любая асинхронная библиотека
с расширяемым реактором

38. Threads
class IReactorThreads(Interface):
def getThreadPool():
"Return the threadpool used by L{callInThread}."
def callInThread(callable, *args, **kwargs):
"Run the callable object in a separate thread."
def callFromThread(callable, *args, **kw):
"Cause a function to be executed by the reactor."
def suggestThreadPoolSize(size):
"Suggest the size of the internal threadpool."
Helpers:
● blockingCallFromThread(reactor, f, *a, **kw)
● deferToThread(f, *args, **kwargs)
● callMultipleInThread(tupleList)

39. adbapi
from twisted.enterprise import adbapi
from twisted.internet import defer, task
dbpool = adbapi.ConnectionPool('sqlite3',
"file.db", check_same_thread=False)
@defer.inlineCallbacks
def useless_work():
yield dbpool.runOperation(
"CREATE TABLE Users (name, nick)")
yield dbpool.runOperation(
"INSERT INTO Users (name, nick) VALUES (?, ?)",
["Andrei", "anjensan"])
nick = yield dbpool.runQuery(
"SELECT nick FROM Users WHERE name = ?",
["Andrei"])
print(nick)

40. adbapi
def txn_work_with_cursor(cursor, values):
# …
cursor.executemany(
"INSERT INTO Users (name, nick) VALUES (?, ?)",
values,
)
threads.blockingCallFromThread(reactor,
lambda: task.deferLater(reactor, 3, int))
# …
cursor.execute("SELECT COUNT(*) FROM Users")
return cursor.fetchone()[0]
@defer.inlineCallbacks
def useless_work2():
row = yield dbpool.runInteraction(
txn_work_with_cursor,
[("Ivan", "ivann"), ("Petr", "pettr")])
print(row)

41. Twisted Web
● HTTP server
– конейнер для WSGI (threads!)
– сервер и клиент XMLRPC
– статические файлы
– проксирование
– распределенный (pb)
● HTML шаблонный движок (асинхронный)
● HTTP клиент

42. from twisted.internet import reactor
from twisted.application import service, internet
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET, Site
import time
class ClockResource(Resource):
isLeaf = True
def render_GET(self, request):
reactor.callLater(2, self._delayed_render, request)
return NOT_DONE_YET
def _delayed_render(self, request):
request.write("%s" % time.ctime())
request.finish()
resource = ClockResource()
site = Site(resource)
application = service.Application("hello-world")
internet.TCPServer(8080, site).setServiceParent(application)

43. class ClockResource(Resource):
def _delayed_body(self, requset):
return task.deferLater(reactor, 2, lambda: str(time.ctime()))
def send_response(self, body, request):
request.write(body)
request.finish()
def fail_response(self, fail, request):
if fail.check(defer.CancelledError):
return
request.setResponseCode(501)
request.write(str(fail))
request.finish()
def response_failed(self, err, d):
d.cancel()
err.trap(error.ConnectionDone)
def render_GET(self, request):
d = self._delayed_body(request)
request.notifyFinish().addErrback(self.response_failed, d)
d.addCallback(self.send_response, request)
d.addErrback(self.fail_response, request)
return NOT_DONE_YET

44. Resource tree
Site (extends HTTPFactory)
/foo
'foo'
/bar
/bar/ham
/bar/fam/.../.....
'bar'
'ham'
'fam'
/foo/<dyn>
getChild(path)
/foo/abc
'abc'
/bar/
''
isLeaf = 1
/
''

45. from twisted.internet import reactor
from twisted.application import internet, service
from twisted.web import static, server, twcgi, wsgi, proxy
def wsgi_hello_world(environ, start_response):
start_response('200 OK', [('Content-type','text/plain')])
return ["Hello World!"]
root = static.File("/var/www/htdocs")
root.putChild("about", static.Data("lorem ipsum", 'text/plain'))
root.putChild("doc", static.File("/usr/share/doc"))
root.putChild("cgi-bin", twcgi.CGIDirectory("/var/www/cgi-bin"))
root.putChild("tut.by", proxy.ReverseProxyResource("tut.by", 80, ""))
hw_resource = wsgi.WSGIResource(
reactor, reactor.getThreadPool(), wsgi_hello_world)
root.putChild("hello-world", hw_resource)
site = server.Site(root)
ss = internet.TCPServer(8080, site)
application = service.Application('web')
ss.setServiceParent(application)

46. ~> twistd web --wsgi my_module.wsgi_app
~> twistd web --path ~/my/directory/
Запуск WSGI приложения
Отдача статических файлов + *.{rpy,epy}
~> twistd web --resource-script myfile.rpy
Запуск отдельного Resource (дебаг) через *.rpy
~> twistd web --class my_module.ResourceClass
Запуск отдельного Resource (дебаг) через класс

47. import sys
from StringIO import StringIO
from twisted.internet import reactor, task, defer
from twisted.web import client
@defer.inlineCallbacks
def send_post(reactor, url, data):
agent = client.Agent(reactor)
request_body = client.FileBodyProducer(StringIO(data))
response = yield agent.request(
'POST', url, body_producer=request_body)
response_body = yield client.readBody(response)
print("code:", response.code)
print("response_body:", response_body)
if __name__ == '__main__':
task.react(send_post, sys.argv[1:])
HTTP клиент

48. import sys
from twisted.internet import defer, task
from twisted.web import template
from twisted.python.filepath import FilePath
class MyElement(template.Element):
loader = template.XMLFile(FilePath("template.xml"))
@template.renderer
def title(self, request, tag):
return tag("Title")
@template.renderer
def widgets(self, request, tag):
for wn in ["first", "second", "third"]:
yield task.deferLater(
reactor, 1,
lambda: tag.clone().fillSlots(widget_name=wn))
def main(reactor):
return template.flatten(None, MyElement(), sys.stdout.write)
task.react(main)
<p xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<p t:render="title" />
<ul>
<li t:render="widgets">
<b><t:slot name="widget_name"/></b>
</li>
</ul>
</p>
Twisted web templates

49. Twisted projects
● Core – Asynchronous event loop and networking framework.
● Web – An HTTP protocol implementation.
● Conch – An SSH and SFTP protocol implementation.
● Mail – An SMTP, IMAP and POP protocol implementation.
● Names – A DNS protocol implementation with client and server.
● News – An NNTP protocol implementation with client and server.
● Words – Chat and Instant Messaging.
● Runner – Process management, including an inetd server.
● Pair – Low-level networking transports and utilities.
● Trial – Unittest-compatible automated testing.

50. In a Nutshell, Twisted...
has had 16,823 commits made by 86 contributors
representing 196,503 lines of code
took an estimated 51 years of effort (COCOMO model)
starting with its first commit in July, 2001
http://ohloh.net/p/twisted