A survey of tools and techniques for automated testing of Python projects. Given at the Portland Python Users Group, 2009-08-11.

1. Testing My Patience
Automated Testing for Python Projects
Adam Lowry
Portland Python Users Group
2009-08-11

2. • Unit Testing
• Integration/Functional Testing
• Web Testing

3. Maurice Koop: http://www.flickr.com/photos/mauricekoop/1491529630/
Avoid This

4. michelle_p: http://www.flickr.com/photos/catalan/3656255672/
Unit Testing

5. unittest
import unittest
class TestMath(unittest.TestCase):
def test_addition(self):
"""Simple addition must be correct."""
self.assertEquals(2 + 2, 4)
assert 3 + 3 == 6
if __name__ == '__main__':
unittest.main()

6. Running
adam@rye:~/projects/pdxpython$ python test-examples.py --verbose
Simple addition must be correct. ... ok
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK

7. nose
for sanity’s sake
adam@rye:~/projects/pdxpython$ nosetests -v
Simple addition must be correct. ... ok
----------------------------------------------------------------------
Ran 1 test in 0.010 seconds
OK
(?:^|[b_.-])[Tt]est

8. py.test
worth considering
(pdxpython)adam@rye:~/projects/pdxpython$ py.test -v test-examples.py
============================= test session starts ==============================
python: platform darwin -- Python 2.5.2 -- /Users/adam/projects/pdxpython/bin/
python2.5 -- pytest-1.0.0
test object 1: /Users/adam/projects/pdxpython/test-examples.py
test-examples.py:6: TestMath.test_addition PASS
=========================== 1 passed in 0.03 seconds ===========================

9. import nose
def test_add():
assert 2 + 2 == 3
def test_add2():
nose.tools.assert_equals(2 + 2, 3)
(pdxpython)adam@rye:~/projects/pdxpython$ nosetests -v nose-examples.py
nose-examples.test_add ... FAIL
nose-examples.test_add2 ... FAIL
======================================================================
FAIL: nose-examples.test_add
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/adam/projects/pdxpython/lib/python2.5/site-packages/nose-0.11.1-py2.5.egg/nose/case.py",
line 183, in runTest
self.test(*self.arg)
File "/Users/adam/projects/pdxpython/nose-examples.py", line 5, in test_add
assert 2 + 2 == 3
AssertionError
======================================================================
FAIL: nose-examples.test_add2
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/adam/projects/pdxpython/lib/python2.5/site-packages/nose-0.11.1-py2.5.egg/nose/case.py",
line 183, in runTest
self.test(*self.arg)
File "/Users/adam/projects/pdxpython/nose-examples.py", line 9, in test_add2
nose.tools.assert_equals(2 + 2, 3)
AssertionError: 4 != 3
----------------------------------------------------------------------
Ran 2 tests in 0.005 seconds
FAILED (failures=2)

10. class TestMath(unittest.TestCase):
def setUp(self):
self.target = 4
def test_addition(self):
"""Simple addition must be correct."""
self.assertEquals(2 + 2, self.target)
assert 3 + 3 == 6
target = None
def set_target():
global target
target = 4
@nose.with_setup(set_target)
def test_add():
nose.tools.assert_equals(2 + 2, target)

11. Doctest
class Outer(object):
"""Outside worker
>>> outer = Outer()
>>> outer.outer_work()
True
"""
def __init__(self):
self.inner = Inner()
def outer_work(self):
try:
return self.inner.inner_work()
except Exception:
return False
(pdxpython)adam@rye:~/projects/pdxpython$ nosetests -v --with-doctest worker.py
Doctest: worker.Outer ... ok
----------------------------------------------------------------------
Ran 1 test in 0.006 seconds
OK

12. Mocks and Stubs
http://martinfowler.com/articles/mocksArentStubs.html
Sakurako Kitsa Andrei Z
http://www.flickr.com/photos/kitsa_sakurako/1117063708/ http://www.flickr.com/photos/andreiz/330632238/

13. Mock:
“action→assertion”
Mox:
“record→replay”

14. class Inner(object):
def inner_work(self):
return True
class Outer(object):
def __init__(self):
self.inner = Inner()
def outer_work(self):
try:
return self.inner.inner_work()
except Exception:
return False
Goal: test Outer isolated from Inner

15. mock.patch()
@mock.patch('worker.Inner.inner_work', lambda x: True)
def test_outer_w_mock():
outer = worker.Outer()
nose.tools.assert_equal(outer.outer_work(), True)
@mock.patch('worker.Inner.inner_work', lambda x: False)
def test_outer_inner_fails_w_mock():
outer = worker.Outer()
nose.tools.assert_equal(outer.outer_work(), False)

16. mox.StubOutWithMock()
def test_outer_w_mox():
moxer = mox.Mox()
moxer.StubOutWithMock(worker.Inner, 'inner_work')
worker.Inner.inner_work().AndReturn(True)
moxer.ReplayAll()
outer = worker.Outer()
nose.tools.assert_equal(outer.outer_work(), True)
moxer.VerifyAll()
moxer.UnsetStubs()

17. class TestOuterWMox(mox.MoxTestBase):
def test_outer_fails(self):
self.mox.StubOutWithMock(worker.Inner, 'inner_work')
worker.Inner.inner_work().AndReturn(False)
self.mox.ReplayAll()
outer = worker.Outer()
nose.tools.assert_equal(outer.outer_work(), False)
self.mox.VerifyAll()

18. Testing Exceptions
def raiser(obj):
raise Exception("Oops!")
@mock.patch('worker.Inner.inner_work', raiser)
def test_outer_exc_w_mock():
outer = worker.Outer()
nose.tools.assert_equal(outer.outer_work(), False)
def test_outer_exception(self):
self.mox.StubOutWithMock(worker.Inner, 'inner_work')
worker.Inner.inner_work().AndRaise(Exception)
self.mox.ReplayAll()
outer = worker.Outer()
nose.tools.assert_equal(outer.outer_work(), False)
self.mox.VerifyAll()

19. This is your application
AriCee: http://www.flickr.com/photos/aricee/64358127/

20. class User(object):
"""User object, connected to the user_table
>>> user = User(password='password')
>>> db.session.flush()
>>> user # doctest: +ELLIPSIS
<User ...>
>>> user.is_anonymous()
False
>>> User.query.filter_by(id=user.id).one() is user
True
>>> user.id
1
"""
def __init__(self, **kwargs):

21. Solution: Custom Nose
Plugin
from nose.plugins import Plugin
class RollbackPlugin(Plugin):
"""Rollback transaction after each test"""
name = 'rollback'
def beforeTest(self, test):
from myapp import test_utils
test_utils.set_up_app()
def afterTest(self, test):
from myapp import db
db.session.rollback()
db.session.close()

22. setup.py
try:
import ez_setup
ez_setup.use_setuptools()
except ImportError:
pass
from setuptools import setup
setup(
name='RollbackPlugin',
description='Nose plugin to rollback transaction after each test',
py_modules=['rollback'],
version='0.1',
zip_safe=False,
entry_points={
'nose.plugins': [
'rollback = rollback:RollbackPlugin'
]
}
)
nosetests -v --with-doctest --with-rollback

23. Integration/Functional
Testing
fotoopa: http://www.flickr.com/photos/fotoopa_hs/3103153894/

24. # in tests/__init__.py
server_proc = None
def setup_package():
global server_proc
subprocess.call("dropdb myapp-test")
subprocess.call("createdb myapp-test")
server_proc = subprocess.Popen("myapp/manage.py runserver --noreload")
def teardown_package():
server_proc.kill()

25. Web Testing

26. Twill
go $home
code 200
find "Don't have an account yet?" # login screen
fv 1 username $username
fv 1 password $password
submit
code 200
notfind 'Sorry'
from twill.commands import *
go(home)
code(200)
find("Don't have an account yet?") # login screen
fv(1, "username", username)
fv(1, "password", password)
submit()
code(200)
notfind('Sorry')

27. Twill Plugins
def enter_password(email, password):
"""Enters email and password in login form."""
import twill.commands as cmds
cmds.fv('email', 'email', email)
cmds.fv('email', 'password', password)
cmds.fv('email', 'has_password', 'yes')
cmds.submit()
cmds.code(200)
twill.commands.extend_with('myapp.twill_extensions')

28. Javascript?
Windmill

29. References
http://docs.python.org/library/unittest.html
http://docs.python.org/library/doctest.html
nose: http://www.somethingaboutorange.com/mrl/projects/nose/
py.test: http://codespeak.net/py/dist/test/test.html
Mock: http://www.voidspace.org.uk/python/mock/
Mox: http://code.google.com/p/pymox/
Twill: http://twill.idyll.org/
Windmill: http://www.getwindmill.com/

30. Thanks!
• http://adam.therobots.org/
• http://twitter.com/robotdam
• http://urbanairship.com
• adam@therobots.org