YouTube: https://www.youtube.com/watch?v=gtjzjiTI96c At Netflix we run hundreds of A/B tests every year. Maintaining multivariate experiences quickly adds strain to any UI engineering team. In this talk, Alex Liu explores the patterns we’ve built in Node.js to tame this beast - ultimately enabling quick feature development and rapid test iteration on our service used by over 50 million people around the world. This is the condensed 20 minute version given at NodeConfEu 2014.

1. Scaling A/B testing
on Netflix.com with
_________ Alex Liu
@stinkydofu

2. data driven
product development

6. Test 1 Test 2 Test 3 Test 4 Test 5 Test 6 Test 7
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G

7. 2,097,152
unique experiences across
seven tests

8. hundreds
of new A/B tests per year

9. 43351892955034
94860861172181
85493567650…
72061153709996

10. 2105 566 685
templates CSS JS

11. 2.5M
unique packages
every week

12. problem:
conditional dependencies

13. Packaging

15. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

16. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

17. app.js
import jquery from 'jquery';
import oldSearch from 'oldSearch';
import newSearch from 'newSearch';
export ...

18. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

20. 685 files…?
2.5M packages…?

21. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

22. problem:
conditional dependencies

24. build request

25. require('nf-include-when')

26. oldSearch.js
/*
* @includewhen rule.notInNewSearch
*/

27. newSearch.js
/*
* @includewhen rule.inNewSearch
*/

28. anatomy of a rule
var Rule = require('nf-rule-infrastructure'),
inNewSearch;
inNewSearch = new Rule('inNewSearch', function(context, cb) {
var test = context.abtests.get(1534);
cb(test && test.cell(1));
});
module.exports = inNewSearch;

29. require('nf-asset-registry')

30. app.js
import jquery from 'jquery';
import oldSearch from 'oldSearch';
import newSearch from 'newSearch';
export ...

31. app.js
jquery
oldSearch.js
newSearch.js
registry

32. "app.js": {
"deps": [
"jquery",
"oldSearch.js",
"newSearch.js",
],
"depsFull": [
"jquery",
"oldSearchDep2.js",
"oldSearchDep1.js",
"oldSearch.js",
"newSearchDep2.js",
"newSearchDep1.js",
"newSearch.js"
],
"fileSize": "4.41 kB",
"fileSizeFull": "120.52 kB"
}

33. "newSearch.js": {
"rule": "inNewSearch",
"deps": [
"jquery",
"newSearchDep2.js",
"newSearchDep1.js",
],
"depsFull": [
"jquery",
"newSearchSubDep3.js",
"newSearchSubDep2.js"
"newSearchSubDep1.js"
"newSearchDep2.js",
"newSearchDep1.js"
],
"fileSize": "10.41 kB",
"fileSizeFull": "40.52 kB"
}
nf-include-when

34. require('nf-packager')

35. var packager = require('nf-packager'),
includeWhen = require('nf-include-when'),
registries = require('nf-asset-registry');
function getScriptUrl()
return packager.getPackageDefinition('app.js',
registries,
includeWhen);
}

36. "app.js": {
"deps": [
"jquery",
"oldSearch.js",
"newSearch.js",
],
"depsFull": [
"jquery",
"oldSearchDep2.js",
"oldSearchDep1.js",
"oldSearch.js",
"newSearchDep2.js",
"newSearchDep1.js",
"newSearch.js"
],
"fileSize": "4.41 kB",
"fileSizeFull": "120.52 kB"
}
Step 1:
Get the full dependency tree for the
requested package from the registry.

37. [
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 2:
Determine which files have rules.

38. [
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 3:
Run the rules. Filter out all deps that
resolved false.
✓

39. [
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 4:
Filter out all extraneous
sub deps.
✓

40. Step 5:
Concatenate the files.
[
"jquery", /* no rule */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]

41. build
javascript
registry

42. request registry
packager rules

43. Bonus Round

44. be creative
with the registry

45. "account/bb/models/ratingHistoryModel.js": {
"rule": null,
"deps": [...],
"depsFull": [...],
"depsCount": {
"underscore": 2,
"backbone": 1,
"jquery": 2,
"common/requirejs-plugins.js": 4,
"requirejs-text": 4,
"utils/contextData.js": 1,
"common/nfNamespace.js": 1
},
"hash": "dd23b163",
"fileSize": "1.21 kB",
"fileSizeFull": "173.04 kB"
}
dependency
counting
dependency
pruning
file sizes

46. @import modules
@import (reference) "/common/_nf_defs.less";
@import (reference) "/member/memberCore.less";
@import (reference) "/components/menu.less";
@import (reference) "/components/breadcrumbs.less";

47. "account/containerResponsive.css": {
"rule": null,
"deps": [...],
"depsFull": [...],
"depsCount": [...],
"hash": "65a431f3",
"fileSize": "709 B",
"fileSizeFull": "709 B",
"css": {
"selectors": 8,
"declarationBlocks": 6,
"declarations": 17,
"mediaQueries": 3
}
}
css
analysis

48. the
best part

49. "cache": {
"account/pin.js": "define('account/pin.js', ['member/memberC…",
"account/bb/models/changePlanModel.js": "define('account/b…",
"account/bb/models/ratingHistoryModel.js": "define('account…",
"account/bb/models/viewingActivityModel.js": "define('account…",
"account/bb/views/changePlanView.js": "define('account/bb/vi…",
"account/bb/views/changePlanView.js": "define('account/bb/vi…",
"account/bb/views/emailSubView.js": "define('account/bb/views…",
"account/bb/views/viewingActivityView.js": "define('account…",
"common/UITracking.js": "define('common/UITracking.js, ['me…",
"common/UITrackingOverlay.js": "define('common/UITrackingOve…",
…
…
…

50. templates templates
mappings
javascript
css
mappings
javascript
css

51. templates
mappings
javascript
css
UI Bundle

53. deploy UI bundles
anytime

54. never
touch the file system

55. < 5ms
package response times

56. Takeaways
▶ leverage the server
▶ static analysis FTW
▶ divide and conquer with modules

57. Our Learnings

58. learn by doing

59. fail fast
move faster

60. “I have not failed.
I’ve just found 10,000 ways
that won’t work.”
Thomas Edison

61. simplify

62. thanks! come find us!
Alex Liu
@stinkydofu
Chris Saint-Amant
@csaintamant
Micah Ransdell
@mjr578
Kris Baxter
@kristoferbaxter