MongoDB's oplog is possibly its most underrated feature. The oplog is vital as the basis on which replication is built, but its value doesn't stop there. Unlike the MySQL binlog, which is poorly documented and not directly exposed to MySQL clients, the oplog is a well-documented, structured format for changes that is query-able through the same mechanisms as your data. This allows many types of powerful, application-driven streaming or transformation. At Stripe, we've used the MongoDB oplog to create PostgresSQL, HBase, and ElasticSearch mirrors of our data. We've built a simple real-time trigger mechanism for detecting new data. And we've even used it to recover data. In this talk, we'll show you how we use the MongoDB oplog, and how you can build powerful reactive streaming data applications on top of it. If you'd like to see the presentation with presenter's notes, I've published my Google Docs presentation at https://docs.google.com/presentation/d/19NcoFI9BG7PwLoBV7zvidjs2VLgQWeVVcUd7Xc7NoV0/pub Originally given at MongoDB World 2014 in New York

1. MongoDB and
the Oplog
EVAN BRODER @ebroder

2. AGENDA
INTRO TO THE OPLOG
EXAMPLE APPLICATIONS

3. INTRO
TO THE OPLOG

4. PRIMARY
SECONDARIES
APPLICATION

5. APPLICATION
save
{_id: 1, a: 2}

6. THINGS I’VE DONE:
- save {_id: 1, a: 2}

7. APPLICATION
update where
{a: 2},
{$set: {a: 3}}

8. THINGS I’VE DONE:
- save {_id: 1, a: 2}
- update {_id: 1},
{$set: {a: 3}}

9. THINGS I’VE DONE:
- save {_id: 1, a: 2}
- update {_id: 1},
{$set: {a: 3}}
- insert…
- delete…
- delete…
- save…
- update…

10. THINGS I’VE
DONE:
save
…
THINGS I’VE
DONE:

11. THINGS I’VE
DONE:
save
…
THINGS I’VE
DONE:
save
…

20. TRIGGERS

21. GOAL:
EVENT PROCESSING

25. GOAL:
DETECT INSERTIONS

27. WARNING
THIS CODE IS NOT
PRODUCTION-READY

28. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

29. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

30. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

31. oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end

32. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

33. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

34. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

35. oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

42. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end

43. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']

44. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

45. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

46. DATA
TRANSFORMATIONS

47. GOAL:
MONGODB TO POSTGRESQL

48. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

49. start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end

50. cursor.each do |op|
puts op['o']['_id']
end

51. cursor.each do |op|
case op['op']
when 'i'
puts op['o']['_id']
else
# ¯_(ツ)_/¯
end
end

52. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
else
# ¯_(ツ)_/¯
end
end

56. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

60. query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id="
query += op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

61. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') + ') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id=" + op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

62. github.com/stripe/mosql

63. github.com/stripe/zerowing

64. cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') + ') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id=" + op['o2']['_id'].inspect
else
# ¯_(ツ)_/¯
end
end

65. DISASTER
RECOVERY

66. task = collection.find_one({'finished' => nil}
# do something with task…
collection.update({'_id' => task.id},
{'$set' => {'finished' => Time.now.to_i}})

67. loop do
collection.remove(
{'finished' => {'$lt' => Time.now.to_i - 30}})
sleep(10)
end

68. evan@caron:~$ mongo
MongoDB shell version: 2.4.10
connecting to: test
normal:PRIMARY> null < (Date.now() / 1000) - 30
true

71. THINGS I’VE
DONE:
insert
delete
…
THINGS I’VE
DONE:

72. > db.getReplicationInfo()
{
"logSizeMB" : 48964.3541015625,
"usedMB" : 46116.4,
"timeDiff" : 316550,
"timeDiffHours" : 87.93,
"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",
"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",
"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"
}

73. > db.getReplicationInfo()
{
"logSizeMB" : 48964.3541015625,
"usedMB" : 46116.4,
"timeDiff" : 316550,
"timeDiffHours" : 87.93,
"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",
"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",
"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"
}

74. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == nil
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end

75. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == false
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end

76. new_oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.each do |op|
if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks'
old_task = old_tasks.find_one({'_id' => op['o']['_id']})
if old_task['finished'] == false
# found one!
# save old_task to a file, and we'll re-queue it later
end
end
old_connection['admin'].command({'applyOps' => [op]})
end
end

77. THINGS I’VE
DONE:
save
…
THINGS I’VE
DONE:
save
…

82. QUESTIONS?