This document provides an overview and examples of various MongoDB tools and techniques, including full-text indexing, geospatial queries, data aggregation, creating a job queue, and using tailable cursors. It demonstrates how to create indexes, perform searches, aggregate data, select and process jobs asynchronously, and consume the oplog in real-time applications. The document is intended to help readers explore the MongoDB toolbox.

1. RAIDINGTHE
MONGODB
TOOLBOX
Jeremy Mikola
jmikola

2. Agenda
Full-text Indexing
Geospatial Queries
Data Aggregation
Creating a Job Queue
Tailable Cursors

3. Full-text
Indexing

4. YouhaveanawesomePHP blog
{
"_id":ObjectId("544fd63860dab3b12521379b"),
"title":"TenSecretsAboutPSR-7YouWon'tBelieve!",
"content":"PhilSturgeoncausedquiteastironthePHP-FIG
mailinglistthismorningwhenheunanimously
passedMatthewWeierO'Phinney'scontroversial
PSR-7specification.PHP-FIGmemberswereoutraged
astheself-proclaimedGordonRamsayofPHP…",
"published":true,
"created_at":ISODate("2014-10-28T17:46:36.065Z")
}

5. We’dliketosearchthecontent
Store arrays of keyword strings
Query with regular expressions
Sync data to Solr, Elasticsearch, etc.
Create a full-text index

6. Creatingafull-textindex
$collection->createIndex(
['content'=>'text']
);
Compound indexing with other fields
$collection->createIndex(
['content'=>'text','created_at'=>1]
);
Indexing multiple string fields
$collection->createIndex(
['content'=>'text','title'=>'text']
);

7. Step1: Tokenization
[
Phil,Sturgeon,caused,quite,a,stir,
on,the,PHP-FIG,mailing,list,this,
morning,when,he,unanimously,passed,
…
]

8. Step2: Trimstop-words
[
Phil,Sturgeon,caused,quite,stir,
PHP-FIG,mailing,list,
morning,unanimously,passed,
…
]

9. Step3: Stemming
[
Phil,Sturgeon,cause,quite,stir,
PHP-FIG,mail,list,
morning,unanimous,pass,
…
]

10. Step4: Profit?

11. Queryingatextindex
$cursor=$collection->find(
['$text'=>['$search'=>'PhilSturgeon']]
);
foreach($cursoras$document){
echo$document['content']."nn";
}
↓
PhilSturgeoncausedquiteastironthePHP-FIG…
PhilJerkson,betterknownas@phpjerkonTwitter…

12. andPhrases negations
$cursor=$collection->find(
['$text'=>['$search'=>'PHP-"PhilSturgeon"']]
);
foreach($cursoras$document){
echo$document['content']."nn";
}
↓
BepreparedforthelatestandgreatestversionofPHPwith…

13. Sortingbythematchscore
$cursor=$collection->find(
['$text'=>['$search'=>'PhilSturgeon']],
['score'=>['$meta'=>'textScore']]
);
$cursor->sort(['score'=>['$meta'=>'textScore']]);
foreach($cursoras$document){
printf("%.6f:%snn",$document['score'],$document['content']);
}
↓
1.035714:PhilSturgeoncausedquiteastironthePHP-FIG…
0.555556:PhilJerkson,betterknownas@phpjerkonTwitter…

14. Supportingmultiplelanguages
$collection->createIndex(
['content'=>'text'],
['default_language'=>'en']
);
$collection->insert([
'content'=>'Weareplanningahotdogconference',
]);
$collection->insert([
'content'=>'DieKonferenzwirdWurstConbenanntwerden',
'language'=>'de',
]);
$collection->find(
['$text'=>['$search'=>'saucisse','$language'=>'fr']],
);

15. GeospatialQueries

16. Becausesomeofus ❤maps

17. Geospatialindexes
, , and
2dspherefor earth-like geometry
Supports objects
2dfor flat geometry
Legacy point format: [x,y]
Inclusion proximity intersection
GeoJSON

18. inanutshellGeoJSON
{"type":"Point","coordinates":[100.0,0.0]}
{"type":"LineString","coordinates":[[100.0,0.0],[101.0,1.0]]}
{"type":"Polygon",
"coordinates":[
[[100.0,0.0],[101.0,0.0],[101.0,1.0],[100.0,1.0],[100.0,0.0]],
[[100.2,0.2],[100.8,0.2],[100.8,0.8],[100.2,0.8],[100.2,0.2]]
]
}
{"type":"MultiPolygon",
"coordinates":[
[[[102,2],[103,2],[103,3],[102,3],[102,2]]],
[[[100,0],[101,0],[101,1],[100,1],[100,0.0]]]
]
}
{"type":"GeometryCollection",
"geometries":[{…},{…}]
}

19. ARRAYS
ARRAYSEVERYWHERE

21. Indexingsomeplaces ofinterest
$collection->insert([
'name'=>'HyattRegencySantaClara',
'type'=>'hotel',
'loc' =>[
'type'=>'Point',
'coordinates'=>[-121.976557,37.404977],
],
]);
$collection->insert([
'name'=>'In-N-OutBurger',
'type'=>'restaurant',
'loc' =>[
'type'=>'Point',
'coordinates'=>[-121.982102,37.387993],
]
]);
$collection->ensureIndex(['loc'=>'2dsphere']);

22. Inclusionqueries
//DefineaGeoJSONpolgyon
$polygon=[
'type'=>'Polygon',
'coordinates'=>[
[
[-121.976557,37.404977],//HyattRegency
[-121.982102,37.387993],//In-N-OutBurger
[-121.992311,37.404385],//Rabbit'sFootMeadery
[-121.976557,37.404977],
],
],
];
//Finddocumentswithinthepolygon'sbounds
$collection->find(['loc'=>['$geoWithin'=>$polygon]]);
//Finddocumentswithincircularbounds
$collection->find(['loc'=>['$geoWithin'=>['$centerSphere'=>[
[-121.976557,37.404977],//Centercoordinate
5/3959, //Convertmilestoradians
]]]]);

23. Sorted proximityqueries
$point=[
'type'=>'Point',
'coordinates'=>[-121.976557,37.404977]
];
//Findlocationsnearestapoint
$collection->find(['loc'=>['$near'=>$point]]);
//Findthenearest50restaurantswithin5km
$collection->find([
'loc'=>['$near'=>$point,'$maxDistance'=>5000],
'type'=>'restuarant',
])->limit(50);

25. DataAggregation

26. Wehavesomecommands forthis
aggregate
count
distinct
group
mapReduce

27. Count
$collection->insert(['code'=>'A123','num'=>500]);
$collection->insert(['code'=>'A123','num'=>250]);
$collection->insert(['code'=>'B212','num'=>200]);
$collection->insert(['code'=>'A123','num'=>300]);
$collection->count();//Returns4
$collection->count(['num'=>['$gte'=>250]]);//Returns3

28. Distinct
$collection->insert(['code'=>'A123','num'=>500]);
$collection->insert(['code'=>'A123','num'=>250]);
$collection->insert(['code'=>'B212','num'=>200]);
$collection->insert(['code'=>'A123','num'=>300]);
$collection->distinct('code');//Returns["A123","B212"]

29. Group
$collection->insert(['code'=>'A123','num'=>500]);
$collection->insert(['code'=>'A123','num'=>250]);
$collection->insert(['code'=>'B212','num'=>200]);
$collection->insert(['code'=>'A123','num'=>300]);
$result=$collection->group(
['code'=>1],//field(s)onwhichtogroup
['sum'=>0],//initialaggregatevalue
newMongoCode('function(cur,agg){agg.sum+=cur.num}')
);
foreach($result['retval']as$grouped){
printf("%s:%dn",$grouped['code'],$grouped['sum']);
}
↓
A123:1050
B212:200

30. MapReduce
Extremely versatile, powerful
Intended for complex data analysis
Overkill for simple aggregation tasks
e.g. averages, summation, grouping
Incremental data processing

31. Aggregatingqueryprofileroutput
{
"op":"query",
"ns":"db.collection",
"query":{
"code":"A123",
"num":{
"$gt":225
}
},
"ntoreturn":0,
"ntoskip":0,
"nscanned":11426,
"lockStats":{…},
"nreturned":0,
"responseLength":20,
"millis":12,
"ts":ISODate("2013-05-23T21:24:39.327Z"),
}

32. Constructingaqueryskeleton
{
"code":"A123",
"num":{
"$gt":225
}
}
↓
{
"code":<string>,
"num":{
"$gt":<number>
}
}
Aggregate stats for similar queries
(e.g. execution time, index performance)

34. Aggregationframework
Process a stream of documents
Original input is a collection
Outputs one or more result documents
Series of operators
Filter or transform data
Input/output chain
ps ax | grep mongod | head -n 1

35. Executinganaggregationpipeline
$collection->aggregateCursor([
['$match'=>['status'=>'A']],
['$group'=>['_id'=>'$cust_id','total'=>['$sum'=>'$amount']]]
]);

36. Pipelineoperators
$match
$geoNear
$project
$group
$unwind
$sort
$limit
$skip
$redact
$out

37. Aggregatingfractals

38. Solvingsymbolicequations andcalculus

39. Creating aJobQueue

40. Things nottodoinyourcontrollers
Send email messages
Upload files to S3
Blocking API calls
Heavy data processing
Mining cryptocurrency

41. Creatingajob
$collection->insert([
'data'=>[…],
'processed'=>false,
'createdAt'=>newMongoDate,
]);
$collection->createIndex(
['processed'=>1,'createdAt'=>1]
);

42. Selectingajob
$job=$collection->findAndModify(
['processed'=>false],
['$set'=>['processed'=>true,'receivedAt'=>newMongoDate]],
null,//fieldprojection(ifany)
[
'sort'=>['createdAt'=>1],
'new'=>true,
]
);
↓
{
"_id":ObjectId("54515e16ba5a4da1b15a1766"),
"data":{…},
"processed":true,
"createdAt":ISODate("2014-10-29T21:37:26.405Z"),
"receivedAt":ISODate("2014-10-29T21:37:33.118Z")
}

43. Schedulejobs inthefuture
$collection->insert([
'data'=>[…],
'processed'=>false,
'createdAt'=>newMongoDate,
'scheduledAt'=>newMongoDate(strtotime('1hour')),
]);
↓
$now=newMongoDate;
$job=$collection->findAndModify(
['processed'=>false,'scheduledAt'=>['$lt'=>$now]],
['$set'=>['processed'=>true,'receivedAt'=>$now]],
null,
[
'sort'=>['createdAt'=>1],
'new'=>true,
]
);

44. Prioritizejobselection
$collection->insert([
'data'=>[…],
'processed'=>false,
'createdAt'=>newMongoDate,
'priority'=>0,
]);
//Index:{"processed":1,"priority":-1,"createdAt":1}
↓
$now=newMongoDate;
$job=$collection->findAndModify(
['processed'=>false],
['$set'=>['processed'=>true,'receivedAt'=>$now]],
null,
[
'sort'=>['priority'=>-1,'createdAt'=>1],
'new'=>true,
]
);

45. Gracefullyhandlefailedjobs
$collection->insert([
'data'=>[…],
'processed'=>false,
'createdAt'=>newMongoDate,
'attempts'=>0,
]);
↓
$now=newMongoDate;
$job=$collection->findAndModify(
['processed'=>false],
[
'$set'=>['processed'=>true,'receivedAt'=>$now],
'$inc'=>['attempts'=>1],
],
null,
[
'sort'=>['createdAt'=>1],
'new'=>true,
]
);

46. TailableCursors

47. Cappedcollections
$database->createCollection(
'tailme',
[
'capped'=>true,
'size'=>16777216,//16MiB
'max'=>1000,
]
);

48. Producer
for($i=0;++$i;){
$collection->insert(['x'=>$i]);
printf("Inserted:%dn",$i);
sleep(1);
}
↓
Inserted:1
Inserted:2
Inserted:3
Inserted:4
Inserted:5
…

49. Consumer
$cursor=$collection->find();
$cursor->tailable(true);
$cursor->awaitData(true);
while(true){
if($cursor->dead()){
break;
}
if(!$cursor->hasNext()){
continue;
}
printf("Consumed:%dn",$cursor->getNext()['x']);
}
↓
Consumed:1
Consumed:2
…

50. Replicasetoplog
$collection->insert([
'x'=>1,
]);
↓
{
"ts":Timestamp(1414624929,1),
"h":NumberLong("2631382894387434484"),
"v":2,
"op":"i",
"ns":"test.foo",
"o":{
"_id":ObjectId("545176a14ab5c0c999da70f0"),
"x":1
}
}

51. Replicasetoplog
$collection->update(
['x'=>1],
['$inc'=>['x'=>1]]
);
↓
{
"ts":Timestamp(1414624962,1),
"h":NumberLong("5079425106850550701"),
"v":2,
"op":"u",
"ns":"test.foo",
"o2":{
"_id":ObjectId("545176a14ab5c0c999da70f0")
},
"o":{
"$set":{
"x":2
}
}
}

52. FunwithMongoDB’s oplog
Syncing MongoDB to Solr with PHP
MongoDB river plugin for Elasticsearch
Building real-time systems @ Stripe
Scalable oplog tailing @ Meteor

53. THANKS!
Questions?

54. ImageCredits
Books designed by from the
Aggregator designed by from the
Register designed by from the
Ouroboros designed by from the
Catherine Please Noun Project
stuart mcmorris Noun Project
Wilson Joseph Noun Project
Silas Reeves Noun Project
http://mariompittore.com/wp-content/uploads/2013/08/Social-Gnomes1.png