(Presented by Antonio Piccolboni to Strata 2012 Conference, Feb 29 2012). Rhadoop is an open source project spearheaded by Revolution Analytics to grant data scientists access to Hadoop’s scalability from their favorite language, R. RHadoop is comprised of three packages. - rhdfs provides file level manipulation for HDFS, the Hadoop file system - rhbase provides access to HBASE, the hadoop database - rmr allows to write mapreduce programs in R rmr allows R developers to program in the mapreduce framework, and to all developers provides an alternative way to implement mapreduce programs that strikes a delicate compromise betwen power and usability. It allows to write general mapreduce programs, offering the full power and ecosystem of an existing, established programming language. It doesn’t force you to replace the R interpreter with a special run-time—it is just a library. You can write logistic regression in half a page and even understand it. It feels and behaves almost like the usual R iteration and aggregation primitives. It is comprised of a handful of functions with a modest number of arguments and sensible defaults that combine in many useful ways. But there is no way to prove that an API works: one can only show examples of what it allows to do and we will do that covering a few from machine learning and statistics. Finally, we will discuss how to get involved.

1. RHadoop, Hadoop for R

3. Scholarly Activity 05-09 change
50%
37.5%
25%
12.5%
0%
-12.5%
-25%
-37.5%
R SAS SPSS S-Plus Stata

4. Scholarly Activity 05-09 change
50%
37.5%
25%
12.5%
Packages
0%
10000 -12.5%
-25%
1000 -37.5%
R SAS SPSS S-Plus Stata
100
10
1
2002 2004 2006 2008 2010

5. Scholarly Activity 05-09 change
50%
37.5%
25%
12.5%
Packages
0%
10000 -12.5%
-25%
1000 -37.5%
R SAS SPSS S-Plus Stata
100
10
http://r4stats.com/popularity
1
2002 2004 2006 2008 2010

7. David Champagne, CTO

14. f s
h d
r

15. rh d f s
rhb
ase

16. rh d f s
rhb
ase
rm
r

17. rmr

20. sapply(data, function)
mapreduce(data, map = function)

21. library(rmr)
mapreduce(…)

22. Rmr

23. Rmr
Java, C++

24. Rmr
Cascading,
Java, C++
Crunch

25. Rmr, Rhipe, Dumbo,
Rmr
Pydoop, Hadoopy
Cascading,
Java, C++
Crunch

26. Rmr, Rhipe, Dumbo,
Rmr
Pydoop, Hadoopy
Cascading,
Java, C++
Crunch

27. Expose MR Hide MR
Rmr, Rhipe, Dumbo,
Rmr
Pydoop, Hadoopy
Cascading,
Java, C++
Crunch

28. Expose MR Hide MR
Hive, Pig
Rmr, Rhipe, Dumbo,
Rmr
Pydoop, Hadoopy
Cascading,
Java, C++
Crunch

29. Expose MR Hide MR
Hive, Pig
Rmr, Rhipe, Dumbo,
Rmr Cascalog,
Pydoop, Hadoopy Scalding, Scrunch
Cascading,
Java, C++
Crunch

30. mapreduce(input, output, map, reduce)

31. mapreduce(input, output, map, reduce)

32. mapreduce(input, output, map, reduce)

33. mapreduce(input, output, map, reduce)

34. mapreduce(input, output, map, reduce)

35. mapreduce(input, output, map, reduce)

36. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

37. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

38. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

39. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

40. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

41. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

42. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

43. map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)
reduce = function(k, vv) keyval(k, length(vv))

44. condition = function(x) x > 10

45. condition = function(x) x > 10
out = mapreduce(

46. condition = function(x) x > 10
out = mapreduce(
input = input,

47. condition = function(x) x > 10
out = mapreduce(
input = input,
map = function(k,v)

48. condition = function(x) x > 10
out = mapreduce(
input = input,
map = function(k,v)
if (condition(v)) keyval(k,v))

49. condition = function(x) x > 10
out = mapreduce(
input = input,
map = function(k,v)
if (condition(v)) keyval(k,v))

50. x = from.dfs(hdfs.object)
hdfs.object = to.dfs(x)

51. INSERT OVERWRITE TABLE pv_gender_sum
SELECT pv_users.gender, count (DISTINCT pv_users.userid)
FROM pv_users
GROUP BY pv_users.gender;

52. INSERT OVERWRITE TABLE pv_gender_sum
SELECT pv_users.gender, count (DISTINCT pv_users.userid)
FROM pv_users
GROUP BY pv_users.gender;
mapreduce(input =
mapreduce(input = "pv_users",
map = function(k, v) keyval(v['userid'], v['gender']),
reduce = function(uid, genders)
lapply(unique(genders), function(g) keyval(NULL, g)),
output = "pv_gender_sum",
map = function(x, gender) keyval(gender, 1)
reduce = function(gender,counts)
keyval(k,sum(unlist(counts)))

53. kmeans =
function(points, ncenters, iterations = 10,
distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) {
newCenters = kmeans.iter(points, distfun, ncenters = ncenters)
for(i in 1:iterations) {
newCenters = kmeans.iter(points, distfun, centers = newCenters)}
newCenters}

54. kmeans =
function(points, ncenters, iterations = 10,
distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) {
newCenters = kmeans.iter(points, distfun, ncenters = ncenters)
for(i in 1:iterations) {
newCenters = kmeans.iter(points, distfun, centers = newCenters)}
newCenters}
kmeans.iter =
function(points, distfun, ncenters = dim(centers)[1], centers = NULL) {
from.dfs(
mapreduce(
input = points,
map = if (is.null(centers)) {
function(k,v) keyval(sample(1:ncenters,1),v)}
else {
function(k,v) {
distances = apply(centers, 1, function(c) distfun(c,v))
keyval(centers[which.min(distances),], v)}},
reduce = function(k,vv) keyval(NULL, apply(do.call(rbind, vv), 2, mean))),
to.data.frame = T)}

55. kmeans =
function(points, ncenters, iterations = 10,
distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) {
newCenters = kmeans.iter(points, distfun, ncenters = ncenters)
for(i in 1:iterations) {
newCenters = kmeans.iter(points, distfun, centers = newCenters)}
newCenters}
kmeans.iter =
function(points, distfun, ncenters = dim(centers)[1], centers = NULL) {
from.dfs(
mapreduce(
input = points,
map = if (is.null(centers)) {
function(k,v) keyval(sample(1:ncenters,1),v)}
else {
function(k,v) {
distances = apply(centers, 1, function(c) distfun(c,v))
keyval(centers[which.min(distances),], v)}},
reduce = function(k,vv) keyval(NULL, apply(do.call(rbind, vv), 2, mean))),
to.data.frame = T)}

56. kmeans =
function(points, ncenters, iterations = 10,
distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) {
newCenters = kmeans.iter(points, distfun, ncenters = ncenters)
for(i in 1:iterations) {
newCenters = kmeans.iter(points, distfun, centers = newCenters)}
newCenters}
kmeans.iter =
function(points, distfun, ncenters = dim(centers)[1], centers = NULL) {
from.dfs(
mapreduce(
input = points,
map = if (is.null(centers)) {
function(k,v) keyval(sample(1:ncenters,1),v)}
else {
function(k,v) {
distances = apply(centers, 1, function(c) distfun(c,v))
keyval(centers[which.min(distances),], v)}},
reduce = function(k,vv) keyval(NULL, apply(do.call(rbind, vv), 2, mean))),
to.data.frame = T)}

57. kmeans =
function(points, ncenters, iterations = 10,
distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) {
newCenters = kmeans.iter(points, distfun, ncenters = ncenters)
for(i in 1:iterations) {
newCenters = kmeans.iter(points, distfun, centers = newCenters)}
newCenters}
kmeans.iter =
function(points, distfun, ncenters = dim(centers)[1], centers = NULL) {
from.dfs(
mapreduce(
input = points,
map = if (is.null(centers)) {
function(k,v) keyval(sample(1:ncenters,1),v)}
else {
function(k,v) {
distances = apply(centers, 1, function(c) distfun(c,v))
keyval(centers[which.min(distances),], v)}},
reduce = function(k,vv) keyval(NULL, apply(do.call(rbind, vv), 2, mean))),
to.data.frame = T)}

58. #!/usr/bin/python
import sys
from math import fabs
from org.apache.pig.scripting import Pig
filename = "student.txt"
k = 4
tolerance = 0.01
MAX_SCORE = 4
MIN_SCORE = 0
MAX_ITERATION = 100
# initial centroid, equally divide the space
initial_centroids = ""
last_centroids = [None] * k
for i in range(k):
last_centroids[i] = MIN_SCORE + float(i)/k*(MAX_SCORE-MIN_SCORE)
initial_centroids = initial_centroids + str(last_centroids[i])
if i!=k-1:
initial_centroids = initial_centroids + ":"
P = Pig.compile("""register udf.jar
DEFINE find_centroid FindCentroid('$centroids');
raw = load 'student.txt' as (name:chararray, age:int, gpa:double);
centroided = foreach raw generate gpa, find_centroid(gpa) as centroid;
grouped = group centroided by centroid;
result = foreach grouped generate group, AVG(centroided.gpa);
store result into 'output';
""")
converged = False
iter_num = 0
while iter_num<MAX_ITERATION:
Q = P.bind({'centroids':initial_centroids})
results = Q.runSingle()

59. if results.isSuccessful() == "FAILED":
raise "Pig job failed"
iter = results.result("result").iterator()
centroids = [None] * k
distance_move = 0
# get new centroid of this iteration, caculate the moving distance with last iteration
for i in range(k):
tuple = iter.next()
centroids[i] = float(str(tuple.get(1)))
distance_move = distance_move + fabs(last_centroids[i]-centroids[i])
distance_move = distance_move / k;
Pig.fs("rmr output")
print("iteration " + str(iter_num))
print("average distance moved: " + str(distance_move))
if distance_move<tolerance:
sys.stdout.write("k-means converged at centroids: [")
sys.stdout.write(",".join(str(v) for v in centroids))
sys.stdout.write("]n")
converged = True
break
last_centroids = centroids[:]
initial_centroids = ""
for i in range(k):
initial_centroids = initial_centroids + str(last_centroids[i])
if i!=k-1:
initial_centroids = initial_centroids + ":"
iter_num += 1
if not converged:
print("not converge after " + str(iter_num) + " iterations")
sys.stdout.write("last centroids: [")
sys.stdout.write(",".join(str(v) for v in last_centroids))
sys.stdout.write("]n")

60. import java.io.IOException;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;
public class FindCentroid extends EvalFunc<Double> {
double[] centroids;
public FindCentroid(String initialCentroid) {
String[] centroidStrings = initialCentroid.split(":");
centroids = new double[centroidStrings.length];
for (int i=0;i<centroidStrings.length;i++)
centroids[i] = Double.parseDouble(centroidStrings[i]);
}
@Override
public Double exec(Tuple input) throws IOException {
double min_distance = Double.MAX_VALUE;
double closest_centroid = 0;
for (double centroid : centroids) {
double distance = Math.abs(centroid - (Double)input.get(0));
if (distance < min_distance) {
min_distance = distance;
closest_centroid = centroid;
}
}
return closest_centroid;
}
}

61. mapreduce(mapreduce(…

62. mapreduce(mapreduce(…
mapreduce(input = c(input1, input2), …)

63. mapreduce(mapreduce(…
mapreduce(input = c(input1, input2), …)
equijoin = function(
left.input, right.input, input,
output,
outer,
map.left, map.right,
reduce, reduce.all)

64. out1 = mapreduce(…)
mapreduce(input = out1, <xyz>)
mapreduce(input = out1, <abc>)

65. out1 = mapreduce(…)
mapreduce(input = out1, <xyz>)
mapreduce(input = out1, <abc>)
abstract.job = function(input, output, …) {
…
result = mapreduce(input = input,
output = output)
…
result}

66. input.format, output.format, format

67. input.format, output.format, format
combine

68. input.format, output.format, format
combine
reduce.on.data.frame

69. input.format, output.format, format
combine
reduce.on.data.frame
local, hadoop backends

70. input.format, output.format, format
combine
reduce.on.data.frame
local, hadoop backends
backend.parameters

71. input.format, output.format, format
combine
reduce.on.data.frame
local, hadoop backends
backend.parameters
profiling

72. input.format, output.format, format
combine
reduce.on.data.frame
local, hadoop backends
backend.parameters
profiling
verbose

76. RHADOOP USER
ONE FAT CLUSTER AVE.
HYDROPOWER CITY, OR 0x0000
RHADOOP@
REVOLUTIONANALYTICS.COM