An introductory course in Scalding, which I gave at a Scala meetup.

1. TUPLESTUPLES
ALLALL
THE WAYTHE WAY
DOWNDOWN

2. HIHI

5. ABOUT THIS TALKABOUT THIS TALK
Scala
Hadoop
Cascading
Scalding

6. SCALASCALA

7. IMPLICITIMPLICIT
CONVERSIONSCONVERSIONS

8. scala> "blah and blah"
res4: java.lang.String = blah and blah
scala> "blah and blah".urlencode()
error: value urlencode is not a member of java.lang.String

9. class WrappedString(val x:String) {
def urlencode() = URLEncoder.encode(x)
}

10. implicit def stringToWrappedString(x:String): WrappedString =
new WrappedString(x)

11. scala> "blah and blah".urlencode()
res2: java.lang.String = blah+and+blah!
scala> stringToWrappedString(“blah and blah”).urlencode()
res2: java.lang.String = blah+and+blah!

12. IMPLICIT PARAMETERSIMPLICIT PARAMETERS

13. def emailSignature(name:String)(implicit organization:String) =
"Sincerely Yours,n" + name + "n" + organization
scala> implicit val organization = "Etsy, Inc."
organization: java.lang.String = Etsy, Inc.
scala> emailSignature("Giovanni Fernandez-Kincade")
res3: java.lang.String =
Sincerely Yours,
Giovanni Fernandez-Kincade
Etsy, Inc.

14. WORD COUNTWORD COUNT
FIRST CHARACTERFIRST CHARACTER
COUNTCOUNT

15. val words = List("one", "two", "three")
words
.map {
word =>
Map(word.substring(0,1) -> 1)
}
.reduce {
(a, b) =>
a ++ b.map {
mapTuple =>
val (key, count) = mapTuple
(key, a.getOrElse(key, 0) + count)
}
}

16. val words = List("one", "two", "three")
words
.map {
word =>
Map(word.substring(0,1) -> 1)
}
List(Map(o -> 1), Map(t -> 1), Map(t -> 1))

17. List(Map(o -> 1), Map(t -> 1), Map(t -> 1))
.reduce {
(a, b) =>
a ++ b.map {
mapTuple =>
val (key, count) = mapTuple
(key, a.getOrElse(key, 0) + count)
}
}
Map(o -> 1, t -> 2)

18. HADOOPHADOOP

19. ~> echo "onentwonthree" > ~/foo.txt
~> hadoop fs -put ~/foo.txt

22. SORT OFSORT OF
void map(K1 key, V1 value, OutputCollector<K2,V2> output)
void reduce(K2 key, Iterator<V2> values,
OutputCollector<K3,V3> output)

23. CASCADINGCASCADING

24. cascading.tuple.Tuple

25. val tuple = new Tuple(
1.asInstanceOf[Object],
"gigi".asInstanceOf[Object],
"miami".asInstanceOf[Object]
)
val fields = new Fields(
"user_id",
"user_name",
"location"
)
val one = new TupleEntry(
fields,
tuple
)

26. > one.get("user_id")
res4: java.lang.Comparable[_] = 1
> one.get("user_name")
res5: java.lang.Comparable[_] = gigi

27. FIRST CHARACTERFIRST CHARACTER
COUNT 2.0COUNT 2.0

29. def cascadingTuple[T](fieldName:String, entry:T): TupleEntry =
new TupleEntry(
new Fields(fieldName),
new Tuple(entry.asInstanceOf[Object])
)

30. val words = List(
cascadingTuple("word", "one"),
cascadingTuple("word", "two"),
cascadingTuple("word", "three")
)

31. words
.map {
tuple =>
cascadingTuple(
"map",
Map(tuple.getString("word").substring(0,1) -> 1)
)
}

32. .reduce {
(tupleA, tupleB) =>
val (a,b) = (
tupleA.getObject("map").asInstanceOf[Map[String,Int]],
tupleB.getObject("map").asInstanceOf[Map[String,Int]],
)
val result = a ++ b.map {
mapTuple =>
val (key, count) = mapTuple
(key, a.getOrElse(key, 0) + count)
}
cascadingTuple("map", result)
}

33. > run
fields: ['map'] tuple: ['Map(o -> 1, t -> 2)']

34. FIRST CHARACTERFIRST CHARACTER
COUNT 1.2COUNT 1.2

35. List("one", "two", "three")
.groupBy(_.substring(0,1))
res1: Map(o -> List(one), t -> List(two, three))

36. List("one", "two", "three")
.groupBy(_.substring(0,1))
.map {
(tuple) =>
val (key, value) = tuple
(key, value.size)
}
res1: Map(o -> 1, t -> 2)

37. SCALDINGSCALDING

38. import com.twitter.scalding._
class ExampleJob(args: Args)
extends Job(args) {
TextLine("data/words.txt")
.map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
.groupBy('first_character) {
(g: GroupBuilder) =>
g.size('size)
}
.write(Tsv("data/output/characters.tsv"))
}

39. class ExampleJob(args: Args)
extends Job(args) {

40. TextLine("data/words.txt")
fields: ['line'] tuple: ['one']
fields: ['line'] tuple: ['two']
fields: ['line'] tuple: ['three']

41. .map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
fields: ['line', 'first_character'] tuple: ['one', 'o']
fields: ['line', 'first_character'] tuple: ['two', 't']
fields: ['line', 'first_character'] tuple: ['three', 't']

42. .groupBy('first_character) {
(g: GroupBuilder) =>
g.size('size)
}
fields: ['first_character', 'size'] tuple: ['o', 1]
fields: ['first_character', 'size'] tuple: ['t', 2]

43. .groupBy('first_character) {
(g: GroupBuilder) =>
g.size('size)
}
fields: ['first_character', ‘size’] tuple: ['o', ‘1’]
fields: ['first_character', ‘size’] tuple: ['t', ‘2’]
.write(Tsv("data/output/characters.tsv"))
o 1
t 2

44. BEHIND THE SCENESBEHIND THE SCENES

45. TextLine("data/words.txt")
.map('line -> 'first_character) {

46. HUH?HUH?

48. class Job(val args: Args)
extends FieldConversions
with java.io.Serializable {
implicit def pipeToRichPipe(pipe: Pipe): RichPipe =
new RichPipe(pipe)
implicit def sourceToRichPipe(src: Source): RichPipe =
new RichPipe(src.read)

49. def map[A, T](fs: (Fields, Fields))
(fn: A => T)
(implicit conv: TupleConverter[A],
setter: TupleSetter[T]): Pipe

50. .map('line -> 'first_character) {
def map[A, T](fs: (Fields, Fields))
implicit def symbolToFields(x: Symbol) = {
if (x == '*) {
Fields.ALL
} else {
new Fields(x.name)
}
}

51. .map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
def map[A, T](fs: (Fields, Fields))
(fn: A => T)

52. implicit def tuple1Converter[A](implicit gA: TupleGetter[A]): TupleCo
implicit object StringGetter extends TupleGetter[String] {
override def get(tup: CTuple, i: Int) = tup.getString(i)
}
.map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
def map[A, T](fs: (Fields, Fields))
(fn: A => T)
(implicit conv: TupleConverter[A],
setter: TupleSetter[T]): Pipe
implicit def singleSetter[A]: TupleSetter[A] = new TupleSetter[A] {
override def apply(arg: A) = {
val tup = CTuple.size(1)
tup.set(0, arg)
tup
}

53. WHAT HAPPENS WHENWHAT HAPPENS WHEN
YOU RUN IT?YOU RUN IT?
> runMain com.twitter.scalding.Tool
com.giokincade.scalding.ExampleJob --local
hadoop jar foo.jar com.giokincade.scalding.ExampleJob --hdfs

54. COMPILE TIMECOMPILE TIME

55. COMPOSITION TIMECOMPOSITION TIME

58. JOB RUN TIMEJOB RUN TIME

59. TYPED PIPETYPED PIPE

60. TypedPipe.from(
TextLine("data/words.txt")
)
.map(line => line.substring(0,1))
.groupBy(character => character)
.size
.write(
TypedTsv[(String, Long)]("data/output/typed-characters.tsv")
)

61. RESOURCESRESOURCES
github.com/giokincade/scalding-talk-examples
github.com/twitter/scalding

62. MEME
gio@etsy.com
@giokincade

63. FINFIN