Scala PySpark Big Data in Practice: The Code and Commands That Really Matter

// Type these commands one by one in the spark-shell:

scala> import spark.implicits._
// already imported automatically, but useful if you create your own SparkSession

scala> val data = Seq(("Java", "20000"), ("Python", "100000"), ("Scala", "3000"))
// data: Seq[(String, String)] = List((Java,20000), (Python,100000), (Scala,3000))

scala> val df = data.toDF()
// df: org.apache.spark.sql.DataFrame = [_1: string, _2: string]

scala> df.show()
// +------+------+
// |    _1|    _2|
// +------+------+
// |  Java| 20000|
// |Python|100000|
// | Scala|  3000|
// +------+------+

scala> val df = data.toDF("language", "offers")
scala> df.show()
// +-------+------+
// |language|offers|
// +-------+------+
// |   Java| 20000|
// | Python|100000|
// |  Scala|  3000|
// +-------+------+

// Display the full help (list of all special commands)
scala> :help

// Step 1: type :paste and press Enter
scala> :paste
// Entering paste mode (ctrl-D to finish)

// Step 2: paste your multi-line code
val names = Seq("Alice", "Bob", "Charlie")
val rdd = sc.parallelize(names)
val upper = rdd.map(_.toUpperCase)
upper.collect()

// Step 3: press Ctrl+D to execute
// Exiting paste mode, now interpreting.

// names: Seq[String] = List(Alice, Bob, Charlie)
// rdd: org.apache.spark.rdd.RDD[String] = ...
// upper: org.apache.spark.rdd.RDD[String] = ...
// res0: Array[String] = Array(ALICE, BOB, CHARLIE)

val day = "Wednesday"

val dayType = day match {
  case "Monday" | "Tuesday" | "Wednesday" | "Thursday" | "Friday" =>
    "Weekday"
  case "Saturday" | "Sunday" =>
    "Weekend"
  case _ =>
    "Unknown day"  // _ = the “Other” box of postal sorting
}

println(dayType)  // "Weekday"

val grade = 15

val mention = grade match {
  case 20                 => "Perfect"
  case 16 | 17 | 18 | 19 => "Very good"
  case 14 | 15            => "Good"
  case 12 | 13            => "Fairly good"
  case 10 | 11            => "Pass"
  case _                  => "Fail"
}

println(mention)  // "Good"

// Scala: match is an expression, it can be assigned directly
val category = age match {
  case a if a < 18  => "Minor"
  case a if a < 65  => "Adult"
  case _            => "Senior"
}

// It can also be used as a function argument
println(age match {
  case a if a < 18 => "Minor"
  case _           => "Adult"
})

// Or inside a string interpolation
val message = s"Status: ${age match {
  case a if a < 18 => "Minor"
  case _           => "Adult"
}}"

// Java: the switch (before Java 14) does not return a value
// You had to write:
String category;
switch (age) {
  case ...: category = "Minor"; break;
  default:  category = "Adult";
}
// Java 14+ added switch expressions to fill this gap

# Python 3.10+ structural pattern matching
day = "Wednesday"
match day:
    case "Monday" | "Tuesday" | "Wednesday" | "Thursday" | "Friday":
        day_type = "Weekday"
    case "Saturday" | "Sunday":
        day_type = "Weekend"
    case _:
        day_type = "Unknown day"

# Difference: in Python, match is a statement (no direct assignment)
# In Scala: val x = value match { ... }  <-- expression that returns a value
# In Python: you must assign in each branch manually

# Python < 3.10: no match/case, we use if/elif/else
day = "Wednesday"

if day in ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday"):
    day_type = "Weekday"
elif day in ("Saturday", "Sunday"):
    day_type = "Weekend"
else:
    day_type = "Unknown day"