Scaling Up & Out. Haidar Osman
|
|
|
- Ilene Cole
- 5 years ago
- Views:
Transcription
1 Scaling Up & Out Haidar Osman
2 1- Crash course in Scala - Classes - Objects 2- Actors - The Actor Model - Examples I, II, III, IV 3- Apache Spark - RDD & DAG - Word Count Example 2
3 1- Crash course in Scala - Classes - Objects 2- Actors - The Actor Model - Examples I, II, III, IV 3- Apache Spark - RDD & DAG - Word Count Example 3
4 public class Person { private String name; private String address; public Person(String name, String address) { this.name = name; this.address = address; System.out.println("Person object is constructed"); public String getname() { return name; public String getaddress() { return address;
5 public class Person { private String name; private String address; public Person(String name, String address) { this.name = name; this.address = address; System.out.println("Person object is constructed"); public String getname() { return name; public String getaddress() { return address; class Person {
6 public class Person { private String name; private String address; public Person(String name, String address) { this.name = name; this.address = address; System.out.println("Person object is constructed"); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null
7 public class Person { private String name; private String address; public Person(String name, String address) { this.name = name; this.address = address; System.out.println("Person object is constructed"); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null def this(name: String, address: String) { this() this.name = name this.address = address println("person object is constructed")
8 public class Person { private String name; private String address; public Person(String name, String address) { this.name = name; this.address = address; System.out.println("Person object is constructed"); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null def this(name: String, address: String) { this() this.name = name this.address = address println("person object is constructed") def getname(): String = name def getaddress(): String = address
9 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null def this(name: String, address: String) { this() this.name = name this.address = address println("person object is constructed") def getname(): String = name def getaddress(): String = address
10 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null private val welcomenote: String = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname(): String = name def getaddress(): String = address
11 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null private val welcomenote: String = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname(): String = name def getaddress(): String = address
12 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null private val welcomenote: String = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname: String = name def getaddress: String = address
13 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null private val welcomenote: String = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname: String = name def getaddress: String = address
14 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null private val welcomenote = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname = name def getaddress = address
15 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { private var name: String = null private var address: String = null private val welcomenote = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname = name def getaddress = address
16 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { var name: String = null var address: String = null val welcomenote = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname = name def getaddress = address
17 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person { var name: String = null var address: String = null val welcomenote = "Person object is constructed" def this(name: String, address: String) { this() this.name = name this.address = address println(welcomenote) def getname = name def getaddress = address
18 public class Person { private String name; private String address; private final String welcomenote = "Person object is constructed"; public Person(String name, String address) { this.name = name; this.address = address; System.out.println(welcomeNote); public String getname() { return name; public String getaddress() { return address; class Person (var name: String, var address: String){ val welcomenote = "Person object is constructed" println(welcomenote)
19 class Person (var name: String, var address: String){ val welcomenote = "Person object is constructed" println(welcomenote)
20 class Person (var name: String, var address: String){ Person.welcome object Person { def welcome = { val welcomenote = "Person object is constructed" println(welcomenote)
21 public class MainClass { public static void main(string[] args){ System.out.println("Hello World"); object MainClass { def main(args: Array[String]): Unit = { println("hello World")
22 class Person (var name: String, var address: String){ Person.welcome object Person { def welcome = { val welcomenote = "Person object is constructed" println(welcomenote)
23 case class Person (var name: String, var address: String){ Person.welcome object Person { def welcome = { val welcomenote = "Person object is constructed" println(welcomenote)
24 case class Person (var name: String, var address: String){ Person.welcome object Person { def welcome = { val welcomenote = "Person object is constructed" println(welcomenote) object MainClass { def main(args: Array[String]): Unit = { val person = new Person("John","My Address") person match { case Person("John", address) => println("john's adress: " + address) case _ => println("not John")
25 1- Crash course in Scala - Classes - Objects 2- Actors - The Actor Model - Examples I, II, III, IV 3- Apache Spark - RDD & DAG - Word Count Example 25
26 What are actors? > The Actors Model is another model for expressing computational problems. It is equivalent to Turing Machine and Lambda Calculus > Actors are not new. It is a solid and scientifically robust idea (Carl Hewitt 1973). 26
27 What are actors? Actors are fundamental units of computation that embody: Processing (behavior) Storage (state) Communication (messages) When an actor receives a message, it can: create more actors send messages to other actors designate what to do with the next message Actor Behavior State 27
28 Rules of bare-metal actors > Every actor has an address. > An actor can process one message at a time. > Everything in an actor system is an actor. > Message delivery is best-effort > There are no guarantees that messages are received in the same order they are sent. > Messages to actors should always be immutable to avoid shared state. 28
29 Example 1 - Simple Actor Definition import akka.actor.actor class Counter extends Actor { var count = 0 def receive = { case "increase" => count += 1 case "get" => sender()! count case _ => println("unknown message") 29
30 Example 1 - Simple Actor Definition import akka.actor.actor class StatelessCounter extends Actor { def count(n:int): Receive = { case "increase" => context.become(count(n+1)) case "get" => sender()! n def receive = count(0) 30
31 Example 1 - Simple Actor Definition import akka.actor.actor class StatelessCounter extends Actor { def count(n:int): Receive = { case "increase" => context.become(count(n+1)) case "get" => sender()! n def receive = count(0) for the next message, receive = count(n+1) 31
32 Example 1 - Simple Actor Definition import akka.actor.actor class StatelessCounter extends Actor { def count(n:int): Receive = { case "increase" => context.become(count(n+1)) case "get" => sender()! n def receive = count(0) type Receive = PartialFunction[Any, Unit] 32
33 Example 1 - Simple Actor Definition import akka.actor.{actorsystem, Props object Application extends App { val system = ActorSystem("SimpleActorSystem") val counter = system.actorof(props[counter], "counter") counter! "increase" counter! "increase" counter! "get" 33
34 Example 2 - Ping Pong import akka.actor.{actor, ActorRef, ActorSystem, PoisonPill, Props import scala.language.postfixops case object Ping case object Pong class Pinger extends Actor { var countdown = 5 def receive = { case Pong => println(s"${self.path received pong, count down $countdown") if (countdown > 0) { countdown -= 1 sender()! Ping else { sender()! PoisonPill self! PoisonPill 34
35 Example 2 - Ping Pong class Ponger(pinger: ActorRef) extends Actor { def receive = { case Ping => println(s"${self.path received ping") pinger! Pong object PingPongApplication extends App { val system = ActorSystem("PingPong") val pinger = system.actorof(props[pinger], "pinger") val ponger = system.actorof(props(classof[ponger], pinger), "ponger") ponger! Ping 35
36 Example 2 - Ping Pong akka://pingpong/user/ponger received ping akka://pingpong/user/pinger received pong, count down 5 akka://pingpong/user/ponger received ping akka://pingpong/user/pinger received pong, count down 4 akka://pingpong/user/ponger received ping akka://pingpong/user/pinger received pong, count down 3 akka://pingpong/user/ponger received ping akka://pingpong/user/pinger received pong, count down 2 akka://pingpong/user/ponger received ping akka://pingpong/user/pinger received pong, count down 1 akka://pingpong/user/ponger received ping akka://pingpong/user/pinger received pong, count down 0 36
37 Example 3 - Approximating π using Leibniz formula = π / 4 π = 4 * (1-1/3 + 1/5-1/7 + 1/9-1/11 + 1/13-1/15 + ) 37
38 Example 3 - Approximating π using Leibniz formula = π / 4 π = 4 * (1-1/3 + 1/5-1/7 + 1/9-1/11 + 1/13-1/15 + ) 38
39 Example 3 - Designing The Actor System Master Workers 39
40 Example 3 - Designing The Actor System Calculate Master Workers 40
41 Example 3 - Designing The Actor System Work Work Master Work Workers 41
42 Example 3 - Designing The Actor System Result Result Master Result Workers 42
43 Example 3 - Designing The Actor System PiApproximation Listener Master Workers 43
44 Example 3 - Approximating π using Leibniz formula π = 4 * (1-1/3 + 1/5-1/7 + 1/9-1/11 + 1/13-1/15 + ) 44
45 Example 3 - Approximating π using Leibniz formula π = 4 * (1-1/3 + 1/5-1/7 + 1/9-1/11 + 1/13-1/15 + ) worker1 worker2 worker3 worker1 45
46 Example 3 - Approximating π using Leibniz formula System configurations: Number of workers Number of messages Number of elements per message π = 4 * (1-1/3 + 1/5-1/7 + 1/9-1/11 + 1/13-1/15 + ) worker1 worker2 worker3 worker1 46
47 Example 3 - Messages Calculate PiApproximation Work Result Work Result worker1 worker2 Listener Master Work Result worker3 47
48 Example 3 - Worker 48
49 Example 3 - Worker = π / 4 49
50 Example 3 - Master 1/2 50
51 Example 3 - Master 2/2 51
52 Example 3 - Listener 52
53 Example 3 - Putting It Together 53
54 With actors > It is easier to build scalable software Scalability is a deployment problem > It is a convenient abstraction for modeling Similarly to OOP > Fault tolerance can be implemented on top Actors can fail and get replaced at run time 54
55 1- Crash course in Scala - Classes - Objects 2- Actors - The Actor Model - Examples I, II, III, IV 3- Apache Spark - RDD & DAG - Word Count Example 55
56
57
58 High-Level Spark Architecture 58
59 Resilient Distributed Dataset > RDD is a partitioned collection of records > RDD is read-only (immutable) > RDDs can only be created through deterministic operations on: > data from stable storage, or > other RDDs Transformation Transformation RDD RDD RDD 59
60 We call these deterministic operations transformations. Examples of transformations include map, filter, and join. RDDs do not need to be materialized at all times. Instead, an RDD has enough information about how it was derived from other datasets (its lineage) to compute its partitions from data in stable storage.
61 Directed Acyclic Graph (DAG) Transformation Action RDD RDD RDD data RDD RDD RDD data RDD RDD 61
62 Word Count Example val conf = new SparkConf().setMaster( local[2]").setappname("wordcounter") val sparkcontext = new SparkContext(conf) val lines = sparkcontext.textfile("/var/log/commerce.log") val words = lines.flatmap(line => line.split(" ")) val pairs = words.map(word => (word, 1)) val wordcounts = pairs.reducebykey((count1,count2)=>count1+count2) val results = wordcounts.take(100) results.foreach(resultpair => println(resultpair)) RDD[String] RDD[(String, Int)] Array[(String, Int)] word lines words pairs results Counts RDD[String] RDD[(String, Int)]
63 Word Count Example val conf = new SparkConf().setMaster( local[2]").setappname("wordcounter") val sparkcontext = new SparkContext(conf) val lines = sparkcontext.textfile("/var/log/commerce.log") val words = lines.flatmap(line => line.split(" ")) val pairs = words.map(word => (word, 1)) val wordcounts = pairs.reducebykey((count1,count2)=>count1+count2) val results = wordcounts.take(100) results.foreach(resultpair => println(resultpair)) RDD[String] RDD[(String, Int)] Array[(String, Int)] word lines words pairs results Counts RDD[String] RDD[(String, Int)]
64 Word Count Example val conf = new SparkConf().setMaster( local[2]").setappname("wordcounter") val sparkcontext = new SparkContext(conf) val lines = sparkcontext.textfile("/var/log/commerce.log") val words = lines.flatmap(line => line.split(" ")) val pairs = words.map(word => (word, 1)) val wordcounts = pairs.reducebykey((count1,count2)=>count1+count2) val results = wordcounts.take(100) results.foreach(resultpair => println(resultpair)) RDD[String] RDD[(String, Int)] Array[(String, Int)] word lines words pairs results Counts RDD[String] RDD[(String, Int)]
65 Word Count Example val conf = new SparkConf().setMaster( local[2]").setappname("wordcounter") val sparkcontext = new SparkContext(conf) val lines = sparkcontext.textfile("/var/log/commerce.log") val words = lines.flatmap(line => line.split(" ")) val pairs = words.map(word => (word, 1)) val wordcounts = pairs.reducebykey((count1,count2)=>count1+count2) val results = wordcounts.take(100) results.foreach(resultpair => println(resultpair)) RDD[String] RDD[(String, Int)] Array[(String, Int)] word lines words pairs results Counts RDD[String] RDD[(String, Int)]
66 References > The original actor paper [Hewitt73]: A universal modular ACTOR formalism for artificial intelligence > Hewitt explaining actors: Actor-Model-everything-you-wanted-to-know-but-were-afraid-to-ask > Akka Tutorial: > Scaling up and out with actors: > RDD original paper: Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In- Memory Cluster Computing 66
67 Exercise > Implement a naïve actor framework in Java where: each actor is a monitor Java class each actor has a mailbox where messages from other actors are saved messages are asynchronous > Using your framework, reimplement the three examples in the lecture. 67
Reactive App using Actor model & Apache Spark. Rahul Kumar Software
Reactive App using Actor model & Apache Spark Rahul Kumar Software Developer @rahul_kumar_aws About Sigmoid We build realtime & big data systems. OUR CUSTOMERS Agenda Big Data - Intro Distributed Application
CS Programming Languages: Scala
CS 3101-2 - Programming Languages: Scala Lecture 6: Actors and Concurrency Daniel Bauer (bauer@cs.columbia.edu) December 3, 2014 Daniel Bauer CS3101-2 Scala - 06 - Actors and Concurrency 1/19 1 Actors
Beyond MapReduce: Apache Spark Antonino Virgillito
Beyond MapReduce: Apache Spark Antonino Virgillito 1 Why Spark? Most of Machine Learning Algorithms are iterative because each iteration can improve the results With Disk based approach each iteration
Data Engineering. How MapReduce Works. Shivnath Babu
Data Engineering How MapReduce Works Shivnath Babu Lifecycle of a MapReduce Job Map function Reduce function Run this program as a MapReduce job Lifecycle of a MapReduce Job Map function Reduce function
The detailed Spark programming guide is available at:
Aims This exercise aims to get you to: Analyze data using Spark shell Monitor Spark tasks using Web UI Write self-contained Spark applications using Scala in Eclipse Background Spark is already installed
Spark and Spark SQL. Amir H. Payberah. SICS Swedish ICT. Amir H. Payberah (SICS) Spark and Spark SQL June 29, / 71
Spark and Spark SQL Amir H. Payberah amir@sics.se SICS Swedish ICT Amir H. Payberah (SICS) Spark and Spark SQL June 29, 2016 1 / 71 What is Big Data? Amir H. Payberah (SICS) Spark and Spark SQL June 29,
Big Data Analytics with Apache Spark. Nastaran Fatemi
Big Data Analytics with Apache Spark Nastaran Fatemi Apache Spark Throughout this part of the course we will use the Apache Spark framework for distributed data-parallel programming. Spark implements a
RDDs are the primary abstraction in Spark RDDs are distributed collections of objects spread across the nodes of a clusters
1 RDDs are the primary abstraction in Spark RDDs are distributed collections of objects spread across the nodes of a clusters They are split in partitions Each node of the cluster that is running an application
Spark, Shark and Spark Streaming Introduction
Spark, Shark and Spark Streaming Introduction Tushar Kale tusharkale@in.ibm.com June 2015 This Talk Introduction to Shark, Spark and Spark Streaming Architecture Deployment Methodology Performance References
08/04/2018. RDDs. RDDs are the primary abstraction in Spark RDDs are distributed collections of objects spread across the nodes of a clusters
are the primary abstraction in Spark are distributed collections of objects spread across the nodes of a clusters They are split in partitions Each node of the cluster that is running an application contains
Spark Streaming. Guido Salvaneschi
Spark Streaming Guido Salvaneschi 1 Spark Streaming Framework for large scale stream processing Scales to 100s of nodes Can achieve second scale latencies Integrates with Spark s batch and interactive
2/26/2017. RDDs. RDDs are the primary abstraction in Spark RDDs are distributed collections of objects spread across the nodes of a clusters
are the primary abstraction in Spark are distributed collections of objects spread across the nodes of a clusters They are split in partitions Each node of the cluster that is used to run an application
Spark Overview. Professor Sasu Tarkoma.
Spark Overview 2015 Professor Sasu Tarkoma www.cs.helsinki.fi Apache Spark Spark is a general-purpose computing framework for iterative tasks API is provided for Java, Scala and Python The model is based
An Introduction to Apache Spark
An Introduction to Apache Spark Anastasios Skarlatidis @anskarl Software Engineer/Researcher IIT, NCSR "Demokritos" Outline Part I: Getting to know Spark Part II: Basic programming Part III: Spark under
CSE 444: Database Internals. Lecture 23 Spark
CSE 444: Database Internals Lecture 23 Spark References Spark is an open source system from Berkeley Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In-Memory Cluster Computing. Matei
COSC 6339 Big Data Analytics. Introduction to Spark. Edgar Gabriel Fall What is SPARK?
COSC 6339 Big Data Analytics Introduction to Spark Edgar Gabriel Fall 2018 What is SPARK? In-Memory Cluster Computing for Big Data Applications Fixes the weaknesses of MapReduce Iterative applications
Reliable Asynchronous Communication in Distributed Systems
Master thesis in Secure and Reliable communication at Simula@UiB Reliable Asynchronous Communication in Distributed Systems Author : Espen Johnsen February 15, 2018 Acknowledgement I would like to thank
Object-oriented programming
Object-oriented programming HelloWorld The following code print Hello World on the console object HelloWorld { def main(args: Array[String]): Unit = { println("hello World") 2 1 object The keyword object
Temporal Random Testing for Spark Streaming
Temporal Random Testing for Spark Streaming A. Riesco and J. Rodríguez-Hortalá Universidad Complutense de Madrid, Madrid, Spain 12th International Conference on integrated Formal Methods, ifm 2016 June
An Introduction to Apache Spark
An Introduction to Apache Spark Amir H. Payberah amir@sics.se SICS Swedish ICT Amir H. Payberah (SICS) Apache Spark Feb. 2, 2016 1 / 67 Big Data small data big data Amir H. Payberah (SICS) Apache Spark
Chapter 4: Apache Spark
Chapter 4: Apache Spark Lecture Notes Winter semester 2016 / 2017 Ludwig-Maximilians-University Munich PD Dr. Matthias Renz 2015, Based on lectures by Donald Kossmann (ETH Zürich), as well as Jure Leskovec,
Distributed Computing with Spark and MapReduce
Distributed Computing with Spark and MapReduce Reza Zadeh @Reza_Zadeh http://reza-zadeh.com Traditional Network Programming Message-passing between nodes (e.g. MPI) Very difficult to do at scale:» How
Scala. Introduction. Scala
Scala Introduction 1 Scala Scala was proposed by Professor Martin Odersky and his group at EPFL in 2003 to provide a highperformance, concurrent-ready environment for functional programming and object-oriented
RESILIENT DISTRIBUTED DATASETS: A FAULT-TOLERANT ABSTRACTION FOR IN-MEMORY CLUSTER COMPUTING
RESILIENT DISTRIBUTED DATASETS: A FAULT-TOLERANT ABSTRACTION FOR IN-MEMORY CLUSTER COMPUTING Matei Zaharia, Mosharaf Chowdhury, Tathagata Das, Ankur Dave, Justin Ma, Murphy McCauley, Michael J. Franklin,
Index. Raul Estrada and Isaac Ruiz 2016 R. Estrada and I. Ruiz, Big Data SMACK, DOI /
Index A ACID, 251 Actor model Akka installation, 44 Akka logos, 41 OOP vs. actors, 42 43 thread-based concurrency, 42 Agents server, 140, 251 Aggregation techniques materialized views, 216 probabilistic
Lecture 30: Distributed Map-Reduce using Hadoop and Spark Frameworks
COMP 322: Fundamentals of Parallel Programming Lecture 30: Distributed Map-Reduce using Hadoop and Spark Frameworks Mack Joyner and Zoran Budimlić {mjoyner, zoran}@rice.edu http://comp322.rice.edu COMP
Processing of big data with Apache Spark
Processing of big data with Apache Spark JavaSkop 18 Aleksandar Donevski AGENDA What is Apache Spark? Spark vs Hadoop MapReduce Application Requirements Example Architecture Application Challenges 2 WHAT
Spark Tutorial. General Instructions
CS246: Mining Massive Datasets Winter 2018 Spark Tutorial Due Thursday January 25, 2018 at 11:59pm Pacific time General Instructions The purpose of this tutorial is (1) to get you started with Spark and
High-level Parallel Programming: Scala on the JVM
High-level Parallel Programming: Scala on the JVM Contents of Lecture 4 The Scala programming language Parallel programming using actors Jonas Skeppstedt (jonasskeppstedt.net) Lecture 4 2017 1 / 40 Purpose
Big data systems 12/8/17
Big data systems 12/8/17 Today Basic architecture Two levels of scheduling Spark overview Basic architecture Cluster Manager Cluster Cluster Manager 64GB RAM 32 cores 64GB RAM 32 cores 64GB RAM 32 cores
Covers Scala 2.10 IN ACTION. Nilanjan Raychaudhuri. FOREWORD BY Chad Fowler MANNING
Covers Scala 2.10 IN ACTION Nilanjan Raychaudhuri FOREWORD BY Chad Fowler MANNING SAMPLE CHAPTER Scala in Action by Nilanjan Raychaudhuri Chapter 9 Copyright 2013 Manning Publications brief contents PART
Fast, Interactive, Language-Integrated Cluster Computing
Spark Fast, Interactive, Language-Integrated Cluster Computing Matei Zaharia, Mosharaf Chowdhury, Tathagata Das, Ankur Dave, Justin Ma, Murphy McCauley, Michael Franklin, Scott Shenker, Ion Stoica www.spark-project.org
25/05/2018. Spark Streaming is a framework for large scale stream processing
25/5/18 Spark Streaming is a framework for large scale stream processing Scales to s of nodes Can achieve second scale latencies Provides a simple batch-like API for implementing complex algorithm Can
Akka. Reactive Applications made easy
Akka Reactive Applications made easy Michael Pisula JavaLand, 2014-03-25 Akka Akka mountain by Arvelius What are reactive applications? New requirements demand new technologies - The Reactive Manifesto
Apache Spark Internals
Apache Spark Internals Pietro Michiardi Eurecom Pietro Michiardi (Eurecom) Apache Spark Internals 1 / 80 Acknowledgments & Sources Sources Research papers: https://spark.apache.org/research.html Presentations:
MapReduce, Hadoop and Spark. Bompotas Agorakis
MapReduce, Hadoop and Spark Bompotas Agorakis Big Data Processing Most of the computations are conceptually straightforward on a single machine but the volume of data is HUGE Need to use many (1.000s)
15.1 Data flow vs. traditional network programming
CME 323: Distributed Algorithms and Optimization, Spring 2017 http://stanford.edu/~rezab/dao. Instructor: Reza Zadeh, Matroid and Stanford. Lecture 15, 5/22/2017. Scribed by D. Penner, A. Shoemaker, and
CMPT 435/835 Tutorial 1 Actors Model & Akka. Ahmed Abdel Moamen PhD Candidate Agents Lab
CMPT 435/835 Tutorial 1 Actors Model & Akka Ahmed Abdel Moamen PhD Candidate Agents Lab ama883@mail.usask.ca 1 Content Actors Model Akka (for Java) 2 Actors Model (1/7) Definition A general model of concurrent
Data-intensive computing systems
Data-intensive computing systems University of Verona Computer Science Department Damiano Carra Acknowledgements q Credits Part of the course material is based on slides provided by the following authors
Getting Started with Kotlin. Commerzbank Java Developer Day
Getting Started with Kotlin Commerzbank Java Developer Day 30.11.2017 Hello! Alexander Hanschke Hello! Alexander Hanschke CTO at techdev Solutions GmbH in Berlin Hello! Alexander Hanschke CTO at techdev
Overview. Prerequisites. Course Outline. Course Outline :: Apache Spark Development::
Title Duration : Apache Spark Development : 4 days Overview Spark is a fast and general cluster computing system for Big Data. It provides high-level APIs in Scala, Java, Python, and R, and an optimized
An Introduction to Big Data Analysis using Spark
An Introduction to Big Data Analysis using Spark Mohamad Jaber American University of Beirut - Faculty of Arts & Sciences - Department of Computer Science May 17, 2017 Mohamad Jaber (AUB) Spark May 17,
COMP 322: Fundamentals of Parallel Programming. Lecture 37: Distributed Computing, Apache Spark
COMP 322: Fundamentals of Parallel Programming Lecture 37: Distributed Computing, Apache Spark Vivek Sarkar, Shams Imam Department of Computer Science, Rice University vsarkar@rice.edu, shams@rice.edu
Goals Pragmukko features:
Pragmukko Pragmukko Pragmukko is an Akka-based framework for distributed computing. Originally, it was developed for building an IoT solution but it also fit well for many other cases. Pragmukko treats
Outline. CS-562 Introduction to data analysis using Apache Spark
Outline Data flow vs. traditional network programming What is Apache Spark? Core things of Apache Spark RDD CS-562 Introduction to data analysis using Apache Spark Instructor: Vassilis Christophides T.A.:
Lecture 11 Hadoop & Spark
Lecture 11 Hadoop & Spark Dr. Wilson Rivera ICOM 6025: High Performance Computing Electrical and Computer Engineering Department University of Puerto Rico Outline Distributed File Systems Hadoop Ecosystem
Parallel Processing Spark and Spark SQL
Parallel Processing Spark and Spark SQL Amir H. Payberah amir@sics.se KTH Royal Institute of Technology Amir H. Payberah (KTH) Spark and Spark SQL 2016/09/16 1 / 82 Motivation (1/4) Most current cluster
An Introduction to Apache Spark
An Introduction to Apache Spark 1 History Developed in 2009 at UC Berkeley AMPLab. Open sourced in 2010. Spark becomes one of the largest big-data projects with more 400 contributors in 50+ organizations
Introduction to Apache Spark
Introduction to Apache Spark Bu eğitim sunumları İstanbul Kalkınma Ajansı nın 2016 yılı Yenilikçi ve Yaratıcı İstanbul Mali Destek Programı kapsamında yürütülmekte olan TR10/16/YNY/0036 no lu İstanbul
Big Data Infrastructures & Technologies
Big Data Infrastructures & Technologies Spark and MLLIB OVERVIEW OF SPARK What is Spark? Fast and expressive cluster computing system interoperable with Apache Hadoop Improves efficiency through: In-memory
Introduction to Apache Spark. Patrick Wendell - Databricks
Introduction to Apache Spark Patrick Wendell - Databricks What is Spark? Fast and Expressive Cluster Computing Engine Compatible with Apache Hadoop Efficient General execution graphs In-memory storage
Spark. Cluster Computing with Working Sets. Matei Zaharia, Mosharaf Chowdhury, Michael Franklin, Scott Shenker, Ion Stoica.
Spark Cluster Computing with Working Sets Matei Zaharia, Mosharaf Chowdhury, Michael Franklin, Scott Shenker, Ion Stoica UC Berkeley Background MapReduce and Dryad raised level of abstraction in cluster
CS Spark. Slides from Matei Zaharia and Databricks
CS 5450 Spark Slides from Matei Zaharia and Databricks Goals uextend the MapReduce model to better support two common classes of analytics apps Iterative algorithms (machine learning, graphs) Interactive
Massive Online Analysis - Storm,Spark
Massive Online Analysis - Storm,Spark presentation by R. Kishore Kumar Research Scholar Department of Computer Science & Engineering Indian Institute of Technology, Kharagpur Kharagpur-721302, India (R
Cloud Computing 3. CSCI 4850/5850 High-Performance Computing Spring 2018
Cloud Computing 3 CSCI 4850/5850 High-Performance Computing Spring 2018 Tae-Hyuk (Ted) Ahn Department of Computer Science Program of Bioinformatics and Computational Biology Saint Louis University Learning
2/4/2019 Week 3- A Sangmi Lee Pallickara
Week 3-A-0 2/4/2019 Colorado State University, Spring 2019 Week 3-A-1 CS535 BIG DATA FAQs PART A. BIG DATA TECHNOLOGY 3. DISTRIBUTED COMPUTING MODELS FOR SCALABLE BATCH COMPUTING SECTION 1: MAPREDUCE PA1
Programming Systems for Big Data
Programming Systems for Big Data CS315B Lecture 17 Including material from Kunle Olukotun Prof. Aiken CS 315B Lecture 17 1 Big Data We ve focused on parallel programming for computational science There
Distributed Systems. 22. Spark. Paul Krzyzanowski. Rutgers University. Fall 2016
Distributed Systems 22. Spark Paul Krzyzanowski Rutgers University Fall 2016 November 26, 2016 2015-2016 Paul Krzyzanowski 1 Apache Spark Goal: generalize MapReduce Similar shard-and-gather approach to
Evolution of an Apache Spark Architecture for Processing Game Data
Evolution of an Apache Spark Architecture for Processing Game Data Nick Afshartous WB Analytics Platform May 17 th 2017 May 17 th, 2017 About Me nafshartous@wbgames.com WB Analytics Core Platform Lead
Executive Summary. It is important for a Java Programmer to understand the power and limitations of concurrent programming in Java using threads.
Executive Summary. It is important for a Java Programmer to understand the power and limitations of concurrent programming in Java using threads. Poor co-ordination that exists in threads on JVM is bottleneck
Microservices. Webservices with Scala (II) Microservices
Microservices Webservices with Scala (II) Microservices 2018 1 Content Deep Dive into Play2 1. Database Access with Slick 2. Database Migration with Flyway 3. akka 3.1. overview 3.2. akka-http (the http
BSP, Pregel and the need for Graph Processing
BSP, Pregel and the need for Graph Processing Patrizio Dazzi, HPC Lab ISTI - CNR mail: patrizio.dazzi@isti.cnr.it web: http://hpc.isti.cnr.it/~dazzi/ National Research Council of Italy A need for Graph
Introduction to Spark
Introduction to Spark Outlines A brief history of Spark Programming with RDDs Transformations Actions A brief history Limitations of MapReduce MapReduce use cases showed two major limitations: Difficulty
CS/ENGRD 2110 SPRING Lecture 5: Local vars; Inside-out rule; constructors
1 CS/ENGRD 2110 SPRING 2017 Lecture 5: Local vars; Inside-out rule; constructors http://courses.cs.cornell.edu/cs2110 Announcements 2 1. Writing tests to check that the code works when the precondition
Big Streaming Data Processing. How to Process Big Streaming Data 2016/10/11. Fraud detection in bank transactions. Anomalies in sensor data
Big Data Big Streaming Data Big Streaming Data Processing Fraud detection in bank transactions Anomalies in sensor data Cat videos in tweets How to Process Big Streaming Data Raw Data Streams Distributed
Matrix Computations and " Neural Networks in Spark
Matrix Computations and " Neural Networks in Spark Reza Zadeh Paper: http://arxiv.org/abs/1509.02256 Joint work with many folks on paper. @Reza_Zadeh http://reza-zadeh.com Training Neural Networks Datasets
CS 2340 Objects and Design - Scala
CS 2340 Objects and Design - Scala Objects and Operators Christopher Simpkins chris.simpkins@gatech.edu Chris Simpkins (Georgia Tech) CS 2340 Objects and Design - Scala Objects and Operators 1 / 13 Classes
Akka Streams and Bloom Filters
Akka Streams and Bloom Filters Or: How I learned to stop worrying and love resilient elastic distributed real-time transaction processing using space efficient probabilistic data structures The Situation
State and DStreams. Big Data Analysis with Scala and Spark Heather Miller
State and DStreams Big Data Analysis with Scala and Spark Heather Miller State? So far, we ve approached Spark Streaming in the same way we have approached regular Spark. Assumption so far: Functional
Stream Processing on IoT Devices using Calvin Framework
Stream Processing on IoT Devices using Calvin Framework by Ameya Nayak A Project Report Submitted in Partial Fulfillment of the Requirements for the Degree of Master of Science in Computer Science Supervised
Scalable task distribution with Scala, Akka and Mesos. Dario
Scalable task distribution with Scala, Akka and Mesos Dario Rexin @evonox What is Mesos? 2 What is Mesos? Apache open source project Distributed systems kernel Multi resource scheduler (CPU, Memory, Ports,
Pyspark standalone code
COSC 6339 Big Data Analytics Introduction to Spark (II) Edgar Gabriel Spring 2017 Pyspark standalone code from pyspark import SparkConf, SparkContext from operator import add conf = SparkConf() conf.setappname(
...something useful to do with the JVM.
Scala Finally... ...something useful to do with the JVM. Image source: http://www.tripadvisor.com/locationphotos-g187789-lazio.html Young Developed in 2003 by Martin Odersky at EPFL Martin also brought
Introduction to Apache Spark
1 Introduction to Apache Spark Thomas Ropars thomas.ropars@univ-grenoble-alpes.fr 2017 2 References The content of this lectures is inspired by: The lecture notes of Yann Vernaz. The lecture notes of Vincent
@h2oai presents. Sparkling Water Meetup
@h2oai & @mmalohlava presents Sparkling Water Meetup User-friendly API for data transformation Large and active community Memory efficient Performance of computation Platform components - SQL Machine learning
Apache Spark: Hands-on Session A.A. 2016/17
Università degli Studi di Roma Tor Vergata Dipartimento di Ingegneria Civile e Ingegneria Informatica Apache Spark: Hands-on Session A.A. 2016/17 Matteo Nardelli Laurea Magistrale in Ingegneria Informatica
Machine Learning With Spark
Ons Dridi R&D Engineer 13 Novembre 2015 Centre d Excellence en Technologies de l Information et de la Communication CETIC Presentation - An applied research centre in the field of ICT - The knowledge developed
DATA SCIENCE USING SPARK: AN INTRODUCTION
DATA SCIENCE USING SPARK: AN INTRODUCTION TOPICS COVERED Introduction to Spark Getting Started with Spark Programming in Spark Data Science with Spark What next? 2 DATA SCIENCE PROCESS Exploratory Data
Apache Spark: Hands-on Session A.A. 2017/18
Università degli Studi di Roma Tor Vergata Dipartimento di Ingegneria Civile e Ingegneria Informatica Apache Spark: Hands-on Session A.A. 2017/18 Matteo Nardelli Laurea Magistrale in Ingegneria Informatica
Databases and Big Data Today. CS634 Class 22
Databases and Big Data Today CS634 Class 22 Current types of Databases SQL using relational tables: still very important! NoSQL, i.e., not using relational tables: term NoSQL popular since about 2007.
Analytic Cloud with. Shelly Garion. IBM Research -- Haifa IBM Corporation
Analytic Cloud with Shelly Garion IBM Research -- Haifa 2014 IBM Corporation Why Spark? Apache Spark is a fast and general open-source cluster computing engine for big data processing Speed: Spark is capable
Spark.jl: Resilient Distributed Datasets in Julia
Spark.jl: Resilient Distributed Datasets in Julia Dennis Wilson, Martín Martínez Rivera, Nicole Power, Tim Mickel [dennisw, martinmr, npower, tmickel]@mit.edu INTRODUCTION 1 Julia is a new high level,
Higher level data processing in Apache Spark
Higher level data processing in Apache Spark Pelle Jakovits 12 October, 2016, Tartu Outline Recall Apache Spark Spark DataFrames Introduction Creating and storing DataFrames DataFrame API functions SQL
Using the SDACK Architecture to Build a Big Data Product. Yu-hsin Yeh (Evans Ye) Apache Big Data NA 2016 Vancouver
Using the SDACK Architecture to Build a Big Data Product Yu-hsin Yeh (Evans Ye) Apache Big Data NA 2016 Vancouver Outline A Threat Analytic Big Data product The SDACK Architecture Akka Streams and data
EPL660: Information Retrieval and Search Engines Lab 11
EPL660: Information Retrieval and Search Engines Lab 11 Παύλος Αντωνίου Γραφείο: B109, ΘΕΕ01 University of Cyprus Department of Computer Science Introduction to Apache Spark Fast and general engine for
The Actor Model. Towards Better Concurrency. By: Dror Bereznitsky
The Actor Model Towards Better Concurrency By: Dror Bereznitsky 1 Warning: Code Examples 2 Agenda Agenda The end of Moore law? Shared state concurrency Message passing concurrency Actors on the JVM More
Guidelines For Hadoop and Spark Cluster Usage
Guidelines For Hadoop and Spark Cluster Usage Procedure to create an account in CSX. If you are taking a CS prefix course, you already have an account; to get an initial password created: 1. Login to https://cs.okstate.edu/pwreset
Corpus methods in linguistics and NLP Lecture 7: Programming for large-scale data processing
Corpus methods in linguistics and NLP Lecture 7: Programming for large-scale data processing Richard Johansson December 1, 2015 today's lecture as you've seen, processing large corpora can take time! for
Distributed Systems. Distributed computation with Spark. Abraham Bernstein, Ph.D.
Distributed Systems Distributed computation with Spark Abraham Bernstein, Ph.D. Course material based on: - Based on slides by Reynold Xin, Tudor Lapusan - Some additions by Johannes Schneider How good
Big Data Analytics with Hadoop and Spark at OSC
Big Data Analytics with Hadoop and Spark at OSC 09/28/2017 SUG Shameema Oottikkal Data Application Engineer Ohio SuperComputer Center email:soottikkal@osc.edu 1 Data Analytics at OSC Introduction: Data
CS435 Introduction to Big Data FALL 2018 Colorado State University. 10/24/2018 Week 10-B Sangmi Lee Pallickara
10/24/2018 CS435 Introduction to Big Data - FALL 2018 W10B00 CS435 Introduction to Big Data 10/24/2018 CS435 Introduction to Big Data - FALL 2018 W10B1 FAQs Programming Assignment 3 has been posted Recitations
Exercise 6 Multiple Inheritance, Multiple Dispatch and Linearization November 3, 2017
Concepts of Object-Oriented Programming AS 2017 Exercise 6 Multiple Inheritance, Multiple Dispatch and Linearization November 3, 2017 Task 1 Consider the following C++ program: class X X(int p) : fx(p)
CS60021: Scalable Data Mining. Sourangshu Bhattacharya
CS60021: Scalable Data Mining Sourangshu Bhattacharya In this Lecture: Outline: Scala Var and Val Classes and objects Functions and higher order functions Lists SCALA Scala Scala is both functional and
Favoring Isolated Mutability The Actor Model of Concurrency. CSCI 5828: Foundations of Software Engineering Lecture 24 04/11/2012
Favoring Isolated Mutability The Actor Model of Concurrency CSCI 5828: Foundations of Software Engineering Lecture 24 04/11/2012 1 Goals Review the material in Chapter 8 of the Concurrency textbook that
A Tutorial on Apache Spark
A Tutorial on Apache Spark A Practical Perspective By Harold Mitchell The Goal Learning Outcomes The Goal Learning Outcomes NOTE: The setup, installation, and examples assume Windows user Learn the following:
In-Memory Processing with Apache Spark. Vincent Leroy
In-Memory Processing with Apache Spark Vincent Leroy Sources Resilient Distributed Datasets, Henggang Cui Coursera IntroducBon to Apache Spark, University of California, Databricks Datacenter OrganizaBon
Akka: Simpler Concurrency, Scalability & Fault-tolerance through Actors. Jonas Bonér Viktor Klang
Akka: Simpler Concurrency, Scalability & Fault-tolerance through Actors Jonas Bonér Viktor Klang We believe that... Writing correct concurrent applications is too hard Scaling out applications is too hard
Spark. In- Memory Cluster Computing for Iterative and Interactive Applications
Spark In- Memory Cluster Computing for Iterative and Interactive Applications Matei Zaharia, Mosharaf Chowdhury, Tathagata Das, Ankur Dave, Justin Ma, Murphy McCauley, Michael Franklin, Scott Shenker,
Cloud Computing & Visualization
Cloud Computing & Visualization Workflows Distributed Computation with Spark Data Warehousing with Redshift Visualization with Tableau #FIUSCIS School of Computing & Information Sciences, Florida International
Spark Streaming. Professor Sasu Tarkoma.
Spark Streaming 2015 Professor Sasu Tarkoma www.cs.helsinki.fi Spark Streaming Spark extension of accepting and processing of streaming high-throughput live data streams Data is accepted from various sources