{"id":"https://openalex.org/W2613808630","doi":"https://doi.org/10.1145/3041021.3051108","title":"Large Scale Distributed Data Science from scratch using Apache Spark 2.0","display_name":"Large Scale Distributed Data Science from scratch using Apache Spark 2.0","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2613808630","doi":"https://doi.org/10.1145/3041021.3051108","mag":"2613808630"},"language":"en","primary_location":{"id":"doi:10.1145/3041021.3051108","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3041021.3051108","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th International Conference on World Wide Web Companion - WWW '17 Companion","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3041021.3051108","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113808513","display_name":"James G. Shanahan","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Shanahan","raw_affiliation_strings":["University of California, Berkeley, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California, Berkeley, San Francisco, CA, USA","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101988313","display_name":"Liang Dai","orcid":"https://orcid.org/0000-0002-5959-6828"},"institutions":[{"id":"https://openalex.org/I185103710","display_name":"University of California, Santa Cruz","ror":"https://ror.org/03s65by71","country_code":"US","type":"education","lineage":["https://openalex.org/I185103710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liang Dai","raw_affiliation_strings":["University of California Santa Cruz, Santa Cruz, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California Santa Cruz, Santa Cruz, CA, USA","institution_ids":["https://openalex.org/I185103710"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"955","last_page":"957"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10538","display_name":"Data Mining Algorithms and Applications","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.851665735244751},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.7052567601203918},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.6634182929992676},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.6289312243461609},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6132926344871521},{"id":"https://openalex.org/keywords/scala","display_name":"Scala","score":0.6107563972473145},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5583691000938416},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.45813632011413574},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.4487454295158386},{"id":"https://openalex.org/keywords/programming-paradigm","display_name":"Programming paradigm","score":0.4371471107006073},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3603721261024475},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.30501848459243774},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.269167423248291}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.851665735244751},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.7052567601203918},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.6634182929992676},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.6289312243461609},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6132926344871521},{"id":"https://openalex.org/C109701466","wikidata":"https://www.wikidata.org/wiki/Q460584","display_name":"Scala","level":3,"score":0.6107563972473145},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5583691000938416},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.45813632011413574},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.4487454295158386},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.4371471107006073},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3603721261024475},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.30501848459243774},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.269167423248291}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3041021.3051108","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3041021.3051108","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th International Conference on World Wide Web Companion - WWW '17 Companion","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3041021.3051108","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3041021.3051108","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th International Conference on World Wide Web Companion - WWW '17 Companion","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W2061460268","https://openalex.org/W2125816831","https://openalex.org/W2231104193","https://openalex.org/W2271840356","https://openalex.org/W2498778289"],"related_works":["https://openalex.org/W2763674625","https://openalex.org/W4232952072","https://openalex.org/W4252019479","https://openalex.org/W3040669356","https://openalex.org/W2337519567","https://openalex.org/W2548921709","https://openalex.org/W4211064201","https://openalex.org/W4252076541","https://openalex.org/W2620395718","https://openalex.org/W2950624501"],"abstract_inverted_index":{"Apache":[0],"Spark":[1,29,254,278,281,290,292,294,311,348],"is":[2,46,174,207,267],"an":[3,241],"open-source":[4,148],"cluster":[5],"computing":[6],"framework.":[7],"It":[8,227,266],"has":[9,73,119],"emerged":[10],"as":[11,317,329,336],"the":[12,25,83,147,176,272,289,299,353,373],"next":[13],"generation":[14],"big":[15,26],"data":[16,27,81,85,203,250,263,287],"processing":[17,140,326],"engine,":[18],"overtaking":[19],"Hadoop":[20],"MapReduce":[21],"which":[22,189],"helped":[23],"ignite":[24],"revolution.":[28],"maintains":[30],"MapReduce's":[31],"linear":[32],"scalability":[33],"and":[34,70,78,108,141,156,195,216,249,252,255,261,297,308,340,345,371],"fault":[35],"tolerance,":[36],"but":[37],"extends":[38],"it":[39,45,89],"in":[40,59,65,129,164,214,383],"a":[41,97,165,191,208,231,364],"few":[42],"important":[43],"ways:":[44],"much":[47,55,166],"faster":[48,51],"(100":[49],"times":[50],"for":[52,199],"certain":[53],"applications),":[54],"easier":[56,167],"to":[57,61,95,122,146,244,253,258,362],"program":[58],"due":[60],"its":[62,79,256],"rich":[63],"APIs":[64],"Python,":[66],"Java,":[67],"Scala,":[68],"SQL":[69],"R":[71],"(MapReduce":[72],"2":[74],"core":[75,80],"calls)":[76],",":[77],"abstraction,":[82],"distributed":[84,246],"frame.":[86],"In":[87],"addition,":[88],"goes":[90],"far":[91],"beyond":[92],"batch":[93],"applications":[94,344],"support":[96,337],"variety":[98],"of":[99,114,218,222,347],"compute-intensive":[100],"tasks,":[101],"including":[102,280],"interactive":[103],"queries,":[104],"streaming,":[105],"machine":[106,247],"learning,":[107,320],"graph":[109,325],"processing.":[110],"With":[111],"massive":[112],"amounts":[113],"computational":[115],"power,":[116],"deep":[117,161,181,358,365],"learning":[118,162,182,248,359,366],"been":[120],"shown":[121],"produce":[123],"state-of-the-art":[124],"results":[125],"on":[126,220,233,304,375],"various":[127],"tasks":[128],"different":[130],"fields":[131],"like":[132],"computer":[133],"vision,":[134],"automatic":[135],"speech":[136],"recognition,":[137],"natural":[138],"language":[139],"online":[142],"advertising":[143],"targeting.":[144],"Thanks":[145],"frameworks,":[149],"e.g.":[150],"Torch,":[151],"Theano,":[152],"Caffe,":[153],"MxNet,":[154],"Keras":[155,206,370],"TensorFlow,":[157,369],"we":[158],"can":[159],"build":[160],"model":[163,367,374],"way.":[168],"Among":[169],"all":[170],"these":[171],"framework,":[172],"TensorFlow":[173,184,224],"probably":[175],"most":[177],"popular":[178],"open":[179],"source":[180],"library.":[183],"1.0":[185],"was":[186,228],"released":[187],"recently,":[188],"provide":[190,240],"more":[192],"stable,":[193],"flexible":[194],"powerful":[196],"computation":[197,201],"tool":[198],"numerical":[200],"using":[202],"flow":[204],"graphs.":[205],"high-level":[209],"neural":[210],"networks":[211],"library,":[212],"written":[213],"Python":[215],"capable":[217],"running":[219],"top":[221],"either":[223],"or":[225],"Theano.":[226],"developed":[229],"with":[230,310],"focus":[232,303],"enabling":[234],"fast":[235],"experimentation.":[236],"This":[237],"tutorial":[238],"will":[239,275,302,349,356,379],"accessible":[242],"introduction":[243],"large-scale":[245],"mining,":[251],"potential":[257],"revolutionize":[259],"academic":[260],"commercial":[262],"science":[264],"practices.":[265],"divided":[268],"into":[269],"three":[270],"parts:":[271],"first":[273],"part":[274,301,355],"cover":[276],"fundamental":[277],"concepts,":[279,360],"Core,":[282],"functional":[283],"programming":[284],"ala":[285],"map-reduce,":[286],"frames,":[288],"Shell,":[291],"Streaming,":[293],"SQL,":[295],"MLlib,":[296],"more;":[298],"second":[300],"hands-on":[305],"algorithmic":[306],"design":[307],"development":[309],"(developing":[312],"algorithms":[313,327,334],"from":[314],"scratch":[315],"such":[316,328,335],"decision":[318],"tree":[319],"association":[321],"rule":[322],"mining":[323],"(aPriori),":[324],"pagerank/shortest":[330],"path,":[331],"gradient":[332],"descent":[333],"vectors":[338],"machines":[339],"matrix":[341],"factorization.":[342],"Industrial":[343],"deployments":[346],"also":[350],"be":[351,380],"presented.;":[352],"third":[354],"introduce":[357],"how":[361],"implement":[363],"through":[368],"run":[372],"Spark.":[376],"Example":[377],"code":[378],"made":[381],"available":[382],"python":[384],"(pySpark)":[385],"notebooks.":[386]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":1}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}