{"id":"https://openalex.org/W4210603285","doi":"https://doi.org/10.1109/nca53618.2021.9685833","title":"QSpark: Distributed Execution of Batch & Streaming Analytics in Spark Platform","display_name":"QSpark: Distributed Execution of Batch & Streaming Analytics in Spark Platform","publication_year":2021,"publication_date":"2021-11-23","ids":{"openalex":"https://openalex.org/W4210603285","doi":"https://doi.org/10.1109/nca53618.2021.9685833"},"language":"en","primary_location":{"id":"doi:10.1109/nca53618.2021.9685833","is_oa":false,"landing_page_url":"https://doi.org/10.1109/nca53618.2021.9685833","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE 20th International Symposium on Network Computing and Applications (NCA)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063735421","display_name":"M. Reza HoseinyFarahabady","orcid":"https://orcid.org/0000-0002-7851-9377"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"M.Reza HoseinyFarahabady","raw_affiliation_strings":["The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW,Australia","The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW,Australia","institution_ids":["https://openalex.org/I129604602"]},{"raw_affiliation_string":"The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048747481","display_name":"Javid Taheri","orcid":"https://orcid.org/0000-0001-9194-010X"},"institutions":[{"id":"https://openalex.org/I43968019","display_name":"Karlstad University","ror":"https://ror.org/05s754026","country_code":"SE","type":"education","lineage":["https://openalex.org/I43968019"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Javid Taheri","raw_affiliation_strings":["Karlstad University,Department of Mathematics and Computer Science,Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Karlstad University,Department of Mathematics and Computer Science,Sweden","institution_ids":["https://openalex.org/I43968019"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015993565","display_name":"Albert Y. Zomaya","orcid":"https://orcid.org/0000-0002-3090-1059"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Albert Y. Zomaya","raw_affiliation_strings":["The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW,Australia","The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW,Australia","institution_ids":["https://openalex.org/I129604602"]},{"raw_affiliation_string":"The University of Sydney, School of Computer Science, Center for Distributed and High Performance Computing, NSW, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054836950","display_name":"Zahir Tari","orcid":"https://orcid.org/0000-0002-1235-9673"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Zahir Tari","raw_affiliation_strings":["RMIT University, School of Computing Technologies,Victoria,Australia","RMIT University, School of Computing Technologies, Victoria, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, School of Computing Technologies,Victoria,Australia","institution_ids":["https://openalex.org/I82951845"]},{"raw_affiliation_string":"RMIT University, School of Computing Technologies, Victoria, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2538,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.54248862,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"36","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9779000282287598,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8355789184570312},{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.5897698998451233},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5667764544487},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.5511212348937988},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5170467495918274},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.514907956123352},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.4832335412502289},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.463750422000885},{"id":"https://openalex.org/keywords/batch-processing","display_name":"Batch processing","score":0.4567486047744751},{"id":"https://openalex.org/keywords/quality-of-service","display_name":"Quality of service","score":0.44064342975616455},{"id":"https://openalex.org/keywords/throughput","display_name":"Throughput","score":0.43169859051704407},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.2617269456386566},{"id":"https://openalex.org/keywords/wireless","display_name":"Wireless","score":0.20416080951690674},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.18911051750183105}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8355789184570312},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.5897698998451233},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5667764544487},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.5511212348937988},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5170467495918274},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.514907956123352},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4832335412502289},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.463750422000885},{"id":"https://openalex.org/C172658912","wikidata":"https://www.wikidata.org/wiki/Q661613","display_name":"Batch processing","level":2,"score":0.4567486047744751},{"id":"https://openalex.org/C5119721","wikidata":"https://www.wikidata.org/wiki/Q220501","display_name":"Quality of service","level":2,"score":0.44064342975616455},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.43169859051704407},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.2617269456386566},{"id":"https://openalex.org/C555944384","wikidata":"https://www.wikidata.org/wiki/Q249","display_name":"Wireless","level":2,"score":0.20416080951690674},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.18911051750183105},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/nca53618.2021.9685833","is_oa":false,"landing_page_url":"https://doi.org/10.1109/nca53618.2021.9685833","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE 20th International Symposium on Network Computing and Applications (NCA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1988215189","display_name":"Resource Allocation for High-Volume Streaming Data in Data Centers","funder_award_id":"DP200100005","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320320966","display_name":"University of Sydney","ror":"https://ror.org/0384j8v12"},{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1831560769","https://openalex.org/W1967436269","https://openalex.org/W2023214828","https://openalex.org/W2076984429","https://openalex.org/W2127455097","https://openalex.org/W2150478767","https://openalex.org/W2244876453","https://openalex.org/W2307190521","https://openalex.org/W2509341006","https://openalex.org/W2542459869","https://openalex.org/W2566979091","https://openalex.org/W2724427576","https://openalex.org/W2735078864","https://openalex.org/W2898352726","https://openalex.org/W2913854892","https://openalex.org/W2916596859","https://openalex.org/W3009076569","https://openalex.org/W3017800872","https://openalex.org/W4232900580","https://openalex.org/W6698037560","https://openalex.org/W6731596640","https://openalex.org/W6740335196"],"related_works":["https://openalex.org/W4390608645","https://openalex.org/W4394895745","https://openalex.org/W2960264696","https://openalex.org/W2766461310","https://openalex.org/W4247566972","https://openalex.org/W4388692845","https://openalex.org/W3202731209","https://openalex.org/W3211874991","https://openalex.org/W4236613945","https://openalex.org/W3042827706"],"abstract_inverted_index":{"A":[0,102],"significant":[1],"portion":[2],"of":[3,33,68,148,171,211,217,220,233,264,277,294,303,342],"research":[4,54],"work":[5],"in":[6,65,109,138,178,186,199,223,254],"the":[7,58,66,129,146,169,179,187,192,218,230,240,261,268,292,295,300,311,325,335,340],"past":[8],"decade":[9],"has":[10],"been":[11],"devoted":[12],"on":[13],"developing":[14],"resource":[15,118,125,161,176,194,271,345],"allocation":[16,119,219,272,346],"and":[17,182,190,284,318],"task":[18],"scheduling":[19],"solutions":[20],"for":[21,50,77,80,214,242,274,332],"large-scale":[22],"data":[23,34,82,235],"processing":[24,83,236],"platforms.":[25],"Such":[26],"algorithms":[27],"are":[28,86],"designed":[29,87],"to":[30,89,105,114,144,166],"facilitate":[31],"deployment":[32],"analytic":[35],"applications":[36,143,198,306,333],"across":[37],"either":[38],"conventional":[39],"cluster":[40],"computing":[41,221],"systems":[42],"or":[43,99],"modern":[44,81,202],"virtualized":[45],"data-centers.":[46],"The":[47,205],"main":[48],"reason":[49],"such":[51,69,110],"a":[52,62,73,111,152,160,200,209,224,275],"huge":[53],"effort":[55],"stems":[56],"from":[57],"fact":[59],"that":[60,85,121,226],"even":[61],"slight":[63],"improvement":[64],"performance":[67,131,174,263],"platforms":[70],"can":[71,122,228,323],"bring":[72],"considerable":[74],"monetary":[75],"savings":[76],"vendors,":[78],"especially":[79],"engines":[84],"solely":[88],"perform":[90],"high":[91,304],"throughput":[92,241],"or/and":[93],"low-latency":[94],"computations":[95],"over":[96],"massive-scale":[97],"batch":[98,180,317],"streaming":[100,188],"data.":[101],"challenging":[103],"question":[104],"be":[106],"yet":[107],"answered":[108],"context":[112],"is":[113,151],"design":[115],"an":[116],"effective":[117],"solution":[120,207,266,297],"prevent":[123],"low":[124,173],"utilization":[126,177],"while":[127,238,321],"meeting":[128],"enforced":[130],"level":[132,328],"(such":[133],"as":[134],"99-th":[135],"latency":[136,302],"percentile)":[137],"circumstances":[139],"where":[140],"contention":[141],"among":[142,196],"obtain":[145],"capacity":[147],"shared":[149,193],"resources":[150],"non":[153],"negligible":[154],"performance-limiting":[155],"parameter.":[156],"This":[157],"paper":[158],"proposes":[159],"controller":[162],"system,":[163],"called":[164],"QSpark,":[165],"cope":[167],"with":[168,334,339],"problem":[170],"(i)":[172],"(i.e.,":[175],"mode":[181],"p-99":[183,301],"response":[184],"time":[185],"mode),":[189],"(ii)":[191],"interference":[195],"collocated":[197],"multi-tenancy":[201],"Spark":[203,257,270,344],"platform.":[204],"proposed":[206,265,296],"leverages":[208],"set":[210],"controlling":[212],"mechanisms":[213],"dynamic":[215],"partitioning":[216],"resources,":[222],"way":[225],"it":[227,322],"fulfill":[229],"QoS":[231,326],"re-quirements":[232],"latency-critical":[234],"applications,":[237],"enhancing":[239],"all":[243],"working":[244],"nodes":[245],"without":[246],"reaching":[247],"their":[248],"saturation":[249],"points.":[250],"Through":[251],"extensive":[252],"experiments":[253],"our":[255],"in-house":[256],"cluster,":[258],"we":[259],"compared":[260],"achieved":[262],"against":[267],"default":[269,343],"policy":[273],"variety":[276],"Machine":[278],"Learning":[279,286],"(ML),":[280],"Artificial":[281],"Intelligence":[282],"(AI),":[283],"Deep":[285],"(DL)":[287],"applications.":[288],"Experimental":[289],"results":[290,341],"show":[291],"effectiveness":[293],"by":[298,307,329],"reducing":[299],"priority":[305,337],"32":[308],"%":[309,331],"during":[310],"burst":[312],"traffic":[313],"periods":[314],"(for":[315],"both":[316],"stream":[319],"modes),":[320],"enhance":[324],"satisfaction":[327],"65":[330],"highest":[336],"(compared":[338],"strategy).":[347]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}