{"id":"https://openalex.org/W2784332964","doi":"https://doi.org/10.1109/bigdata.2017.8258262","title":"A novel compression algorithm decision method for spark shuffle process","display_name":"A novel compression algorithm decision method for spark shuffle process","publication_year":2017,"publication_date":"2017-12-01","ids":{"openalex":"https://openalex.org/W2784332964","doi":"https://doi.org/10.1109/bigdata.2017.8258262","mag":"2784332964"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata.2017.8258262","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8258262","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030585626","display_name":"Shanshan Huang","orcid":"https://orcid.org/0000-0001-8305-7966"},"institutions":[{"id":"https://openalex.org/I37796252","display_name":"Beijing University of Technology","ror":"https://ror.org/037b1pp87","country_code":"CN","type":"education","lineage":["https://openalex.org/I37796252"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shanshan Huang","raw_affiliation_strings":["Faculty of Information Technology, Beijing University of Technology, Beijing, China","School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Beijing University of Technology, Beijing, China","institution_ids":["https://openalex.org/I37796252"]},{"raw_affiliation_string":"School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016973102","display_name":"Jungang Xu","orcid":"https://orcid.org/0000-0002-3994-1401"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jungang Xu","raw_affiliation_strings":["School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030903490","display_name":"Renfeng Liu","orcid":"https://orcid.org/0000-0003-2527-7420"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Renfeng Liu","raw_affiliation_strings":["School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer and Control Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083318950","display_name":"Husheng Liao","orcid":null},"institutions":[{"id":"https://openalex.org/I37796252","display_name":"Beijing University of Technology","ror":"https://ror.org/037b1pp87","country_code":"CN","type":"education","lineage":["https://openalex.org/I37796252"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Husheng Liao","raw_affiliation_strings":["Faculty of Information Technology, Beijing University of Technology, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Beijing University of Technology, Beijing, China","institution_ids":["https://openalex.org/I37796252"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.4315,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.6479264,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"1","issue":null,"first_page":"2931","last_page":"2940"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spark","display_name":"SPARK (programming language)","score":0.8044974207878113},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7992333173751831},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.7727009057998657},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5737019777297974},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.5654287934303284},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5521841049194336},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.5520739555358887},{"id":"https://openalex.org/keywords/compression-ratio","display_name":"Compression ratio","score":0.5150918364524841},{"id":"https://openalex.org/keywords/compression","display_name":"Compression (physics)","score":0.47083425521850586},{"id":"https://openalex.org/keywords/transmission","display_name":"Transmission (telecommunications)","score":0.4435841739177704},{"id":"https://openalex.org/keywords/factor","display_name":"Factor (programming language)","score":0.42614176869392395},{"id":"https://openalex.org/keywords/data-compression-ratio","display_name":"Data compression ratio","score":0.4197317957878113},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.39174380898475647},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.35519739985466003},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.34256553649902344},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3139018416404724},{"id":"https://openalex.org/keywords/image-compression","display_name":"Image compression","score":0.20552456378936768},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.19098779559135437},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.1534806191921234},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.12228432297706604},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09319692850112915},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.08471289277076721}],"concepts":[{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.8044974207878113},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7992333173751831},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.7727009057998657},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5737019777297974},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.5654287934303284},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5521841049194336},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.5520739555358887},{"id":"https://openalex.org/C25797200","wikidata":"https://www.wikidata.org/wiki/Q828137","display_name":"Compression ratio","level":3,"score":0.5150918364524841},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.47083425521850586},{"id":"https://openalex.org/C761482","wikidata":"https://www.wikidata.org/wiki/Q118093","display_name":"Transmission (telecommunications)","level":2,"score":0.4435841739177704},{"id":"https://openalex.org/C2781039887","wikidata":"https://www.wikidata.org/wiki/Q1391724","display_name":"Factor (programming language)","level":2,"score":0.42614176869392395},{"id":"https://openalex.org/C94835093","wikidata":"https://www.wikidata.org/wiki/Q3113333","display_name":"Data compression ratio","level":5,"score":0.4197317957878113},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.39174380898475647},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.35519739985466003},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.34256553649902344},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3139018416404724},{"id":"https://openalex.org/C13481523","wikidata":"https://www.wikidata.org/wiki/Q412438","display_name":"Image compression","level":4,"score":0.20552456378936768},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.19098779559135437},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.1534806191921234},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.12228432297706604},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09319692850112915},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.08471289277076721},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C511840579","wikidata":"https://www.wikidata.org/wiki/Q12757","display_name":"Internal combustion engine","level":2,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C171146098","wikidata":"https://www.wikidata.org/wiki/Q124192","display_name":"Automotive engineering","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata.2017.8258262","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata.2017.8258262","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8899999856948853,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1582576952","https://openalex.org/W1794174336","https://openalex.org/W1834532152","https://openalex.org/W1980376525","https://openalex.org/W2131975293","https://openalex.org/W2150478767","https://openalex.org/W2173213060","https://openalex.org/W2175075972","https://openalex.org/W2221106743","https://openalex.org/W2294316975","https://openalex.org/W2608558840","https://openalex.org/W3088315585","https://openalex.org/W3104065274","https://openalex.org/W4297814130","https://openalex.org/W6638329389","https://openalex.org/W6639007902","https://openalex.org/W6679815717","https://openalex.org/W6783532148"],"related_works":["https://openalex.org/W2237980836","https://openalex.org/W2279512187","https://openalex.org/W2364879890","https://openalex.org/W4200061735","https://openalex.org/W2152893502","https://openalex.org/W2380817024","https://openalex.org/W1996863089","https://openalex.org/W3165542721","https://openalex.org/W2784332964","https://openalex.org/W4288059664"],"abstract_inverted_index":{"With":[0],"the":[1,19,33,59,65,80,98,101,117,124,127,159,175,185,192,213,225,237,244],"wide":[2],"application":[3,13,218],"of":[4,18,28,32,36,45,55,67,76,91,103,126,188,234],"Spark":[5,29,133,200],"big":[6,49,68],"data":[7,50,69,107,128],"platform,":[8],"some":[9,46],"problems":[10,21],"in":[11,152,205],"practical":[12],"are":[14,120],"exposed,":[15],"and":[16,38,89,113,116,138,155,236],"one":[17,31,90,195],"main":[20,73],"is":[22,30,40,58,94,162,203],"performance":[23,66,245],"optimization.":[24],"The":[25,53,72,105,220],"Shuffle":[26,56,77,193,201],"module":[27,44,57],"core":[34],"modules":[35],"Spark,":[37],"it":[39],"also":[41],"an":[42,232],"important":[43],"other":[47],"distributed":[48],"computing":[51,70],"frameworks.":[52],"design":[54],"key":[60],"factor":[61],"that":[62,224],"directly":[63],"determines":[64],"framework.":[71],"optimization":[74,197,240],"parameters":[75],"process":[78,202],"involve":[79],"CPU":[81,118],"utilization,":[82],"I/O":[83,111],"literacy":[84],"rate,":[85,88],"network":[86,106],"transmission":[87,108],"these":[92],"factors":[93],"likely":[95],"to":[96,144,183,211],"be":[97,179],"bottleneck":[99],"during":[100],"execution":[102],"application.":[104],"time":[109],"consumption,":[110],"read":[112],"write":[114],"time,":[115],"utilization":[119],"closely":[121],"related":[122],"with":[123],"size":[125],"processing.":[129],"As":[130],"a":[131],"result,":[132],"provides":[134],"compression":[135,140,147,153,156,189,215,229],"configuration":[136,161,177,187,216,230],"options":[137],"different":[139,150,172],"algorithms":[141,148],"for":[142,191,199,228],"users":[143,167,210],"select.":[145],"Different":[146],"have":[149],"effects":[151],"rate":[154],"ratio,":[157],"but":[158],"default":[160],"usually":[163],"selected":[164],"by":[165,246],"all":[166],"even":[168],"though":[169],"they":[170],"run":[171],"applications,":[173],"so":[174],"optimal":[176,186],"cannot":[178],"achieved.":[180],"In":[181],"order":[182],"achieve":[184],"algorithm":[190],"process,":[194],"cost":[196,239],"model":[198,227,241],"proposed":[204,238],"this":[206],"paper,":[207],"which":[208],"enables":[209],"get":[212],"best":[214],"before":[217],"execution.":[219],"experimental":[221],"results":[222],"show":[223],"prediction":[226],"has":[231],"accuracy":[233],"58.3%,":[235],"can":[242],"improve":[243],"48.9%.":[247]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}