{"id":"https://openalex.org/W4411800494","doi":"https://doi.org/10.1109/ccgrid64434.2025.00052","title":"rInfer: A Generic and High-Performance Framework for Remote Inference with Heterogeneous Accelerators","display_name":"rInfer: A Generic and High-Performance Framework for Remote Inference with Heterogeneous Accelerators","publication_year":2025,"publication_date":"2025-05-19","ids":{"openalex":"https://openalex.org/W4411800494","doi":"https://doi.org/10.1109/ccgrid64434.2025.00052"},"language":"en","primary_location":{"id":"doi:10.1109/ccgrid64434.2025.00052","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ccgrid64434.2025.00052","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 25th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100779363","display_name":"Zhen Jin","orcid":"https://orcid.org/0000-0002-9763-707X"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhen Jin","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053247827","display_name":"Yiquan Chen","orcid":"https://orcid.org/0009-0004-8714-6949"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yiquan Chen","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100780980","display_name":"Yijing Wang","orcid":"https://orcid.org/0000-0002-2430-7213"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yijing Wang","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051629728","display_name":"Du Yin","orcid":"https://orcid.org/0000-0002-2345-0683"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yin Du","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016945447","display_name":"Keyao Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Keyao Zhang","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069329526","display_name":"Jiexiong Xu","orcid":"https://orcid.org/0000-0002-2491-1381"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiexiong Xu","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065265972","display_name":"Wenhai Lin","orcid":"https://orcid.org/0000-0002-0642-9073"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenhai Lin","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101269459","display_name":"Jingchang Qin","orcid":null},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingchang Qin","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068364613","display_name":"Kanghua Fang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kanghua Fang","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I4210095624"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101562847","display_name":"Wenzhi Chen","orcid":"https://orcid.org/0000-0003-1674-4701"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenzhi Chen","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.13744946,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"363","last_page":"372"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.9796000123023987,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.780854344367981},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6701482534408569},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4114844799041748},{"id":"https://openalex.org/keywords/computer-architecture","display_name":"Computer architecture","score":0.33332836627960205},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24810129404067993}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.780854344367981},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6701482534408569},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4114844799041748},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.33332836627960205},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24810129404067993}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ccgrid64434.2025.00052","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ccgrid64434.2025.00052","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 25th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W1508947956","https://openalex.org/W1996212904","https://openalex.org/W2005574683","https://openalex.org/W2010220217","https://openalex.org/W2033242260","https://openalex.org/W2327501763","https://openalex.org/W2345057472","https://openalex.org/W2606722458","https://openalex.org/W2734941459","https://openalex.org/W2883929540","https://openalex.org/W2907351266","https://openalex.org/W2984100107","https://openalex.org/W3036601975","https://openalex.org/W3138930359","https://openalex.org/W3174399477","https://openalex.org/W3180530691","https://openalex.org/W4205402639","https://openalex.org/W4214624153","https://openalex.org/W4285504009","https://openalex.org/W4288083766","https://openalex.org/W4295312788","https://openalex.org/W4297808460","https://openalex.org/W4313156423","https://openalex.org/W4372262787","https://openalex.org/W4400234037","https://openalex.org/W6678204122","https://openalex.org/W6747759466","https://openalex.org/W6766978945","https://openalex.org/W6780218876","https://openalex.org/W6989568545","https://openalex.org/W7067822191"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Inference":[0],"applications":[1,19],"leverage":[2],"various":[3,178],"heterogeneous":[4,56,112],"accelerators,":[5],"including":[6],"GPUs,":[7],"TPUs,":[8],"and":[9,20,34,58,77,92,105,127,142,156,172,192,200,248,253],"FPGAs,":[10],"to":[11,22,53,60,120,196,203,217,233],"achieve":[12],"remarkable":[13],"performance.":[14],"Remote":[15],"inference,":[16],"which":[17],"enables":[18],"accelerators":[21,57],"reside":[23],"on":[24,258],"different":[25],"nodes,":[26],"can":[27],"effectively":[28],"address":[29],"low":[30],"hardware":[31],"resource":[32,38],"utilization":[33],"enhance":[35,204],"flexibility":[36],"in":[37,88,194,241],"allocation.":[39],"However,":[40],"existing":[41],"APIremoting":[42],"solutions":[43],"face":[44],"compatibility":[45,176],"issues,":[46],"as":[47,170],"they":[48],"demand":[49,73],"extensive":[50],"development":[51],"efforts":[52],"support":[54],"diverse":[55,166],"adapt":[59],"rapidly":[61],"evolving":[62],"device":[63],"runtimes.":[64],"On":[65,131,159],"the":[66,89,122,132,151,160,188,221,245,259],"other":[67],"hand,":[68],"current":[69],"inference":[70,110,125,167],"service":[71],"systems":[72],"cumbersome":[74],"cross-server":[75],"configuration":[76,155],"suffer":[78],"from":[79,150],"suboptimal":[80],"data":[81,85,184,198,205],"transmission":[82,185],"performance,":[83],"overlooking":[84],"copy":[86],"overhead":[87],"network":[90,129,157,183],"stack":[91],"not":[93],"leveraging":[94],"high-performance":[95],"RDMA.":[96],"In":[97],"this":[98],"paper,":[99],"we":[100,181],"present":[101],"rInfer,":[102],"a":[103,226,237],"generic":[104],"highperformance":[106],"framework":[107],"for":[108,190,220,244],"remote":[109,124],"with":[111,165,177,224],"accelerators.":[113,179],"The":[114],"key":[115],"idea":[116],"of":[117,153,229],"rInfer":[118,135,140,145,163,212],"is":[119],"abstract":[121],"entire":[123],"process":[126],"optimize":[128,182],"transmission.":[130],"client":[133],"node,":[134,162],"offers":[136],"users":[137,149],"an":[138,143],"inference-oriented":[139],"Device":[141],"ease-of-use":[144],"programming":[146],"model,":[147,223],"shielding":[148],"complexities":[152],"server":[154,161,260],"communication.":[158],"integrates":[164],"frameworks,":[168],"such":[169],"TensorRT":[171],"Torch,":[173],"ensuring":[174],"high":[175],"Furthermore,":[180],"by":[186],"eliminating":[187],"need":[189],"serialization":[191],"deserialization":[193],"Protobuf":[195],"minimize":[197],"copying":[199],"adopt":[201],"RDMA":[202],"transfer":[206],"efficiency.":[207],"Experimental":[208],"results":[209],"demonstrate":[210],"that":[211],"achieves":[213,236],"performance":[214],"nearly":[215],"equivalent":[216],"local":[218],"execution":[219,242],"BERT":[222],"only":[225],"minor":[227],"difference":[228],"1.5":[230],"%.":[231],"Compared":[232],"TorchServe,":[234],"rInfer-TCP":[235],"51.6":[238],"%":[239],"reduction":[240],"time":[243],"ResNet":[246],"models":[247],"consumes":[249],"37.3%":[250],"less":[251,255],"CPU":[252],"61.2%":[254],"memory":[256],"bandwidth":[257],"node.":[261]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}