{"id":"https://openalex.org/W4413917725","doi":"https://doi.org/10.1109/icra55743.2025.11127987","title":"Beyond Sight: Finetuning Generalist Robot Policies with Heterogeneous Sensors via Language Grounding","display_name":"Beyond Sight: Finetuning Generalist Robot Policies with Heterogeneous Sensors via Language Grounding","publication_year":2025,"publication_date":"2025-05-19","ids":{"openalex":"https://openalex.org/W4413917725","doi":"https://doi.org/10.1109/icra55743.2025.11127987"},"language":"en","primary_location":{"id":"doi:10.1109/icra55743.2025.11127987","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra55743.2025.11127987","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042339520","display_name":"Joshua Jones","orcid":null},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joshua Jones","raw_affiliation_strings":["Berkeley AI Research (BAIR),UC Berkeley,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Berkeley AI Research (BAIR),UC Berkeley,USA","institution_ids":["https://openalex.org/I134446601"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054505204","display_name":"Oier Mees","orcid":"https://orcid.org/0000-0001-6020-9744"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oier Mees","raw_affiliation_strings":["Berkeley AI Research (BAIR),UC Berkeley,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Berkeley AI Research (BAIR),UC Berkeley,USA","institution_ids":["https://openalex.org/I134446601"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041119232","display_name":"Carmelo Sferrazza","orcid":"https://orcid.org/0000-0002-7432-7634"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carmelo Sferrazza","raw_affiliation_strings":["Berkeley AI Research (BAIR),UC Berkeley,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Berkeley AI Research (BAIR),UC Berkeley,USA","institution_ids":["https://openalex.org/I134446601"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063799945","display_name":"Kyle Stachowicz","orcid":"https://orcid.org/0000-0002-9880-7261"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kyle Stachowicz","raw_affiliation_strings":["Berkeley AI Research (BAIR),UC Berkeley,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Berkeley AI Research (BAIR),UC Berkeley,USA","institution_ids":["https://openalex.org/I134446601"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049349154","display_name":"Pieter Abbeel","orcid":null},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pieter Abbeel","raw_affiliation_strings":["Berkeley AI Research (BAIR),UC Berkeley,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Berkeley AI Research (BAIR),UC Berkeley,USA","institution_ids":["https://openalex.org/I134446601"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026322200","display_name":"Sergey Levine","orcid":"https://orcid.org/0000-0001-6764-2743"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sergey Levine","raw_affiliation_strings":["Berkeley AI Research (BAIR),UC Berkeley,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Berkeley AI Research (BAIR),UC Berkeley,USA","institution_ids":["https://openalex.org/I134446601"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I134446601"],"apc_list":null,"apc_paid":null,"fwci":14.9945,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.99191946,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"5961","last_page":"5968"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9431999921798706,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9431999921798706,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6517770290374756},{"id":"https://openalex.org/keywords/sight","display_name":"Sight","score":0.5420005321502686},{"id":"https://openalex.org/keywords/generalist-and-specialist-species","display_name":"Generalist and specialist species","score":0.519587516784668},{"id":"https://openalex.org/keywords/ground","display_name":"Ground","score":0.5003859996795654},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.48739391565322876},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2721993029117584},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10931172966957092},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.07317808270454407}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6517770290374756},{"id":"https://openalex.org/C1517167","wikidata":"https://www.wikidata.org/wiki/Q1134322","display_name":"Sight","level":2,"score":0.5420005321502686},{"id":"https://openalex.org/C45371612","wikidata":"https://www.wikidata.org/wiki/Q3058587","display_name":"Generalist and specialist species","level":3,"score":0.519587516784668},{"id":"https://openalex.org/C168993435","wikidata":"https://www.wikidata.org/wiki/Q6501125","display_name":"Ground","level":2,"score":0.5003859996795654},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.48739391565322876},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2721993029117584},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10931172966957092},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.07317808270454407},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C185933670","wikidata":"https://www.wikidata.org/wiki/Q52105","display_name":"Habitat","level":2,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icra55743.2025.11127987","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icra55743.2025.11127987","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4531707580","display_name":null,"funder_award_id":"N000142212121","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W2074936002","https://openalex.org/W2566550653","https://openalex.org/W2760103357","https://openalex.org/W2804941773","https://openalex.org/W2952866402","https://openalex.org/W3012366945","https://openalex.org/W3094502228","https://openalex.org/W3108655343","https://openalex.org/W3130633756","https://openalex.org/W3176445421","https://openalex.org/W3176646219","https://openalex.org/W3196974791","https://openalex.org/W3201065055","https://openalex.org/W4281887079","https://openalex.org/W4383097638","https://openalex.org/W4383108296","https://openalex.org/W4383109478","https://openalex.org/W4385430679","https://openalex.org/W4401415761","https://openalex.org/W4401416238","https://openalex.org/W4401417123","https://openalex.org/W4402353985","https://openalex.org/W4402354011","https://openalex.org/W4402354047","https://openalex.org/W4402726993"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3135453479","https://openalex.org/W2306541987","https://openalex.org/W2947952289","https://openalex.org/W4322625948","https://openalex.org/W1986194836","https://openalex.org/W2896365120","https://openalex.org/W1986504168"],"abstract_inverted_index":{"Interacting":[0],"with":[1,117],"the":[2,128,174,197],"world":[3,199],"is":[4,37,177,203],"a":[5,41,43,81,107,113,118,154],"multi-sensory":[6],"experience:":[7],"achieving":[8],"effective":[9],"general-purpose":[10],"interaction":[11],"requires":[12],"making":[13],"use":[14],"of":[15,50,130,166],"all":[16,214],"available":[17,101],"modalities":[18,93,146],"-":[19,25],"including":[20,184],"vision,":[21,149],"touch,":[22,150],"and":[23,52,72,151,164,189],"audio":[24],"to":[26,65,123,179,205,213],"fill":[27],"in":[28,153,196],"gaps":[29],"from":[30,70],"partial":[31],"observation.":[32],"For":[33],"example,":[34],"when":[35],"vision":[36],"occluded":[38],"reaching":[39],"into":[40],"bag,":[42],"robot":[44,57,67,131],"should":[45],"rely":[46],"on":[47,62,90],"its":[48],"senses":[49],"touch":[51],"sound.":[53],"However,":[54],"state-of-the-art":[55],"generalist":[56,88,182,187],"policies":[58,89,188],"are":[59,98],"typically":[60],"trained":[61],"large":[63,96,190],"datasets":[64,97],"predict":[66],"actions":[68],"solely":[69],"visual":[71],"proprioceptive":[73],"observations.":[74],"In":[75,127],"this":[76],"work,":[77],"we":[78,133],"propose":[79],"FuSe,":[80],"novel":[82],"approach":[83],"that":[84,135,141,173,201],"enables":[85,137],"finetuning":[86],"visuomotor":[87],"heterogeneous":[91],"sensor":[92],"for":[94],"which":[95],"not":[99],"readily":[100],"by":[102,209],"leveraging":[103],"natural":[104],"language":[105,120],"as":[106,148,158],"common":[108],"cross-modal":[109,162],"grounding.":[110],"We":[111,171],"combine":[112],"multimodal":[114,159],"contrastive":[115],"loss":[116,122],"sensory-grounded":[119],"generation":[121],"encode":[124],"high-level":[125],"semantics.":[126],"context":[129],"manipulation,":[132],"show":[134,172,200],"FuSe":[136,202],"performing":[138],"challenging":[139],"tasks":[140],"require":[142],"reasoning":[143],"jointly":[144],"over":[145,210],"such":[147,157],"sound":[152],"zero-shot":[155],"setting,":[156],"prompting,":[160,163],"compositional":[161],"descriptions":[165],"objects":[167],"it":[168],"interacts":[169],"with.":[170],"same":[175],"recipe":[176],"applicable":[178],"widely":[180],"different":[181],"policies,":[183],"both":[185],"diffusion-based":[186],"vision-language-action":[191],"(VLA)":[192],"models.":[193],"Extensive":[194],"experiments":[195],"real":[198],"able":[204],"increase":[206],"success":[207],"rates":[208],"20%":[211],"compared":[212],"considered":[215],"baselines.":[216]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":4}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}