{"id":"https://openalex.org/W4416048756","doi":"https://doi.org/10.1109/iccv51701.2025.00407","title":"Zero-Shot Vision Encoder Grafting via LLM Surrogates","display_name":"Zero-Shot Vision Encoder Grafting via LLM Surrogates","publication_year":2025,"publication_date":"2025-10-19","ids":{"openalex":"https://openalex.org/W4416048756","doi":"https://doi.org/10.1109/iccv51701.2025.00407"},"language":"en","primary_location":{"id":"doi:10.1109/iccv51701.2025.00407","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.00407","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.22664","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027880686","display_name":"Kaiyu Yue","orcid":"https://orcid.org/0000-0002-1820-3223"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kaiyu Yue","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083826260","display_name":"Vasu Singla","orcid":null},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vasu Singla","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047506888","display_name":"Menglin Jia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Menglin Jia","raw_affiliation_strings":["Meta"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025859844","display_name":"John Kirchenbauer","orcid":null},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Kirchenbauer","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099241666","display_name":"Rifaa Qadri","orcid":null},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rifaa Qadri","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063977912","display_name":"Zikui Cai","orcid":"https://orcid.org/0000-0003-1663-9493"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zikui Cai","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081506338","display_name":"Abhinav Bhatel\u00e9","orcid":"https://orcid.org/0000-0003-3069-3701"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abhinav Bhatele","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091135797","display_name":"Furong Huang","orcid":"https://orcid.org/0000-0001-8760-439X"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Furong Huang","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060687985","display_name":"Tom Goldstein","orcid":"https://orcid.org/0000-0003-1660-9307"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tom Goldstein","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.37370194,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4275","last_page":"4284"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2750999927520752,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2750999927520752,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.15459999442100525,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1281999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.796999990940094},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.6700000166893005},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5764999985694885},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5611000061035156},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.48080000281333923},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.4514999985694885},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.44609999656677246}],"concepts":[{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.796999990940094},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7422000169754028},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.6700000166893005},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5764999985694885},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5656999945640564},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5647000074386597},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5611000061035156},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.48080000281333923},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.4514999985694885},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.44609999656677246},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.44130000472068787},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4163999855518341},{"id":"https://openalex.org/C5339829","wikidata":"https://www.wikidata.org/wiki/Q1425977","display_name":"Machine vision","level":2,"score":0.3709000051021576},{"id":"https://openalex.org/C2775937945","wikidata":"https://www.wikidata.org/wiki/Q222958","display_name":"Grafting","level":3,"score":0.32409998774528503},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.272599995136261},{"id":"https://openalex.org/C131675550","wikidata":"https://www.wikidata.org/wiki/Q7646884","display_name":"Surrogate model","level":2,"score":0.26969999074935913},{"id":"https://openalex.org/C77660490","wikidata":"https://www.wikidata.org/wiki/Q244916","display_name":"Intermediate language","level":3,"score":0.2574999928474426}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/iccv51701.2025.00407","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.00407","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.22664","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.22664","pdf_url":"https://arxiv.org/pdf/2505.22664","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.22664","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.22664","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.22664","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.22664","pdf_url":"https://arxiv.org/pdf/2505.22664","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3442346314","display_name":"CIF: RI: Medium: Design principles and theory for data augmentation","funder_award_id":"2212182","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4078776429","display_name":null,"funder_award_id":"IIS-2212182","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320315389","display_name":"Open Philanthropy Project","ror":"https://ror.org/004d1k391"},{"id":"https://openalex.org/F4320333591","display_name":"Multidisciplinary University Research Initiative","ror":null},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416048756.pdf","grobid_xml":"https://content.openalex.org/works/W4416048756.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Vision":[0,79],"language":[1,14,45,67],"models":[2],"(VLMs)":[3],"typically":[4],"pair":[5,111,115],"a":[6,12,31,43,94],"modestly":[7],"sized":[8],"vision":[9,40],"encoder":[10,41],"with":[11,124,128],"large":[13,52,70],"model":[15,46],"(LLM),":[16],"e.g.,":[17],"Llama-70B,":[18],"making":[19],"the":[20,22,39,51,61,69,83,91,105,109,113,129,148],"decoder":[21,126],"primary":[23],"computational":[24],"burden":[25],"during":[26],"training.":[27],"To":[28],"reduce":[29],"costs,":[30],"potential":[32],"promising":[33],"strategy":[34],"is":[35,152],"to":[36,50,90],"first":[37],"train":[38],"using":[42,145],"small":[44,56],"before":[47],"transferring":[48],"it":[49],"one.":[53],"We":[54],"construct":[55],"\"surrogate":[57],"models\"":[58],"that":[59],"share":[60],"same":[62],"embedding":[63],"space":[64],"and":[65],"representation":[66],"as":[68,147],"target":[71,107,130],"LLM":[72],"by":[73,142],"directly":[74,88,103],"inheriting":[75],"its":[76],"shallow":[77],"layers.":[78],"encoders":[80],"trained":[81],"on":[82,117,122],"surrogate":[84,134],"can":[85],"then":[86],"be":[87],"transferred":[89],"larger":[92],"model,":[93],"process":[95],"we":[96],"call":[97],"zero-shot":[98],"grafting":[99],"--":[100],"when":[101,144],"plugged":[102],"into":[104],"full-size":[106],"LLM,":[108],"grafted":[110],"surpasses":[112],"encoder-surrogate":[114],"and,":[116],"some":[118],"benchmarks,":[119],"even":[120],"performs":[121],"par":[123],"full":[125],"training":[127,135,140],"LLM.":[131],"Furthermore,":[132],"our":[133],"approach":[136],"reduces":[137],"overall":[138],"VLM":[139],"costs":[141],"~45%":[143],"Llama-70B":[146],"decoder.":[149],"The":[150],"code":[151],"at":[153],"https://github.com/facebookresearch/zero.":[154]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}