{"id":"https://openalex.org/W7164885466","doi":"https://doi.org/10.48550/arxiv.2606.15308","title":"Forced Deferral: Manipulating Routing Decisions in Multimodal LLM Cascades","display_name":"Forced Deferral: Manipulating Routing Decisions in Multimodal LLM Cascades","publication_year":2026,"publication_date":"2026-06-13","ids":{"openalex":"https://openalex.org/W7164885466","doi":"https://doi.org/10.48550/arxiv.2606.15308"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.15308","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15308","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.15308","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089751051","display_name":"Zhongye Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zhongye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101292344","display_name":"Yaopei Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Yaopei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138689976","display_name":"Yurui Chang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Yurui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5110853658","display_name":"L C-C Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Lu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.3077000081539154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.3077000081539154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2298000007867813,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.13590000569820404,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversary","display_name":"Adversary","score":0.7405999898910522},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6940000057220459},{"id":"https://openalex.org/keywords/forcing","display_name":"Forcing (mathematics)","score":0.6485999822616577},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.5127000212669373},{"id":"https://openalex.org/keywords/margin","display_name":"Margin (machine learning)","score":0.5049999952316284},{"id":"https://openalex.org/keywords/threat-model","display_name":"Threat model","score":0.49000000953674316},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.4481000006198883}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7610999941825867},{"id":"https://openalex.org/C41065033","wikidata":"https://www.wikidata.org/wiki/Q2825412","display_name":"Adversary","level":2,"score":0.7405999898910522},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6940000057220459},{"id":"https://openalex.org/C197115733","wikidata":"https://www.wikidata.org/wiki/Q1003136","display_name":"Forcing (mathematics)","level":2,"score":0.6485999822616577},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.5127000212669373},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.5049999952316284},{"id":"https://openalex.org/C140547941","wikidata":"https://www.wikidata.org/wiki/Q7797194","display_name":"Threat model","level":2,"score":0.49000000953674316},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.4481000006198883},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.39820000529289246},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3894999921321869},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3504999876022339},{"id":"https://openalex.org/C149810388","wikidata":"https://www.wikidata.org/wiki/Q5374873","display_name":"Emulation","level":2,"score":0.3253999948501587},{"id":"https://openalex.org/C2778281098","wikidata":"https://www.wikidata.org/wiki/Q885383","display_name":"Deferral","level":2,"score":0.30489999055862427},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2930000126361847},{"id":"https://openalex.org/C2777317252","wikidata":"https://www.wikidata.org/wiki/Q18393516","display_name":"Rare events","level":2,"score":0.26989999413490295},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.25450000166893005}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.15308","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15308","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.15308","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15308","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"While":[0],"multimodal":[1],"large":[2,14],"language":[3],"models":[4],"(MLLMs)":[5],"have":[6],"shown":[7],"strong":[8,39,79,110],"visual":[9],"reasoning":[10],"abilities,":[11],"serving":[12],"a":[13,30,38,61,114,120],"model":[15,34,40,145],"for":[16],"every":[17],"query":[18],"is":[19,46],"computationally":[20],"expensive.":[21],"MLLM":[22,165],"cascades":[23,104,166],"mitigate":[24],"this":[25,83],"cost":[26],"by":[27,82,118],"first":[28],"querying":[29],"weak":[31,43,51,99,127],"but":[32],"cheaper":[33],"and":[35,102,147,158],"deferring":[36],"to":[37,77,105,108,169],"when":[41],"the":[42,50,78,87,98,109,126],"model's":[44,52,100,128],"output":[45],"unconfident.":[47],"However,":[48],"since":[49],"confidence":[53,69,101],"directly":[54,180],"controls":[55],"compute":[56,173],"allocation,":[57,174],"these":[58],"systems":[59],"expose":[60],"new":[62],"attack":[63,95],"surface:":[64],"an":[65,92],"adversary":[66],"can":[67],"manipulate":[68,172],"so":[70],"that":[71,96,164,171],"their":[72],"queries":[73,107],"are":[74,167],"consistently":[75,151],"deferred":[76],"model.":[80,111],"Motivated":[81],"vulnerability,":[84],"we":[85],"introduce":[86],"Forced":[88],"Deferral":[89],"Attack":[90],"(FDA),":[91],"adversarial":[93],"image":[94],"lowers":[97],"causes":[103],"route":[106],"FDA":[112,150],"learns":[113],"universal":[115],"border":[116],"trigger":[117],"optimizing":[119],"temperature-flattened":[121],"objective.":[122],"This":[123],"objective":[124],"pushes":[125],"token":[129],"distribution":[130],"on":[131],"triggered":[132],"inputs":[133],"toward":[134],"less":[135],"concentrated":[136],"targets":[137],"constructed":[138],"from":[139],"its":[140],"clean":[141],"responses.":[142],"Across":[143],"datasets,":[144],"families,":[146],"deferral":[148],"metrics,":[149],"increases":[152],"strong-model":[153,177],"routing":[154],"while":[155],"outperforming":[156],"image-perturbation":[157],"prompt-injection":[159],"baselines.":[160],"These":[161],"results":[162],"show":[163],"vulnerable":[168],"attacks":[170],"forcing":[175],"unintended":[176],"usage":[178],"without":[179],"targeting":[181],"answer":[182],"correctness.":[183]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-17T00:00:00"}