{"id":"https://openalex.org/W7214404372","doi":"https://doi.org/10.48550/arxiv.2609.29278","title":"Reasoning Instructions Can Break Answer Decoding in Vision--Language Models","display_name":"Reasoning Instructions Can Break Answer Decoding in Vision--Language Models","publication_year":2026,"publication_date":"2026-09-24","ids":{"openalex":"https://openalex.org/W7214404372","doi":"https://doi.org/10.48550/arxiv.2609.29278"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.29278","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.29278","pdf_url":"https://arxiv.org/pdf/2609.29278","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.29278","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5152855591","display_name":"Zeyan Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Zeyan","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064648538","display_name":"Siyuan Qiu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiu, Siyuan","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5133557388","display_name":"Jianfeng Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu, Jianfeng","raw_affiliation_strings":["Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chain-of-thought-prompting","display_name":"chain-of-thought prompting","score":0.843999981880188},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.8320000171661377},{"id":"https://openalex.org/keywords/linear-probing","display_name":"linear probing","score":0.6079999804496765},{"id":"https://openalex.org/keywords/scienceqa","display_name":"ScienceQA","score":0.7110000252723694},{"id":"https://openalex.org/keywords/hidden-state-representations","display_name":"hidden state representations","score":0.33799999952316284}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7386000156402588},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.593999981880188},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.5631999969482422},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.554099977016449},{"id":"https://openalex.org/C141603448","wikidata":"https://www.wikidata.org/wiki/Q134830","display_name":"Prefix","level":2,"score":0.5329999923706055},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.499099999666214},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4609000086784363},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.4584999978542328},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.366100013256073},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.35659998655319214},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.320499986410141},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3183000087738037},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.30410000681877136},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28780001401901245},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.25450000166893005},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25360000133514404}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.29278","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.29278","pdf_url":"https://arxiv.org/pdf/2609.29278","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.29278","doi":"https://doi.org/10.48550/arxiv.2609.29278","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.29278","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.29278","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.29278","pdf_url":"https://arxiv.org/pdf/2609.29278","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214404372.pdf","grobid_xml":"https://content.openalex.org/works/W7214404372.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7166810306","https://openalex.org/W7160968614","https://openalex.org/W7102624217","https://openalex.org/W7133310718","https://openalex.org/W7161353660","https://openalex.org/W7162796762","https://openalex.org/W7165675192","https://openalex.org/W4404089382","https://openalex.org/W7169849130","https://openalex.org/W7166663151"],"abstract_inverted_index":{"Chain-of-thought":[0],"(CoT)":[1],"instructions":[2],"can":[3,120],"distort":[4],"multiple-choice":[5],"VLM":[6],"evaluation":[7],"when":[8],"a":[9,12],"scorer":[10],"appends":[11],"reasoning":[13],"cue":[14],"but":[15],"reads":[16],"answer-label":[17],"logits":[18],"before":[19],"the":[20,48,57,68,72,75,84,133],"model":[21,122],"generates":[22],"any":[23],"rationale.":[24],"We":[25],"call":[26],"this":[27],"CoT-prefix":[28,45,118],"scoring.":[29],"On":[30],"ScienceQA,":[31],"Qwen2.5-VL-7B":[32],"drops":[33],"from":[34,56],"80.76%":[35],"to":[36],"45.48%,":[37],"and":[38,74,80,109,128,135],"across":[39,107],"five":[40],"option-content":[41],"permutations":[42],"93.54%":[43],"of":[44],"predictions":[46],"select":[47],"first":[49],"slot.":[50],"Condition-matched":[51],"linear":[52],"probes":[53],"recover":[54],"78.94%":[55],"same":[58],"hidden":[59],"states,":[60],"while":[61,92],"free":[62],"generation":[63],"restores":[64],"75.24%,":[65],"showing":[66],"that":[67,117],"answer":[69,93],"often":[70],"survives":[71],"prefix":[73],"immediate":[76],"readout":[77],"fails.":[78],"Vocabulary":[79],"layer":[81],"diagnostics":[82],"explain":[83],"mismatch:":[85],"probability":[86],"mass":[87],"moves":[88],"toward":[89],"continuation":[90],"tokens,":[91],"information":[94],"remains":[95],"linearly":[96],"accessible":[97],"in":[98],"late":[99],"layers.":[100],"The":[101],"effect":[102],"recurs":[103],"with":[104,124],"varying":[105],"severity":[106],"datasets":[108],"models,":[110],"though":[111],"not":[112],"universally.":[113],"These":[114],"results":[115],"show":[116],"scoring":[119],"confound":[121],"knowledge":[123],"an":[125],"evaluation-interface":[126],"mismatch":[127],"should":[129],"be":[130],"avoided":[131],"unless":[132],"requested":[134],"scored":[136],"output":[137],"events":[138],"are":[139],"aligned.":[140]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-26T00:00:00"}