{"id":"https://openalex.org/W7225200264","doi":"https://doi.org/10.48550/arxiv.2611.03377","title":"Same Outputs, Different Scores: Auditing the Unchecked Graders of Multimodal LLM Benchmarks","display_name":"Same Outputs, Different Scores: Auditing the Unchecked Graders of Multimodal LLM Benchmarks","publication_year":2026,"publication_date":"2026-11-02","ids":{"openalex":"https://openalex.org/W7225200264","doi":"https://doi.org/10.48550/arxiv.2611.03377"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2611.03377","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03377","pdf_url":"https://arxiv.org/pdf/2611.03377","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2611.03377","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5160501734","display_name":"Ahmad Jarjis Hasan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hasan, Ahmad Jarjis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5055672869","display_name":"Jonathan Lwowski","orcid":"https://orcid.org/0000-0001-7568-4265","observed_orcids":["https://orcid.org/0000-0001-7568-4265"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lwowski, Jonathan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9843999743461609,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9843999743461609,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scoring-rules","display_name":"scoring rules","score":0.734000027179718},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.824999988079071},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.7419999837875366}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7215999960899353},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7200000286102295},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.6107000112533569},{"id":"https://openalex.org/C61783943","wikidata":"https://www.wikidata.org/wiki/Q1970348","display_name":"Luck","level":2,"score":0.5626999735832214},{"id":"https://openalex.org/C204434341","wikidata":"https://www.wikidata.org/wiki/Q357789","display_name":"Adjudication","level":2,"score":0.5379999876022339},{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.5350000262260437},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.489300012588501},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4350000023841858},{"id":"https://openalex.org/C63002673","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring rule","level":2,"score":0.42899999022483826},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.41119998693466187},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.3840999901294708},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3449999988079071},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34310001134872437},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.3206000030040741},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.3125},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.29030001163482666},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2741999924182892},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C3020493868","wikidata":"https://www.wikidata.org/wiki/Q55631277","display_name":"Real world data","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.25760000944137573}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2611.03377","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03377","pdf_url":"https://arxiv.org/pdf/2611.03377","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2611.03377","doi":"https://doi.org/10.48550/arxiv.2611.03377","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.03377","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2611.03377","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03377","pdf_url":"https://arxiv.org/pdf/2611.03377","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7225200264.pdf","grobid_xml":"https://content.openalex.org/works/W7225200264.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7143510439","https://openalex.org/W7223661311","https://openalex.org/W4415280678","https://openalex.org/W7123672319","https://openalex.org/W7166852497","https://openalex.org/W7162988157","https://openalex.org/W7124151122","https://openalex.org/W4391673253","https://openalex.org/W4416781454","https://openalex.org/W7134034217"],"abstract_inverted_index":{"Every":[0],"multimodal-LLM":[1],"multiple-choice":[2],"benchmark":[3,129],"ends":[4],"with":[5,130,152,159,207],"a":[6,11,19,24,66,73,104,149,175,202,226],"scoring":[7,87,197],"rule":[8,35,88,171],"that":[9,64,79,124],"converts":[10],"model's":[12],"response,":[13],"often":[14],"free-form":[15],"text":[16],"rather":[17,209],"than":[18,210],"clean":[20,107],"option":[21,108,180],"letter,":[22],"into":[23],"right/wrong":[25],"verdict,":[26],"buried":[27],"in":[28,193],"its":[29,143,164,178],"code.":[30],"Unlike":[31],"the":[32,46,51,82,118,122,126,128,131,166,219],"dataset,":[33],"this":[34],"is":[36,117,172],"chosen":[37],"without":[38],"justification":[39],"and":[40,71,120,195,218,221,229],"left":[41],"unexamined,":[42],"yet":[43,100,185],"it":[44,111],"sets":[45],"published":[47],"number.":[48],"We":[49,213],"reproduce":[50],"official":[52],"scorers":[53],"of":[54,139,147,163],"eight":[55],"peer-reviewed":[56],"benchmarks":[57,223],"on":[58,97,113],"five":[59],"open":[60],"vision-language":[61],"models":[62],"(VLMs)":[63],"fit":[65],"single":[67,203],"16":[68],"GB":[69],"GPU,":[70],"introduce":[72],"blinded":[74],"human-plus-LLM-judge":[75],"adjudication":[76],"(Cohen's":[77],"kappa=0.956)":[78],"recovers":[80],"what":[81],"model":[83,105,151],"actually":[84],"answered.":[85],"The":[86],"alone":[89],"moves":[90],"accuracy":[91],"by":[92],"up":[93],"to":[94,177],"22":[95],"points":[96,158],"identical":[98],"outputs,":[99],"exactly":[101,140],"0":[102],"wherever":[103],"emits":[106],"letters,":[109],"so":[110,201],"hides":[112],"strong-model":[114],"leaderboards.":[115],"It":[116],"fallback":[119],"not":[121,234],"rambling":[123],"does":[125],"damage:":[127],"highest":[132],"non-compliance":[133,153,227],"we":[134],"measure":[135],"has":[136],"an":[137],"artifact":[138],"zero,":[141],"because":[142],"scorer":[144],"abstains":[145],"instead":[146],"guessing;":[148],"reasoning":[150],"0.000":[154],"still":[155],"swings":[156],"9.5":[157],"which":[160],"regex":[161],"match":[162],"trace":[165],"parser":[167],"keeps.":[168],"No":[169],"standard":[170],"reliable:":[173],"snapping":[174],"non-answer":[176],"closest":[179],"credits":[181],"answers":[182,191],"never":[183],"given":[184],"buries":[186],"correct":[187],"ones,":[188],"abstention":[189],"discards":[190],"stated":[192],"words,":[194],"likelihood":[196],"follows":[198],"answer-token":[199],"priors,":[200],"grade":[204],"conflates":[205],"capability":[206],"luck":[208],"measuring":[211],"either.":[212],"release":[214],"all":[215],"code,":[216],"labels,":[217],"protocol,":[220],"argue":[222],"should":[224],"report":[225],"rate":[228],"at":[230],"least":[231],"two":[232],"rules,":[233],"one":[235],"unexamined":[236],"grade.":[237]},"counts_by_year":[],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2026-11-04T00:00:00"}