{"id":"https://openalex.org/W7220714928","doi":"https://doi.org/10.48550/arxiv.2610.06170","title":"MS-Exam-Gen: An Automatic Benchmark Construction Toolkit for Evaluating LLMs on Multiple Sclerosis-MRI Imaging Knowledge","display_name":"MS-Exam-Gen: An Automatic Benchmark Construction Toolkit for Evaluating LLMs on Multiple Sclerosis-MRI Imaging Knowledge","publication_year":2026,"publication_date":"2026-10-05","ids":{"openalex":"https://openalex.org/W7220714928","doi":"https://doi.org/10.48550/arxiv.2610.06170"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.06170","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.06170","pdf_url":"https://arxiv.org/pdf/2610.06170","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.06170","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5157732159","display_name":"Abdul Basit","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I120250893","display_name":"New York University Abu Dhabi","ror":"https://ror.org/00e5k0821","country_code":"AE","type":"education","lineage":["https://openalex.org/I120250893","https://openalex.org/I57206974"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Basit, Abdul","raw_affiliation_strings":["eBRAIN Lab , Division of Engineering , New York University Abu Dhabi (NYUAD) , Abu Dhabi , UAE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"eBRAIN Lab , Division of Engineering , New York University Abu Dhabi (NYUAD) , Abu Dhabi , UAE","institution_ids":["https://openalex.org/I120250893"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100647460","display_name":"Muhammad Abdullah Hanif","orcid":"https://orcid.org/0000-0001-9841-6132","observed_orcids":["https://orcid.org/0000-0001-9841-6132"]},"institutions":[{"id":"https://openalex.org/I120250893","display_name":"New York University Abu Dhabi","ror":"https://ror.org/00e5k0821","country_code":"AE","type":"education","lineage":["https://openalex.org/I120250893","https://openalex.org/I57206974"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Hanif, Muhammad Abdullah","raw_affiliation_strings":["eBRAIN Lab , Division of Engineering , New York University Abu Dhabi (NYUAD) , Abu Dhabi , UAE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"eBRAIN Lab , Division of Engineering , New York University Abu Dhabi (NYUAD) , Abu Dhabi , UAE","institution_ids":["https://openalex.org/I120250893"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005190949","display_name":"Muhammad Shafique","orcid":"https://orcid.org/0000-0002-2607-8135","observed_orcids":["https://orcid.org/0000-0002-2607-8135"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shafique, Muhammad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.8579000234603882,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.8579000234603882,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.034699998795986176,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.01209999993443489,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark-design","display_name":"benchmark design","score":0.6600000262260437},{"id":"https://openalex.org/keywords/multiple-choice-question-generation","display_name":"multiple-choice question generation","score":0.8960000276565552},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.8230000138282776},{"id":"https://openalex.org/keywords/diagnostic-criteria","display_name":"diagnostic criteria","score":0.5699999928474426},{"id":"https://openalex.org/keywords/mri-reporting","display_name":"MRI reporting","score":0.44200000166893005},{"id":"https://openalex.org/keywords/differential-diagnosis","display_name":"differential diagnosis","score":0.6959999799728394},{"id":"https://openalex.org/keywords/lesion-morphology","display_name":"lesion morphology","score":0.578000009059906}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6990000009536743},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6388999819755554},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6068999767303467},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.5367000102996826},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.49900001287460327},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49399998784065247},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45320001244544983},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.4399999976158142},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.43459999561309814},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.41110000014305115},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.40869998931884766},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.38499999046325684},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38499999046325684},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.37689998745918274},{"id":"https://openalex.org/C46304622","wikidata":"https://www.wikidata.org/wiki/Q374814","display_name":"Certification","level":2,"score":0.3734000027179718},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.36719998717308044},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.35100001096725464},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.328900009393692},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.32409998774528503},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.32019999623298645},{"id":"https://openalex.org/C204434341","wikidata":"https://www.wikidata.org/wiki/Q357789","display_name":"Adjudication","level":2,"score":0.30480000376701355},{"id":"https://openalex.org/C31601959","wikidata":"https://www.wikidata.org/wiki/Q931309","display_name":"Medical imaging","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C2777898490","wikidata":"https://www.wikidata.org/wiki/Q17157236","display_name":"Writing assessment","level":2,"score":0.2757999897003174},{"id":"https://openalex.org/C71901391","wikidata":"https://www.wikidata.org/wiki/Q7126699","display_name":"Upload","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.2651999890804291},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26510000228881836},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2606000006198883}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.06170","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.06170","pdf_url":"https://arxiv.org/pdf/2610.06170","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.06170","doi":"https://doi.org/10.48550/arxiv.2610.06170","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.06170","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.06170","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.06170","pdf_url":"https://arxiv.org/pdf/2610.06170","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.5683000087738037,"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220714928.pdf","grobid_xml":"https://content.openalex.org/works/W7220714928.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7160488224","https://openalex.org/W6902018440","https://openalex.org/W4400642441","https://openalex.org/W4389210014","https://openalex.org/W2067001863","https://openalex.org/W4412945732","https://openalex.org/W4416693324","https://openalex.org/W4412886673","https://openalex.org/W7147202400","https://openalex.org/W6910560910"],"abstract_inverted_index":{"Biomedical":[0],"large":[1],"language":[2],"model":[3],"(LLM)":[4],"evaluation":[5],"requires":[6,28],"auditable":[7],"assessment":[8],"of":[9,44,152],"narrow,":[10],"evolving,":[11],"source-grounded":[12,76,215],"subspecialty":[13],"knowledge.":[14],"Multiple":[15],"sclerosis":[16],"MRI":[17,71],"(MS-MRI)":[18],"provides":[19],"a":[20,50,57,97,105,116,141,203],"high-stakes":[21],"textual-knowledge":[22],"test":[23],"case":[24],"because":[25],"correct":[26],"reasoning":[27],"current":[29],"diagnostic":[30],"criteria,":[31,77],"standardized":[32],"acquisition":[33],"and":[34,42,55,80,101,124,137,193,218,224],"reporting":[35],"knowledge,":[36],"longitudinal":[37],"monitoring":[38],"concepts,":[39],"lesion":[40],"morphology,":[41],"recognition":[43],"difficult":[45],"mimics.":[46],"We":[47],"present":[48],"MS-Exam-Gen,":[49],"reproducible":[51,219],"framework":[52,84],"for":[53,63,222],"constructing":[54],"auditing":[56],"text-based":[58],"multiple-choice":[59],"question":[60],"(MCQ)":[61],"benchmark":[62,120,217],"MS-MRI":[64],"knowledge;":[65],"it":[66],"does":[67],"not":[68,202],"evaluate":[69],"direct":[70],"image":[72],"interpretation.":[73],"MS-Exam-Gen":[74,200],"targets":[75],"protocols,":[78],"reporting,":[79],"differential":[81],"diagnosis.":[82],"The":[83],"combines":[85],"expert-source":[86],"indexing,":[87],"exam-oriented":[88],"topic":[89],"induction,":[90],"evidence-grounded":[91],"MCQ":[92,176],"generation,":[93],"automated":[94],"quality":[95],"audits,":[96],"same-family":[98],"consistency":[99],"screen,":[100],"empirical":[102],"calibration.":[103],"From":[104],"66-source":[106],"corpus":[107],"indexed":[108],"into":[109],"4,289":[110],"retrieval":[111],"chunks,":[112],"the":[113],"pipeline":[114],"produced":[115],"locked":[117],"3,058-item":[118],"candidate":[119,216],"spanning":[121],"16":[122],"topics":[123],"53":[125],"subtopics.":[126],"Evaluation":[127],"across":[128],"12":[129],"primary":[130],"LLM":[131,226],"endpoints":[132],"yielded":[133],"36,696":[134],"item-level":[135,223],"predictions":[136],"separated":[138],"performance":[139],"over":[140],"42.8-percentage-point":[142],"accuracy":[143],"range":[144],"(89.7%":[145],"to":[146],"46.9%).":[147],"Across":[148],"these":[149],"endpoints,":[150],"25.5%":[151],"items":[153],"were":[154],"missed":[155],"by":[156],"at":[157],"least":[158],"four.":[159],"Post-generation":[160],"audits":[161],"showed":[162,173],"that":[163,174],"refreshed":[164],"construction":[165,181],"reduced":[166],"measurable":[167],"answer":[168],"cues,":[169],"while":[170],"option-order":[171,195],"testing":[172],"absolute":[175],"scores":[177],"remain":[178,183,197],"position-sensitive.":[179],"Generated":[180],"labels":[182],"metadata":[184],"rather":[185],"than":[186],"validated":[187],"psychometric":[188],"categories.":[189],"Because":[190],"expert":[191],"adjudication":[192],"full":[194],"counterbalancing":[196],"future":[198],"work,":[199],"is":[201],"clinically":[204],"certified":[205],"examination.":[206],"It":[207],"should":[208],"be":[209],"interpreted":[210],"as":[211],"an":[212],"automatically":[213],"filtered,":[214],"audit":[220],"workflow":[221],"topic-specific":[225],"evaluation.":[227]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-10-07T00:00:00"}