{"id":"https://openalex.org/W7138103932","doi":"https://doi.org/10.48550/arxiv.2603.13691","title":"QuarkMedBench: A Real-World Scenario Driven Benchmark for Evaluating Large Language Models","display_name":"QuarkMedBench: A Real-World Scenario Driven Benchmark for Evaluating Large Language Models","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138103932","doi":"https://doi.org/10.48550/arxiv.2603.13691"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.13691","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13691","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.13691","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129641467","display_name":"Yao Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129729608","display_name":"Kangping Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Kangping","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129679413","display_name":"Liang Dong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129642734","display_name":"Zhenxin Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Zhenxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129706387","display_name":"Shuting Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Shuting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Xuehai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xuehai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006561266","display_name":"Yuxuan Jiang","orcid":"https://orcid.org/0000-0002-2279-6199"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Yuxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129660932","display_name":"Tingting Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Tingting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062716187","display_name":"Yunqing Hong","orcid":"https://orcid.org/0000-0001-9748-6489"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hong, Yunqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129730024","display_name":"Jiayi Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiayi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129651691","display_name":"Rianzhe Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Rianzhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113483104","display_name":"Shuxin Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Shuxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129733091","display_name":"Haiping Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Haiping","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129726547","display_name":"Wen Shang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shang, Wen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129665217","display_name":"Jian Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129675100","display_name":"Guanjun Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Guanjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.49939998984336853,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.49939998984336853,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.362199991941452,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.02879999950528145,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.807200014591217},{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.7379999756813049},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5945000052452087},{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.5740000009536743},{"id":"https://openalex.org/keywords/yardstick","display_name":"Yardstick","score":0.516700029373169},{"id":"https://openalex.org/keywords/concordance","display_name":"Concordance","score":0.3781000077724457},{"id":"https://openalex.org/keywords/pruning","display_name":"Pruning","score":0.33180001378059387}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.807200014591217},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.762499988079071},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.7379999756813049},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5945000052452087},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.5740000009536743},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5501000285148621},{"id":"https://openalex.org/C82261069","wikidata":"https://www.wikidata.org/wiki/Q1925365","display_name":"Yardstick","level":2,"score":0.516700029373169},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4359999895095825},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4284000098705292},{"id":"https://openalex.org/C160798450","wikidata":"https://www.wikidata.org/wiki/Q4230870","display_name":"Concordance","level":2,"score":0.3781000077724457},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.33180001378059387},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2985000014305115},{"id":"https://openalex.org/C69505689","wikidata":"https://www.wikidata.org/wiki/Q455338","display_name":"Unified Medical Language System","level":2,"score":0.29820001125335693},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.29760000109672546},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C148524875","wikidata":"https://www.wikidata.org/wiki/Q6975395","display_name":"F1 score","level":2,"score":0.2605000138282776},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.25130000710487366}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.13691","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13691","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.13691","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13691","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6293725371360779}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"While":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"excel":[5],"on":[6,27,161,194],"standardized":[7],"medical":[8,21,58,119,156],"exams,":[9],"high":[10,129],"scores":[11],"often":[12],"fail":[13],"to":[14,16,31,100,206],"translate":[15],"high-quality":[17],"responses":[18],"for":[19,56,190],"real-world":[20,57,173],"queries.":[22],"Current":[23],"evaluations":[24,160],"rely":[25],"heavily":[26],"multiple-choice":[28],"questions,":[29],"failing":[30],"capture":[32],"the":[33,128,140,177],"unstructured,":[34],"ambiguous,":[35],"and":[36,71,78,114,123,131],"long-tail":[37],"complexities":[38],"inherent":[39],"in":[40],"genuine":[41],"user":[42],"inquiries.":[43],"To":[44,82],"bridge":[45],"this":[46,162],"gap,":[47],"we":[48,87],"introduce":[49],"QuarkMedBench,":[50],"an":[51,89],"ecologically":[52],"valid":[53],"benchmark":[54,163,208],"tailored":[55],"LLM":[59,192],"assessment.":[60],"We":[61],"compiled":[62],"a":[63,144,186],"massive":[64],"dataset":[65],"spanning":[66],"Clinical":[67],"Care,":[68],"Wellness":[69],"Health,":[70],"Professional":[72],"Inquiry,":[73],"comprising":[74],"20,821":[75],"single-turn":[76],"queries":[77],"3,853":[79],"multi-turn":[80],"sessions.":[81],"objectively":[83],"evaluate":[84],"open-ended":[85],"answers,":[86],"propose":[88],"automated":[90],"scoring":[91,105],"framework":[92,200],"that":[93,139],"integrates":[94],"multi-model":[95],"consensus":[96],"with":[97,148],"evidence-based":[98],"retrieval":[99],"dynamically":[101],"generate":[102],"220,617":[103],"fine-grained":[104],"rubrics":[106,142],"(~9.8":[107],"per":[108],"query).":[109],"During":[110],"evaluation,":[111],"hierarchical":[112],"weighting":[113],"safety":[115],"constraints":[116],"structurally":[117],"quantify":[118],"accuracy,":[120],"key-point":[121],"coverage,":[122],"risk":[124],"interception,":[125],"effectively":[126],"mitigating":[127],"costs":[130],"subjectivity":[132],"of":[133,179],"human":[134],"grading.":[135],"Experimental":[136],"results":[137],"demonstrate":[138],"generated":[141],"achieve":[143],"91.8%":[145],"concordance":[146],"rate":[147],"clinical":[149,174],"expert":[150],"blind":[151],"audits,":[152],"establishing":[153],"highly":[154],"dependable":[155],"reliability.":[157],"Crucially,":[158],"baseline":[159],"reveal":[164],"significant":[165],"performance":[166,193],"disparities":[167],"among":[168],"state-of-the-art":[169],"models":[170],"when":[171],"navigating":[172],"nuances,":[175],"highlighting":[176],"limitations":[178],"conventional":[180],"exam-based":[181],"metrics.":[182],"Ultimately,":[183],"QuarkMedBench":[184],"establishes":[185],"rigorous,":[187],"reproducible":[188],"yardstick":[189],"measuring":[191],"complex":[195],"health":[196],"issues,":[197],"while":[198],"its":[199],"inherently":[201],"supports":[202],"dynamic":[203],"knowledge":[204],"updates":[205],"prevent":[207],"obsolescence.":[209]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-18T00:00:00"}