{"id":"https://openalex.org/W7224601685","doi":"https://doi.org/10.18653/2026.emnlp-main.62","title":"The Evaluation Scaling Law: Toward More Reliable LLM Assessment via Multi-Prompt Diversity","display_name":"The Evaluation Scaling Law: Toward More Reliable LLM Assessment via Multi-Prompt Diversity","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7224601685","doi":"https://doi.org/10.18653/2026.emnlp-main.62"},"language":"en","primary_location":{"id":"doi:10.18653/2026.emnlp-main.62","doi":"https://doi.org/10.18653/2026.emnlp-main.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.62","pdf_url":"https://aclanthology.org/2026.emnlp-main.62.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.emnlp-main.62.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069329865","display_name":"Hongzhi Luan","orcid":"https://orcid.org/0009-0002-4168-4640","observed_orcids":["https://orcid.org/0009-0002-4168-4640"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hongzhi Luan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001638913","display_name":"Zhaoxin Huan","orcid":"https://orcid.org/0000-0002-3611-0901","observed_orcids":["https://orcid.org/0000-0002-3611-0901"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhaoxin Huan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160109460","display_name":"Linfeng Shi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Linfeng Shi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160113805","display_name":"Longfei Zheng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Longfei Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160068417","display_name":"Xiaolu Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaolu Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5160128867","display_name":"Jun Zhou","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jun Zhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.90977511,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1419","last_page":"1433"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6211000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6211000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3041999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.010200000368058681,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.8610000014305115},{"id":"https://openalex.org/keywords/prompt-diversity","display_name":"prompt diversity","score":0.4429999887943268}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5016000270843506},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.41600000858306885},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.35499998927116394},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.3203999996185303},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.29809999465942383},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2946999967098236},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.27070000767707825},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.26910001039505005},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.263700008392334},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/2026.emnlp-main.62","doi":"https://doi.org/10.18653/2026.emnlp-main.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.62","pdf_url":"https://aclanthology.org/2026.emnlp-main.62.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/2026.emnlp-main.62","doi":"https://doi.org/10.18653/2026.emnlp-main.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.62","pdf_url":"https://aclanthology.org/2026.emnlp-main.62.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7224601685.pdf","grobid_xml":"https://content.openalex.org/works/W7224601685.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4412887089","https://openalex.org/W4402667036","https://openalex.org/W4415798641","https://openalex.org/W4404783034","https://openalex.org/W7106323100","https://openalex.org/W4416037937","https://openalex.org/W4416034661","https://openalex.org/W7219157097","https://openalex.org/W7224563552","https://openalex.org/W4416037872"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Model":[2],"(LLM)":[3],"evaluation":[4,29,71,83],"is":[5],"frequently":[6],"challenged":[7],"by":[8],"prompt":[9,41,64],"sensitivity,":[10],"which":[11],"limits":[12],"the":[13,61,87,101],"reliability":[14,72],"of":[15,63,89,103,119,133],"benchmarks":[16],"relying":[17],"on":[18],"a":[19,27,38,55,96,106,123],"single":[20],"template.To":[21],"address":[22],"this,":[23],"we":[24,53,69],"systematically":[25],"investigate":[26],"multi-prompt":[28],"framework":[30,114],"that":[31,82,128],"aggregates":[32],"performance":[33],"across":[34],"diverse":[35],"templates,":[36],"leveraging":[37],"Quality-Diversity":[39],"(QD)":[40],"selection":[42],"mechanism":[43],"to":[44,74,100],"ensure":[45],"both":[46],"variety":[47],"and":[48,66,136],"fidelity.Through":[49],"extensive":[50],"empirical":[51],"analysis,":[52],"uncover":[54],"novel":[56],"Evaluation":[57],"Scaling":[58],"Law:":[59],"within":[60],"range":[62],"counts":[65],"model":[67],"families":[68],"tested,":[70],"tends":[73],"improve":[75],"with":[76],"template":[77],"diversity.Specifically,":[78],"our":[79,113],"experiments":[80],"demonstrate":[81],"variance":[84],"diminishes":[85],"as":[86],"scale":[88],"prompts":[90],"increases,":[91],"while":[92],"ranking":[93],"monotonicity":[94],"exhibits":[95],"log-linear":[97],"improvement":[98],"relative":[99],"number":[102],"templates.By":[104],"decoupling":[105],"model's":[107],"intrinsic":[108],"capability":[109],"from":[110],"template-specific":[111],"fluctuations,":[112],"facilitates":[115],"more":[116,130],"reliable":[117],"assessments":[118],"LLMs,":[120],"thereby":[121],"providing":[122],"solid":[124],"foundation":[125],"for":[126],"benchmarking":[127],"enables":[129],"effective":[131],"monitoring":[132],"training":[134],"dynamics":[135],"rigorous":[137],"comparisons":[138],"among":[139],"state-of-the-art":[140],"models.":[141]},"counts_by_year":[],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2026-10-30T00:00:00"}