{"id":"https://openalex.org/W7164404662","doi":"https://doi.org/10.48550/arxiv.2606.12117","title":"Soft-Prompt Tuning for Fair and Efficient LLM Benchmark Evaluation","display_name":"Soft-Prompt Tuning for Fair and Efficient LLM Benchmark Evaluation","publication_year":2026,"publication_date":"2026-06-10","ids":{"openalex":"https://openalex.org/W7164404662","doi":"https://doi.org/10.48550/arxiv.2606.12117"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.12117","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12117","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.12117","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138450496","display_name":"Selen Erkan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Erkan, Selen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049893678","display_name":"Bastian Boll","orcid":"https://orcid.org/0000-0002-3490-3350"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Boll, Bastian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138404200","display_name":"Kristian Kersting","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kersting, Kristian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138451711","display_name":"Bj\u00f6rn Deiseroth","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deiseroth, Bj\u00f6rn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138412769","display_name":"Letitia Parcalabescu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Parcalabescu, Letitia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3758000135421753,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3758000135421753,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.27970001101493835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.06239999830722809,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8755999803543091},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8519999980926514},{"id":"https://openalex.org/keywords/knowledge-base","display_name":"Knowledge base","score":0.5400999784469604},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.4602000117301941},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.413100004196167},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.40549999475479126}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8755999803543091},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8519999980926514},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7537000179290771},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5507000088691711},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.5400999784469604},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.4602000117301941},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41990000009536743},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.413100004196167},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C2778775528","wikidata":"https://www.wikidata.org/wiki/Q5135432","display_name":"Closing (real estate)","level":2,"score":0.37229999899864197},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3628999888896942},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.36250001192092896},{"id":"https://openalex.org/C88006597","wikidata":"https://www.wikidata.org/wiki/Q690117","display_name":"Disk formatting","level":2,"score":0.3456999957561493},{"id":"https://openalex.org/C2779714256","wikidata":"https://www.wikidata.org/wiki/Q25305062","display_name":"Multiple Models","level":2,"score":0.29919999837875366},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.29840001463890076}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.12117","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12117","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.12117","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12117","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Benchmark":[0],"scores":[1],"often":[2],"misrepresent":[3],"a":[4,73,77,204,223,232],"large":[5],"language":[6],"model's":[7,16],"(LLM's)":[8],"knowledge,":[9,229],"because":[10],"they":[11],"rely,":[12],"e.g.,":[13],"on":[14,119],"the":[15,31,36,122],"ability":[17,37],"to":[18,44,84,106,181,237],"follow":[19],"specific":[20,85],"formatting":[21],"requirements.":[22],"This":[23,103],"especially":[24],"penalizes":[25],"base":[26,110,165,189],"models":[27,83,111,133,176],"that":[28,94,140,168,172,186],"may":[29],"know":[30],"correct":[32],"answers":[33],"but":[34],"lack":[35],"--":[38,43,112,118],"typically":[39],"introduced":[40],"in":[41,90,100,243],"post-training":[42],"structure":[45],"them":[46],"as":[47],"instructed.":[48],"To":[49],"overcome":[50],"this,":[51],"we":[52,81],"propose":[53],"soft-prompt":[54,67,129,142,156],"tuning,":[55],"an":[56],"efficient,":[57,154],"fair,":[58],"and":[59,92,134,161,185,200,219,230,234],"architecture-agnostic":[60],"model":[61,166,190,194,209],"evaluation.":[62],"By":[63],"optimizing":[64],"only":[65],"10":[66],"vectors":[68],"(roughly":[69],"0.0006%":[70],"parameters":[71],"for":[72,124,207],"7B":[74],"model)":[75],"over":[76],"short":[78],"tuning":[79,130,143,157],"period,":[80],"adapt":[82],"benchmark":[86,101],"formats,":[87],"closing":[88],"gaps":[89],"format-following":[91,145,218],"ensuring":[93],"underlying":[95],"knowledge":[96,167,220],"is":[97],"accurately":[98],"reflected":[99],"scores.":[102],"allows":[104],"one":[105],"fairly":[107],"compare":[108],"different":[109],"trained":[113],"with":[114],"various":[115],"pre-training":[116,240],"recipes":[117],"benchmarks":[120],"without":[121],"need":[123],"full":[125],"post-training.":[126],"We":[127],"evaluated":[128],"across":[131],"7":[132,135],"datasets.":[136],"The":[137],"results":[138],"show":[139],"(a)":[141],"saturates":[144],"within":[146],"80":[147],"steps":[148],"(~640":[149],"samples)":[150],"making":[151],"it":[152],"highly":[153],"(b)":[155],"significantly":[158],"outperforms":[159],"zero-":[160,199],"few-shot":[162,201],"prompting,":[163],"surfacing":[164],"standard":[169],"prompting":[170],"misses,":[171],"(c)":[173],"even":[174],"post-trained":[175,193],"can":[177],"benefit":[178],"from":[179],"soft-prompts":[180],"maximize":[182],"format":[183],"compliance,":[184],"(d)":[187],"soft-prompted":[188],"performance":[191],"predicts":[192],"rankings":[195],"more":[196],"reliably":[197],"than":[198],"baselines,":[202],"offering":[203],"low-cost":[205],"proxy":[206],"downstream":[208],"quality.":[210],"Our":[211],"contributions":[212],"include":[213],"(1)":[214],"metrics":[215],"which":[216],"disentangle":[217],"accuracy,":[221],"(2)":[222],"fairer":[224],"benchmarking":[225],"protocol":[226],"of":[227],"LLM":[228,244],"(3)":[231],"cost-":[233],"memory-effective":[235],"recipe":[236],"identify":[238],"optimal":[239],"strategies":[241],"early":[242],"development.":[245]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-12T00:00:00"}