{"id":"https://openalex.org/W4412888226","doi":"https://doi.org/10.18653/v1/2025.findings-acl.611","title":"DOVE: A Large-Scale Multi-Dimensional Predictions Dataset Towards Meaningful LLM Evaluation","display_name":"DOVE: A Large-Scale Multi-Dimensional Predictions Dataset Towards Meaningful LLM Evaluation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888226","doi":"https://doi.org/10.18653/v1/2025.findings-acl.611"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.611","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.611","pdf_url":"https://aclanthology.org/2025.findings-acl.611.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.611.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091921041","display_name":"Eliya Habba","orcid":"https://orcid.org/0009-0001-3472-0650"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eliya Habba","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071439989","display_name":"Ofir Arviv","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ofir Arviv","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017452853","display_name":"Itay Itzhak","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Itay Itzhak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015325814","display_name":"Yotam Perlitz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yotam Perlitz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002225429","display_name":"Elron Bandel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elron Bandel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040286212","display_name":"Leshem Choshen","orcid":"https://orcid.org/0000-0002-0085-6496"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leshem Choshen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051536706","display_name":"Michal Shmueli-Scheuer","orcid":"https://orcid.org/0000-0002-6386-8726"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michal Shmueli-Scheuer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5082136238","display_name":"Gabriel Stanovsky","orcid":"https://orcid.org/0000-0002-2420-8979"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gabriel Stanovsky","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"11744","last_page":"11763"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9174000024795532,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9174000024795532,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dove","display_name":"Dove","score":0.9182395935058594},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6565870642662048},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.6538766622543335},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3873014748096466},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3523027300834656},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3275851011276245},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.130190908908844},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.11121320724487305}],"concepts":[{"id":"https://openalex.org/C109703566","wikidata":"https://www.wikidata.org/wiki/Q962709","display_name":"Dove","level":2,"score":0.9182395935058594},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6565870642662048},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.6538766622543335},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3873014748096466},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3523027300834656},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3275851011276245},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.130190908908844},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.11121320724487305},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.611","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.611","pdf_url":"https://aclanthology.org/2025.findings-acl.611.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.01622","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.01622","pdf_url":"https://arxiv.org/pdf/2503.01622","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.611","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.611","pdf_url":"https://aclanthology.org/2025.findings-acl.611.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888226.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Recent":[0],"work":[1],"found":[2],"that":[3,100],"LLMs":[4],"are":[5,109],"sensitive":[6],"to":[7,53,89,131],"a":[8,41,133],"wide":[9],"range":[10],"of":[11,18,47,69,77,117],"arbitrary":[12],"prompt":[13,45,121],"dimensions,":[14,73],"including":[15,92],"the":[16,66,143],"type":[17],"delimiters,":[19],"answer":[20],"enumerators,":[21],"instruction":[22],"wording,":[23],"and":[24,64,105,123,139,146],"more.":[25],"This":[26],"throws":[27],"into":[28],"question":[29],"popular":[30],"single-prompt":[31],"evaluation":[32,49],"practices.":[33],"We":[34,81],"present":[35],"DOVE":[36,115],"(Dataset":[37],"Of":[38],"Variation":[39],"Evaluation)":[40],"large-scale":[42],"dataset":[43],"containing":[44],"perturbations":[46,70,78,122],"various":[48,72],"benchmarks.":[50],"In":[51],"contrast":[52],"previous":[54],"work,":[55],"we":[56,127],"examine":[57],"LLM":[58],"sensitivity":[59],"from":[60],"an":[61],"holistic":[62],"perspective,":[63],"assess":[65],"joint":[67],"effects":[68],"along":[71],"resulting":[74],"in":[75],"thousands":[76],"per":[79],"instance.":[80],"evaluate":[82],"several":[83,90],"model":[84,124],"families":[85],"against":[86],"DOVE,":[87],"leading":[88],"findings,":[91],"efficient":[93,140],"methods":[94],"for":[95],"choosing":[96],"well-performing":[97],"prompts,":[98],"observing":[99],"few-shot":[101],"examples":[102],"reduce":[103],"sensitivity,":[104],"identifying":[106],"instances":[107],"which":[108,126],"inherently":[110],"hard":[111],"across":[112],"all":[113],"perturbations.":[114],"consists":[116],"more":[118],"than":[119],"250M":[120],"outputs,":[125],"make":[128],"publicly":[129],"available":[130],"spur":[132],"community-wide":[134],"effort":[135],"toward":[136],"meaningful,":[137],"robust,":[138],"evaluation.":[141],"Browse":[142],"data,":[144],"contribute,":[145],"more:":[147],"https://slab-nlp.github.io/DOVE/":[148]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}