{"id":"https://openalex.org/W4412887880","doi":"https://doi.org/10.18653/v1/2025.findings-acl.954","title":"Trick or Neat: Adversarial Ambiguity and Language Model Evaluation","display_name":"Trick or Neat: Adversarial Ambiguity and Language Model Evaluation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412887880","doi":"https://doi.org/10.18653/v1/2025.findings-acl.954"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.954","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.954","pdf_url":"https://aclanthology.org/2025.findings-acl.954.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.954.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002734392","display_name":"Antonia Karamolegkou","orcid":"https://orcid.org/0000-0002-6458-0986"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Antonia Karamolegkou","raw_affiliation_strings":["University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011718238","display_name":"Oliver Eberle","orcid":"https://orcid.org/0000-0002-6967-9950"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Oliver Eberle","raw_affiliation_strings":["University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007759955","display_name":"Phillip Rust","orcid":"https://orcid.org/0000-0001-5123-821X"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Phillip Rust","raw_affiliation_strings":["University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024327650","display_name":"Carina Kauf","orcid":"https://orcid.org/0000-0002-2718-1978"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Carina Kauf","raw_affiliation_strings":["University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018138946","display_name":"Anders S\u00f8gaard","orcid":"https://orcid.org/0000-0001-5250-4276"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Anders S\u00f8gaard","raw_affiliation_strings":["University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen Technische Universitt Berlin Massachusetts Institute of Technology Aleph Alpha Research","institution_ids":["https://openalex.org/I4577782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4577782"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"18542","last_page":"18561"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8567000031471252,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8567000031471252,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8015999794006348,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.9064348936080933},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.8515802621841431},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6982402801513672},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47277238965034485},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4370628893375397},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2946324348449707}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.9064348936080933},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.8515802621841431},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6982402801513672},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47277238965034485},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4370628893375397},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2946324348449707}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.954","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.954","pdf_url":"https://aclanthology.org/2025.findings-acl.954.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:publications/a803f996-8713-4d8c-92c1-2f8e5766121c","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/a803f996-8713-4d8c-92c1-2f8e5766121c","pdf_url":"https://curis.ku.dk/ws/files/533615359/Trick_or_Neat.pdf","source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.954","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.954","pdf_url":"https://aclanthology.org/2025.findings-acl.954.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412887880.pdf","grobid_xml":"https://content.openalex.org/works/W4412887880.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2502115930","https://openalex.org/W2482350142","https://openalex.org/W4246396837","https://openalex.org/W3126451824","https://openalex.org/W1561927205","https://openalex.org/W3191453585","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Detecting":[0],"ambiguity":[1,22,27,67,86],"is":[2],"important":[3],"for":[4],"language":[5,18,83,162],"understanding,":[6],"including":[7],"uncertainty":[8],"estimation,":[9],"humour":[10],"detection,":[11],"and":[12,33,45,81,94,129],"processing":[13],"garden":[14],"path":[15],"sentences.We":[16],"assess":[17],"models'":[19],"sensitivity":[20],"to":[21,54,119,147,159],"by":[23,133],"introducing":[24],"an":[25],"adversarial":[26,38],"dataset":[28],"that":[29,50,124],"includes":[30],"syntactic,":[31],"lexical,":[32],"phonological":[34],"ambiguities":[35,123],"along":[36],"with":[37,68],"variations":[39],"(e.g.,":[40],"word-order":[41],"changes,":[42],"synonym":[43],"replacements,":[44],"randombased":[46],"alterations).Our":[47],"findings":[48],"show":[49],"direct":[51],"prompting":[52,79],"fails":[53],"robustly":[55],"identify":[56],"ambiguity,":[57],"while":[58],"linear":[59],"probes":[60],"trained":[61],"on":[62],"model":[63],"representations":[64],"can":[65],"decode":[66],"high":[69],"accuracy,":[70],"sometimes":[71],"exceeding":[72],"90%.Our":[73],"results":[74],"offer":[75],"insights":[76],"into":[77],"the":[78,100,145],"paradigm":[80],"how":[82,107],"models":[84],"encode":[85],"at":[87],"different":[88],"layers.We":[89],"release":[90],"both":[91],"our":[92],"code":[93],"data:coastalcph/lm_ambiguity.1":[95],"Except,":[96],"of":[97],"course,":[98],"if":[99],"dress":[101],"has":[102],"a":[103,156],"mirror.Such":[104],"exceptions":[105],"illustrate":[106],"pragmatics":[108],"may":[109],"override":[110],"semantics":[111],"(Morris,":[112],"1946),":[113],"but":[114],"LLMs":[115],"must":[116],"be":[117],"sensitive":[118],"distinctions":[120],"between":[121],"syntactic":[122],"support":[125],"multiple":[126],"conventional":[127],"readings":[128],"those":[130],"typically":[131],"resolved":[132],"context.":[134],"2":[135],"spaces/lmsys/chatbot-arena-leaderboard":[136],"3":[137],"jndiogo/LLM-chat-templates":[138],"4":[139],"We":[140],"define":[141],"consistency":[142],"here":[143],"as":[144],"ability":[146],"make":[148],"reliable":[149],"decisions":[150],"in":[151],"semantically":[152],"similar":[153],"contexts":[154],"demonstrating":[155],"systematic":[157],"capacity":[158],"generalize":[160],"across":[161],"variations\"":[163],"(Elazar":[164],"et":[165],"al.,":[166],"2021).saahowwwnem":[167],"ttwahimitonteehmaww":[168],"w":[169,175],"tntaihotnemh":[170],"i":[171],"wttttah":[172],"h":[173],"maehinntllehtew":[174],"s":[176],"sttceecitihtolhte":[177],"e":[178],"heep":[179],"stttelcheeh.oelseepcto":[180],"steecp.loes.lepsec.ope.cope.sub":[181],"sju_bsysonym":[182],"|":[183,186],"t":[184,195],"suj_nbjyn_raonndyomThm":[185],"eJe":[187],"mtneoatcheer":[188],"ssa":[189],"omhea":[190],"ntee":[191],"wle":[192],"tsteelec":[193],"ihcloess":[194],"tpheco.oppee":[196]},"counts_by_year":[],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}