{"id":"https://openalex.org/W7166828384","doi":"https://doi.org/10.18653/v1/2026.acl-srw.103","title":"Linguistically-Informed Evaluation of LLMs on Acceptability Judgments in a Forced-Choice Paradigm","display_name":"Linguistically-Informed Evaluation of LLMs on Acceptability Judgments in a Forced-Choice Paradigm","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166828384","doi":"https://doi.org/10.18653/v1/2026.acl-srw.103"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.acl-srw.103","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-srw.103","pdf_url":"https://aclanthology.org/2026.acl-srw.103.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 4: Student Research Workshop)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-srw.103.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104345547","display_name":"Ziyue Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ziyue Liu","raw_affiliation_strings":["Department of Digital Humanities University of Cologne"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Digital Humanities University of Cologne","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5139733206","display_name":"Nils Reiter","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nils Reiter","raw_affiliation_strings":["Department of Digital Humanities University of Cologne"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Digital Humanities University of Cologne","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.74074074,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1177","last_page":"1189"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.03909999877214432,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.03909999877214432,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.03759999945759773,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03440000116825104,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.2854999899864197},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.2639000117778778},{"id":"https://openalex.org/keywords/ethnic-group","display_name":"Ethnic group","score":0.25929999351501465},{"id":"https://openalex.org/keywords/government","display_name":"Government (linguistics)","score":0.258899986743927},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.25440001487731934}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.531000018119812},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.4772000014781952},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.29319998621940613},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.2854999899864197},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C137403100","wikidata":"https://www.wikidata.org/wiki/Q41710","display_name":"Ethnic group","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.2556999921798706},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.25440001487731934},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-srw.103","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-srw.103","pdf_url":"https://aclanthology.org/2026.acl-srw.103.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 4: Student Research Workshop)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-srw.103","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-srw.103","pdf_url":"https://aclanthology.org/2026.acl-srw.103.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 4: Student Research Workshop)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.5576865077018738,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G452762234","display_name":null,"funder_award_id":"SFB1252","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G6263430500","display_name":"SFB 1252: Prominenz in Sprache","funder_award_id":"281511265","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320324811","display_name":"Universit\u00e4t zu K\u00f6ln","ror":"https://ror.org/00rcxh774"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166828384.pdf","grobid_xml":"https://content.openalex.org/works/W7166828384.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"the":[1,18],"grammatical":[2],"abilities":[3],"of":[4,20,36],"large":[5,21],"language":[6,22],"models":[7,23,64],"(LLMs)":[8,24],"is":[9],"important":[10],"for":[11],"both":[12],"NLP":[13],"and":[14,47,65,77],"linguistic":[15,51],"theory.We":[16],"investigate":[17],"ability":[19],"to":[25],"perform":[26],"acceptability":[27],"judgments":[28],"in":[29],"a":[30,34],"forcedchoice":[31],"paradigm.We":[32],"evaluate":[33],"subset":[35],"LLMs":[37,57],"on":[38,71,75],"150":[39],"minimal":[40,83],"sentence":[41],"pairs":[42],"sampled":[43],"from":[44],"Linguistic":[45],"Inquiry":[46],"categorized":[48],"using":[49],"BLiMP":[50],"phenomena.Our":[52],"results":[53],"show":[54],"that":[55],"while":[56],"approximate":[58],"human":[59],"judgments,":[60],"performance":[61],"varies":[62],"across":[63],"phenomenon":[66],"types,":[67],"with":[68],"stronger":[69],"alignment":[70],"morphosyntactic":[72],"phenomena":[73],"than":[74],"linguistically":[76],"semantically":[78],"demanding":[79],"phenomena.Prompting":[80],"strategies":[81],"have":[82],"impact.":[84]},"counts_by_year":[],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2026-07-02T00:00:00"}