{"id":"https://openalex.org/W4389519937","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.514","title":"Are Language Models Worse than Humans at Following Prompts? It\u2019s Complicated","display_name":"Are Language Models Worse than Humans at Following Prompts? It\u2019s Complicated","alternate_titles":[],"publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389519937","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.514"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.514","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.514","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.514","pdf_url":"https://aclanthology.org/2023.findings-emnlp.514.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4389519937"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-emnlp.514.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056219662","display_name":"Albert Webson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Albert Webson","raw_affiliation_strings":["Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","Brown University","\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047002689","display_name":"Alyssa Marie Loo","orcid":"https://orcid.org/0009-0004-0656-2888","observed_orcids":["https://orcid.org/0009-0004-0656-2888"]},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Alyssa Loo","raw_affiliation_strings":["Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","Brown University","\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010977706","display_name":"Qinan Yu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Qinan Yu","raw_affiliation_strings":["Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","Brown University","\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053850863","display_name":"Ellie Pavlick","orcid":"https://orcid.org/0000-0002-7155-5420","observed_orcids":["https://orcid.org/0000-0002-7155-5420"]},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Ellie Pavlick","raw_affiliation_strings":["Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","Brown University","\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"Brown University","institution_ids":["https://openalex.org/I27804330"]},{"raw_affiliation_string":"\u03ba Department of Computer Science \u03d5 Department of Philosophy \u03bb Program in Linguistics","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I27804330"],"apc_list":null,"apc_paid":null,"fwci":1.6363,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.89418407,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"7662","last_page":"7686"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6151999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6151999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.25200000405311584,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.05490000173449516,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.6970000267028809},{"id":"https://openalex.org/keywords/human-behavior","display_name":"human behavior","score":0.4819999933242798},{"id":"https://openalex.org/keywords/zero-shot-learning","display_name":"zero-shot learning","score":0.7689999938011169},{"id":"https://openalex.org/keywords/few-shot-learning","display_name":"few-shot learning","score":0.9909999966621399},{"id":"https://openalex.org/keywords/instruction-following","display_name":"instruction following","score":0.3409999907016754}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7465012073516846},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6911383271217346},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.576734721660614},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5407590270042419},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34840673208236694},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2805996239185333},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.findings-emnlp.514","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.514","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.514","pdf_url":"https://aclanthology.org/2023.findings-emnlp.514.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2301.07085","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2301.07085","pdf_url":"https://arxiv.org/pdf/2301.07085","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2301.07085","doi":"https://doi.org/10.48550/arxiv.2301.07085","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2301.07085","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.514","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.514","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.514","pdf_url":"https://aclanthology.org/2023.findings-emnlp.514.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389519937.pdf","grobid_xml":"https://content.openalex.org/works/W4389519937.grobid-xml"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W2042492008","https://openalex.org/W2130158090","https://openalex.org/W2170240176","https://openalex.org/W2607892599","https://openalex.org/W2891555348","https://openalex.org/W2990241049","https://openalex.org/W2996728628","https://openalex.org/W3003804331","https://openalex.org/W3030163527","https://openalex.org/W3198002980","https://openalex.org/W3198599617","https://openalex.org/W4281483047","https://openalex.org/W4281690148","https://openalex.org/W4307079201","https://openalex.org/W4400681340","https://openalex.org/W7133184757","https://openalex.org/W7133224126"],"related_works":["https://openalex.org/W3198599617","https://openalex.org/W7162763763","https://openalex.org/W4297412056","https://openalex.org/W4283798643","https://openalex.org/W7164529227","https://openalex.org/W4312107542","https://openalex.org/W4385571325","https://openalex.org/W7164928628","https://openalex.org/W7161994925","https://openalex.org/W4403788929"],"abstract_inverted_index":{"Prompts":[0],"have":[1],"been":[2],"the":[3,80,90,105],"center":[4],"of":[5,87],"progress":[6],"in":[7,51],"advancing":[8],"language":[9],"models'":[10],"zero-shot":[11],"and":[12,73,125],"fewshot":[13],"performance.However,":[14],"recent":[15],"work":[16],"finds":[17],"that":[18,37,54,64,114],"models":[19,109,131],"can":[20],"perform":[21,57,77],"surprisingly":[22],"well":[23,78],"when":[24,59,98],"given":[25,60,99],"intentionally":[26],"irrelevant":[27,71],"or":[28,129],"misleading":[29,101],"prompts.Such":[30],"results":[31],"may":[32],"be":[33],"interpreted":[34],"as":[35,122],"evidence":[36],"model":[38],"behavior":[39],"is":[40],"not":[41,118,127],"\"human":[42],"like\".In":[43],"this":[44],"study,":[45],"we":[46],"challenge":[47],"a":[48,123],"central":[49],"assumption":[50],"such":[52],"work:":[53],"humans":[55,65,103],"would":[56],"badly":[58],"pathological":[61],"instructions.We":[62],"find":[63],"are":[66,93],"able":[67],"to":[68,96,132],"reliably":[69],"ignore":[70],"instructions":[72,106],"thus,":[74],"like":[75],"models,":[76],"on":[79],"underlying":[81],"task":[82,91],"despite":[83],"an":[84],"apparent":[85],"lack":[86],"signal":[88],"regarding":[89],"they":[92],"being":[94],"asked":[95],"do.However,":[97],"deliberately":[100],"instructions,":[102],"follow":[104],"faithfully,":[107],"whereas":[108],"do":[110],"not.Our":[111],"findings":[112],"caution":[113],"future":[115],"research":[116],"should":[117,126],"idealize":[119],"human":[120],"behaviors":[121,137,142],"monolith":[124],"train":[128],"evaluate":[130],"mimic":[133],"assumptions":[134],"about":[135],"these":[136],"without":[138],"first":[139],"validating":[140],"humans'":[141],"empirically.":[143]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4389519937","counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":7}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2023-12-11T00:00:00"}
