{"id":"https://openalex.org/W7133187004","doi":"https://doi.org/10.52202/075280-0913","title":"The Goldilocks of Pragmatic Understanding: Fine-Tuning Strategy Matters for Implicature Resolution by LLMs","display_name":"The Goldilocks of Pragmatic Understanding: Fine-Tuning Strategy Matters for Implicature Resolution by LLMs","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W7133187004","doi":"https://doi.org/10.52202/075280-0913"},"language":"en","primary_location":{"id":"doi:10.52202/075280-0913","doi":"https://doi.org/10.52202/075280-0913","is_oa":false,"landing_page_url":"https://doi.org/10.52202/075280-0913","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 36","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2210.14986","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060010430","display_name":"Laura Ruis","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laura Ruis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084386381","display_name":"Akbir Khan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akbir Khan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076733732","display_name":"Stella Biderman","orcid":"https://orcid.org/0000-0001-8228-1042","observed_orcids":["https://orcid.org/0000-0001-8228-1042"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stella Biderman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078850040","display_name":"Sara Hooker","orcid":"https://orcid.org/0000-0002-0190-6459","observed_orcids":["https://orcid.org/0000-0002-0190-6459"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sara Hooker","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079315903","display_name":"Tim Rockt\u00e4schel","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tim Rockt\u00e4schel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5023508792","display_name":"Edward Grefenstette","orcid":"https://orcid.org/0000-0003-1164-8809","observed_orcids":["https://orcid.org/0000-0003-1164-8809"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Edward Grefenstette","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.3447,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":{"value":0.95895612,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"20827","last_page":"20905"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3508000075817108,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3508000075817108,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10034","display_name":"Syntax, Semantics, Linguistic Variation","score":0.31470000743865967,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08730000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pragmatic-understanding","display_name":"pragmatic understanding","score":0.7860000133514404},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9179999828338623},{"id":"https://openalex.org/keywords/fine-tuning-strategies","display_name":"fine-tuning strategies","score":0.8600000143051147},{"id":"https://openalex.org/keywords/pragmatic-reasoning","display_name":"pragmatic reasoning","score":0.3720000088214874},{"id":"https://openalex.org/keywords/conversational-implicature","display_name":"conversational implicature","score":0.6129999756813049}],"concepts":[{"id":"https://openalex.org/C177321328","wikidata":"https://www.wikidata.org/wiki/Q13580479","display_name":"Goldilocks principle","level":2,"score":0.9139999747276306},{"id":"https://openalex.org/C2779899686","wikidata":"https://www.wikidata.org/wiki/Q571227","display_name":"Implicature","level":3,"score":0.6394000053405762},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.39739999175071716},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.39169999957084656},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.34619998931884766},{"id":"https://openalex.org/C11693617","wikidata":"https://www.wikidata.org/wiki/Q181839","display_name":"Pragmatics","level":2,"score":0.34599998593330383},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3368000090122223},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.3285999894142151},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.313400000333786},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.29179999232292175},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.2831999957561493},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.271699994802475},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.2669000029563904},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.2660999894142151},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.262800008058548},{"id":"https://openalex.org/C2780339515","wikidata":"https://www.wikidata.org/wiki/Q3074698","display_name":"Arrow","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.2596000134944916},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.52202/075280-0913","doi":"https://doi.org/10.52202/075280-0913","is_oa":false,"landing_page_url":"https://doi.org/10.52202/075280-0913","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 36","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2210.14986","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.14986","pdf_url":"https://arxiv.org/pdf/2210.14986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:10216732","doi":null,"is_oa":true,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10216732/","pdf_url":"https://discovery.ucl.ac.uk/10216732/1/2210.14986v2.pdf","source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"In: Oh, A and Neumann, T and Globerson, A and Saenko, K and Hardt, M and Levine, S, (eds.) Proceedings of the 37th Conference on Neural Information Processing Systems (NeurIPS). Neural Information Processing Systems Foundation, Inc. (NeurIPS): New Orleans, LA, USA. (2023)","raw_type":"Proceedings paper"},{"id":"doi:10.48550/arxiv.2210.14986","doi":"https://doi.org/10.48550/arxiv.2210.14986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2210.14986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2210.14986","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.14986","pdf_url":"https://arxiv.org/pdf/2210.14986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4583412171","display_name":"UKRI AI Centre for Doctoral Training in Foundational Artificial Intelligence","funder_award_id":"EP/S021566/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Despite":[0],"widespread":[1],"use":[2],"of":[3,9,17,69,84,108,164,170],"LLMs":[4,61,116,154],"as":[5,55,72,144],"conversational":[6],"agents,":[7],"evaluations":[8],"performance":[10],"fail":[11],"to":[12,48,65,113,160],"capture":[13],"a":[14,77,99],"crucial":[15],"aspect":[16],"communication:":[18],"interpreting":[19],"language":[20,29,156],"in":[21,106,138,157],"context":[22,158],"--":[23],"incorporating":[24],"its":[25],"pragmatics.":[26],"Humans":[27],"interpret":[28,155],"using":[30],"beliefs":[31],"and":[32,80,159,167],"prior":[33],"knowledge":[34],"about":[35],"the":[36,43,49,63,119,145,162],"world.":[37],"For":[38],"example,":[39],"we":[40,75],"intuitively":[41],"understand":[42],"response":[44],"\"I":[45],"wore":[46],"gloves\"":[47],"question":[50],"\"Did":[51],"you":[52],"leave":[53],"fingerprints?\"":[54],"meaning":[56],"\"No\".":[57],"To":[58],"investigate":[59],"whether":[60],"have":[62],"ability":[64],"make":[66],"this":[67],"type":[68],"inference,":[70],"known":[71],"an":[73],"implicature,":[74],"design":[76],"simple":[78],"task":[79],"evaluate":[81],"four":[82],"categories":[83,110],"widely":[85],"used":[86],"state-of-the-art":[87],"models.":[88,139],"We":[89,140],"find":[90],"that,":[91],"despite":[92],"only":[93],"evaluating":[94,152],"on":[95],"utterances":[96],"that":[97,127],"require":[98],"binary":[100],"inference":[101],"(yes":[102],"or":[103],"no),":[104],"models":[105,169],"three":[107],"these":[109],"perform":[111,121],"close":[112],"random.":[114],"However,":[115],"instruction-tuned":[117],"at":[118,134],"example-level":[120],"significantly":[122],"better.":[123],"These":[124],"results":[125],"suggest":[126],"certain":[128],"fine-tuning":[129],"strategies":[130],"are":[131],"far":[132],"better":[133],"inducing":[135],"pragmatic":[136,166],"understanding":[137],"present":[141],"our":[142],"findings":[143],"starting":[146],"point":[147],"for":[148],"further":[149],"research":[150],"into":[151],"how":[153],"drive":[161],"development":[163],"more":[165],"useful":[168],"human":[171],"discourse.":[172]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":10}],"updated_date":"2026-10-13T10:08:12.033832","created_date":"2026-03-03T00:00:00"}