{"id":"https://openalex.org/W4402640107","doi":"https://doi.org/10.1162/tacl_a_00689","title":"Do Language Models Enjoy Their Own Stories? Prompting Large Language Models for Automatic Story Evaluation","display_name":"Do Language Models Enjoy Their Own Stories? Prompting Large Language Models for Automatic Story Evaluation","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402640107","doi":"https://doi.org/10.1162/tacl_a_00689"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00689","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00689","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/tacl_a_00689","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049135414","display_name":"Cyril Chhun","orcid":"https://orcid.org/0000-0002-3820-1225"},"institutions":[{"id":"https://openalex.org/I12356871","display_name":"T\u00e9l\u00e9com Paris","ror":"https://ror.org/01naq7912","country_code":"FR","type":"education","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210165912","display_name":"Laboratoire Traitement et Communication de l\u2019Information","ror":"https://ror.org/057er4c39","country_code":"FR","type":"facility","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102","https://openalex.org/I4210165912"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Cyril Chhun","raw_affiliation_strings":["LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. cyril.chhun@telecom-paris.fr"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. cyril.chhun@telecom-paris.fr","institution_ids":["https://openalex.org/I12356871","https://openalex.org/I4210165912"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034994700","display_name":"Fabian M. Suchanek","orcid":"https://orcid.org/0000-0001-7189-2796"},"institutions":[{"id":"https://openalex.org/I12356871","display_name":"T\u00e9l\u00e9com Paris","ror":"https://ror.org/01naq7912","country_code":"FR","type":"education","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210165912","display_name":"Laboratoire Traitement et Communication de l\u2019Information","ror":"https://ror.org/057er4c39","country_code":"FR","type":"facility","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102","https://openalex.org/I4210165912"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Fabian M. Suchanek","raw_affiliation_strings":["LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. fabian.suchanek@telecom-paris.fr"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. fabian.suchanek@telecom-paris.fr","institution_ids":["https://openalex.org/I12356871","https://openalex.org/I4210165912"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064876801","display_name":"Chlo\u00e9 Clavel","orcid":"https://orcid.org/0000-0003-4850-3398"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Chlo\u00e9 Clavel","raw_affiliation_strings":["ALMAnaCH, INRIA Paris. chloe.clavel@inria.fr"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALMAnaCH, INRIA Paris. chloe.clavel@inria.fr","institution_ids":["https://openalex.org/I1326498283"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.3444,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":{"value":0.9613643,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"12","issue":null,"first_page":"1122","last_page":"1142"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8366552591323853},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.7292210459709167},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5505009293556213},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4141392409801483},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.342606782913208}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8366552591323853},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.7292210459709167},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5505009293556213},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4141392409801483},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.342606782913208},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/tacl_a_00689","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00689","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:HAL:hal-04593961v2","is_oa":true,"landing_page_url":"https://hal.science/hal-04593961","pdf_url":"https://hal.science/hal-04593961v2/file/tacl_a_00689%20%281%29.pdf","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, 2024, 12, pp.1122-1142. ⟨10.1162/tacl_a_00689⟩","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:db10d04e93b24181a980bcd948162344","is_oa":true,"landing_page_url":"https://doaj.org/article/db10d04e93b24181a980bcd948162344","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 12 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00689","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00689","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.6000000238418579,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G5848506337","display_name":"Socio-Inspired Neural Networks for Conversational Systems","funder_award_id":"ANR-23-CE23-0033","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G8442890028","display_name":"Modeling and Extracting Complex Information from Natural Language Text","funder_award_id":"ANR-20-CHIA-0012","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320326256","display_name":"Grand \u00c9quipement National De Calcul Intensif","ror":"https://ror.org/0010d1q40"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":106,"referenced_works":["https://openalex.org/W25648700","https://openalex.org/W1970728244","https://openalex.org/W1975879668","https://openalex.org/W1996290308","https://openalex.org/W1996445490","https://openalex.org/W2020212032","https://openalex.org/W2027151576","https://openalex.org/W2036887984","https://openalex.org/W2053154970","https://openalex.org/W2061504941","https://openalex.org/W2088062078","https://openalex.org/W2097137621","https://openalex.org/W2101105183","https://openalex.org/W2110065044","https://openalex.org/W2112524255","https://openalex.org/W2155243985","https://openalex.org/W2161898721","https://openalex.org/W2250234233","https://openalex.org/W2250342921","https://openalex.org/W2739546170","https://openalex.org/W2758081422","https://openalex.org/W2889326414","https://openalex.org/W2901906577","https://openalex.org/W2922853138","https://openalex.org/W2924950986","https://openalex.org/W2963096510","https://openalex.org/W2963672599","https://openalex.org/W2965373594","https://openalex.org/W2970641574","https://openalex.org/W2970791445","https://openalex.org/W2973049837","https://openalex.org/W2973226110","https://openalex.org/W2979826702","https://openalex.org/W2983485493","https://openalex.org/W3014521650","https://openalex.org/W3035628162","https://openalex.org/W3036120435","https://openalex.org/W3037013468","https://openalex.org/W3098968529","https://openalex.org/W3101017384","https://openalex.org/W3101652466","https://openalex.org/W3102401511","https://openalex.org/W3119497443","https://openalex.org/W3160638507","https://openalex.org/W3170894870","https://openalex.org/W3174264667","https://openalex.org/W3174519801","https://openalex.org/W3198402388","https://openalex.org/W3205819385","https://openalex.org/W3213720306","https://openalex.org/W4205336585","https://openalex.org/W4205509257","https://openalex.org/W4205737716","https://openalex.org/W4212774754","https://openalex.org/W4221143046","https://openalex.org/W4224308101","https://openalex.org/W4226399820","https://openalex.org/W4239181501","https://openalex.org/W4253561749","https://openalex.org/W4283026156","https://openalex.org/W4285107714","https://openalex.org/W4285251400","https://openalex.org/W4286976894","https://openalex.org/W4299959846","https://openalex.org/W4311140163","https://openalex.org/W4321649710","https://openalex.org/W4322718191","https://openalex.org/W4360836968","https://openalex.org/W4379539933","https://openalex.org/W4384918448","https://openalex.org/W4385571411","https://openalex.org/W4385573386","https://openalex.org/W4392669868","https://openalex.org/W4392935324","https://openalex.org/W4396833426","https://openalex.org/W6601869879","https://openalex.org/W6676442675","https://openalex.org/W6682631176","https://openalex.org/W6713553520","https://openalex.org/W6761205521","https://openalex.org/W6763701032","https://openalex.org/W6766673545","https://openalex.org/W6768028577","https://openalex.org/W6778883912","https://openalex.org/W6780673046","https://openalex.org/W6791446462","https://openalex.org/W6797816506","https://openalex.org/W6802052463","https://openalex.org/W6802937455","https://openalex.org/W6809646742","https://openalex.org/W6810081322","https://openalex.org/W6810738896","https://openalex.org/W6811129797","https://openalex.org/W6837989031","https://openalex.org/W6838461927","https://openalex.org/W6838865847","https://openalex.org/W6842361329","https://openalex.org/W6845576796","https://openalex.org/W6846556436","https://openalex.org/W6850202480","https://openalex.org/W6850625674","https://openalex.org/W6850936240","https://openalex.org/W6853381761","https://openalex.org/W6854866820","https://openalex.org/W6857162936","https://openalex.org/W6857316800"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Abstract":[0],"Storytelling":[1],"is":[2],"an":[3,80],"integral":[4],"part":[5],"of":[6,83,100,108],"human":[7,39,74,93],"experience":[8],"and":[9,22,45,92,95,105],"plays":[10],"a":[11],"crucial":[12],"role":[13],"in":[14,28],"social":[15],"interactions.":[16],"Thus,":[17],"Automatic":[18],"Story":[19],"Evaluation":[20],"(ASE)":[21],"Generation":[23],"(ASG)":[24],"could":[25],"benefit":[26],"society":[27],"multiple":[29],"ways,":[30],"but":[31,124],"they":[32],"are":[33],"challenging":[34],"tasks":[35],"which":[36],"require":[37],"high-level":[38],"abilities":[40],"such":[41],"as":[42,71],"creativity,":[43],"reasoning,":[44],"deep":[46],"understanding.":[47],"Meanwhile,":[48],"Large":[49],"Language":[50],"Models":[51],"(LLMs)":[52],"now":[53],"achieve":[54],"state-of-the-art":[55],"performance":[56],"on":[57,102],"many":[58],"NLP":[59],"tasks.":[60],"In":[61],"this":[62],"paper,":[63],"we":[64,96,113],"study":[65],"whether":[66],"LLMs":[67,116],"can":[68],"be":[69],"used":[70],"substitutes":[72],"for":[73,76,121,131],"annotators":[75],"ASE.":[77],"We":[78],"perform":[79],"extensive":[81],"analysis":[82],"the":[84,98,103,106],"correlations":[85],"between":[86],"LLM":[87,109],"ratings,":[88],"other":[89],"automatic":[90,119],"measures,":[91],"annotations,":[94],"explore":[97],"influence":[99],"prompting":[101],"results":[104],"explainability":[107],"behaviour.":[110],"Most":[111],"notably,":[112],"find":[114],"that":[115],"outperform":[117],"current":[118],"measures":[120],"system-level":[122],"evaluation":[123],"still":[125],"struggle":[126],"at":[127],"providing":[128],"satisfactory":[129],"explanations":[130],"their":[132],"answers.":[133]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":2}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}