{"id":"https://openalex.org/W4402640107","doi":"https://doi.org/10.1162/tacl_a_00689","title":"Do Language Models Enjoy Their Own Stories? Prompting Large Language Models for Automatic Story Evaluation","display_name":"Do Language Models Enjoy Their Own Stories? Prompting Large Language Models for Automatic Story Evaluation","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402640107","doi":"https://doi.org/10.1162/tacl_a_00689"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00689","doi":"https://doi.org/10.1162/tacl_a_00689","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00689","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["ccf-b","cwts-core","doaj","dongbi-b","jufo-1","norway-2","poland-20","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["arxiv","crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/tacl_a_00689","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049135414","display_name":"Cyril Chhun","orcid":"https://orcid.org/0000-0002-3820-1225","observed_orcids":["https://orcid.org/0000-0002-3820-1225"]},"institutions":[{"id":"https://openalex.org/I12356871","display_name":"T\u00e9l\u00e9com Paris","ror":"https://ror.org/01naq7912","country_code":"FR","type":"education","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210165912","display_name":"Laboratoire Traitement et Communication de l\u2019Information","ror":"https://ror.org/057er4c39","country_code":"FR","type":"facility","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102","https://openalex.org/I4210165912"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Cyril Chhun","raw_affiliation_strings":["LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. cyril.chhun@telecom-paris.fr"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. cyril.chhun@telecom-paris.fr","institution_ids":["https://openalex.org/I12356871","https://openalex.org/I4210165912"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034994700","display_name":"Fabian M. Suchanek","orcid":"https://orcid.org/0000-0001-7189-2796","observed_orcids":["https://orcid.org/0000-0001-7189-2796"]},"institutions":[{"id":"https://openalex.org/I12356871","display_name":"T\u00e9l\u00e9com Paris","ror":"https://ror.org/01naq7912","country_code":"FR","type":"education","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210165912","display_name":"Laboratoire Traitement et Communication de l\u2019Information","ror":"https://ror.org/057er4c39","country_code":"FR","type":"facility","lineage":["https://openalex.org/I12356871","https://openalex.org/I205703379","https://openalex.org/I4210145102","https://openalex.org/I4210165912"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Fabian M. Suchanek","raw_affiliation_strings":["LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. fabian.suchanek@telecom-paris.fr"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LTCI, T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris. fabian.suchanek@telecom-paris.fr","institution_ids":["https://openalex.org/I12356871","https://openalex.org/I4210165912"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064876801","display_name":"Chlo\u00e9 Clavel","orcid":"https://orcid.org/0000-0003-4850-3398","observed_orcids":["https://orcid.org/0000-0003-4850-3398"]},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I4405261730","display_name":"ALMANACH: Mod\u00e9lisation et analyse linguistique automatique et humanit\u00e9s computationnelles","ror":"https://ror.org/05rnwy495","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1326498283","https://openalex.org/I4405257605","https://openalex.org/I4405261730"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Chlo\u00e9 Clavel","raw_affiliation_strings":["ALMAnaCH, INRIA Paris. chloe.clavel@inria.fr"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALMAnaCH, INRIA Paris. chloe.clavel@inria.fr","institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I4405261730"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.8517,"has_fulltext":true,"cited_by_count":26,"citation_normalized_percentile":{"value":0.94616004,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"12","issue":null,"first_page":"1122","last_page":"1142"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6445000171661377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6445000171661377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11259999871253967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.08479999750852585,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-story-generation","display_name":"automatic story generation","score":0.7649999856948853},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9850000143051147},{"id":"https://openalex.org/keywords/prompting","display_name":"prompting","score":0.5139999985694885},{"id":"https://openalex.org/keywords/explainability","display_name":"explainability","score":0.7839999794960022},{"id":"https://openalex.org/keywords/storytelling","display_name":"storytelling","score":0.41100001335144043}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8366552591323853},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.7292210459709167},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5505009293556213},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4141392409801483},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.342606782913208},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"doi:10.1162/tacl_a_00689","doi":"https://doi.org/10.1162/tacl_a_00689","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00689","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["ccf-b","cwts-core","doaj","dongbi-b","jufo-1","norway-2","poland-20","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2405.13769","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.13769","pdf_url":"https://arxiv.org/pdf/2405.13769","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-04593961v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-04593961","pdf_url":"https://hal.science/hal-04593961v1/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://hal.science/hal-04593961","raw_type":"info:eu-repo/semantics/preprint"},{"id":"pmh:oai:HAL:hal-04593961v2","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-04593961","pdf_url":"https://hal.science/hal-04593961v2/file/tacl_a_00689%20%281%29.pdf","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, 2024, 12, pp.1122-1142. ⟨10.1162/tacl_a_00689⟩","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:db10d04e93b24181a980bcd948162344","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/db10d04e93b24181a980bcd948162344","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 12 (2024)","raw_type":"article"},{"id":"doi:10.48550/arxiv.2405.13769","doi":"https://doi.org/10.48550/arxiv.2405.13769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.13769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00689","doi":"https://doi.org/10.1162/tacl_a_00689","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00689","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["ccf-b","cwts-core","doaj","dongbi-b","jufo-1","norway-2","poland-20","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.6000000238418579}],"study_designs":[],"awards":[{"id":"https://openalex.org/G5848506337","display_name":"Socio-Inspired Neural Networks for Conversational Systems","funder_award_id":"ANR-23-CE23-0033","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G8442890028","display_name":"Modeling and Extracting Complex Information from Natural Language Text","funder_award_id":"ANR-20-CHIA-0012","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320326256","display_name":"Grand \u00c9quipement National De Calcul Intensif","ror":"https://ror.org/0010d1q40"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":96,"referenced_works":["https://openalex.org/W25648700","https://openalex.org/W46135758","https://openalex.org/W1970728244","https://openalex.org/W1975879668","https://openalex.org/W1985514943","https://openalex.org/W1996290308","https://openalex.org/W1996445490","https://openalex.org/W2020212032","https://openalex.org/W2027151576","https://openalex.org/W2036887984","https://openalex.org/W2053154970","https://openalex.org/W2061504941","https://openalex.org/W2088062078","https://openalex.org/W2097137621","https://openalex.org/W2101105183","https://openalex.org/W2110065044","https://openalex.org/W2112524255","https://openalex.org/W2154652894","https://openalex.org/W2155243985","https://openalex.org/W2161898721","https://openalex.org/W2250234233","https://openalex.org/W2250342921","https://openalex.org/W2405762604","https://openalex.org/W2737041661","https://openalex.org/W2739546170","https://openalex.org/W2758081422","https://openalex.org/W2889326414","https://openalex.org/W2901906577","https://openalex.org/W2922853138","https://openalex.org/W2924950986","https://openalex.org/W2936695845","https://openalex.org/W2950813464","https://openalex.org/W2963096510","https://openalex.org/W2964910501","https://openalex.org/W2965373594","https://openalex.org/W2970641574","https://openalex.org/W2970791445","https://openalex.org/W2973049837","https://openalex.org/W2973226110","https://openalex.org/W2979826702","https://openalex.org/W2983485493","https://openalex.org/W3008926737","https://openalex.org/W3014521650","https://openalex.org/W3030163527","https://openalex.org/W3035628162","https://openalex.org/W3037013468","https://openalex.org/W3093429745","https://openalex.org/W3101652466","https://openalex.org/W3102401511","https://openalex.org/W3119497443","https://openalex.org/W3130319171","https://openalex.org/W3132736064","https://openalex.org/W3170894870","https://openalex.org/W3174007554","https://openalex.org/W3174264667","https://openalex.org/W3176456866","https://openalex.org/W3196820561","https://openalex.org/W3198402388","https://openalex.org/W3200377598","https://openalex.org/W3205819385","https://openalex.org/W3210320629","https://openalex.org/W3213720306","https://openalex.org/W4205336585","https://openalex.org/W4205509257","https://openalex.org/W4212774754","https://openalex.org/W4221143046","https://openalex.org/W4221145126","https://openalex.org/W4224308101","https://openalex.org/W4226399820","https://openalex.org/W4253561749","https://openalex.org/W4281483047","https://openalex.org/W4283026156","https://openalex.org/W4285251400","https://openalex.org/W4293333030","https://openalex.org/W4299959846","https://openalex.org/W4303648559","https://openalex.org/W4308244910","https://openalex.org/W4311140163","https://openalex.org/W4321649710","https://openalex.org/W4322718191","https://openalex.org/W4360836968","https://openalex.org/W4379539933","https://openalex.org/W4384918448","https://openalex.org/W4385571411","https://openalex.org/W4385573386","https://openalex.org/W4387985331","https://openalex.org/W4392669868","https://openalex.org/W4392935324","https://openalex.org/W4396833426","https://openalex.org/W6766673545","https://openalex.org/W6850936240","https://openalex.org/W6853381761","https://openalex.org/W6854866820","https://openalex.org/W7133211372","https://openalex.org/W7133224126","https://openalex.org/W7133231369"],"related_works":["https://openalex.org/W4402704281","https://openalex.org/W4385571232","https://openalex.org/W4416057729","https://openalex.org/W4387560265","https://openalex.org/W4389518874","https://openalex.org/W4411120559","https://openalex.org/W4306798653","https://openalex.org/W7160985808","https://openalex.org/W4404388589","https://openalex.org/W4403582514"],"abstract_inverted_index":{"Abstract":[0],"Storytelling":[1],"is":[2],"an":[3,80],"integral":[4],"part":[5],"of":[6,83,100,108],"human":[7,39,74,93],"experience":[8],"and":[9,22,45,92,95,105],"plays":[10],"a":[11],"crucial":[12],"role":[13],"in":[14,28],"social":[15],"interactions.":[16],"Thus,":[17],"Automatic":[18],"Story":[19],"Evaluation":[20],"(ASE)":[21],"Generation":[23],"(ASG)":[24],"could":[25],"benefit":[26],"society":[27],"multiple":[29],"ways,":[30],"but":[31,124],"they":[32],"are":[33],"challenging":[34],"tasks":[35],"which":[36],"require":[37],"high-level":[38],"abilities":[40],"such":[41],"as":[42,71],"creativity,":[43],"reasoning,":[44],"deep":[46],"understanding.":[47],"Meanwhile,":[48],"Large":[49],"Language":[50],"Models":[51],"(LLMs)":[52],"now":[53],"achieve":[54],"state-of-the-art":[55],"performance":[56],"on":[57,102],"many":[58],"NLP":[59],"tasks.":[60],"In":[61],"this":[62],"paper,":[63],"we":[64,96,113],"study":[65],"whether":[66],"LLMs":[67,116],"can":[68],"be":[69],"used":[70],"substitutes":[72],"for":[73,76,121,131],"annotators":[75],"ASE.":[77],"We":[78],"perform":[79],"extensive":[81],"analysis":[82],"the":[84,98,103,106],"correlations":[85],"between":[86],"LLM":[87,109],"ratings,":[88],"other":[89],"automatic":[90,119],"measures,":[91],"annotations,":[94],"explore":[97],"influence":[99],"prompting":[101],"results":[104],"explainability":[107],"behaviour.":[110],"Most":[111],"notably,":[112],"find":[114],"that":[115],"outperform":[117],"current":[118],"measures":[120],"system-level":[122],"evaluation":[123],"still":[125],"struggle":[126],"at":[127],"providing":[128],"satisfactory":[129],"explanations":[130],"their":[132],"answers.":[133]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":3}],"updated_date":"2026-10-25T09:29:13.331239","created_date":"2024-09-20T00:00:00"}