{"id":"https://openalex.org/W3086963535","doi":"https://doi.org/10.1109/access.2020.3023421","title":"Can Machines Tell Stories? A Comparative Study of Deep Neural Language Models and Metrics","display_name":"Can Machines Tell Stories? A Comparative Study of Deep Neural Language Models and Metrics","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3086963535","doi":"https://doi.org/10.1109/access.2020.3023421","mag":"3086963535"},"language":"en","primary_location":{"id":"doi:10.1109/access.2020.3023421","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3023421","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09194709.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09194709.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065790431","display_name":"Avisha Das","orcid":"https://orcid.org/0000-0002-5015-2665"},"institutions":[{"id":"https://openalex.org/I44461941","display_name":"University of Houston","ror":"https://ror.org/048sx0r50","country_code":"US","type":"education","lineage":["https://openalex.org/I44461941"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Avisha Das","raw_affiliation_strings":["Department of Computer Science, University of Houston, Houston, TX, USA"],"raw_orcid":"https://orcid.org/0000-0002-5015-2665","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Houston, Houston, TX, USA","institution_ids":["https://openalex.org/I44461941"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050592031","display_name":"Rakesh Verma","orcid":"https://orcid.org/0000-0002-7466-7823"},"institutions":[{"id":"https://openalex.org/I44461941","display_name":"University of Houston","ror":"https://ror.org/048sx0r50","country_code":"US","type":"education","lineage":["https://openalex.org/I44461941"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rakesh M. Verma","raw_affiliation_strings":["Department of Computer Science, University of Houston, Houston, TX, USA"],"raw_orcid":"https://orcid.org/0000-0002-7466-7823","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Houston, Houston, TX, USA","institution_ids":["https://openalex.org/I44461941"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5065790431"],"corresponding_institution_ids":["https://openalex.org/I44461941"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":1.9039,"has_fulltext":true,"cited_by_count":26,"citation_normalized_percentile":{"value":0.89001188,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"8","issue":null,"first_page":"181258","last_page":"181292"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7924705147743225},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6552181839942932},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6027372479438782},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5999404788017273},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.5780590772628784},{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.5197017788887024},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4997982978820801},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.46790820360183716},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.4663962721824646},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.43018943071365356},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.41668400168418884}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7924705147743225},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6552181839942932},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6027372479438782},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5999404788017273},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.5780590772628784},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.5197017788887024},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4997982978820801},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.46790820360183716},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.4663962721824646},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.43018943071365356},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41668400168418884},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2020.3023421","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3023421","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09194709.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:da4beb725c0b4c8dafac7ccf92099a74","is_oa":true,"landing_page_url":"https://doaj.org/article/da4beb725c0b4c8dafac7ccf92099a74","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 8, Pp 181258-181292 (2020)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2020.3023421","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2020.3023421","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8948470/09194709.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8799999952316284,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G1666312413","display_name":null,"funder_award_id":"DGE 1433817","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3366966419","display_name":null,"funder_award_id":"W911NF-16-1","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G6303296854","display_name":null,"funder_award_id":"1433817","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6523399184","display_name":null,"funder_award_id":"W911NF-20-1-0254","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G657448715","display_name":null,"funder_award_id":"W911NF-16-1-","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G6577779489","display_name":null,"funder_award_id":"1950297","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7452299184","display_name":null,"funder_award_id":"W911NF","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G7628991547","display_name":null,"funder_award_id":"W911NF-16-1-0422","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G7972136562","display_name":null,"funder_award_id":"CCF 1950297","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3086963535.pdf","grobid_xml":"https://content.openalex.org/works/W3086963535.grobid-xml"},"referenced_works_count":78,"referenced_works":["https://openalex.org/W196214544","https://openalex.org/W1507711477","https://openalex.org/W1590378318","https://openalex.org/W1810943226","https://openalex.org/W1982897610","https://openalex.org/W2068390867","https://openalex.org/W2072471709","https://openalex.org/W2130942839","https://openalex.org/W2135046866","https://openalex.org/W2143017621","https://openalex.org/W2153579005","https://openalex.org/W2250539671","https://openalex.org/W2251291469","https://openalex.org/W2561658355","https://openalex.org/W2563845258","https://openalex.org/W2598692538","https://openalex.org/W2605035112","https://openalex.org/W2622385665","https://openalex.org/W2752337926","https://openalex.org/W2784823820","https://openalex.org/W2792210438","https://openalex.org/W2807738734","https://openalex.org/W2807791032","https://openalex.org/W2807925339","https://openalex.org/W2808064329","https://openalex.org/W2810732773","https://openalex.org/W2888779557","https://openalex.org/W2889009749","https://openalex.org/W2896457183","https://openalex.org/W2914949666","https://openalex.org/W2936695845","https://openalex.org/W2938704169","https://openalex.org/W2943553207","https://openalex.org/W2949615070","https://openalex.org/W2962739339","https://openalex.org/W2962784628","https://openalex.org/W2962821399","https://openalex.org/W2963096510","https://openalex.org/W2963167310","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963544700","https://openalex.org/W2963672599","https://openalex.org/W2964110616","https://openalex.org/W2964213788","https://openalex.org/W2967126358","https://openalex.org/W2970597249","https://openalex.org/W2971008823","https://openalex.org/W2979826702","https://openalex.org/W2983962589","https://openalex.org/W2992347006","https://openalex.org/W2996287690","https://openalex.org/W3034287667","https://openalex.org/W4294170691","https://openalex.org/W4299567010","https://openalex.org/W4385245566","https://openalex.org/W6607974698","https://openalex.org/W6638273328","https://openalex.org/W6679436768","https://openalex.org/W6682691769","https://openalex.org/W6691493741","https://openalex.org/W6738966219","https://openalex.org/W6739901393","https://openalex.org/W6743477263","https://openalex.org/W6747826347","https://openalex.org/W6749351710","https://openalex.org/W6755207826","https://openalex.org/W6761205521","https://openalex.org/W6761551260","https://openalex.org/W6761999480","https://openalex.org/W6763240421","https://openalex.org/W6763473550","https://openalex.org/W6763701032","https://openalex.org/W6766654264","https://openalex.org/W6769243733","https://openalex.org/W6776403474","https://openalex.org/W6950536863","https://openalex.org/W7017743087"],"related_works":["https://openalex.org/W2064074028","https://openalex.org/W4281476908","https://openalex.org/W4288365749","https://openalex.org/W2936497627","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W3098382480","https://openalex.org/W4287598411","https://openalex.org/W3100913109","https://openalex.org/W3198458223"],"abstract_inverted_index":{"Massive":[0],"textual":[1,121],"content":[2,37,66,226],"has":[3,19],"enabled":[4],"rapid":[5],"advances":[6],"in":[7,144,235,256],"natural":[8,22],"language":[9,17,23,83,250],"modeling.":[10],"The":[11,200],"use":[12],"of":[13,81,96,103,149,195,212,240],"pre-trained":[14,115,180,217],"deep":[15],"neural":[16],"models":[18,50,69,84,106,183,219,251],"significantly":[20],"improved":[21],"understanding":[24],"tasks.":[25],"However,":[26],"the":[27,64,101,134,138,145,150,157,178,189,205,213,253],"extent":[28],"to":[29,36,119,223],"which":[30],"these":[31,49,68],"systems":[32],"can":[33,51],"be":[34],"applied":[35],"generation":[38,88],"is":[39,58],"unclear.":[40],"While":[41],"a":[42,93],"few":[43],"informal":[44],"studies":[45],"have":[46,252],"claimed":[47],"that":[48,177,247],"generate":[52],"`high":[53],"quality'":[54],"readable":[55],"content,":[56],"there":[57],"no":[59],"prior":[60],"study":[61,148,239],"on":[62,71,188,204,227],"analyzing":[63],"generated":[65,184,220],"from":[67,89,259],"based":[70],"sampling":[72,164],"and":[73,111,113,117,141,153,160,181,197,233,244],"fine-tuning":[74],"hyperparameters.":[75],"We":[76],"conduct":[77],"an":[78,192],"in-depth":[79],"comparison":[80],"several":[82],"for":[85],"open-ended":[86],"story":[87],"given":[90],"prompts.":[91],"Using":[92],"diverse":[94],"set":[95],"automated":[97],"metrics,":[98],"we":[99],"compare":[100],"performance":[102,245],"transformer-based":[104],"generative":[105],"-":[107,133],"OpenAI's":[108],"GPT2":[109,249],"(pre-trained":[110],"fine-tuned)":[112],"Google's":[114],"TransformerXL":[116],"XLNet":[118],"human-written":[120],"references.":[122],"Studying":[123],"inter-metric":[124],"correlation":[125,136],"along":[126,215],"with":[127,191,216],"metric":[128,257],"ranking":[129],"reveals":[130],"interesting":[131],"insights":[132],"high":[135],"between":[137,156],"readability":[139],"scores":[140,158,258],"word":[142],"usage":[143],"text.":[146],"A":[147,238],"statistical":[151],"significance":[152],"empirical":[154],"evaluations":[155],"(human":[159],"machine-generated)":[161],"at":[162],"higher":[163],"hyperparameter":[165],"combinations":[166],"(t":[167],"=":[168,172],"{0.75,":[169],"1.0},":[170],"k":[171],"{100,":[173],"150,":[174],"250})":[175],"reveal":[176],"top":[179],"fine-tuned":[182,203,248],"samples":[185,221],"condition":[186],"well":[187],"prompt":[190],"increased":[193],"occurrence":[194],"unique":[196],"difficult":[198],"words.":[199],"GPT2-medium":[201],"model":[202,242],"1024":[206],"Byte-pair":[207],"Encoding":[208],"(BPE)":[209],"tokenized":[210],"version":[211],"dataset":[214],"Transformer-XL":[218],"close":[222],"human":[224,260],"written":[225],"three":[228],"metrics:":[229],"prompt-based":[230],"overlap,":[231],"coherence,":[232],"variation":[234],"sentence":[236],"length.":[237],"overall":[241],"stability":[243],"shows":[246],"least":[254],"deviation":[255],"performance.":[261]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":2}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-10-10T00:00:00"}