{"id":"https://openalex.org/W4389518751","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.324","title":"COHESENTIA: A Novel Benchmark of Incremental versus Holistic Assessment of Coherence in Generated Texts","display_name":"COHESENTIA: A Novel Benchmark of Incremental versus Holistic Assessment of Coherence in Generated Texts","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389518751","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.324"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.emnlp-main.324","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.324","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.324","pdf_url":"https://aclanthology.org/2023.emnlp-main.324.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.emnlp-main.324.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092998404","display_name":"Aviya Maimon","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Aviya Maimon","raw_affiliation_strings":["Department of Computer Science, Bar Ilan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Bar Ilan University","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063283689","display_name":"Reut Tsarfaty","orcid":"https://orcid.org/0000-0001-6406-6568","observed_orcids":["https://orcid.org/0000-0001-6406-6568"]},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Reut Tsarfaty","raw_affiliation_strings":["Department of Computer Science, Bar Ilan University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Bar Ilan University","institution_ids":["https://openalex.org/I13955877"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I13955877"],"apc_list":null,"apc_paid":null,"fwci":0.707,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.72349586,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7904999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7904999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11680000275373459,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.028300000354647636,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/textual-coherence","display_name":"text coherence","score":0.8709999918937683},{"id":"https://openalex.org/keywords/coherence-assessment","display_name":"coherence assessment","score":0.6359999775886536},{"id":"https://openalex.org/keywords/manual-annotation","display_name":"manual annotation","score":0.4729999899864197},{"id":"https://openalex.org/keywords/inter-annotator-agreement","display_name":"inter-annotator agreement","score":0.9449999928474426},{"id":"https://openalex.org/keywords/coherence-detection","display_name":"coherence detection","score":0.5329999923706055},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7269999980926514}],"concepts":[{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.8610937595367432},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7985300421714783},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.634267270565033},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6320945620536804},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6105974912643433},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5889285802841187},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.4339742064476013},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10952877998352051},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10433483123779297},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.emnlp-main.324","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.324","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.324","pdf_url":"https://aclanthology.org/2023.emnlp-main.324.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.16329","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.16329","pdf_url":"https://arxiv.org/pdf/2310.16329","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.16329","doi":"https://doi.org/10.48550/arxiv.2310.16329","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.16329","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.emnlp-main.324","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.324","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.324","pdf_url":"https://aclanthology.org/2023.emnlp-main.324.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1142999508","display_name":"Natural Language Programming: Turning Text into Executable Code","funder_award_id":"677352","funder_id":"https://openalex.org/F4320338335","funder_display_name":"H2020 European Research Council"}],"funders":[{"id":"https://openalex.org/F4320338335","display_name":"H2020 European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389518751.pdf","grobid_xml":"https://content.openalex.org/works/W4389518751.grobid-xml"},"referenced_works_count":23,"referenced_works":["https://openalex.org/W41621595","https://openalex.org/W1497300277","https://openalex.org/W2045738181","https://openalex.org/W2101105183","https://openalex.org/W2118370253","https://openalex.org/W2118426465","https://openalex.org/W2120465901","https://openalex.org/W2123849094","https://openalex.org/W2133330855","https://openalex.org/W2140676672","https://openalex.org/W2153222072","https://openalex.org/W2159757335","https://openalex.org/W2250952041","https://openalex.org/W2251951653","https://openalex.org/W2264742718","https://openalex.org/W2324044204","https://openalex.org/W2398973842","https://openalex.org/W2896457183","https://openalex.org/W2962790423","https://openalex.org/W2963482033","https://openalex.org/W2970785793","https://openalex.org/W3211686893","https://openalex.org/W4307079201"],"related_works":["https://openalex.org/W4387323696","https://openalex.org/W4411120206","https://openalex.org/W1503071992","https://openalex.org/W4386473595","https://openalex.org/W3018006905","https://openalex.org/W2577027179","https://openalex.org/W7212314043","https://openalex.org/W4393160172","https://openalex.org/W2963482033","https://openalex.org/W2411759152"],"abstract_inverted_index":{"Coherence":[0],"is":[1,36,114,124,174],"a":[2,37,98,117],"linguistic":[3],"term":[4],"that":[5,147,166,185],"refers":[6],"to":[7,24,40,70,200],"the":[8,18,25,27,43,60,67,74,122,167,171,178,196,210],"relations":[9],"between":[10],"small":[11],"textual":[12],"units":[13],"(sentences,":[14],"propositions),":[15],"which":[16],"make":[17],"text":[19,138],"logically":[20],"consistent":[21],"and":[22,65,121,140,153,181],"meaningful":[23],"reader.With":[26],"advances":[28],"of":[29,46,62,84,101,104],"generative":[30],"foundational":[31],"models":[32,205],"in":[33,158,170,177,202],"NLP,":[34],"there":[35],"pressing":[38],"need":[39,211],"automatically":[41,47,105],"assess":[42],"human-perceived":[44],"coherence":[45,61,89,103,119,190,218],"generated":[48,63,106],"texts.Up":[49],"until":[50],"now,":[51],"little":[52],"work":[53,72],"has":[54],"been":[55],"done":[56],"on":[57,73,195],"explicitly":[58],"assessing":[59],"texts":[64],"analyzing":[66],"factors":[68,198],"contributing":[69,199],"(in)coherence.Previous":[71],"topic":[75],"used":[76],"other":[77,123],"tasks,":[78],"e.g.,":[79],"sentence":[80,127],"reordering,":[81],"as":[82],"proxies":[83],"coherence,":[85],"rather":[86],"than":[87,176],"approaching":[88],"detection":[90,191],"heads":[91],"on.In":[92],"this":[93],"paper,":[94],"we":[95],"introduce":[96],"COHESENTIA,":[97],"novel":[99],"benchmark":[100,149],"humanperceived":[102],"texts.Our":[107],"annotation":[108],"protocol":[109],"reflects":[110],"two":[111],"perspectives;":[112],"one":[113],"global,":[115],"assigning":[116],"single":[118],"score,":[120],"incremental,":[125],"scoring":[126],"by":[128,161],"sentence.The":[129],"incremental":[130,172],"method":[131],"produces":[132],"an":[133],"(in)coherence":[134],"score":[135],"for":[136,144,189,212,217],"each":[137,156],"fragment":[139],"also":[141],"pinpoints":[142],"reasons":[143],"incoherence":[145],"at":[146],"point.Our":[148],"contains":[150],"500":[151],"automatically-generated":[152],"human-annotated":[154],"paragraphs,":[155],"annotated":[157],"both":[159],"methods,":[160],"multiple":[162],"raters.Our":[163],"analysis":[164],"shows":[165],"inter-annotator":[168],"agreement":[169],"mode":[173],"higher":[175],"holistic":[179],"alternative,":[180],"our":[182],"experiments":[183],"show":[184,192],"standard":[186],"LMs":[187],"fine-tuned":[188],"varied":[193],"performance":[194],"different":[197],"(in)coherence.All":[201],"all,":[203],"these":[204],"yield":[206],"unsatisfactory":[207],"performance,":[208],"emphasizing":[209],"developing":[213],"more":[214],"reliable":[215],"methods":[216],"assessment.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2023-12-10T00:00:00"}