{"id":"https://openalex.org/W4396574605","doi":"https://doi.org/10.1162/coli_a_00518","title":"Aligning Human and Computational Coherence Evaluations","display_name":"Aligning Human and Computational Coherence Evaluations","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4396574605","doi":"https://doi.org/10.1162/coli_a_00518"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00518","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00518","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/coli_a_00518","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109711628","display_name":"Jia Peng Lim","orcid":null},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":true,"raw_author_name":"Jia Peng Lim","raw_affiliation_strings":["Singapore Management University, School of Computer and Information Systems, PreferredAI Research Group. jiapeng.lim.2021@smu.edu.sg"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Singapore Management University, School of Computer and Information Systems, PreferredAI Research Group. jiapeng.lim.2021@smu.edu.sg","institution_ids":["https://openalex.org/I79891267"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065067629","display_name":"Hady W. Lauw","orcid":"https://orcid.org/0000-0002-8245-8677"},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":true,"raw_author_name":"Hady W. Lauw","raw_affiliation_strings":["Singapore Management University, School of Computer and Information Systems, PreferredAI Research Group. hadywlauw@smu.edu.sg"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Singapore Management University, School of Computer and Information Systems, PreferredAI Research Group. hadywlauw@smu.edu.sg","institution_ids":["https://openalex.org/I79891267"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5065067629","https://openalex.org/A5109711628"],"corresponding_institution_ids":["https://openalex.org/I79891267"],"apc_list":null,"apc_paid":null,"fwci":1.2215,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.77932915,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"50","issue":"3","first_page":"893","last_page":"952"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9510999917984009,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.946399986743927,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6965394020080566},{"id":"https://openalex.org/keywords/coherence","display_name":"Coherence (philosophical gambling strategy)","score":0.6101983189582825},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34979167580604553},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.08301001787185669},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07924401760101318}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6965394020080566},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.6101983189582825},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34979167580604553},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.08301001787185669},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07924401760101318}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1162/coli_a_00518","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00518","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-10427","is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/9427","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://doi.org/10.1162/coli_a_00518","raw_type":"Journal Article"},{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-11543","is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/10541","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://doi.org/10.1162/coli_a_00518","raw_type":"Journal Article"},{"id":"pmh:oai:doaj.org/article:5737c7d0e67a4cad9d0ce2c3a3198deb","is_oa":false,"landing_page_url":"https://doaj.org/article/5737c7d0e67a4cad9d0ce2c3a3198deb","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 50, Iss 3 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00518","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00518","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8100000023841858,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":90,"referenced_works":["https://openalex.org/W136732505","https://openalex.org/W146900863","https://openalex.org/W1492638184","https://openalex.org/W1554540371","https://openalex.org/W1593045043","https://openalex.org/W1608124888","https://openalex.org/W1972978214","https://openalex.org/W2001082470","https://openalex.org/W2003902046","https://openalex.org/W2016404778","https://openalex.org/W2037170747","https://openalex.org/W2038043464","https://openalex.org/W2055245094","https://openalex.org/W2055431313","https://openalex.org/W2081580037","https://openalex.org/W2106378689","https://openalex.org/W2108818539","https://openalex.org/W2123402141","https://openalex.org/W2154851992","https://openalex.org/W2168745915","https://openalex.org/W2250533720","https://openalex.org/W2460159515","https://openalex.org/W2543728926","https://openalex.org/W2604738573","https://openalex.org/W2614322402","https://openalex.org/W2759136286","https://openalex.org/W2787993231","https://openalex.org/W2788135508","https://openalex.org/W2803437449","https://openalex.org/W2893300601","https://openalex.org/W2911966198","https://openalex.org/W2912280546","https://openalex.org/W2914584698","https://openalex.org/W2915480215","https://openalex.org/W2944931850","https://openalex.org/W2962943175","https://openalex.org/W2962966012","https://openalex.org/W2976420234","https://openalex.org/W2997136781","https://openalex.org/W3003265726","https://openalex.org/W3010694149","https://openalex.org/W3012593603","https://openalex.org/W3042602466","https://openalex.org/W3045464143","https://openalex.org/W3103817618","https://openalex.org/W3104097132","https://openalex.org/W3105538385","https://openalex.org/W3155457426","https://openalex.org/W3166170409","https://openalex.org/W3166574703","https://openalex.org/W4214538687","https://openalex.org/W4224315235","https://openalex.org/W4229009238","https://openalex.org/W4231510805","https://openalex.org/W4233063693","https://openalex.org/W4250868687","https://openalex.org/W4252684946","https://openalex.org/W4289533829","https://openalex.org/W4290874888","https://openalex.org/W4317038703","https://openalex.org/W4385570305","https://openalex.org/W4385570337","https://openalex.org/W4385570685","https://openalex.org/W4385571483","https://openalex.org/W4385572928","https://openalex.org/W4385574240","https://openalex.org/W4386839891","https://openalex.org/W4389519076","https://openalex.org/W4389520774","https://openalex.org/W4390684394","https://openalex.org/W6605449825","https://openalex.org/W6615650610","https://openalex.org/W6629396621","https://openalex.org/W6635456232","https://openalex.org/W6637762568","https://openalex.org/W6639619044","https://openalex.org/W6640963894","https://openalex.org/W6678830454","https://openalex.org/W6679482899","https://openalex.org/W6683333316","https://openalex.org/W6684489972","https://openalex.org/W6685356407","https://openalex.org/W6691363302","https://openalex.org/W6721336710","https://openalex.org/W6734688555","https://openalex.org/W6751854905","https://openalex.org/W6768219389","https://openalex.org/W6797676422","https://openalex.org/W6803519869","https://openalex.org/W6857017968"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Abstract":[0],"Automated":[1],"coherence":[2,56],"metrics":[3,57,188,221],"constitute":[4],"an":[5,70],"efficient":[6],"and":[7,94,158,170,189,196],"popular":[8],"way":[9],"to":[10,34,44,76,99,183,213],"evaluate":[11],"topic":[12,36,127],"models.":[13],"Previous":[14],"work":[15,28],"presents":[16],"a":[17,30,39,65,78,160,174,181],"mixed":[18],"picture":[19],"of":[20,52,72,91,125,134,151,164,204,239],"their":[21,122],"presumed":[22],"correlation":[23,185,215],"with":[24,120],"human":[25,83,137,152,165,190,202],"judgment.":[26],"This":[27,86,145],"proposes":[29,64],"novel":[31,66,161],"sampling":[32,101],"approach":[33],"mining":[35],"representations":[37],"at":[38,167],"large":[40,59],"scale":[41],"while":[42],"seeking":[43],"mitigate":[45],"bias":[46],"from":[47],"sampling,":[48],"enabling":[49],"the":[50,82,89,100,103,132,142,149,156,168,201,230],"investigation":[51,178],"widely":[53],"used":[54],"automated":[55],"via":[58],"corpora.":[60],"Additionally,":[61],"this":[62,226],"article":[63,146,227],"user":[67,96,104],"study":[68,105,111,117],"design,":[69],"amalgamation":[71],"different":[73,116],"proxy":[74],"tasks,":[75],"derive":[77,197],"finer":[79],"insight":[80],"into":[81,114,200],"decision-making":[84],"processes.":[85],"design":[87],"subsumes":[88],"purpose":[90],"simple":[92],"rating":[93],"outlier-detection":[95],"studies.":[97],"Similar":[98],"approach,":[102],"conducted":[106],"is":[107],"extensive,":[108],"comprising":[109],"40":[110],"participants":[112],"split":[113],"eight":[115],"groups":[118],"tasked":[119],"evaluating":[121],"respective":[123],"set":[124],"100":[126],"representations.":[128],"Usually,":[129],"when":[130],"substantiating":[131],"use":[133],"these":[135,187,217,220,240],"metrics,":[136],"responses":[138],"are":[139],"treated":[140],"as":[141],"gold":[143],"standard.":[144],"further":[147,198,228],"investigates":[148,229],"reliability":[150],"judgment":[153],"by":[154],"flipping":[155],"comparison":[157],"conducting":[159],"extended":[162],"analysis":[163],"response":[166],"group":[169],"individual":[171],"level":[172],"against":[173],"generic":[175,194],"corpus.":[176],"The":[177],"results":[179],"show":[180],"moderate":[182,212],"good":[184,214],"between":[186,233],"judgment,":[191],"especially":[192],"for":[193],"corpora,":[195,234],"insights":[199],"perception":[203],"coherence.":[205],"Analyzing":[206],"inter-metric":[207],"correlations":[208],"across":[209],"corpora":[210],"shows":[211],"among":[216],"metrics.":[218,241],"As":[219],"depend":[222],"on":[223],"corpus":[224],"statistics,":[225],"topical":[231],"differences":[232],"revealing":[235],"nuances":[236],"in":[237],"applications":[238]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-18T07:49:30.821534","created_date":"2025-10-10T00:00:00"}