{"id":"https://openalex.org/W4389159637","doi":"https://doi.org/10.1145/3611643.3616258","title":"Testing Coreference Resolution Systems without Labeled Test Sets","display_name":"Testing Coreference Resolution Systems without Labeled Test Sets","publication_year":2023,"publication_date":"2023-11-30","ids":{"openalex":"https://openalex.org/W4389159637","doi":"https://doi.org/10.1145/3611643.3616258"},"language":"en","primary_location":{"id":"doi:10.1145/3611643.3616258","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3611643.3616258","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053372458","display_name":"Jialun Cao","orcid":"https://orcid.org/0000-0003-4892-6294"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4210159029","display_name":"Guangzhou HKUST Fok Ying Tung Research Institute","ror":"https://ror.org/05cvbj479","country_code":"CN","type":"facility","lineage":["https://openalex.org/I200769079","https://openalex.org/I4210159029"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":true,"raw_author_name":"Jialun Cao","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China / Guangzhou HKUST Fok Ying Tung Research Institute, Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China / Guangzhou HKUST Fok Ying Tung Research Institute, Guangzhou, China","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4210159029","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103090910","display_name":"Yaojie Lu","orcid":"https://orcid.org/0000-0002-5842-7715"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaojie Lu","raw_affiliation_strings":["Chinese Institute of Software at Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Chinese Institute of Software at Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003031253","display_name":"Ming Wen","orcid":"https://orcid.org/0000-0001-5588-9618"},"institutions":[{"id":"https://openalex.org/I4210157617","display_name":"Huazhong University of Science and Technology Hospital","ror":"https://ror.org/05f9vfg11","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210157617"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ming Wen","raw_affiliation_strings":["Huazhong University of Science and Technology, Hubei, China"],"affiliations":[{"raw_affiliation_string":"Huazhong University of Science and Technology, Hubei, China","institution_ids":["https://openalex.org/I4210157617"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034057959","display_name":"Shing-Chi Cheung","orcid":"https://orcid.org/0000-0002-3508-7172"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4210159029","display_name":"Guangzhou HKUST Fok Ying Tung Research Institute","ror":"https://ror.org/05cvbj479","country_code":"CN","type":"facility","lineage":["https://openalex.org/I200769079","https://openalex.org/I4210159029"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Shing-Chi Cheung","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, China / Guangzhou HKUST Fok Ying Tung Research Institute, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China / Guangzhou HKUST Fok Ying Tung Research Institute, Hong Kong, China","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4210159029","https://openalex.org/I889458895"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5053372458"],"corresponding_institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4210159029","https://openalex.org/I889458895"],"apc_list":null,"apc_paid":null,"fwci":0.5175,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.72812835,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"107","last_page":"119"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/coreference","display_name":"Coreference","score":0.9624075889587402},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.843974769115448},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.6833206415176392},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6501278281211853},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6286396384239197},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5142252445220947},{"id":"https://openalex.org/keywords/false-positive-paradox","display_name":"False positive paradox","score":0.5135726928710938},{"id":"https://openalex.org/keywords/resolution","display_name":"Resolution (logic)","score":0.444564551115036},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.42337003350257874},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4101512134075165},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.40903037786483765},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1754874289035797}],"concepts":[{"id":"https://openalex.org/C28076734","wikidata":"https://www.wikidata.org/wiki/Q63087","display_name":"Coreference","level":3,"score":0.9624075889587402},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.843974769115448},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.6833206415176392},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6501278281211853},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6286396384239197},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5142252445220947},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.5135726928710938},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.444564551115036},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.42337003350257874},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4101512134075165},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40903037786483765},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1754874289035797},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3611643.3616258","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3611643.3616258","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-135992","is_oa":false,"landing_page_url":"https://repository.hkust.edu.hk/ir/Record/1783.1-135992","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7599999904632568}],"awards":[{"id":"https://openalex.org/G1100817172","display_name":null,"funder_award_id":"619320","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1716284841","display_name":null,"funder_award_id":"62141210","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1808308080","display_name":null,"funder_award_id":"R5034-18","funder_id":"https://openalex.org/F4320307285","funder_display_name":"Impact Fund"},{"id":"https://openalex.org/G1888249465","display_name":null,"funder_award_id":"61932021","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2802911279","display_name":null,"funder_award_id":"Young","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3085993365","display_name":null,"funder_award_id":"(Grant No.","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G351720009","display_name":null,"funder_award_id":"2002125","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G391238517","display_name":null,"funder_award_id":", and","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5939423041","display_name":null,"funder_award_id":"Technology","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6163997753","display_name":null,"funder_award_id":"62002125","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6665774762","display_name":null,"funder_award_id":"2021QNRC001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7726157001","display_name":null,"funder_award_id":"Grant No.","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8714109953","display_name":null,"funder_award_id":"QNRC001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307285","display_name":"Impact Fund","ror":"https://ror.org/00jb20j87"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320326427","display_name":"Innovation and Technology Fund","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W138910040","https://openalex.org/W2050273484","https://openalex.org/W2081580037","https://openalex.org/W2149956050","https://openalex.org/W2157944021","https://openalex.org/W2516621648","https://openalex.org/W2604710187","https://openalex.org/W2806718802","https://openalex.org/W2810169973","https://openalex.org/W2892245540","https://openalex.org/W2964150020","https://openalex.org/W2970641574","https://openalex.org/W2981902456","https://openalex.org/W3013520104","https://openalex.org/W3015001695","https://openalex.org/W3090988182","https://openalex.org/W3091407209","https://openalex.org/W3100869665","https://openalex.org/W3105604018","https://openalex.org/W3109744225","https://openalex.org/W3163644064","https://openalex.org/W3169117666","https://openalex.org/W3194983542","https://openalex.org/W3203135439","https://openalex.org/W4205221547","https://openalex.org/W4220891756","https://openalex.org/W4280605341","https://openalex.org/W4284675318","https://openalex.org/W6949549905"],"related_works":["https://openalex.org/W2139373276","https://openalex.org/W2464784130","https://openalex.org/W3041549465","https://openalex.org/W2735770104","https://openalex.org/W2380610138","https://openalex.org/W3171444480","https://openalex.org/W4206648670","https://openalex.org/W1495336436","https://openalex.org/W3212412177","https://openalex.org/W3173436410"],"abstract_inverted_index":{"Coreference":[0],"resolution":[1],"(CR)":[2],"is":[3,23,64,75,99],"a":[4,24,68,85,88,177],"task":[5],"to":[6,16,77,84,101,125,132,141,167,207],"resolve":[7],"different":[8],"expressions":[9],"(e.g.,":[10],"named":[11],"entities,":[12],"pronouns)":[13],"that":[14,31,91,187],"refer":[15],"the":[17,51,65,106,133,143,211],"same":[18,134],"real-world":[19],"en-":[20],"tity/event.":[21],"It":[22],"core":[25],"natural":[26],"language":[27],"processing":[28],"(NLP)":[29],"component":[30],"underlies":[32],"and":[33,44,121,164],"empowers":[34],"major":[35,62],"downstream":[36],"NLP":[37],"applications":[38],"such":[39],"as":[40,81],"machine":[41],"translation,":[42],"chatbots,":[43],"question-answering.":[45],"De-":[46],"spite":[47],"its":[48],"broad":[49],"impact,":[50],"problem":[52],"of":[53,67,128],"testing":[54,113],"CR":[55,86,116,162],"systems":[56],"has":[57],"rarely":[58],"been":[59],"studied.":[60],"A":[61],"difficulty":[63],"shortage":[66],"labeled":[69],"dataset":[70,179],"for":[71,115,147,180],"testing.":[72,149],"While":[73],"it":[74],"possible":[76],"feed":[78],"arbitrary":[79],"sentences":[80,172],"test":[82,89,95,129,156],"inputs":[83,130],"system,":[87],"oracle":[90],"captures":[92],"their":[93],"expected":[94],"outputs":[96],"(coreference":[97],"relations)":[98],"hard":[100],"define":[102,142],"automatically.":[103],"To":[104],"address":[105],"challenge,":[107],"we":[108],"propose":[109],"Crest,":[110],"an":[111],"automated":[112],"methodology":[114],"systems.":[117],"Crest":[118,152,188,196],"uses":[119],"constituency":[120],"depen-":[122],"dency":[123],"relations":[124,137],"construct":[126],"pairs":[127],"subject":[131],"coreference.":[135],"These":[136],"can":[138],"be":[139],"leveraged":[140],"meta-":[144],"morphic":[145],"relation":[146],"metamorphic":[148],"We":[150],"compare":[151],"with":[153,205],"five":[154],"state-of-the-art":[155],"generation":[157],"baselines":[158,190],"on":[159],"two":[160],"popular":[161,178],"systems,":[163],"apply":[165],"them":[166],"generate":[168],"tests":[169],"from":[170,175],"1,000":[171],"randomly":[173],"sampled":[174],"CoNLL-2012,":[176],"corefer-":[181],"ence":[182],"resolution.":[183],"Experimental":[184],"results":[185],"show":[186],"outperforms":[189],"significantly.":[191],"The":[192],"issues":[193],"reported":[194],"by":[195,210],"are":[197],"all":[198],"true":[199],"positives":[200],"(i.e.,":[201],"100%":[202],"precision),":[203],"compared":[204],"63%":[206],"75%":[208],"achieved":[209],"baselines.":[212]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}