{"id":"https://openalex.org/W2978031110","doi":"https://doi.org/10.1109/ijcnn.2019.8851951","title":"A Crowdsourcing Based Human-in-the-Loop Framework for Denoising UUs in Relation Extraction Tasks","display_name":"A Crowdsourcing Based Human-in-the-Loop Framework for Denoising UUs in Relation Extraction Tasks","publication_year":2019,"publication_date":"2019-07-01","ids":{"openalex":"https://openalex.org/W2978031110","doi":"https://doi.org/10.1109/ijcnn.2019.8851951","mag":"2978031110"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn.2019.8851951","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn.2019.8851951","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100643013","display_name":"Mengting Li","orcid":"https://orcid.org/0000-0002-0144-870X"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Mengting Li","raw_affiliation_strings":["School of Computer Science and Software Engineering, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Software Engineering, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101463994","display_name":"Jian Jin","orcid":"https://orcid.org/0000-0002-2109-4961"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Jin","raw_affiliation_strings":["School of Computer Science and Software Engineering, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Software Engineering, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084590062","display_name":"Wen Wu","orcid":"https://orcid.org/0000-0002-2132-5993"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wen Wu","raw_affiliation_strings":["School of Computer Science and Software Engineering, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Software Engineering, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056644602","display_name":"Yan Yang","orcid":"https://orcid.org/0000-0001-9922-2508"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan Yang","raw_affiliation_strings":["School of Computer Science and Software Engineering, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Software Engineering, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010540039","display_name":"Liang He","orcid":"https://orcid.org/0000-0002-4723-5486"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang He","raw_affiliation_strings":["School of Computer Science and Software Engineering, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Software Engineering, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100459985","display_name":"Jing Yang","orcid":"https://orcid.org/0000-0002-5973-248X"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Yang","raw_affiliation_strings":["School of Computer Science and Software Engineering, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Software Engineering, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100643013"],"corresponding_institution_ids":["https://openalex.org/I66867065"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.11584912,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"1","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8790797591209412},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7919682264328003},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.6749443411827087},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6485357284545898},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.5563993453979492},{"id":"https://openalex.org/keywords/relationship-extraction","display_name":"Relationship extraction","score":0.4828553795814514},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4745284616947174},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4660395681858063},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.43684375286102295},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33926326036453247},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3297341465950012},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.2768913507461548},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.12391388416290283}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8790797591209412},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7919682264328003},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.6749443411827087},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6485357284545898},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.5563993453979492},{"id":"https://openalex.org/C153604712","wikidata":"https://www.wikidata.org/wiki/Q7310755","display_name":"Relationship extraction","level":3,"score":0.4828553795814514},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4745284616947174},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4660395681858063},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.43684375286102295},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33926326036453247},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3297341465950012},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2768913507461548},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.12391388416290283},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn.2019.8851951","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn.2019.8851951","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W1604644367","https://openalex.org/W1889268436","https://openalex.org/W2030408698","https://openalex.org/W2034368206","https://openalex.org/W2053238041","https://openalex.org/W2059362837","https://openalex.org/W2068948573","https://openalex.org/W2071571444","https://openalex.org/W2091582906","https://openalex.org/W2094728533","https://openalex.org/W2102267996","https://openalex.org/W2131744502","https://openalex.org/W2155454737","https://openalex.org/W2250521169","https://openalex.org/W2251135946","https://openalex.org/W2251847161","https://openalex.org/W2469104253","https://openalex.org/W2511964075","https://openalex.org/W2515462165","https://openalex.org/W2583689529","https://openalex.org/W2604610161","https://openalex.org/W2759996146","https://openalex.org/W2760600531","https://openalex.org/W2776652360","https://openalex.org/W2897058782","https://openalex.org/W2951274974","https://openalex.org/W2962939608","https://openalex.org/W2964217331","https://openalex.org/W2964317478","https://openalex.org/W6639364127","https://openalex.org/W6675186304","https://openalex.org/W6679775712","https://openalex.org/W6691723933","https://openalex.org/W6732987123","https://openalex.org/W6735888106"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W2805262146","https://openalex.org/W4379517534"],"abstract_inverted_index":{"In":[0],"relation":[1,142,187],"extraction":[2,188],"tasks,":[3],"distant":[4,35,185],"supervision":[5,36,186],"methods":[6,26,37],"expand":[7],"dataset":[8,166,178],"by":[9,34,72,113],"aligning":[10],"entity":[11],"pairs":[12],"in":[13,44],"different":[14],"knowledge":[15],"bases":[16],"and":[17,69,88,120,148,167,179],"completing":[18],"the":[19,28,45,129,162,177,182],"relations":[20],"between":[21],"two":[22,90,116],"entities.":[23],"However,":[24],"these":[25,109],"ignore":[27],"fact":[29],"that":[30,173],"sentences":[31],"labels":[32],"generated":[33],"with":[38,54,115,155],"high":[39],"confidence":[40],"are":[41,111,139],"often":[42],"incorrect":[43],"real":[46],"world":[47],"called":[48],"Unknown":[49],"Unknowns":[50],"(UUs).":[51],"To":[52],"deal":[53],"this":[55],"challenge,":[56],"we":[57,83],"propose":[58],"a":[59,102,134],"crowdsourcing":[60,73,114],"based":[61,94],"human-in-the-loop":[62],"denoising":[63],"framework":[64,149,175],"which":[65],"iteratively":[66],"discovers":[67],"UUs":[68,110,171],"corrects":[70],"them":[71],"to":[74,104,152],"better":[75],"extract":[76],"relations.":[77],"During":[78],"each":[79],"epoch":[80,154],"of":[81,131,169],"iterations,":[82],"choose":[84],"one":[85],"sentence":[86,157],"bag":[87],"repeat":[89],"steps:":[91],"Firstly,":[92],"attention":[93],"Long":[95],"Short-Term":[96],"Memory":[97],"network":[98],"is":[99],"applied":[100],"as":[101,125,144],"selector":[103,124,132],"discover":[105],"potential":[106,170],"UUs.":[107],"Secondly,":[108],"annotated":[112,137],"answer":[117],"collecting":[118],"strategies":[119],"fed":[121],"back":[122],"into":[123,141],"positive":[126],"samples.":[127],"Until":[128],"accuracy":[130],"reaches":[133],"threshold,":[135],"all":[136,181],"samples":[138],"added":[140],"classifier":[143],"cleaned":[145],"train":[146],"set":[147],"moves":[150],"on":[151,161,184],"next":[153],"new":[156],"bags.":[158],"The":[159],"experiments":[160],"New":[163],"York":[164],"Times":[165],"analysis":[168],"demonstrate":[172],"our":[174],"denoise":[176],"outperforms":[180],"baselines":[183],"tasks.":[189]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}