{"id":"https://openalex.org/W2904982178","doi":"https://doi.org/10.1109/tvcg.2018.2864843","title":"An Interactive Method to Improve Crowdsourced Annotations","display_name":"An Interactive Method to Improve Crowdsourced Annotations","publication_year":2018,"publication_date":"2018-08-20","ids":{"openalex":"https://openalex.org/W2904982178","doi":"https://doi.org/10.1109/tvcg.2018.2864843","mag":"2904982178","pmid":"https://pubmed.ncbi.nlm.nih.gov/30130224"},"language":"en","primary_location":{"id":"doi:10.1109/tvcg.2018.2864843","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2018.2864843","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038775008","display_name":"Shi\u2010Xia Liu","orcid":"https://orcid.org/0000-0001-6104-4320"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Shixia Liu","raw_affiliation_strings":["Tsinghua University, Beijing, Beijing, CN"],"affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, Beijing, CN","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019821536","display_name":"Changjian Chen","orcid":"https://orcid.org/0000-0003-2715-8839"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changjian Chen","raw_affiliation_strings":["Tsinghua University, Beijing, Beijing, CN"],"affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, Beijing, CN","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029506224","display_name":"Yafeng Lu","orcid":"https://orcid.org/0000-0002-3978-9550"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yafeng Lu","raw_affiliation_strings":["Arizona State University, Tempe, AZ, US"],"affiliations":[{"raw_affiliation_string":"Arizona State University, Tempe, AZ, US","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108824539","display_name":"Fangxin Ouyang","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fangxin Ouyang","raw_affiliation_strings":["Tsinghua University, Beijing, Beijing, CN"],"affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, Beijing, CN","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100372368","display_name":"Bin Wang","orcid":"https://orcid.org/0000-0002-5176-9202"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Wang","raw_affiliation_strings":["Tsinghua University, Beijing, Beijing, CN"],"affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, Beijing, CN","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5038775008"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":11.8844,"has_fulltext":false,"cited_by_count":75,"citation_normalized_percentile":{"value":0.98456857,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"25","issue":"1","first_page":"235","last_page":"245"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.9081614017486572},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8552336692810059},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.7048486471176147},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6104894876480103},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5795158743858337},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.5236406326293945},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5220701098442078},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4916089177131653},{"id":"https://openalex.org/keywords/data-visualization","display_name":"Data visualization","score":0.43046319484710693},{"id":"https://openalex.org/keywords/projection","display_name":"Projection (relational algebra)","score":0.4282681941986084},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3508586883544922},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.34710800647735596},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.12402629852294922}],"concepts":[{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.9081614017486572},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8552336692810059},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.7048486471176147},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6104894876480103},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5795158743858337},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.5236406326293945},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5220701098442078},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4916089177131653},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.43046319484710693},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.4282681941986084},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3508586883544922},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34710800647735596},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.12402629852294922},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tvcg.2018.2864843","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2018.2864843","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},{"id":"pmid:30130224","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/30130224","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on visualization and computer graphics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4399999976158142,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[{"id":"https://openalex.org/G3665848037","display_name":null,"funder_award_id":"61761136020","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5207934879","display_name":null,"funder_award_id":"61672308","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":75,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W1766493433","https://openalex.org/W1840640513","https://openalex.org/W1901365844","https://openalex.org/W1953652583","https://openalex.org/W1965701254","https://openalex.org/W1981119726","https://openalex.org/W1992492534","https://openalex.org/W1996034305","https://openalex.org/W2004026774","https://openalex.org/W2007185493","https://openalex.org/W2007241592","https://openalex.org/W2009858410","https://openalex.org/W2033074184","https://openalex.org/W2036431079","https://openalex.org/W2037894060","https://openalex.org/W2064766209","https://openalex.org/W2065547622","https://openalex.org/W2066636486","https://openalex.org/W2071830810","https://openalex.org/W2074275752","https://openalex.org/W2087382273","https://openalex.org/W2099223662","https://openalex.org/W2102427929","https://openalex.org/W2102605133","https://openalex.org/W2108598243","https://openalex.org/W2112250557","https://openalex.org/W2113411758","https://openalex.org/W2115206325","https://openalex.org/W2117088188","https://openalex.org/W2117539524","https://openalex.org/W2117831564","https://openalex.org/W2127058057","https://openalex.org/W2136827748","https://openalex.org/W2141200610","https://openalex.org/W2143326696","https://openalex.org/W2152009989","https://openalex.org/W2155541015","https://openalex.org/W2187089797","https://openalex.org/W2187444452","https://openalex.org/W2196365129","https://openalex.org/W2251434949","https://openalex.org/W2343061342","https://openalex.org/W2412593044","https://openalex.org/W2510750606","https://openalex.org/W2512177572","https://openalex.org/W2513302583","https://openalex.org/W2542889218","https://openalex.org/W2587299461","https://openalex.org/W2597908760","https://openalex.org/W2600599059","https://openalex.org/W2602814102","https://openalex.org/W2602830283","https://openalex.org/W2740265435","https://openalex.org/W2750714283","https://openalex.org/W2751722336","https://openalex.org/W2752257308","https://openalex.org/W2752332392","https://openalex.org/W2753426599","https://openalex.org/W2755457990","https://openalex.org/W2758760553","https://openalex.org/W2801605877","https://openalex.org/W2963707011","https://openalex.org/W3099276202","https://openalex.org/W3124418332","https://openalex.org/W4294375521","https://openalex.org/W6637373629","https://openalex.org/W6677006842","https://openalex.org/W6682675497","https://openalex.org/W6682778277","https://openalex.org/W6686975137","https://openalex.org/W6687549421","https://openalex.org/W6691869317","https://openalex.org/W6735844885","https://openalex.org/W6751408895"],"related_works":["https://openalex.org/W4280506488","https://openalex.org/W2013728941","https://openalex.org/W4225274103","https://openalex.org/W2579659702","https://openalex.org/W2154046714","https://openalex.org/W1965329638","https://openalex.org/W1574055964","https://openalex.org/W2923661510","https://openalex.org/W2189613078","https://openalex.org/W2586219255"],"abstract_inverted_index":{"In":[0,28],"order":[1],"to":[2,36,75,96,108,119,133],"effectively":[3],"infer":[4],"correct":[5],"labels":[6,43,50,112],"from":[7,55],"noisy":[8],"crowdsourced":[9,160],"annotations,":[10,89],"learning-from-crowds":[11,57,82,131],"models":[12],"have":[13,91],"introduced":[14],"expert":[15,66],"validation.":[16,67,142],"However,":[17],"little":[18],"research":[19],"has":[20],"been":[21],"done":[22],"on":[23],"facilitating":[24],"the":[25,48,81,86,98,110,130],"validation":[26,87],"procedure.":[27],"this":[29],"paper,":[30],"we":[31,90],"propose":[32],"an":[33,135,153],"interactive":[34],"method":[35,107,155],"assist":[37],"experts":[38],"in":[39,113],"verifying":[40],"uncertain":[41,111],"instance":[42,49],"and":[44,51,61,78,115,137,158],"unreliable":[45],"workers.":[46],"Given":[47],"worker":[52,121],"reliability":[53],"inferred":[54],"a":[56,93,104,116],"model,":[58],"candidate":[59],"instances":[60,77],"workers":[62,79],"are":[63,126],"selected":[64],"for":[65,101,140,156],"The":[68,123],"influence":[69],"of":[70,88],"verified":[71],"results":[72],"is":[73],"propagated":[74],"relevant":[76],"through":[80],"model.":[83],"To":[84],"facilitate":[85],"developed":[92],"confusion":[94],"visualization":[95,118],"indicate":[97],"confusing":[99],"classes":[100],"further":[102],"exploration,":[103],"constrained":[105],"projection":[106],"show":[109],"context,":[114],"scatter-plot-based":[117],"illustrate":[120],"reliability.":[122],"three":[124],"visualizations":[125],"tightly":[127],"integrated":[128],"with":[129],"model":[132],"provide":[134],"iterative":[136],"progressive":[138],"environment":[139],"data":[141],"Two":[143],"case":[144],"studies":[145],"were":[146],"conducted":[147],"that":[148],"demonstrate":[149],"our":[150],"approach":[151],"offers":[152],"efficient":[154],"validating":[157],"improving":[159],"annotations.":[161]},"counts_by_year":[{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":15},{"year":2019,"cited_by_count":7},{"year":2018,"cited_by_count":2}],"updated_date":"2026-03-14T08:43:22.919905","created_date":"2025-10-10T00:00:00"}