{"id":"https://openalex.org/W2913790076","doi":"https://doi.org/10.1145/3308558.3313599","title":"Scalpel-CD: Leveraging Crowdsourcing and Deep Probabilistic Modeling for Debugging Noisy Training Data","display_name":"Scalpel-CD: Leveraging Crowdsourcing and Deep Probabilistic Modeling for Debugging Noisy Training Data","publication_year":2019,"publication_date":"2019-05-13","ids":{"openalex":"https://openalex.org/W2913790076","doi":"https://doi.org/10.1145/3308558.3313599","mag":"2913790076"},"language":"en","primary_location":{"id":"doi:10.1145/3308558.3313599","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3313599","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The World Wide Web Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3308558.3313599","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029123550","display_name":"Jie Yang","orcid":"https://orcid.org/0000-0002-0350-0313"},"institutions":[{"id":"https://openalex.org/I154338468","display_name":"University of Fribourg","ror":"https://ror.org/022fs9h90","country_code":"CH","type":"education","lineage":["https://openalex.org/I154338468"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Jie Yang","raw_affiliation_strings":["U. of Fribourg, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"U. of Fribourg, Switzerland","institution_ids":["https://openalex.org/I154338468"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028171582","display_name":"Alisa Smirnova","orcid":"https://orcid.org/0000-0002-7108-9917"},"institutions":[{"id":"https://openalex.org/I154338468","display_name":"University of Fribourg","ror":"https://ror.org/022fs9h90","country_code":"CH","type":"education","lineage":["https://openalex.org/I154338468"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Alisa Smirnova","raw_affiliation_strings":["University of Fribourg, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Fribourg, Switzerland","institution_ids":["https://openalex.org/I154338468"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026625771","display_name":"Dingqi Yang","orcid":"https://orcid.org/0000-0002-6831-0422"},"institutions":[{"id":"https://openalex.org/I154338468","display_name":"University of Fribourg","ror":"https://ror.org/022fs9h90","country_code":"CH","type":"education","lineage":["https://openalex.org/I154338468"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Dingqi Yang","raw_affiliation_strings":["U. of Fribourg, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"U. of Fribourg, Switzerland","institution_ids":["https://openalex.org/I154338468"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["The University of Queensland, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Queensland, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100390820","display_name":"Lu Yuan","orcid":"https://orcid.org/0000-0001-7879-0389"},"institutions":[{"id":"https://openalex.org/I4210148881","display_name":"ING Bank","ror":"https://ror.org/059et9981","country_code":"NL","type":"other","lineage":["https://openalex.org/I4210148881"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Yuan Lu","raw_affiliation_strings":["ING Bank, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ING Bank, Netherlands","institution_ids":["https://openalex.org/I4210148881"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028454093","display_name":"Philippe Cudr\u00e9-Mauroux","orcid":"https://orcid.org/0000-0003-2588-4212"},"institutions":[{"id":"https://openalex.org/I154338468","display_name":"University of Fribourg","ror":"https://ror.org/022fs9h90","country_code":"CH","type":"education","lineage":["https://openalex.org/I154338468"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Philippe Cudre-Mauroux","raw_affiliation_strings":["U. of Fribourg, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"U. of Fribourg, Switzerland","institution_ids":["https://openalex.org/I154338468"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.3119,"has_fulltext":false,"cited_by_count":26,"citation_normalized_percentile":{"value":0.95278297,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"2158","last_page":"2168"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7933188676834106},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6958735585212708},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.6636759638786316},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6472710371017456},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6211457848548889},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5836541652679443},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.5203670859336853},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4629102349281311},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4510596692562103},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.44844701886177063},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.421450674533844},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.19746029376983643},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.1166047751903534}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7933188676834106},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6958735585212708},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.6636759638786316},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6472710371017456},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6211457848548889},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5836541652679443},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5203670859336853},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4629102349281311},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4510596692562103},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.44844701886177063},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.421450674533844},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.19746029376983643},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.1166047751903534},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3308558.3313599","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3313599","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The World Wide Web Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:doc.rero.ch:20190404095016-HZ","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400148","display_name":"reroDoc Digital Library","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":{"id":"doi:10.1145/3308558.3313599","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3313599","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The World Wide Web Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W109203355","https://openalex.org/W205373223","https://openalex.org/W398859631","https://openalex.org/W1604644367","https://openalex.org/W1832693441","https://openalex.org/W1995470018","https://openalex.org/W2063103859","https://openalex.org/W2089206172","https://openalex.org/W2100031962","https://openalex.org/W2108598243","https://openalex.org/W2114524997","https://openalex.org/W2117173982","https://openalex.org/W2125943921","https://openalex.org/W2132655161","https://openalex.org/W2134305421","https://openalex.org/W2142518823","https://openalex.org/W2149713870","https://openalex.org/W2154096410","https://openalex.org/W2162815002","https://openalex.org/W2181558882","https://openalex.org/W2187089797","https://openalex.org/W2225156818","https://openalex.org/W2234829506","https://openalex.org/W2251135946","https://openalex.org/W2270414365","https://openalex.org/W2282821441","https://openalex.org/W2295054684","https://openalex.org/W2404161646","https://openalex.org/W2510508396","https://openalex.org/W2539469848","https://openalex.org/W2548122763","https://openalex.org/W2558888286","https://openalex.org/W2560674852","https://openalex.org/W2590501763","https://openalex.org/W2594475271","https://openalex.org/W2597787948","https://openalex.org/W2604221175","https://openalex.org/W2610935556","https://openalex.org/W2739912280","https://openalex.org/W2769041395","https://openalex.org/W2788739741","https://openalex.org/W2807760453","https://openalex.org/W2903158431","https://openalex.org/W2949780682","https://openalex.org/W2950220847","https://openalex.org/W2951004968","https://openalex.org/W2959716049","https://openalex.org/W2963773425","https://openalex.org/W3101380508","https://openalex.org/W3101504385","https://openalex.org/W3212575067","https://openalex.org/W4211096099","https://openalex.org/W4299828299"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"Scalpel-CD,":[3],"a":[4,33,46,66,118,152],"first-of-its-kind":[5],"system":[6,27],"that":[7,73,162],"leverages":[8],"both":[9],"human":[10],"and":[11,145],"machine":[12,23,157],"intelligence":[13],"to":[14,40,91,123,166],"debug":[15],"noisy":[16],"labels":[17,31,87],"from":[18,78,111],"the":[19,42,55,76,82,96,102,109,112,127,139,178],"training":[20,98,134],"data":[21,48,52,67,71,93,99,104,141],"of":[22,45,120,133,138],"learning":[24,158],"systems.":[25],"Our":[26],"identifies":[28],"potentially":[29],"wrong":[30],"using":[32],"deep":[34],"probabilistic":[35],"model,":[36],"which":[37,69],"is":[38,115,164],"able":[39,165],"infer":[41],"latent":[43,57],"class":[44],"high-dimensional":[47],"instance":[49],"by":[50,81,100,170,177],"exploiting":[51,101],"distributions":[53],"in":[54,95,136],"underlying":[56,103,140],"feature":[58],"space.":[59],"To":[60],"minimize":[61],"crowd":[62],"efforts,":[63],"it":[64],"employs":[65],"sampler":[68],"selects":[70],"instances":[72,94,175],"would":[74],"benefit":[75],"most":[77],"being":[79],"inspected":[80,176],"crowd.":[83,113,179],"The":[84],"manually":[85],"verified":[86],"are":[88],"then":[89],"propagated":[90],"similar":[92],"original":[97],"structure,":[105,142],"thus":[106],"scaling":[107],"out":[108],"contribution":[110],"Scalpel-CD":[114,163],"designed":[116],"with":[117,172],"set":[119],"algorithmic":[121],"solutions":[122],"automatically":[124],"search":[125],"for":[126,130],"optimal":[128],"configurations":[129],"different":[131],"types":[132,147],"data,":[135],"terms":[137],"noise":[143,146],"ratio,":[144],"(random":[148],"vs.":[149],"structural).":[150],"In":[151],"real":[153],"deployment":[154],"on":[155],"multiple":[156],"tasks,":[159],"we":[160],"demonstrate":[161],"improve":[167],"label":[168],"quality":[169],"12.9%":[171],"only":[173],"2.8%":[174]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":7},{"year":2018,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}