{"id":"https://openalex.org/W3012736183","doi":"https://doi.org/10.1145/3366423.3380306","title":"Towards Hybrid Human-AI Workflows for Unknown Unknown Detection","display_name":"Towards Hybrid Human-AI Workflows for Unknown Unknown Detection","publication_year":2020,"publication_date":"2020-04-20","ids":{"openalex":"https://openalex.org/W3012736183","doi":"https://doi.org/10.1145/3366423.3380306","mag":"3012736183"},"language":"en","primary_location":{"id":"doi:10.1145/3366423.3380306","doi":"https://doi.org/10.1145/3366423.3380306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380306","pdf_url":null,"source":{"id":"https://openalex.org/S7407088480","display_name":"International World Wide Web Conference (WWW)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3366423.3380306","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Anthony Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anthony Liu","raw_affiliation_strings":["University of Michigan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050566108","display_name":"Santiago Guerra","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I894135719","display_name":"University of Monterrey","ror":"https://ror.org/02arnxw97","country_code":"MX","type":"education","lineage":["https://openalex.org/I894135719"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Santiago Guerra","raw_affiliation_strings":["Universidad de Monterrey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universidad de Monterrey","institution_ids":["https://openalex.org/I894135719"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041997118","display_name":"Isaac Pak-Wah Fung","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Isaac Fung","raw_affiliation_strings":["University of Michigan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048356773","display_name":"Gabriel Matute","orcid":"https://orcid.org/0000-0001-7785-1231","observed_orcids":["https://orcid.org/0000-0001-7785-1231"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gabriel Matute","raw_affiliation_strings":["University of Michigan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028114802","display_name":"Ece Semiha Kamar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ece Kamar","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063574664","display_name":"Walter S. Lasecki","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Walter Lasecki","raw_affiliation_strings":["University of Michigan Computer Science and Engineering"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan Computer Science and Engineering","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7596,"has_fulltext":false,"cited_by_count":33,"citation_normalized_percentile":{"value":0.88050108,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"2432","last_page":"2442"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.6904000043869019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.6904000043869019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.13339999318122864,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.03720000013709068,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/unknown-unknowns","display_name":"unknown unknowns","score":0.8460000157356262},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.7960000038146973},{"id":"https://openalex.org/keywords/pattern-identification","display_name":"pattern identification","score":0.39399999380111694},{"id":"https://openalex.org/keywords/classifier-retraining","display_name":"classifier retraining","score":0.7319999933242798},{"id":"https://openalex.org/keywords/misclassification","display_name":"misclassification","score":0.38600000739097595},{"id":"https://openalex.org/keywords/data-augmentation","display_name":"data augmentation","score":0.3310000002384186},{"id":"https://openalex.org/keywords/model-robustness","display_name":"model robustness","score":0.32499998807907104}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.80875563621521},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.7688630819320679},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6934999227523804},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6676698923110962},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6332740783691406},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.5175639986991882},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.5139667391777039},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.5024127960205078},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.45948681235313416},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.43952956795692444},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40582260489463806},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3366423.3380306","doi":"https://doi.org/10.1145/3366423.3380306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380306","pdf_url":null,"source":{"id":"https://openalex.org/S7407088480","display_name":"International World Wide Web Conference (WWW)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3366423.3380306","doi":"https://doi.org/10.1145/3366423.3380306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380306","pdf_url":null,"source":{"id":"https://openalex.org/S7407088480","display_name":"International World Wide Web Conference (WWW)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W1504725289","https://openalex.org/W1573900212","https://openalex.org/W1673923490","https://openalex.org/W1970381522","https://openalex.org/W1991418309","https://openalex.org/W2022450117","https://openalex.org/W2036166268","https://openalex.org/W2059362837","https://openalex.org/W2073408573","https://openalex.org/W2108598243","https://openalex.org/W2120396827","https://openalex.org/W2143890915","https://openalex.org/W2796885425","https://openalex.org/W2806019599","https://openalex.org/W3098649723","https://openalex.org/W4297971002"],"related_works":["https://openalex.org/W3212513908","https://openalex.org/W2946245775","https://openalex.org/W2583689529","https://openalex.org/W2184059664","https://openalex.org/W2394968026","https://openalex.org/W2540479899","https://openalex.org/W4389956001","https://openalex.org/W4404240566","https://openalex.org/W2788428929","https://openalex.org/W2394962086"],"abstract_inverted_index":{"Predictive":[0],"models":[1],"are":[2],"susceptible":[3],"to":[4,16,54,65,78,81,93,106,111,151,154,161],"errors":[5],"called":[6],"unknown":[7,48,99,138,139],"unknowns,":[8,49],"in":[9,98,127,158],"which":[10],"the":[11,52,119,128,149],"model":[12,36],"assigns":[13],"incorrect":[14],"labels":[15],"instances":[17,46,61],"with":[18,86],"high":[19],"confidence.":[20],"These":[21],"commonly":[22],"arise":[23],"when":[24],"training":[25],"data":[26,117],"does":[27],"not":[28],"represent":[29],"variations":[30],"of":[31,47,59],"a":[32,56],"class":[33],"encountered":[34,123],"at":[35,142],"deployment.":[37],"Prior":[38],"work":[39,147],"showed":[40],"that":[41,74,118,133],"crowd":[42,53,91],"workers":[43,92],"can":[44,62],"identify":[45,55,112,155],"but":[50],"asking":[51],"sufficient":[57],"number":[58],"individual":[60],"be":[63],"costly":[64],"acquire":[66],"[2].":[67],"Instead,":[68],"this":[69],"paper":[70],"presents":[71],"an":[72,108],"approach":[73,135],"leverages":[75],"people\u2019s":[76],"ability":[77],"find":[79],"patterns":[80,97,105,157],"retrain":[82],"classifiers":[83],"more":[84],"effectively":[85],"fewer":[87],"examples.":[88],"We":[89,101],"ask":[90],"suggest":[94],"and":[95],"verify":[96],"unknowns.":[100],"then":[102],"use":[103],"these":[104],"train":[107],"expansion":[109],"classifier":[110,121,144],"additional":[113],"examples":[114],"from":[115],"existing":[116,137],"primary":[120],"has":[122],"(and":[124],"potentially":[125],"misclassified)":[126],"past.":[129],"Our":[130],"experiments":[131],"show":[132],"our":[134],"outperforms":[136],"detection":[140],"methods":[141],"improving":[143],"performance.":[145],"This":[146],"is":[148],"first":[150],"leverage":[152],"crowds":[153],"error":[156],"large":[159],"datasets":[160],"improve":[162],"ML":[163],"training.":[164]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":8}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2020-03-27T00:00:00"}