{"id":"https://openalex.org/W3012736183","doi":"https://doi.org/10.1145/3366423.3380306","title":"Towards Hybrid Human-AI Workflows for Unknown Unknown Detection","display_name":"Towards Hybrid Human-AI Workflows for Unknown Unknown Detection","publication_year":2020,"publication_date":"2020-04-20","ids":{"openalex":"https://openalex.org/W3012736183","doi":"https://doi.org/10.1145/3366423.3380306","mag":"3012736183"},"language":"en","primary_location":{"id":"doi:10.1145/3366423.3380306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380306","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3366423.3380306","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037867922","display_name":"Anthony Liu","orcid":"https://orcid.org/0000-0002-1374-2455"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Anthony Liu","raw_affiliation_strings":["University of Michigan"],"affiliations":[{"raw_affiliation_string":"University of Michigan","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050566108","display_name":"Santiago Guerra","orcid":null},"institutions":[{"id":"https://openalex.org/I4210163479","display_name":"Universidad Ateneo de Monterrey","ror":"https://ror.org/01fqx5d30","country_code":"MX","type":"education","lineage":["https://openalex.org/I4210163479"]},{"id":"https://openalex.org/I894135719","display_name":"University of Monterrey","ror":"https://ror.org/02arnxw97","country_code":"MX","type":"education","lineage":["https://openalex.org/I894135719"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Santiago Guerra","raw_affiliation_strings":["Universidad de Monterrey"],"affiliations":[{"raw_affiliation_string":"Universidad de Monterrey","institution_ids":["https://openalex.org/I4210163479","https://openalex.org/I894135719"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041997118","display_name":"Isaac Pak-Wah Fung","orcid":null},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Isaac Fung","raw_affiliation_strings":["University of Michigan"],"affiliations":[{"raw_affiliation_string":"University of Michigan","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048356773","display_name":"Gabriel Matute","orcid":null},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gabriel Matute","raw_affiliation_strings":["University of Michigan"],"affiliations":[{"raw_affiliation_string":"University of Michigan","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028114802","display_name":"Ece Kamar","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ece Kamar","raw_affiliation_strings":["Microsoft"],"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063574664","display_name":"Walter S. Lasecki","orcid":null},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Walter Lasecki","raw_affiliation_strings":["University of Michigan Computer Science and Engineering"],"affiliations":[{"raw_affiliation_string":"University of Michigan Computer Science and Engineering","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5037867922"],"corresponding_institution_ids":["https://openalex.org/I27837315"],"apc_list":null,"apc_paid":null,"fwci":5.4037,"has_fulltext":false,"cited_by_count":31,"citation_normalized_percentile":{"value":0.9562784,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"2432","last_page":"2442"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.80875563621521},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.7688630819320679},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6934999227523804},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6676698923110962},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6332740783691406},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.5175639986991882},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.5139667391777039},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.5024127960205078},{"id":"https://openalex.org/keywords/labeled-data","display_name":"Labeled data","score":0.45948681235313416},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.43952956795692444},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40582260489463806}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.80875563621521},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.7688630819320679},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6934999227523804},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6676698923110962},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6332740783691406},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.5175639986991882},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.5139667391777039},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.5024127960205078},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.45948681235313416},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.43952956795692444},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40582260489463806},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3366423.3380306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380306","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3366423.3380306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3366423.3380306","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of The Web Conference 2020","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1504725289","https://openalex.org/W1573900212","https://openalex.org/W1970381522","https://openalex.org/W1991418309","https://openalex.org/W2022450117","https://openalex.org/W2036166268","https://openalex.org/W2059362837","https://openalex.org/W2073408573","https://openalex.org/W2108598243","https://openalex.org/W2120396827","https://openalex.org/W2143890915","https://openalex.org/W2806019599","https://openalex.org/W2963483561","https://openalex.org/W3098649723","https://openalex.org/W4297971002"],"related_works":["https://openalex.org/W2129767422","https://openalex.org/W3210196349","https://openalex.org/W4214728004","https://openalex.org/W2950181282","https://openalex.org/W2963261224","https://openalex.org/W2798287483","https://openalex.org/W2913410650","https://openalex.org/W2130553454","https://openalex.org/W4317548404","https://openalex.org/W3022007134"],"abstract_inverted_index":{"Predictive":[0],"models":[1],"are":[2],"susceptible":[3],"to":[4,16,54,65,78,81,93,106,111,151,154,161],"errors":[5],"called":[6],"unknown":[7,48,99,138,139],"unknowns,":[8,49],"in":[9,98,127,158],"which":[10],"the":[11,52,119,128,149],"model":[12,36],"assigns":[13],"incorrect":[14],"labels":[15],"instances":[17,46,61],"with":[18,86],"high":[19],"confidence.":[20],"These":[21],"commonly":[22],"arise":[23],"when":[24],"training":[25],"data":[26,117],"does":[27],"not":[28],"represent":[29],"variations":[30],"of":[31,47,59],"a":[32,56],"class":[33],"encountered":[34,123],"at":[35,142],"deployment.":[37],"Prior":[38],"work":[39,147],"showed":[40],"that":[41,74,118,133],"crowd":[42,53,91],"workers":[43,92],"can":[44,62],"identify":[45,55,112,155],"but":[50],"asking":[51],"sufficient":[57],"number":[58],"individual":[60],"be":[63],"costly":[64],"acquire":[66],"[2].":[67],"Instead,":[68],"this":[69],"paper":[70],"presents":[71],"an":[72,108],"approach":[73,135],"leverages":[75],"people\u2019s":[76],"ability":[77],"find":[79],"patterns":[80,97,105,157],"retrain":[82],"classifiers":[83],"more":[84],"effectively":[85],"fewer":[87],"examples.":[88],"We":[89,101],"ask":[90],"suggest":[94],"and":[95],"verify":[96],"unknowns.":[100],"then":[102],"use":[103],"these":[104],"train":[107],"expansion":[109],"classifier":[110,121,144],"additional":[113],"examples":[114],"from":[115],"existing":[116,137],"primary":[120],"has":[122],"(and":[124],"potentially":[125],"misclassified)":[126],"past.":[129],"Our":[130],"experiments":[131],"show":[132],"our":[134],"outperforms":[136],"detection":[140],"methods":[141],"improving":[143],"performance.":[145],"This":[146],"is":[148],"first":[150],"leverage":[152],"crowds":[153],"error":[156],"large":[159],"datasets":[160],"improve":[162],"ML":[163],"training.":[164]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":8}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}