{"id":"https://openalex.org/W4396832377","doi":"https://doi.org/10.1145/3613904.3642446","title":"Evaluating the Utility of Conformal Prediction Sets for AI-Advised Image Labeling","display_name":"Evaluating the Utility of Conformal Prediction Sets for AI-Advised Image Labeling","publication_year":2024,"publication_date":"2024-05-11","ids":{"openalex":"https://openalex.org/W4396832377","doi":"https://doi.org/10.1145/3613904.3642446"},"language":"en","primary_location":{"id":"doi:10.1145/3613904.3642446","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642446","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642446","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642446","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100609372","display_name":"Dongping Zhang","orcid":"https://orcid.org/0000-0001-9825-1411"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Dongping Zhang","raw_affiliation_strings":["Technology and Social Behavior, Northwestern University, United States"],"raw_orcid":"https://orcid.org/0000-0001-9825-1411","affiliations":[{"raw_affiliation_string":"Technology and Social Behavior, Northwestern University, United States","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070488104","display_name":"Angelos Chatzimparmpas","orcid":"https://orcid.org/0000-0002-9079-2376"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Angelos Chatzimparmpas","raw_affiliation_strings":["Computer Science, Northwestern University, United States"],"raw_orcid":"https://orcid.org/0000-0002-9079-2376","affiliations":[{"raw_affiliation_string":"Computer Science, Northwestern University, United States","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087854473","display_name":"Negar Kamali","orcid":"https://orcid.org/0000-0002-1086-6735"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Negar Kamali","raw_affiliation_strings":["Computer Science, Northwestern University, United States"],"raw_orcid":"https://orcid.org/0000-0002-1086-6735","affiliations":[{"raw_affiliation_string":"Computer Science, Northwestern University, United States","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068008545","display_name":"Jessica Hullman","orcid":"https://orcid.org/0000-0001-6826-3550"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jessica Hullman","raw_affiliation_strings":["Computer Science, Northwestern University, United States"],"raw_orcid":"https://orcid.org/0000-0001-6826-3550","affiliations":[{"raw_affiliation_string":"Computer Science, Northwestern University, United States","institution_ids":["https://openalex.org/I111979921"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100609372"],"corresponding_institution_ids":["https://openalex.org/I111979921"],"apc_list":null,"apc_paid":null,"fwci":1.6557,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.85677282,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"19"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.977400004863739,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9735000133514404,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7136633396148682},{"id":"https://openalex.org/keywords/conformal-map","display_name":"Conformal map","score":0.6886074542999268},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.6443407535552979},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5863970518112183},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5820254683494568},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.5748742818832397},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5223387479782104},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5004353523254395},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4860709011554718},{"id":"https://openalex.org/keywords/black-box","display_name":"Black box","score":0.4516351819038391},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4430598020553589},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.4362598657608032},{"id":"https://openalex.org/keywords/distribution","display_name":"Distribution (mathematics)","score":0.4245690107345581},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.40941518545150757},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1770208775997162},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07260224223136902}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7136633396148682},{"id":"https://openalex.org/C98214594","wikidata":"https://www.wikidata.org/wiki/Q850275","display_name":"Conformal map","level":2,"score":0.6886074542999268},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.6443407535552979},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5863970518112183},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5820254683494568},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.5748742818832397},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5223387479782104},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5004353523254395},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4860709011554718},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.4516351819038391},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4430598020553589},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4362598657608032},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.4245690107345581},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.40941518545150757},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1770208775997162},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07260224223136902},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3613904.3642446","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642446","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642446","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3613904.3642446","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642446","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642446","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.8199999928474426,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4396832377.pdf"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W1499689786","https://openalex.org/W1593532658","https://openalex.org/W1618905105","https://openalex.org/W1989388297","https://openalex.org/W2005534705","https://openalex.org/W2031342017","https://openalex.org/W2038721957","https://openalex.org/W2049951042","https://openalex.org/W2081580037","https://openalex.org/W2108598243","https://openalex.org/W2127875809","https://openalex.org/W2136930489","https://openalex.org/W2161506256","https://openalex.org/W2194775991","https://openalex.org/W2292312835","https://openalex.org/W2398344594","https://openalex.org/W2531563875","https://openalex.org/W2754613478","https://openalex.org/W2772138765","https://openalex.org/W2891628559","https://openalex.org/W2901895173","https://openalex.org/W2902660860","https://openalex.org/W2964137095","https://openalex.org/W2979893369","https://openalex.org/W2999637955","https://openalex.org/W3004483087","https://openalex.org/W3037843601","https://openalex.org/W3099742594","https://openalex.org/W3103751997","https://openalex.org/W3119689140","https://openalex.org/W3122501643","https://openalex.org/W3154246618","https://openalex.org/W3154733909","https://openalex.org/W3160909372","https://openalex.org/W3163443091","https://openalex.org/W3202443572","https://openalex.org/W3203816956","https://openalex.org/W4220891329","https://openalex.org/W4285600961","https://openalex.org/W4287765491","https://openalex.org/W4360991197","https://openalex.org/W4368362764","https://openalex.org/W6779121023"],"related_works":["https://openalex.org/W1971223889","https://openalex.org/W4312601715","https://openalex.org/W4377865163","https://openalex.org/W4392828243","https://openalex.org/W4312822655","https://openalex.org/W3157170264","https://openalex.org/W4298185893","https://openalex.org/W4402427143","https://openalex.org/W2896078964","https://openalex.org/W4281399026"],"abstract_inverted_index":{"As":[0],"deep":[1],"neural":[2],"networks":[3],"are":[4],"more":[5],"commonly":[6],"deployed":[7],"in":[8,40,91,112],"high-stakes":[9],"domains,":[10],"their":[11],"black-box":[12],"nature":[13],"makes":[14],"uncertainty":[15,39],"quantification":[16],"challenging.":[17],"We":[18],"investigate":[19],"the":[20,50,74,83,86,119],"effects":[21],"of":[22,29,52,58,76,85,130],"presenting":[23],"conformal":[24,53,131],"prediction":[25,33,54,77,106,132],"sets\u2014a":[26],"distribution-free":[27],"class":[28],"methods":[30],"for":[31,63,79,103,142],"generating":[32],"sets":[34,55,78,107,133],"with":[35,82,95],"specified":[36],"coverage\u2014to":[37],"express":[38],"AI-advised":[41,64],"decision-making.":[42,144],"Through":[43],"a":[44,68],"large":[45],"online":[46],"experiment,":[47],"we":[48,71],"compare":[49],"utility":[51,75],"to":[56,139],"displays":[57,102],"Top-1":[59,99],"and":[60,100,134],"Top-k":[61,101],"predictions":[62],"image":[65],"labeling.":[66],"In":[67],"pre-registered":[69],"analysis,":[70],"find":[72],"that":[73],"accuracy":[80,92],"varies":[81],"difficulty":[84],"task:":[87],"while":[88],"they":[89],"result":[90],"on":[93,137],"par":[94],"or":[96],"less":[97],"than":[98],"easy":[104],"images,":[105,116],"excel":[108],"at":[109],"assisting":[110],"humans":[111],"labeling":[113],"out-of-distribution":[114],"(OOD)":[115],"especially":[117],"when":[118],"set":[120],"size":[121],"is":[122],"small.":[123],"Our":[124],"results":[125],"empirically":[126],"pinpoint":[127],"practical":[128],"challenges":[129],"provide":[135],"implications":[136],"how":[138],"incorporate":[140],"them":[141],"real-world":[143]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}