{"id":"https://openalex.org/W4380365974","doi":"https://doi.org/10.1145/3593013.3594087","title":"Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies","display_name":"Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies","publication_year":2023,"publication_date":"2023-06-12","ids":{"openalex":"https://openalex.org/W4380365974","doi":"https://doi.org/10.1145/3593013.3594087"},"language":"en","primary_location":{"id":"doi:10.1145/3593013.3594087","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3593013.3594087","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000544794","display_name":"Vivian Lai","orcid":"https://orcid.org/0000-0003-3165-6136"},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Vivian Lai","raw_affiliation_strings":["University of Colorado Boulder, USA"],"affiliations":[{"raw_affiliation_string":"University of Colorado Boulder, USA","institution_ids":["https://openalex.org/I188538660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028711697","display_name":"Chacha Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chacha Chen","raw_affiliation_strings":["University of Chicago, USA"],"affiliations":[{"raw_affiliation_string":"University of Chicago, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101541372","display_name":"Alison Smith","orcid":"https://orcid.org/0000-0002-6600-267X"},"institutions":[{"id":"https://openalex.org/I4401726866","display_name":"Dataminr (United States)","ror":"https://ror.org/024ncvw44","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alison Smith-Renner","raw_affiliation_strings":["Dataminr Inc., USA"],"affiliations":[{"raw_affiliation_string":"Dataminr Inc., USA","institution_ids":["https://openalex.org/I4401726866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024320659","display_name":"Q. Vera Liao","orcid":"https://orcid.org/0000-0003-4543-7196"},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Q. Vera Liao","raw_affiliation_strings":["Microsoft Research, Canada"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Canada","institution_ids":["https://openalex.org/I4210153468"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079270249","display_name":"Chenhao Tan","orcid":"https://orcid.org/0000-0002-3981-2116"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenhao Tan","raw_affiliation_strings":["University of Chicago, USA"],"affiliations":[{"raw_affiliation_string":"University of Chicago, USA","institution_ids":["https://openalex.org/I40347166"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5000544794"],"corresponding_institution_ids":["https://openalex.org/I188538660"],"apc_list":null,"apc_paid":null,"fwci":34.803,"has_fulltext":false,"cited_by_count":137,"citation_normalized_percentile":{"value":0.99764094,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1369","last_page":"1385"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.9480999708175659,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10763","display_name":"Digital Transformation in Industry","score":0.9406999945640564,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/subject","display_name":"Subject (documents)","score":0.7003142833709717},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.6246124505996704},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5931131839752197},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4868442416191101},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.40177011489868164},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.3755740821361542},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3662217855453491},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.20683252811431885},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.15320143103599548},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.09465432167053223},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07446500658988953}],"concepts":[{"id":"https://openalex.org/C2777855551","wikidata":"https://www.wikidata.org/wiki/Q12310021","display_name":"Subject (documents)","level":2,"score":0.7003142833709717},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.6246124505996704},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5931131839752197},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4868442416191101},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.40177011489868164},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3755740821361542},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3662217855453491},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.20683252811431885},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15320143103599548},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09465432167053223},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07446500658988953},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3593013.3594087","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3593013.3594087","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6800000071525574}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":84,"referenced_works":["https://openalex.org/W1501005121","https://openalex.org/W1908150177","https://openalex.org/W1994606570","https://openalex.org/W2037306653","https://openalex.org/W2048657872","https://openalex.org/W2087271627","https://openalex.org/W2154157725","https://openalex.org/W2168747479","https://openalex.org/W2215157235","https://openalex.org/W2237502368","https://openalex.org/W2282821441","https://openalex.org/W2336133126","https://openalex.org/W2367397349","https://openalex.org/W2398344594","https://openalex.org/W2741678970","https://openalex.org/W2769533150","https://openalex.org/W2786242872","https://openalex.org/W2788403449","https://openalex.org/W2796133875","https://openalex.org/W2800068874","https://openalex.org/W2804878416","https://openalex.org/W2897042519","https://openalex.org/W2901481055","https://openalex.org/W2901895173","https://openalex.org/W2905034244","https://openalex.org/W2916797267","https://openalex.org/W2918027211","https://openalex.org/W2918341242","https://openalex.org/W2918497321","https://openalex.org/W2941320055","https://openalex.org/W2941411956","https://openalex.org/W2942073295","https://openalex.org/W2942157335","https://openalex.org/W2974071289","https://openalex.org/W2979893369","https://openalex.org/W2983137008","https://openalex.org/W2983996708","https://openalex.org/W2984353433","https://openalex.org/W3001062618","https://openalex.org/W3005876361","https://openalex.org/W3006326487","https://openalex.org/W3007464329","https://openalex.org/W3007910004","https://openalex.org/W3009578469","https://openalex.org/W3010300896","https://openalex.org/W3014011525","https://openalex.org/W3029900269","https://openalex.org/W3030109276","https://openalex.org/W3030350595","https://openalex.org/W3032816739","https://openalex.org/W3037843601","https://openalex.org/W3094262373","https://openalex.org/W3099742594","https://openalex.org/W3100046612","https://openalex.org/W3100279624","https://openalex.org/W3101792976","https://openalex.org/W3102139770","https://openalex.org/W3104831984","https://openalex.org/W3119527628","https://openalex.org/W3121588992","https://openalex.org/W3121705224","https://openalex.org/W3121921079","https://openalex.org/W3127032480","https://openalex.org/W3134111219","https://openalex.org/W3136933888","https://openalex.org/W3155920732","https://openalex.org/W3156106752","https://openalex.org/W3156666346","https://openalex.org/W3160343688","https://openalex.org/W3160735296","https://openalex.org/W3160909372","https://openalex.org/W3161926791","https://openalex.org/W3162614376","https://openalex.org/W3163128657","https://openalex.org/W3163667721","https://openalex.org/W3207598588","https://openalex.org/W4220820301","https://openalex.org/W4232593342","https://openalex.org/W4241614188","https://openalex.org/W4288083796","https://openalex.org/W4288414189","https://openalex.org/W4296978576","https://openalex.org/W4302612613","https://openalex.org/W4403865180"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W4388258507","https://openalex.org/W2392013855","https://openalex.org/W4318064328","https://openalex.org/W2357926602","https://openalex.org/W3200517220","https://openalex.org/W2374569605","https://openalex.org/W2386062718","https://openalex.org/W4393047559","https://openalex.org/W4386651600"],"abstract_inverted_index":{"AI":[0,63,67,95,146,236],"systems":[1],"are":[2],"adopted":[3],"in":[4,15,53,134,138,162,205],"numerous":[5],"domains":[6,17],"due":[7,30],"to":[8,31,57,83,96,182,186,238,241],"their":[9],"increasingly":[10],"strong":[11],"predictive":[12],"performance.":[13],"However,":[14],"high-stakes":[16],"such":[18],"as":[19,230],"criminal":[20],"justice":[21],"and":[22,34,44,92,101,111,149,168,191,208,218,235,247],"healthcare,":[23],"full":[24],"automation":[25],"is":[26,50],"often":[27],"not":[28],"desirable":[29],"safety,":[32],"ethical,":[33],"legal":[35],"concerns,":[36],"yet":[37],"fully":[38],"manual":[39],"approaches":[40,82],"can":[41,201,212],"be":[42],"inaccurate":[43],"time-consuming.":[45],"As":[46],"a":[47,85,107,170,231],"result,":[48],"there":[49],"growing":[51],"interest":[52],"the":[54,72,129,166,180,189,209,244],"research":[55,104,192,210],"community":[56,211],"augment":[58],"human":[59],"decision":[60,77,114,143,196,252],"making":[61,78],"with":[62,94],"assistance.":[64],"Besides":[65],"developing":[66],"technologies":[68,249],"for":[69,174,188,233,250],"this":[70,125,226],"purpose,":[71],"emerging":[73],"field":[74],"of":[75,88,109,120,165,172,194],"human-AI":[76,113,195,251],"must":[79],"embrace":[80],"empirical":[81,121,245],"form":[84],"foundational":[86],"understanding":[87,110],"how":[89],"humans":[90],"interact":[91],"work":[93,178,217,227,239],"make":[97,169,202],"decisions.":[98],"To":[99],"invite":[100],"help":[102],"structure":[103],"efforts":[105],"towards":[106],"science":[108,246],"improving":[112],"making,":[115,197],"we":[116,156],"survey":[117],"recent":[118],"literature":[119],"human-subject":[122],"studies":[123],"on":[124,214],"topic.":[126],"We":[127,223],"summarize":[128,157],"study":[130,206],"design":[131,190],"choices":[132,204],"made":[133],"over":[135],"100":[136],"papers":[137],"three":[139],"important":[140],"aspects:":[141],"(1)":[142],"tasks,":[144],"(2)":[145],"assistance":[147],"elements,":[148],"(3)":[150],"evaluation":[151],"metrics.":[152],"For":[153],"each":[154,215],"aspect,":[155],"current":[158,163],"trends,":[159],"discuss":[160],"gaps":[161],"practices":[164],"field,":[167],"list":[171],"recommendations":[173],"future":[175],"research.":[176],"Our":[177],"highlights":[179],"need":[181],"develop":[183],"common":[184],"frameworks":[185],"account":[187],"spaces":[193],"so":[198],"that":[199],"researchers":[200],"rigorous":[203],"design,":[207],"build":[213],"other\u2019s":[216],"produce":[219],"generalizable":[220],"scientific":[221],"knowledge.":[222],"also":[224],"hope":[225],"will":[228],"serve":[229],"bridge":[232],"HCI":[234],"communities":[237],"together":[240],"mutually":[242],"shape":[243],"computational":[248],"making.":[253]},"counts_by_year":[{"year":2026,"cited_by_count":19},{"year":2025,"cited_by_count":69},{"year":2024,"cited_by_count":44},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}