{"id":"https://openalex.org/W4385651226","doi":"https://doi.org/10.1177/00187208231190459","title":"When Do Humans Heed AI Agents\u2019 Advice? When Should They?","display_name":"When Do Humans Heed AI Agents\u2019 Advice? When Should They?","publication_year":2023,"publication_date":"2023-08-08","ids":{"openalex":"https://openalex.org/W4385651226","doi":"https://doi.org/10.1177/00187208231190459","pmid":"https://pubmed.ncbi.nlm.nih.gov/37553098"},"language":"en","primary_location":{"id":"doi:10.1177/00187208231190459","is_oa":true,"landing_page_url":"https://doi.org/10.1177/00187208231190459","pdf_url":null,"source":{"id":"https://openalex.org/S83386566","display_name":"Human Factors The Journal of the Human Factors and Ergonomics Society","issn_l":"0018-7208","issn":["0018-7208","1547-8181"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Human Factors: The Journal of the Human Factors and Ergonomics Society","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1177/00187208231190459","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046836112","display_name":"Richard E. Dunning","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Richard E. Dunning","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077250901","display_name":"Baruch Fischhoff","orcid":"https://orcid.org/0000-0002-3030-6874"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Baruch Fischhoff","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111743953","display_name":"Alex Davis","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex L. Davis","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5046836112"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":{"value":1500,"currency":"USD","value_usd":1500},"apc_paid":{"value":1500,"currency":"USD","value_usd":1500},"fwci":1.2933,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.83556818,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"66","issue":"7","first_page":"1914","last_page":"1927"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.303600013256073,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.303600013256073,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09130000323057175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.07580000162124634,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/advice","display_name":"Advice (programming)","score":0.6137758493423462},{"id":"https://openalex.org/keywords/categorical-variable","display_name":"Categorical variable","score":0.5995785593986511},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.582918643951416},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5187880992889404},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.4395720958709717},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3586914837360382},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3281121850013733}],"concepts":[{"id":"https://openalex.org/C2779955035","wikidata":"https://www.wikidata.org/wiki/Q4686785","display_name":"Advice (programming)","level":2,"score":0.6137758493423462},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.5995785593986511},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.582918643951416},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5187880992889404},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.4395720958709717},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3586914837360382},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3281121850013733},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003657","descriptor_name":"Decision Making","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003657","descriptor_name":"Decision Making","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003657","descriptor_name":"Decision Making","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003657","descriptor_name":"Decision Making","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003657","descriptor_name":"Decision Making","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003657","descriptor_name":"Decision Making","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D055815","descriptor_name":"Young Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D055815","descriptor_name":"Young Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D055815","descriptor_name":"Young Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D055815","descriptor_name":"Young Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D055815","descriptor_name":"Young Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D055815","descriptor_name":"Young Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":3,"locations":[{"id":"doi:10.1177/00187208231190459","is_oa":true,"landing_page_url":"https://doi.org/10.1177/00187208231190459","pdf_url":null,"source":{"id":"https://openalex.org/S83386566","display_name":"Human Factors The Journal of the Human Factors and Ergonomics Society","issn_l":"0018-7208","issn":["0018-7208","1547-8181"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Human Factors: The Journal of the Human Factors and Ergonomics Society","raw_type":"journal-article"},{"id":"pmid:37553098","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37553098","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Human factors","raw_type":"Journal Article"},{"id":"pmh:oai:pubmedcentral.nih.gov:11089830","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/11089830","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Hum Factors","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1177/00187208231190459","is_oa":true,"landing_page_url":"https://doi.org/10.1177/00187208231190459","pdf_url":null,"source":{"id":"https://openalex.org/S83386566","display_name":"Human Factors The Journal of the Human Factors and Ergonomics Society","issn_l":"0018-7208","issn":["0018-7208","1547-8181"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Human Factors: The Journal of the Human Factors and Ergonomics Society","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.6399999856948853,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W278488923","https://openalex.org/W1120261353","https://openalex.org/W1642805147","https://openalex.org/W1964768680","https://openalex.org/W1990229920","https://openalex.org/W1992498129","https://openalex.org/W1998643818","https://openalex.org/W2019160338","https://openalex.org/W2020372685","https://openalex.org/W2040490772","https://openalex.org/W2043732361","https://openalex.org/W2051380595","https://openalex.org/W2063052894","https://openalex.org/W2108278623","https://openalex.org/W2110171129","https://openalex.org/W2111940674","https://openalex.org/W2113312317","https://openalex.org/W2135795051","https://openalex.org/W2135907989","https://openalex.org/W2138753347","https://openalex.org/W2145885996","https://openalex.org/W2152905082","https://openalex.org/W2155859548","https://openalex.org/W2257461403","https://openalex.org/W2310425190","https://openalex.org/W2312662668","https://openalex.org/W2531711351","https://openalex.org/W2533523411","https://openalex.org/W2574287525","https://openalex.org/W2602140409","https://openalex.org/W2741375793","https://openalex.org/W2744890050","https://openalex.org/W2770836081","https://openalex.org/W2786358356","https://openalex.org/W2788481061","https://openalex.org/W2897042519","https://openalex.org/W2902907165","https://openalex.org/W2921390920","https://openalex.org/W2921629193","https://openalex.org/W2934399013","https://openalex.org/W2980202986","https://openalex.org/W2982316857","https://openalex.org/W2999637955","https://openalex.org/W3064053140","https://openalex.org/W3100279624","https://openalex.org/W3103751997","https://openalex.org/W3113535890","https://openalex.org/W3118210634","https://openalex.org/W3130937151","https://openalex.org/W3158836591","https://openalex.org/W3165767308","https://openalex.org/W4283781398","https://openalex.org/W4301861531","https://openalex.org/W6736685754"],"related_works":["https://openalex.org/W3090906284","https://openalex.org/W253876680","https://openalex.org/W1987931999","https://openalex.org/W4293797372","https://openalex.org/W4238052600","https://openalex.org/W4386799044","https://openalex.org/W3005176110","https://openalex.org/W162918864","https://openalex.org/W4243730168","https://openalex.org/W2028211458"],"abstract_inverted_index":{"OBJECTIVE:":[0],"We":[1,49,75,199],"manipulate":[2],"the":[3,46,67,88,97,118,142,196],"presence,":[4],"skill,":[5],"and":[6,31,96,122,175,207],"display":[7,153],"of":[8,27,62,71,90,101,110,166,195],"artificial":[9],"intelligence":[10],"(AI)":[11],"recommendations":[12,92],"in":[13],"a":[14,55,79,115,130],"strategy":[15],"game":[16,70],"to":[17,34,178,203],"measure":[18],"their":[19,172,182,209],"effect":[20,157],"on":[21,40,45,141,148,158,171],"users'":[22,159,176],"performance.":[23,210],"BACKGROUND:":[24],"Many":[25],"applications":[26],"AI":[28,32,47,91,98,119,168,187],"require":[29],"humans":[30,43],"agents":[33,61,169],"make":[35],"decisions":[36],"collaboratively.":[37],"Success":[38],"depends":[39],"how":[41],"appropriately":[42],"rely":[44],"agent.":[48,143],"demonstrate":[50,200],"an":[51,201],"evaluation":[52],"method":[53],"for":[54,66],"platform":[56],"that":[57,86],"uses":[58],"neural":[59],"network":[60],"varying":[63],"skill":[64,160,173],"levels":[65],"simple":[68],"strategic":[69],"Connect":[72],"Four.":[73],"METHODS:":[74],"report":[76],"results":[77],"from":[78,181],"2":[80],"\u00d7":[81],"3":[82],"between-subjects":[83],"factorial":[84],"experiment":[85],"varies":[87],"format":[89,154],"(categorical":[93],"or":[94,105,161],"probabilistic)":[95],"agent's":[99,120],"amount":[100],"training":[102],"(low,":[103],"medium,":[104],"high).":[106],"On":[107],"each":[108],"round":[109],"10":[111],"games,":[112],"participants":[113],"proposed":[114],"move,":[116],"saw":[117],"recommendations,":[121],"then":[123],"moved.":[124],"RESULTS:":[125],"Participants'":[126],"performance":[127],"improved":[128],"with":[129],"highly":[131],"skilled":[132,150],"agent,":[133],"but":[134],"quickly":[135],"plateaued,":[136],"as":[137],"they":[138],"relied":[139,145],"uncritically":[140],"Participants":[144],"too":[146],"little":[147],"lower":[149],"agents.":[151],"The":[152,164],"had":[155],"no":[156],"choices.":[162],"CONCLUSIONS:":[163],"value":[165],"these":[167],"depended":[170],"level":[174],"ability":[177],"extract":[179],"lessons":[180],"advice.":[183],"APPLICATION:":[184],"Organizations":[185],"employing":[186],"decision":[188],"support":[189],"systems":[190],"must":[191],"consider":[192],"behavioral":[193],"aspects":[194],"human-agent":[197],"team.":[198],"approach":[202],"evaluating":[204],"competing":[205],"designs":[206],"assessing":[208]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}