{"id":"https://openalex.org/W4396802056","doi":"https://doi.org/10.1145/3630106.3658941","title":"\"I'm Not Sure, But...\": Examining the Impact of Large Language Models' Uncertainty Expression on User Reliance and Trust","display_name":"\"I'm Not Sure, But...\": Examining the Impact of Large Language Models' Uncertainty Expression on User Reliance and Trust","publication_year":2024,"publication_date":"2024-06-03","ids":{"openalex":"https://openalex.org/W4396802056","doi":"https://doi.org/10.1145/3630106.3658941"},"language":"en","primary_location":{"id":"doi:10.1145/3630106.3658941","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3658941","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658941","source":null,"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658941","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061843649","display_name":"Sunnie S. Y. Kim","orcid":"https://orcid.org/0000-0002-8901-7233"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sunnie S. Y. Kim","raw_affiliation_strings":["Princeton University, USA"],"affiliations":[{"raw_affiliation_string":"Princeton University, USA","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024320659","display_name":"Q. Vera Liao","orcid":"https://orcid.org/0000-0003-4543-7196"},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Q. Vera Liao","raw_affiliation_strings":["Microsoft, Canada"],"affiliations":[{"raw_affiliation_string":"Microsoft, Canada","institution_ids":["https://openalex.org/I4210153468"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066017612","display_name":"Mihaela Vorvoreanu","orcid":"https://orcid.org/0000-0002-3322-3548"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mihaela Vorvoreanu","raw_affiliation_strings":["Microsoft, United States of America"],"affiliations":[{"raw_affiliation_string":"Microsoft, United States of America","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088878486","display_name":"Stephanie Ballard","orcid":"https://orcid.org/0000-0002-5174-4654"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephanie Ballard","raw_affiliation_strings":["Microsoft, United States of America"],"affiliations":[{"raw_affiliation_string":"Microsoft, United States of America","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043117896","display_name":"Jennifer Wortman Vaughan","orcid":"https://orcid.org/0000-0002-7807-2018"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jennifer Wortman Vaughan","raw_affiliation_strings":["Microsoft, United States of America"],"affiliations":[{"raw_affiliation_string":"Microsoft, United States of America","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5061843649"],"corresponding_institution_ids":["https://openalex.org/I20089843"],"apc_list":null,"apc_paid":null,"fwci":30.765,"has_fulltext":true,"cited_by_count":92,"citation_normalized_percentile":{"value":0.9978728,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"822","last_page":"835"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5179154276847839},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4988667964935303},{"id":"https://openalex.org/keywords/natural-experiment","display_name":"Natural experiment","score":0.4962833523750305},{"id":"https://openalex.org/keywords/expression","display_name":"Expression (computer science)","score":0.49117061495780945},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.4654823839664459},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.4465906322002411},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.42927467823028564},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.396844744682312},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.38319218158721924},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2124425768852234},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.13630810379981995},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.12361884117126465}],"concepts":[{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5179154276847839},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4988667964935303},{"id":"https://openalex.org/C49630185","wikidata":"https://www.wikidata.org/wiki/Q6980675","display_name":"Natural experiment","level":2,"score":0.4962833523750305},{"id":"https://openalex.org/C90559484","wikidata":"https://www.wikidata.org/wiki/Q778379","display_name":"Expression (computer science)","level":2,"score":0.49117061495780945},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4654823839664459},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.4465906322002411},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42927467823028564},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.396844744682312},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.38319218158721924},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2124425768852234},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.13630810379981995},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.12361884117126465},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3630106.3658941","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3658941","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658941","source":null,"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2405.00623","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.00623","pdf_url":"https://arxiv.org/pdf/2405.00623","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3630106.3658941","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3658941","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658941","source":null,"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2514421424","display_name":null,"funder_award_id":"Award","funder_id":"https://openalex.org/F4320308943","funder_display_name":"Microsoft Research"},{"id":"https://openalex.org/G3631313543","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320308943","funder_display_name":"Microsoft Research"},{"id":"https://openalex.org/G6894402473","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4396802056.pdf"},"referenced_works_count":77,"referenced_works":["https://openalex.org/W1763242134","https://openalex.org/W1979290264","https://openalex.org/W1990302162","https://openalex.org/W1991015565","https://openalex.org/W2000688897","https://openalex.org/W2032568497","https://openalex.org/W2033861818","https://openalex.org/W2108240218","https://openalex.org/W2110405340","https://openalex.org/W2120789618","https://openalex.org/W2137872373","https://openalex.org/W2141708418","https://openalex.org/W2156580739","https://openalex.org/W2157698673","https://openalex.org/W2741007583","https://openalex.org/W2794471005","https://openalex.org/W2800068874","https://openalex.org/W2897042519","https://openalex.org/W2901895173","https://openalex.org/W2942157335","https://openalex.org/W2942367614","https://openalex.org/W2945482067","https://openalex.org/W2970837303","https://openalex.org/W2979864122","https://openalex.org/W2990372444","https://openalex.org/W2998954378","https://openalex.org/W2999637955","https://openalex.org/W3099742594","https://openalex.org/W3100279624","https://openalex.org/W3103751997","https://openalex.org/W3108812612","https://openalex.org/W3119689140","https://openalex.org/W3123102076","https://openalex.org/W3123404044","https://openalex.org/W3123895079","https://openalex.org/W3125701840","https://openalex.org/W3133702157","https://openalex.org/W3139456429","https://openalex.org/W3156106752","https://openalex.org/W3159250634","https://openalex.org/W3163411042","https://openalex.org/W3163443091","https://openalex.org/W3163667721","https://openalex.org/W3173414067","https://openalex.org/W3176309423","https://openalex.org/W3183398589","https://openalex.org/W3199747482","https://openalex.org/W3207598588","https://openalex.org/W4220889970","https://openalex.org/W4221055872","https://openalex.org/W4225095662","https://openalex.org/W4225159586","https://openalex.org/W4231055147","https://openalex.org/W4248588746","https://openalex.org/W4283170666","https://openalex.org/W4288768554","https://openalex.org/W4290994954","https://openalex.org/W4292133617","https://openalex.org/W4294432232","https://openalex.org/W4308411106","https://openalex.org/W4309618893","https://openalex.org/W4309674289","https://openalex.org/W4312407344","https://openalex.org/W4317748910","https://openalex.org/W4319654023","https://openalex.org/W4360991197","https://openalex.org/W4360991389","https://openalex.org/W4366003124","https://openalex.org/W4366594764","https://openalex.org/W4367046989","https://openalex.org/W4376654148","https://openalex.org/W4380365974","https://openalex.org/W4380367493","https://openalex.org/W4386249234","https://openalex.org/W4387344906","https://openalex.org/W4389523793","https://openalex.org/W4393153298"],"related_works":["https://openalex.org/W2149537132","https://openalex.org/W2018871932","https://openalex.org/W641279757","https://openalex.org/W370975646","https://openalex.org/W1670566515","https://openalex.org/W4242022592","https://openalex.org/W596972243","https://openalex.org/W4313230280","https://openalex.org/W69751022","https://openalex.org/W2063534752"],"abstract_inverted_index":{"Widely":[0],"deployed":[1],"large":[2],"language":[3,100,191,209],"models":[4],"(LLMs)":[5],"can":[6,147],"produce":[7],"convincing":[8],"yet":[9],"incorrect":[10,158],"outputs,":[11],"potentially":[12],"misleading":[13],"users":[14,51],"who":[15],"may":[16,195],"rely":[17],"on":[18,157,203],"them":[19],"as":[20],"if":[21],"they":[22],"were":[23],"correct.":[24],"To":[25],"reduce":[26],"such":[27],"overreliance,":[28],"there":[29,43],"have":[30],"been":[31,45],"calls":[32],"for":[33,165,200],"LLMs":[34,221],"to":[35,39,81,130,150],"communicate":[36],"their":[37],"uncertainty":[38,103,166,194],"end":[40],"users.":[41],"However,":[42],"has":[44],"little":[46],"empirical":[47],"work":[48],"examining":[49],"how":[50,97],"perceive":[52],"and":[53,92,108,128,181],"act":[54],"upon":[55],"LLMs\u2019":[56],"expressions":[57,101,116,192],"of":[58,102,193,216],"uncertainty.":[59],"We":[60,112],"explore":[61],"this":[62,145],"question":[63],"through":[64],"a":[65,84,169],"large-scale,":[66],"pre-registered,":[67],"human-subject":[68],"experiment":[69],"(N=404)":[70],"in":[71,125],"which":[72],"participants":[73],"answer":[74],"medical":[75],"questions":[76],"with":[77,132],"or":[78],"without":[79],"access":[80],"responses":[82],"from":[83,168],"fictional":[85],"LLM-infused":[86],"search":[87],"engine.":[88],"Using":[89],"both":[90],"behavioral":[91],"self-reported":[93],"measures,":[94],"we":[95,161],"examine":[96],"different":[98],"natural":[99,190],"impact":[104],"participants\u2019":[105,123,138],"reliance,":[106],"trust,":[107],"overall":[109],"task":[110],"performance.":[111],"find":[113],"that":[114,144,188,206],"first-person":[115],"(e.g.,":[117,172],"\u201cI\u2019m":[118],"not":[119,153,174,182],"sure,":[120],"but...\u201d)":[121],"decrease":[122],"confidence":[124],"the":[126,133,207,214],"system":[127],"tendency":[129],"agree":[131],"system\u2019s":[134],"answers,":[135],"while":[136],"increasing":[137],"accuracy.":[139],"An":[140],"exploratory":[141],"analysis":[142],"suggests":[143],"increase":[146],"be":[148,196],"attributed":[149],"reduced":[151],"(but":[152],"fully":[154],"eliminated)":[155],"overreliance":[156,202],"answers.":[159],"While":[160],"observe":[162],"similar":[163],"effects":[164,178],"expressed":[167],"general":[170],"perspective":[171],"\u201cIt\u2019s":[173],"clear,":[175],"but...\u201d),":[176],"these":[177],"are":[179],"weaker":[180],"statistically":[183],"significant.":[184],"Our":[185],"findings":[186],"suggest":[187],"using":[189],"an":[197],"effective":[198],"approach":[199],"reducing":[201],"LLMs,":[204],"but":[205],"precise":[208],"used":[210],"matters.":[211],"This":[212],"highlights":[213],"importance":[215],"user":[217],"testing":[218],"before":[219],"deploying":[220],"at":[222],"scale.":[223]},"counts_by_year":[{"year":2026,"cited_by_count":32},{"year":2025,"cited_by_count":55},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2024-05-11T00:00:00"}