{"id":"https://openalex.org/W4405144074","doi":"https://doi.org/10.1145/3673791.3698420","title":"AI Can Be Cognitively Biased: An Exploratory Study on Threshold Priming in LLM-Based Batch Relevance Assessment","display_name":"AI Can Be Cognitively Biased: An Exploratory Study on Threshold Priming in LLM-Based Batch Relevance Assessment","publication_year":2024,"publication_date":"2024-12-08","ids":{"openalex":"https://openalex.org/W4405144074","doi":"https://doi.org/10.1145/3673791.3698420"},"language":"en","primary_location":{"id":"doi:10.1145/3673791.3698420","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698420","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698420","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698420","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007524571","display_name":"Nuo Chen","orcid":"https://orcid.org/0000-0001-8600-8203"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Nuo Chen","raw_affiliation_strings":["The Hong Kong Polytechnic University, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-8600-8203","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University, Hong Kong, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088558868","display_name":"Jiqun Liu","orcid":"https://orcid.org/0000-0003-3643-2182"},"institutions":[{"id":"https://openalex.org/I8692664","display_name":"University of Oklahoma","ror":"https://ror.org/02aqsxs83","country_code":"US","type":"education","lineage":["https://openalex.org/I8692664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiqun Liu","raw_affiliation_strings":["The University of Oklahoma, Norman, OK, USA"],"raw_orcid":"https://orcid.org/0000-0003-3643-2182","affiliations":[{"raw_affiliation_string":"The University of Oklahoma, Norman, OK, USA","institution_ids":["https://openalex.org/I8692664"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113194464","display_name":"Xiaoyu Dong","orcid":null},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiaoyu Dong","raw_affiliation_strings":["The Hong Kong Polytechnic University, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-4788-9279","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University, Hong Kong, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019877925","display_name":"Qijiong Liu","orcid":"https://orcid.org/0000-0001-6087-383X"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qijiong Liu","raw_affiliation_strings":["The Hong Kong Polytechnic University, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-6087-383X","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University, Hong Kong, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6720-963X","affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101981128","display_name":"Xiao-Ming Wu","orcid":"https://orcid.org/0000-0002-3130-0554"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiao-Ming Wu","raw_affiliation_strings":["The Hong Kong Polytechnic University, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-3130-0554","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University, Hong Kong, China","institution_ids":["https://openalex.org/I14243506"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"54","last_page":"63"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7302314043045044},{"id":"https://openalex.org/keywords/priming","display_name":"Priming (agriculture)","score":0.6132532954216003},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5628855228424072},{"id":"https://openalex.org/keywords/exploratory-research","display_name":"Exploratory research","score":0.5327866673469543},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.47829610109329224},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.44644686579704285},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3639492988586426}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7302314043045044},{"id":"https://openalex.org/C81444415","wikidata":"https://www.wikidata.org/wiki/Q7243535","display_name":"Priming (agriculture)","level":3,"score":0.6132532954216003},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5628855228424072},{"id":"https://openalex.org/C85973986","wikidata":"https://www.wikidata.org/wiki/Q1091731","display_name":"Exploratory research","level":2,"score":0.5327866673469543},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.47829610109329224},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.44644686579704285},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3639492988586426},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C100701293","wikidata":"https://www.wikidata.org/wiki/Q193838","display_name":"Germination","level":2,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3673791.3698420","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698420","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698420","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3673791.3698420","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698420","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698420","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4405144074.pdf"},"referenced_works_count":59,"referenced_works":["https://openalex.org/W1971070829","https://openalex.org/W1987831316","https://openalex.org/W2009485604","https://openalex.org/W2041946752","https://openalex.org/W2048646785","https://openalex.org/W2068751432","https://openalex.org/W2071446250","https://openalex.org/W2090321376","https://openalex.org/W2106925735","https://openalex.org/W2117503214","https://openalex.org/W2158254843","https://openalex.org/W2165893637","https://openalex.org/W2166558062","https://openalex.org/W2171664132","https://openalex.org/W2338216121","https://openalex.org/W2593591377","https://openalex.org/W2741619064","https://openalex.org/W2783668259","https://openalex.org/W2788493644","https://openalex.org/W2792684907","https://openalex.org/W2895782815","https://openalex.org/W2913897616","https://openalex.org/W2922184570","https://openalex.org/W2952667223","https://openalex.org/W3011445502","https://openalex.org/W3034909182","https://openalex.org/W3043886961","https://openalex.org/W3125913709","https://openalex.org/W3135580307","https://openalex.org/W3155638432","https://openalex.org/W3160200580","https://openalex.org/W3184074550","https://openalex.org/W3184144760","https://openalex.org/W4205480697","https://openalex.org/W4224980442","https://openalex.org/W4284689681","https://openalex.org/W4292157289","https://openalex.org/W4300539796","https://openalex.org/W4319072064","https://openalex.org/W4367046962","https://openalex.org/W4382141248","https://openalex.org/W4384662964","https://openalex.org/W4385688511","https://openalex.org/W4385958572","https://openalex.org/W4386246835","https://openalex.org/W4386928872","https://openalex.org/W4387821331","https://openalex.org/W4388042901","https://openalex.org/W4391766505","https://openalex.org/W4391835702","https://openalex.org/W4392490008","https://openalex.org/W4392612131","https://openalex.org/W4393284545","https://openalex.org/W4393335138","https://openalex.org/W4400526908","https://openalex.org/W4400529573","https://openalex.org/W4401330329","https://openalex.org/W4403432343","https://openalex.org/W4405578323"],"related_works":["https://openalex.org/W2085384747","https://openalex.org/W2088166309","https://openalex.org/W1891216533","https://openalex.org/W4312133475","https://openalex.org/W4238976562","https://openalex.org/W2276587472","https://openalex.org/W2068167342","https://openalex.org/W2091108294","https://openalex.org/W2615795876","https://openalex.org/W2049612369"],"abstract_inverted_index":{"Cognitive":[0],"biases":[1,34,41,48,65,209],"are":[2,78,188],"systematic":[3],"deviations":[4],"in":[5,42,68,85,95,210,216],"thinking":[6],"that":[7,152,181,197],"lead":[8],"to":[9,107,155,159,185],"irrational":[10],"judgments":[11,133],"and":[12,91,114,130,141,148,168,175,195,199,213,219],"problematic":[13],"decision-making,":[14],"extensively":[15],"studied":[16],"across":[17],"various":[18,69],"fields.":[19],"Recently,":[20],"large":[21],"language":[22],"models":[23],"(LLMs)":[24],"have":[25,44,49,165],"shown":[26],"advanced":[27],"understanding":[28],"capabilities":[29],"but":[30],"may":[31],"inherit":[32],"human":[33,186],"from":[35,121],"their":[36],"training":[37],"data.":[38],"While":[39],"social":[40],"LLMs":[43,67,77,153,215],"been":[45],"well-studied,":[46],"cognitive":[47,64,208],"received":[50],"less":[51],"attention,":[52],"with":[53],"existing":[54],"research":[55,93],"focusing":[56],"on":[57,66],"specific":[58],"scenarios.":[59],"The":[60,101],"broader":[61],"impact":[62],"of":[63,172],"decision-making":[70],"contexts":[71],"remains":[72],"underexplored.":[73],"We":[74],"investigated":[75],"whether":[76],"influenced":[79,190],"by":[80,191],"the":[81,96,122,173],"threshold":[82,192],"priming":[83,102,193],"effect":[84,103],"relevance":[86,137],"judgments,":[87,183,187],"a":[88],"core":[89],"task":[90],"widely-discussed":[92],"topic":[94],"Information":[97],"Retrieval":[98],"(IR)":[99],"coummunity.":[100],"occurs":[104],"when":[105],"exposure":[106],"certain":[108],"stimuli":[109],"unconsciously":[110],"affects":[111],"subsequent":[112],"behavior":[113],"decisions.":[115],"Our":[116,178],"experiment":[117],"employed":[118],"10":[119],"topics":[120],"TREC":[123],"2019":[124],"Deep":[125],"Learning":[126],"passage":[127],"track":[128],"collection,":[129],"tested":[131],"AI":[132],"under":[134],"different":[135],"document":[136],"scores,":[138],"batch":[139],"lengths,":[140],"LLM":[142],"models,":[143],"including":[144],"GPT-3.5,":[145],"GPT-4,":[146],"LLaMa2-13B":[147],"LLaMa2-70B.":[149],"Results":[150],"showed":[151],"tend":[154],"give":[156],"lower":[157],"scores":[158],"later":[160],"documents":[161],"if":[162],"earlier":[163],"ones":[164],"high":[166],"relevance,":[167],"vice":[169],"versa,":[170],"regardless":[171],"combination":[174],"model":[176],"used.":[177],"finding":[179],"demonstrates":[180],"LLM's":[182],"similar":[184],"also":[189],"biases,":[194],"suggests":[196],"researchers":[198],"system":[200],"engineers":[201],"should":[202],"take":[203],"into":[204],"account":[205],"potential":[206],"human-like":[207],"designing,":[211],"evaluating,":[212],"auditing":[214],"IR":[217],"tasks":[218],"beyond":[220]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-16T13:24:37.021932","created_date":"2025-10-10T00:00:00"}