{"id":"https://openalex.org/W4366460235","doi":"https://doi.org/10.1145/3544548.3580670","title":"A Field Test of Bandit Algorithms for Recommendations: Understanding the Validity of Assumptions on Human Preferences in Multi-armed Bandits","display_name":"A Field Test of Bandit Algorithms for Recommendations: Understanding the Validity of Assumptions on Human Preferences in Multi-armed Bandits","publication_year":2023,"publication_date":"2023-04-19","ids":{"openalex":"https://openalex.org/W4366460235","doi":"https://doi.org/10.1145/3544548.3580670"},"language":"en","primary_location":{"id":"doi:10.1145/3544548.3580670","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580670","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580670","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580670","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011342505","display_name":"Liu Leqi","orcid":"https://orcid.org/0000-0002-9707-4529"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Liu Leqi","raw_affiliation_strings":["Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039263158","display_name":"Giulio Zhou","orcid":"https://orcid.org/0000-0002-9802-0741"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Giulio Zhou","raw_affiliation_strings":["Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037072174","display_name":"Fatma K\u0131l\u0131n\u00e7-Karzan","orcid":"https://orcid.org/0000-0001-5939-4575"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fatma Kilinc-Karzan","raw_affiliation_strings":["Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029448258","display_name":"Zachary C. Lipton","orcid":"https://orcid.org/0000-0002-3824-4241"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zachary Lipton","raw_affiliation_strings":["Machine Learning Department and Tepper School of Business, Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Machine Learning Department and Tepper School of Business, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004165506","display_name":"Alan L. Montgomery","orcid":"https://orcid.org/0000-0001-8375-3785"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alan Montgomery","raw_affiliation_strings":["Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5011342505"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":0.7465,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.73133736,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"16"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.983299970626831,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.7234957218170166},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.6031134724617004},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5476046204566956},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.38280636072158813},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3462740182876587},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.34144654870033264},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.23234093189239502}],"concepts":[{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.7234957218170166},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.6031134724617004},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5476046204566956},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.38280636072158813},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3462740182876587},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34144654870033264},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.23234093189239502},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3544548.3580670","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580670","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580670","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2304.09088","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.09088","pdf_url":"https://arxiv.org/pdf/2304.09088","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3544548.3580670","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3580670","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3580670","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.7900000214576721,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366460235.pdf","grobid_xml":"https://content.openalex.org/works/W4366460235.grobid-xml"},"referenced_works_count":61,"referenced_works":["https://openalex.org/W105998487","https://openalex.org/W572458652","https://openalex.org/W1552276231","https://openalex.org/W1847765408","https://openalex.org/W1965819279","https://openalex.org/W1986981703","https://openalex.org/W1996235270","https://openalex.org/W1998498767","https://openalex.org/W2018403196","https://openalex.org/W2024930574","https://openalex.org/W2029737723","https://openalex.org/W2048068013","https://openalex.org/W2054141820","https://openalex.org/W2069411646","https://openalex.org/W2094992056","https://openalex.org/W2099471337","https://openalex.org/W2099476974","https://openalex.org/W2112420033","https://openalex.org/W2136891251","https://openalex.org/W2138817229","https://openalex.org/W2139622428","https://openalex.org/W2210543184","https://openalex.org/W2323859140","https://openalex.org/W2390938437","https://openalex.org/W2507134384","https://openalex.org/W2533696134","https://openalex.org/W2558073743","https://openalex.org/W2589932687","https://openalex.org/W2787933113","https://openalex.org/W2878188201","https://openalex.org/W2888023058","https://openalex.org/W2893370267","https://openalex.org/W2902572901","https://openalex.org/W2902847006","https://openalex.org/W2951727499","https://openalex.org/W2953132212","https://openalex.org/W3040804160","https://openalex.org/W3080077280","https://openalex.org/W3088731664","https://openalex.org/W3091868446","https://openalex.org/W3104509044","https://openalex.org/W3116249021","https://openalex.org/W3121330560","https://openalex.org/W3200556911","https://openalex.org/W3210737511","https://openalex.org/W4205621617","https://openalex.org/W4206275166","https://openalex.org/W4210893662","https://openalex.org/W4232791905","https://openalex.org/W4237910115","https://openalex.org/W4240870952","https://openalex.org/W4247915891","https://openalex.org/W4249624378","https://openalex.org/W4252080790","https://openalex.org/W4288110508","https://openalex.org/W4290927958","https://openalex.org/W4293585414","https://openalex.org/W4296604485","https://openalex.org/W4301597078","https://openalex.org/W6630221451","https://openalex.org/W6703949738"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Personalized":[0],"recommender":[1,158],"systems":[2,19,159],"suffuse":[3],"modern":[4],"life,":[5],"shaping":[6],"what":[7,12],"media":[8],"we":[9,14,102,170],"read":[10],"and":[11,58,120,181,194],"products":[13],"consume.":[15],"Algorithms":[16],"powering":[17],"such":[18,26,97],"tend":[20],"to":[21,55,88,95],"consist":[22],"of":[23,34,42,49,91,131],"supervised-learning-based":[24],"heuristics,":[25],"as":[27],"latent":[28],"factor":[29],"models":[30],"with":[31,106,185],"a":[32,104,109,117,172],"variety":[33],"heuristically":[35],"chosen":[36],"prediction":[37],"targets.":[38],"Meanwhile,":[39],"theoretical":[40],"treatments":[41],"recommendation":[43,111],"frequently":[44],"address":[45],"the":[46,50,53,61,89,129,195],"decision-theoretic":[47],"nature":[48],"problem,":[51],"including":[52],"need":[54],"balance":[56],"exploration":[57],"exploitation,":[59],"via":[60],"multi-armed":[62],"bandits":[63],"(MABs)":[64],"framework.":[65],"However,":[66],"MAB-based":[67],"approaches":[68],"rely":[69],"heavily":[70],"on":[71],"assumptions":[72,78],"about":[73],"human":[74,83,135,163,178,186],"preferences.":[75],"These":[76],"preference":[77,164,179],"are":[79,139],"seldom":[80],"tested":[81],"using":[82],"subject":[84],"studies,":[85],"partly":[86],"due":[87],"lack":[90],"publicly":[92],"available":[93],"toolkits":[94],"conduct":[96,103],"studies.":[98],"In":[99],"this":[100],"work,":[101],"study":[105],"crowdworkers":[107],"in":[108],"comics":[110],"MABs":[112,133,183],"setting.":[113],"Each":[114],"arm":[115],"represents":[116],"comic":[118],"category,":[119],"users":[121],"provide":[122,171],"feedback":[123],"after":[124],"each":[125],"recommendation.":[126],"We":[127],"check":[128],"validity":[130],"core":[132],"assumptions\u2014that":[134],"preferences":[136],"(reward":[137],"distributions)":[138],"fixed":[140],"over":[141],"time\u2014and":[142],"find":[143],"that":[144,152],"they":[145],"do":[146],"not":[147],"hold.":[148],"This":[149],"finding":[150],"suggests":[151],"any":[153],"MAB":[154],"algorithm":[155],"used":[156],"for":[157,162,176,190],"should":[160],"account":[161],"dynamics.":[165],"While":[166],"answering":[167],"these":[168],"questions,":[169],"flexible":[173],"experimental":[174,192],"framework":[175,193],"understanding":[177],"dynamics":[180],"testing":[182],"algorithms":[184],"users.":[187],"The":[188],"code":[189],"our":[191],"collected":[196],"data":[197],"can":[198],"be":[199],"found":[200],"at":[201],"https://github.com/HumainLab/human-bandit-evaluation.":[202]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":2}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}