{"id":"https://openalex.org/W4315705277","doi":"https://doi.org/10.1145/3578931","title":"Examining User Heterogeneity in Digital Experiments","display_name":"Examining User Heterogeneity in Digital Experiments","publication_year":2023,"publication_date":"2023-01-12","ids":{"openalex":"https://openalex.org/W4315705277","doi":"https://doi.org/10.1145/3578931"},"language":"en","primary_location":{"id":"doi:10.1145/3578931","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3578931","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007288601","display_name":"Sriram Somanchi","orcid":"https://orcid.org/0000-0002-3153-1248"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sriram Somanchi","raw_affiliation_strings":["University of Notre Dame, Notre Dame, IN, USA"],"affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, IN, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016943900","display_name":"Ahmed Abbasi","orcid":"https://orcid.org/0000-0001-7698-7794"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed Abbasi","raw_affiliation_strings":["University of Notre Dame, Notre Dame, IN, USA"],"affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, IN, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073962125","display_name":"Ken Kelley","orcid":"https://orcid.org/0000-0002-4756-8360"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ken Kelley","raw_affiliation_strings":["University of Notre Dame, USA","University of Notre Dame, Notre Dame, IN, USA"],"affiliations":[{"raw_affiliation_string":"University of Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]},{"raw_affiliation_string":"University of Notre Dame, Notre Dame, IN, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006686080","display_name":"David G. Dobolyi","orcid":"https://orcid.org/0000-0002-9493-3447"},"institutions":[{"id":"https://openalex.org/I2802236040","display_name":"University of Colorado System","ror":"https://ror.org/00jc20583","country_code":"US","type":"education","lineage":["https://openalex.org/I2802236040"]},{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Dobolyi","raw_affiliation_strings":["University of Colorado, USA","University of Colorado, Boulder, CO, USA"],"affiliations":[{"raw_affiliation_string":"University of Colorado, USA","institution_ids":["https://openalex.org/I2802236040"]},{"raw_affiliation_string":"University of Colorado, Boulder, CO, USA","institution_ids":["https://openalex.org/I188538660"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060193104","display_name":"Ted Tao Yuan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210150719","display_name":"eBay (United States)","ror":"https://ror.org/05cnabr44","country_code":"US","type":"company","lineage":["https://openalex.org/I4210150719"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ted Tao Yuan","raw_affiliation_strings":["eBay, USA","eBay, San Jose, CA, USA"],"affiliations":[{"raw_affiliation_string":"eBay, USA","institution_ids":["https://openalex.org/I4210150719"]},{"raw_affiliation_string":"eBay, San Jose, CA, USA","institution_ids":["https://openalex.org/I4210150719"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5007288601"],"corresponding_institution_ids":["https://openalex.org/I107639228"],"apc_list":null,"apc_paid":null,"fwci":4.8034,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.94992267,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9865999817848206,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9811999797821045,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8681222200393677},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.5481661558151245},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.4773597717285156},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.44856876134872437},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.43549636006355286},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.38732457160949707},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21367958188056946}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8681222200393677},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5481661558151245},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.4773597717285156},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.44856876134872437},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.43549636006355286},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.38732457160949707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21367958188056946},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3578931","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3578931","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":66,"referenced_works":["https://openalex.org/W48272409","https://openalex.org/W597471439","https://openalex.org/W1566647102","https://openalex.org/W1618214481","https://openalex.org/W1969348190","https://openalex.org/W1975566260","https://openalex.org/W1985995642","https://openalex.org/W2043679667","https://openalex.org/W2064971756","https://openalex.org/W2077459813","https://openalex.org/W2078658856","https://openalex.org/W2092073019","https://openalex.org/W2109177042","https://openalex.org/W2110065044","https://openalex.org/W2112508839","https://openalex.org/W2126002144","https://openalex.org/W2138123110","https://openalex.org/W2143117649","https://openalex.org/W2144026687","https://openalex.org/W2154516913","https://openalex.org/W2208550830","https://openalex.org/W2305754340","https://openalex.org/W2341636400","https://openalex.org/W2509295096","https://openalex.org/W2583860259","https://openalex.org/W2624816748","https://openalex.org/W2735108298","https://openalex.org/W2746337320","https://openalex.org/W2750779823","https://openalex.org/W2752579606","https://openalex.org/W2791907880","https://openalex.org/W2803901985","https://openalex.org/W2809468631","https://openalex.org/W2883214080","https://openalex.org/W2890118063","https://openalex.org/W2899575547","https://openalex.org/W2905278593","https://openalex.org/W2946144951","https://openalex.org/W2946387282","https://openalex.org/W2962727190","https://openalex.org/W3004561114","https://openalex.org/W3005139125","https://openalex.org/W3010665282","https://openalex.org/W3044871282","https://openalex.org/W3071563765","https://openalex.org/W3102479286","https://openalex.org/W3103988283","https://openalex.org/W3104973090","https://openalex.org/W3106777270","https://openalex.org/W3121411908","https://openalex.org/W3124999902","https://openalex.org/W3171492708","https://openalex.org/W3189979102","https://openalex.org/W3193570035","https://openalex.org/W3196693533","https://openalex.org/W3201966249","https://openalex.org/W3208524908","https://openalex.org/W3210209978","https://openalex.org/W3216418890","https://openalex.org/W3216761404","https://openalex.org/W3217229393","https://openalex.org/W3217672792","https://openalex.org/W4200412261","https://openalex.org/W4206503466","https://openalex.org/W4226280022","https://openalex.org/W4243325401"],"related_works":["https://openalex.org/W1843462531","https://openalex.org/W2036641180","https://openalex.org/W2751656612","https://openalex.org/W2020120615","https://openalex.org/W91919785","https://openalex.org/W1546415126","https://openalex.org/W2167091778","https://openalex.org/W1516841646","https://openalex.org/W2015192756","https://openalex.org/W2088745704"],"abstract_inverted_index":{"Digital":[0],"experiments":[1,41,71,156,225,262],"are":[2,230,263],"routinely":[3],"used":[4,265],"to":[5,13,124,172,192,266],"test":[6],"the":[7,75,104,116,165,199,222,268],"value":[8],"of":[9,52,62,118,145,153,167,194,201,221,224,233,270],"a":[10,14,22,28,31,36,49,59,129],"treatment":[11,77,89],"relative":[12,171],"status":[15],"quo":[16],"control":[17],"setting":[18],"\u2014":[19,87],"for":[20,27,35,64,131,249],"instance,":[21],"new":[23,32,60],"search":[24],"relevance":[25],"algorithm":[26],"website":[29],"or":[30],"results":[33],"layout":[34],"mobile":[37],"app.":[38],"As":[39],"digital":[40,138,256],"have":[42,98,108,246],"become":[43],"increasingly":[44],"pervasive":[45],"in":[46,100,137,190,259],"organizations":[47,102],"and":[48,121,133,161,183,212,226,255],"wide":[50],"variety":[51],"research":[53],"areas,":[54],"their":[55],"growth":[56],"has":[57,237],"prompted":[58],"set":[61],"challenges":[63],"experimentation":[65],"platforms.":[66],"One":[67],"challenge":[68],"is":[69,93],"that":[70,178,236],"often":[72,264],"focus":[73],"on":[74],"average":[76],"effect":[78,90],"(ATE)":[79],"without":[80],"explicitly":[81],"considering":[82,202],"differences":[83],"across":[84],"major":[85],"sub-groups":[86],"heterogeneous":[88],"(HTE).":[91],"This":[92],"especially":[94],"problematic":[95],"because":[96],"ATEs":[97],"decreased":[99],"many":[101],"as":[103],"more":[105],"obvious":[106],"benefits":[107],"already":[109],"been":[110],"realized.":[111],"However,":[112],"questions":[113],"abound":[114],"regarding":[115],"pervasiveness":[117],"user":[119,135,146,203,227,239,252],"HTEs":[120,136,189,240],"how":[122],"best":[123],"detect":[125],"them.":[126],"We":[127,175],"propose":[128],"framework":[130,141],"detecting":[132],"analyzing":[134,206],"experiments.":[139],"Our":[140,244],"combines":[142],"an":[143],"array":[144],"characteristics":[147,185],"with":[148],"double":[149],"machine":[150],"learning.":[151],"Analysis":[152],"27":[154],"real-world":[155,196],"spanning":[157],"1.76":[158],"billion":[159],"sessions":[160],"simulated":[162],"data":[163],"demonstrates":[164],"effectiveness":[166,269],"our":[168,195],"detection":[169],"method":[170],"existing":[173],"techniques.":[174],"also":[176],"find":[177],"transaction,":[179],"demographic,":[180],"engagement,":[181],"satisfaction,":[182],"lifecycle":[184],"exhibit":[186],"statistically":[187],"significant":[188],"10%":[191],"20%":[193],"experiments,":[197],"underscoring":[198],"importance":[200],"heterogeneity":[204],"when":[205],"experiment":[207],"results,":[208],"otherwise":[209],"personalized":[210],"features":[211],"experiences":[213],"cannot":[214],"happen,":[215],"thus":[216],"reducing":[217],"effectiveness.":[218],"In":[219],"terms":[220],"number":[223],"sessions,":[228],"we":[229],"not":[231],"aware":[232],"any":[234],"study":[235],"examined":[238],"at":[241],"this":[242],"scale.":[243],"findings":[245],"important":[247],"implications":[248],"information":[250],"retrieval,":[251],"modeling,":[253],"platforms,":[254],"experience":[257],"contexts,":[258],"which":[260],"online":[261],"evaluate":[267],"design":[271],"artifacts.":[272]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":5},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}