{"id":"https://openalex.org/W4367046962","doi":"https://doi.org/10.1145/3543507.3583551","title":"A Reference-Dependent Model for Web Search Evaluation","display_name":"A Reference-Dependent Model for Web Search Evaluation","publication_year":2023,"publication_date":"2023-04-26","ids":{"openalex":"https://openalex.org/W4367046962","doi":"https://doi.org/10.1145/3543507.3583551"},"language":"en","primary_location":{"id":"doi:10.1145/3543507.3583551","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3543507.3583551","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2023","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007524571","display_name":"Nuo Chen","orcid":"https://orcid.org/0000-0001-8600-8203"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Nuo Chen","raw_affiliation_strings":["Waseda University, Japan"],"raw_orcid":"https://orcid.org/0000-0001-8600-8203","affiliations":[{"raw_affiliation_string":"Waseda University, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088558868","display_name":"Jiqun Liu","orcid":"https://orcid.org/0000-0003-3643-2182"},"institutions":[{"id":"https://openalex.org/I8692664","display_name":"University of Oklahoma","ror":"https://ror.org/02aqsxs83","country_code":"US","type":"education","lineage":["https://openalex.org/I8692664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiqun Liu","raw_affiliation_strings":["The University of Oklahoma, USA"],"raw_orcid":"https://orcid.org/0000-0003-3643-2182","affiliations":[{"raw_affiliation_string":"The University of Oklahoma, USA","institution_ids":["https://openalex.org/I8692664"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6720-963X","affiliations":[{"raw_affiliation_string":"Waseda University, Japan","institution_ids":["https://openalex.org/I150744194"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3396","last_page":"3405"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.9914000034332275,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7063832879066467},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4559228718280792},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.35295456647872925}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7063832879066467},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4559228718280792},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.35295456647872925}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3543507.3583551","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3543507.3583551","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2023","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.7400000095367432,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W32360141","https://openalex.org/W1968927634","https://openalex.org/W1983164881","https://openalex.org/W1987831316","https://openalex.org/W1991774506","https://openalex.org/W2000270827","https://openalex.org/W2009485604","https://openalex.org/W2014721055","https://openalex.org/W2017292914","https://openalex.org/W2022995284","https://openalex.org/W2029009258","https://openalex.org/W2031858317","https://openalex.org/W2041946752","https://openalex.org/W2043787485","https://openalex.org/W2053100920","https://openalex.org/W2058896506","https://openalex.org/W2059120814","https://openalex.org/W2069870183","https://openalex.org/W2071446250","https://openalex.org/W2090035194","https://openalex.org/W2106404250","https://openalex.org/W2106925735","https://openalex.org/W2113640060","https://openalex.org/W2137274315","https://openalex.org/W2143331230","https://openalex.org/W2165893637","https://openalex.org/W2624553223","https://openalex.org/W2740517242","https://openalex.org/W2740916867","https://openalex.org/W2808234013","https://openalex.org/W2810030148","https://openalex.org/W2895782815","https://openalex.org/W2896022497","https://openalex.org/W2952667223","https://openalex.org/W2998372272","https://openalex.org/W3034645139","https://openalex.org/W3034909182","https://openalex.org/W3043886961","https://openalex.org/W3117895913","https://openalex.org/W3125065975","https://openalex.org/W3125913709","https://openalex.org/W3146722042","https://openalex.org/W3155638432","https://openalex.org/W3183835286","https://openalex.org/W3208649236","https://openalex.org/W4205480697","https://openalex.org/W4206655182","https://openalex.org/W4220728222","https://openalex.org/W4233760227","https://openalex.org/W4243617875","https://openalex.org/W4281624652","https://openalex.org/W4283762145","https://openalex.org/W4284689681","https://openalex.org/W4284713556"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Previous":[0],"researches":[1],"demonstrate":[2],"that":[3,104],"users\u2019":[4],"actions":[5],"in":[6,33,89,139,268],"search":[7,39,60,73,111,272],"interaction":[8],"are":[9],"associated":[10],"with":[11,93,151,159],"relative":[12],"gains":[13],"and":[14,127,130,173,221,226,248,274],"losses":[15],"to":[16,192,223,236,258],"reference":[17,22,66,101,154,240],"points,":[18],"known":[19],"as":[20],"the":[21,63,69,77,81,87,106,116,134,165,196,204,234,239,244,260],"dependence":[23,67,241],"effect.":[24],"However,":[25],"this":[26,43,231],"widely":[27],"confirmed":[28],"effect":[29,64,242],"is":[30,209,233],"not":[31],"represented":[32],"most":[34,163],"user":[35,72,94,160,245,263,271,276],"models":[36],"underpinning":[37],"existing":[38,166,193],"evaluation":[40,49,250],"metrics.":[41,251],"In":[42],"study,":[44],"we":[45,85,114,132],"propose":[46],"a":[47,152,199,255],"new":[48],"metric":[50],"framework,":[51,83],"namely":[52],"Reference":[53],"Dependent":[54],"Metric":[55],"(ReDeM),":[56],"for":[57],"assessing":[58],"query-level":[59],"by":[61],"incorporating":[62],"of":[65,71,80,91,96,105,118,137,141,164,207],"into":[68,243],"modelling":[70],"behavior.":[74],"To":[75,228],"test":[76],"overall":[78],"effectiveness":[79],"proposed":[82],"(1)":[84,148],"evaluate":[86],"performance,":[88],"terms":[90,140],"correlation":[92],"satisfaction,":[95],"ReDeMs":[97,119,138,149,186,208],"built":[98],"upon":[99],"different":[100,121],"points":[102],"against":[103],"widely-used":[107],"metrics":[108,194],"on":[109],"three":[110],"datasets;":[112],"(2)":[113,185],"examine":[115],"performance":[117,190],"under":[120],"task":[122,125,128,197],"states,":[123],"like":[124,168],"difficulty":[126],"urgency;":[129],"(3)":[131,203],"analyze":[133],"statistical":[135],"reliability":[136],"discriminative":[142,205],"power.":[143],"Experimental":[144],"results":[145],"indicate":[146],"that:":[147],"integrated":[150],"proper":[153],"point":[155],"achieve":[156],"better":[157,189,269],"correlations":[158],"satisfaction":[161],"than":[162,212,219],"metrics,":[167],"Discounted":[169],"Cumulative":[170],"Gain":[171],"(DCG)":[172],"Rank-Biased":[174],"Precision":[175,220],"(RBP),":[176],"even":[177],"though":[178],"their":[179],"parameters":[180],"have":[181],"already":[182],"been":[183],"well-tuned;":[184],"reach":[187],"relatively":[188],"compared":[191],"when":[195],"triggers":[198],"high-level":[200],"cognitive":[201,266],"load;":[202],"power":[206],"far":[210],"stronger":[211,218],"Expected":[213],"Reciprocal":[214],"Rank":[215],"(ERR),":[216],"slightly":[217],"similar":[222],"DCG,":[224],"RBP":[225],"INST.":[227],"our":[229],"knowledge,":[230],"study":[232],"first":[235],"explicitly":[237],"incorporate":[238],"browsing":[246],"model":[247],"offline":[249],"Our":[252],"work":[253],"illustrates":[254],"promising":[256],"approach":[257],"leveraging":[259],"insights":[261],"about":[262],"biases":[264],"from":[265],"psychology":[267],"evaluating":[270],"experience":[273],"enhancing":[275],"models.":[277]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":9}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}