{"id":"https://openalex.org/W2624553223","doi":"https://doi.org/10.1145/3052768","title":"Incorporating User Expectations and Behavior into the Measurement of Search Effectiveness","display_name":"Incorporating User Expectations and Behavior into the Measurement of Search Effectiveness","publication_year":2017,"publication_date":"2017-06-05","ids":{"openalex":"https://openalex.org/W2624553223","doi":"https://doi.org/10.1145/3052768","mag":"2624553223"},"language":"en","primary_location":{"id":"doi:10.1145/3052768","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3052768","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://figshare.com/articles/journal_contribution/Incorporating_user_expectations_and_behavior_into_the_measurement_of_search_effectiveness/27503421","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081861848","display_name":"Alistair Moffat","orcid":"https://orcid.org/0000-0002-6638-0232"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Alistair Moffat","raw_affiliation_strings":["The University of Melbourne, Victoria, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Victoria, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017503948","display_name":"Peter Bailey","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peter Bailey","raw_affiliation_strings":["Microsoft, Barton, ACT, Australia"],"affiliations":[{"raw_affiliation_string":"Microsoft, Barton, ACT, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Victoria, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Victoria, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046377370","display_name":"Paul Thomas","orcid":"https://orcid.org/0000-0003-2425-3136"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul Thomas","raw_affiliation_strings":["Microsoft, Barton, ACT, Australia"],"affiliations":[{"raw_affiliation_string":"Microsoft, Barton, ACT, Australia","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5081861848"],"corresponding_institution_ids":["https://openalex.org/I165779595"],"apc_list":null,"apc_paid":null,"fwci":14.1585,"has_fulltext":false,"cited_by_count":108,"citation_normalized_percentile":{"value":0.98771015,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"35","issue":"3","first_page":"1","last_page":"38"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9876000285148621,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8566950559616089},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7874883413314819},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7361479997634888},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.6506795287132263},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.6423967480659485},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5878846645355225},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5316897630691528},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.47562816739082336},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.4624692499637604},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4487893283367157},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39164337515830994},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3570953607559204},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0887523889541626}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8566950559616089},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7874883413314819},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7361479997634888},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6506795287132263},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.6423967480659485},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5878846645355225},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5316897630691528},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.47562816739082336},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.4624692499637604},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4487893283367157},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39164337515830994},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3570953607559204},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0887523889541626},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3052768","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3052768","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:alma.61RMIT_INST:11247577510001341","is_oa":false,"landing_page_url":"http://purl.org/au-research/grants/arc/DP140102655","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:figshare.com:article/27503421","is_oa":true,"landing_page_url":"https://figshare.com/articles/journal_contribution/Incorporating_user_expectations_and_behavior_into_the_measurement_of_search_effectiveness/27503421","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/27503421","is_oa":true,"landing_page_url":"https://figshare.com/articles/journal_contribution/Incorporating_user_expectations_and_behavior_into_the_measurement_of_search_effectiveness/27503421","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.41999998688697815}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":82,"referenced_works":["https://openalex.org/W42001192","https://openalex.org/W60779023","https://openalex.org/W92750350","https://openalex.org/W114240708","https://openalex.org/W132882894","https://openalex.org/W208548429","https://openalex.org/W1491092528","https://openalex.org/W1507711745","https://openalex.org/W1548128919","https://openalex.org/W1579063438","https://openalex.org/W1824124613","https://openalex.org/W1965862077","https://openalex.org/W1967907280","https://openalex.org/W1968927634","https://openalex.org/W1970977883","https://openalex.org/W1975879668","https://openalex.org/W1976445139","https://openalex.org/W1977437814","https://openalex.org/W1980104409","https://openalex.org/W1983132692","https://openalex.org/W1983164881","https://openalex.org/W1985709675","https://openalex.org/W1998381498","https://openalex.org/W1999331679","https://openalex.org/W2004918107","https://openalex.org/W2008285050","https://openalex.org/W2021581601","https://openalex.org/W2023217403","https://openalex.org/W2024963031","https://openalex.org/W2029009258","https://openalex.org/W2031858317","https://openalex.org/W2032577510","https://openalex.org/W2034076157","https://openalex.org/W2037140704","https://openalex.org/W2039847928","https://openalex.org/W2040214686","https://openalex.org/W2047610959","https://openalex.org/W2052658973","https://openalex.org/W2053100920","https://openalex.org/W2058896506","https://openalex.org/W2059120814","https://openalex.org/W2068333205","https://openalex.org/W2069049590","https://openalex.org/W2069870183","https://openalex.org/W2070740689","https://openalex.org/W2075893676","https://openalex.org/W2084457609","https://openalex.org/W2090031974","https://openalex.org/W2090035194","https://openalex.org/W2100548715","https://openalex.org/W2102698119","https://openalex.org/W2113640060","https://openalex.org/W2114811766","https://openalex.org/W2116817443","https://openalex.org/W2120308175","https://openalex.org/W2127326269","https://openalex.org/W2132232498","https://openalex.org/W2141175968","https://openalex.org/W2145781566","https://openalex.org/W2150234222","https://openalex.org/W2153252192","https://openalex.org/W2153763032","https://openalex.org/W2158450083","https://openalex.org/W2158890594","https://openalex.org/W2160484851","https://openalex.org/W2161722485","https://openalex.org/W2162821268","https://openalex.org/W2164734124","https://openalex.org/W2231711798","https://openalex.org/W2259592457","https://openalex.org/W2284349988","https://openalex.org/W2336806308","https://openalex.org/W2338477879","https://openalex.org/W2388783363","https://openalex.org/W2399777719","https://openalex.org/W2402441596","https://openalex.org/W2440929485","https://openalex.org/W2469056188","https://openalex.org/W2559401578","https://openalex.org/W2915610764","https://openalex.org/W4247890353","https://openalex.org/W4302322961"],"related_works":["https://openalex.org/W51364034","https://openalex.org/W3127142483","https://openalex.org/W3160516639","https://openalex.org/W4385565564","https://openalex.org/W2138488530","https://openalex.org/W2898073868","https://openalex.org/W4284663758","https://openalex.org/W2971071571","https://openalex.org/W2798835721","https://openalex.org/W2922169395"],"abstract_inverted_index":{"Information":[0],"retrieval":[1],"systems":[2],"aim":[3],"to":[4,32,45,64,99,144,157,212,221,242,286],"help":[5],"users":[6,96],"satisfy":[7],"information":[8,101,299],"needs.":[9],"We":[10,113],"argue":[11],"that":[12,26,34,53,62,81,104,116,137,168,189,253,262,288,302],"the":[13,16,19,22,40,47,54,65,73,88,126,145,153,172,185,191,194,201,206,214,264,267,328],"goal":[14,110,131,188],"of":[15,24,49,75,90,149,171,184,193,266,276,292,309,318,330],"person":[17],"using":[18],"system,":[20],"and":[21,42,79,135,234,283,301,326],"pattern":[23],"behavior":[25,118,209],"they":[27,30],"exhibit":[28],"as":[29,198,241],"proceed":[31],"attain":[33],"goal,":[35],"should":[36],"be":[37,109,142,257],"incorporated":[38],"into":[39,312],"methods":[41],"techniques":[43],"used":[44],"evaluate":[46],"effectiveness":[48,56,106,139,165,223],"IR":[50],"systems,":[51],"so":[52],"resulting":[55],"scores":[57],"have":[58,320],"a":[59,84,163,177,181,249,260,289],"useful":[60,105],"interpretation":[61],"corresponds":[63],"users\u2019":[66],"search":[67,76,187],"experience.":[68],"In":[69,155],"particular,":[70],"we":[71,161,279],"investigate":[72],"role":[74],"task":[77],"complexity,":[78],"show":[80],"it":[82],"has":[83,169],"direct":[85,182],"bearing":[86],"on":[87],"number":[89],"relevant":[91,150],"answer":[92],"documents":[93,151],"sought":[94],"by":[95,125,259,271],"in":[97,152],"response":[98,156],"an":[100],"need,":[102],"suggesting":[103],"metrics":[107,140],"must":[108,141],"sensitive":[111],".":[112],"further":[114],"suggest":[115],"user":[117,208],"while":[119],"scanning":[120],"results":[121],"listings":[122],"is":[123,132,204,210,218,248],"affected":[124],"rate":[127],"at":[128],"which":[129],"their":[130],"being":[133],"realized,":[134],"hence":[136],"appropriate":[138],"adaptive":[143],"presence":[146],"(or":[147],"not)":[148],"ranking.":[154],"these":[158,303],"two":[159],"observations,":[160],"present":[162],"new":[164],"metric,":[166,251],"INST,":[167],"both":[170],"desired":[173],"properties:":[174],"INST":[175,217,247],"employs":[176],"parameter":[178],"T":[179,203],",":[180],"measure":[183],"user\u2019s":[186],"adjusts":[190],"top-weightedness":[192],"evaluation":[195],"score;":[196],"moreover,":[197],"progress":[199],"towards":[200],"target":[202],"made,":[205],"modeled":[207],"adapted,":[211],"reflect":[213],"remaining":[215],"expectations.":[216],"experimentally":[219],"compared":[220],"previous":[222],"metrics,":[224],"including":[225],"Average":[226],"Precision":[227,236],"(AP),":[228],"Normalized":[229],"Discounted":[230],"Cumulative":[231],"Gain":[232],"(NDCG),":[233],"Rank-Biased":[235],"(RBP),":[237],"demonstrating":[238],"our":[239,277],"claims":[240],"INST\u2019s":[243],"usefulness.":[244],"Like":[245],"RBP,":[246],"weighted-precision":[250],"meaning":[252],"each":[254],"score":[255,268,284],"can":[256],"accompanied":[258],"residual":[261,310],"quantifies":[263],"extent":[265],"uncertainty":[269,311],"caused":[270],"unjudged":[272],"documents.":[273],"As":[274],"part":[275],"experimentation,":[278],"use":[280],"crowd-sourced":[281],"data":[282],"residuals":[285],"demonstrate":[287],"wide":[290],"range":[291],"queries":[293,305],"arise":[294],"for":[295,323,327],"even":[296],"quite":[297],"specific":[298],"needs,":[300],"variant":[304],"introduce":[306],"significant":[307],"levels":[308],"typical":[313],"experimental":[314],"evaluations.":[315],"These":[316],"causes":[317],"variability":[319],"wide-reaching":[321],"implications":[322],"experiment":[324],"design,":[325],"construction":[329],"test":[331],"collections.":[332]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":32},{"year":2023,"cited_by_count":18},{"year":2022,"cited_by_count":13},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":10},{"year":2018,"cited_by_count":7},{"year":2017,"cited_by_count":2}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}