{"id":"https://openalex.org/W1998381498","doi":"https://doi.org/10.1145/2661829.2661953","title":"Relevance and Effort","display_name":"Relevance and Effort","publication_year":2014,"publication_date":"2014-11-03","ids":{"openalex":"https://openalex.org/W1998381498","doi":"https://doi.org/10.1145/2661829.2661953","mag":"1998381498"},"language":"en","primary_location":{"id":"doi:10.1145/2661829.2661953","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661953","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076265623","display_name":"Emine Y\u0131lmaz","orcid":"https://orcid.org/0000-0002-3434-8932"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emine Yilmaz","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101503356","display_name":"Manisha Verma","orcid":"https://orcid.org/0000-0001-5281-2527"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Manisha Verma","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055132321","display_name":"Nick Craswell","orcid":"https://orcid.org/0000-0002-9351-8137"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nick Craswell","raw_affiliation_strings":["Microsoft, Seattle, WA, USA","[Microsoft, Seattle, WA, USA]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Seattle, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"[Microsoft, Seattle, WA, USA]","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081692910","display_name":"Filip Radlinski","orcid":"https://orcid.org/0000-0002-1386-9288"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Filip Radlinski","raw_affiliation_strings":["Microsoft Research Cambridge, Cambridge, United Kingdom","Microsoft Research Cambridge, Cambridge, United Kingdom#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Cambridge, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research Cambridge, Cambridge, United Kingdom#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017503948","display_name":"Peter Bailey","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peter Bailey","raw_affiliation_strings":["Microsoft, Canberra, Australia","Microsoft, Canberra , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Canberra, Australia","institution_ids":[]},{"raw_affiliation_string":"Microsoft, Canberra , Australia","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":95,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"91","last_page":"100"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9837999939918518,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.9482147693634033},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6993743181228638},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6279102563858032},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3285515308380127},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.0946044921875}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.9482147693634033},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6993743181228638},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6279102563858032},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3285515308380127},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0946044921875},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2661829.2661953","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661953","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5099999904632568,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W114240708","https://openalex.org/W1507711477","https://openalex.org/W1595665484","https://openalex.org/W1787140601","https://openalex.org/W1967498673","https://openalex.org/W1969587718","https://openalex.org/W1974360117","https://openalex.org/W1974548087","https://openalex.org/W1980036597","https://openalex.org/W1982149803","https://openalex.org/W2006551346","https://openalex.org/W2006811637","https://openalex.org/W2008285050","https://openalex.org/W2020150327","https://openalex.org/W2024230945","https://openalex.org/W2026784708","https://openalex.org/W2029009258","https://openalex.org/W2032577510","https://openalex.org/W2038385982","https://openalex.org/W2061503185","https://openalex.org/W2062236692","https://openalex.org/W2069997576","https://openalex.org/W2074680184","https://openalex.org/W2075227725","https://openalex.org/W2090715008","https://openalex.org/W2094607109","https://openalex.org/W2106576032","https://openalex.org/W2110417303","https://openalex.org/W2119031520","https://openalex.org/W2119461458","https://openalex.org/W2119529697","https://openalex.org/W2120308175","https://openalex.org/W2120966432","https://openalex.org/W2121821044","https://openalex.org/W2123937625","https://openalex.org/W2134440707","https://openalex.org/W2135877659","https://openalex.org/W2143445498","https://openalex.org/W2153763032","https://openalex.org/W2153783077","https://openalex.org/W2156186232","https://openalex.org/W2164734124","https://openalex.org/W2910001684"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W1967370444","https://openalex.org/W2150136235","https://openalex.org/W2157408137","https://openalex.org/W1980762553","https://openalex.org/W4312252109"],"abstract_inverted_index":{"In":[0],"this":[1,55],"paper,":[2],"we":[3],"study":[4],"one":[5],"important":[6],"source":[7],"of":[8,23,47,82],"the":[9,20,32],"mis-match":[10],"between":[11],"user":[12],"data":[13],"and":[14,30],"relevance":[15,39,48,83],"judgments,":[16],"those":[17],"due":[18],"to":[19,28,43,58,87],"high":[21],"degree":[22],"effort":[24],"required":[25],"by":[26],"users":[27,70],"identify":[29],"consume":[31],"information":[33],"in":[34,67],"a":[35],"document.":[36,65],"Information":[37],"retrieval":[38],"judges":[40],"are":[41,71],"trained":[42],"search":[44,69],"for":[45],"evidence":[46,81],"when":[49],"assessing":[50],"documents.":[51],"For":[52],"complex":[53],"documents,":[54],"can":[56],"lead":[57],"judges'":[59],"spending":[60],"substantial":[61],"time":[62],"considering":[63],"each":[64],"However,":[66],"practice,":[68],"often":[72],"much":[73],"more":[74],"impatient:":[75],"if":[76],"they":[77,85],"do":[78],"not":[79],"see":[80],"quickly,":[84],"tend":[86],"give":[88],"up.":[89]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":23},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":11},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":12},{"year":2016,"cited_by_count":15},{"year":2015,"cited_by_count":6}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}