{"id":"https://openalex.org/W2034536997","doi":"https://doi.org/10.1145/1835449.1835675","title":"Low cost evaluation in information retrieval","display_name":"Low cost evaluation in information retrieval","publication_year":2010,"publication_date":"2010-07-19","ids":{"openalex":"https://openalex.org/W2034536997","doi":"https://doi.org/10.1145/1835449.1835675","mag":"2034536997"},"language":"en","primary_location":{"id":"doi:10.1145/1835449.1835675","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1835449.1835675","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},"type":"conference-abstract","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078123101","display_name":"Ben Carterette","orcid":"https://orcid.org/0000-0001-9538-047X"},"institutions":[{"id":"https://openalex.org/I86501945","display_name":"University of Delaware","ror":"https://ror.org/01sbq1a82","country_code":"US","type":"education","lineage":["https://openalex.org/I86501945"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ben Carterette","raw_affiliation_strings":["University of Delaware, Newark, DE, USA","University of Delaware, Newark, DE. USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Delaware, Newark, DE, USA","institution_ids":["https://openalex.org/I86501945"]},{"raw_affiliation_string":"University of Delaware, Newark, DE. USA","institution_ids":["https://openalex.org/I86501945"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055639036","display_name":"Evangelos Kanoulas","orcid":"https://orcid.org/0000-0002-8312-0694"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Evangelos Kanoulas","raw_affiliation_strings":["University of Sheffield, Sheffield, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076265623","display_name":"Emine Y\u0131lmaz","orcid":"https://orcid.org/0000-0002-3434-8932"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Emine Yilmaz","raw_affiliation_strings":["Microsoft Research Cambridge, Cambridge, United Kingdom","Microsoft Research Cambridge, Cambridge, United Kingdom#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Cambridge, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research Cambridge, Cambridge, United Kingdom#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"903","last_page":"903"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.8711000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.8711000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7116767168045044},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.6899092197418213},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6595587134361267},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6446337103843689},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5877589583396912},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5007169246673584},{"id":"https://openalex.org/keywords/web-page","display_name":"Web page","score":0.45236441493034363},{"id":"https://openalex.org/keywords/government","display_name":"Government (linguistics)","score":0.4509236514568329},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.43452197313308716},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.411123126745224},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3526124954223633},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.17286154627799988},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.11058348417282104},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.07855913043022156}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7116767168045044},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.6899092197418213},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6595587134361267},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6446337103843689},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5877589583396912},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5007169246673584},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.45236441493034363},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.4509236514568329},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.43452197313308716},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.411123126745224},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3526124954223633},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.17286154627799988},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.11058348417282104},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.07855913043022156},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/1835449.1835675","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1835449.1835675","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:1421696","is_oa":false,"landing_page_url":"http://discovery.ucl.ac.uk/1421696/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"In: Chen, HH and Efthimiadis, EN and Savoy, J and Crestani, F and MarchandMaillet, S, (eds.) (Proceedings) 33rd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. (pp. p. 903). ASSOC COMPUTING MACHINERY (2010)","raw_type":"Proceedings paper"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.5}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2953234277","https://openalex.org/W2626256601","https://openalex.org/W147410782","https://openalex.org/W2900413183","https://openalex.org/W4390975304","https://openalex.org/W3022252430","https://openalex.org/W4287804464","https://openalex.org/W3103989898","https://openalex.org/W3211292372","https://openalex.org/W803346624"],"abstract_inverted_index":{"Search":[0],"corpora":[1],"are":[2,107],"growing":[3],"larger":[4],"and":[5,59],"larger:":[6],"over":[7,100],"the":[8,12,19,25,29,41,48,90,96,101],"last":[9],"10":[10],"years,":[11],"IR":[13],"research":[14],"community":[15],"has":[16],"moved":[17],"from":[18],"several":[20],"hundred":[21],"thousand":[22],"documents":[23,64],"on":[24],"TREC":[26],"disks":[27],"to":[28,40,65,73,76],"tens":[30],"of":[31,33,38,55],"millions":[32],"U.S.":[34],"government":[35],"web":[36,45],"pages":[37,46],"GOV2":[39],"one":[42],"billion":[43],"general-interest":[44],"in":[47,86,93,105],"new":[49,78],"ClueWeb09":[50],"collection.":[51],"But":[52],"traditional":[53],"means":[54],"acquiring":[56],"relevance":[57],"judgments":[58],"evaluating":[60],"-":[61,69],"e.g.":[62],"pooling":[63],"calculate":[66],"average":[67],"precision":[68],"do":[70],"not":[71],"seem":[72],"scale":[74],"well":[75],"these":[77],"large":[79],"collections.":[80],"They":[81],"require":[82],"substantially":[83],"more":[84],"cost":[85,98],"human":[87],"assessments":[88],"for":[89],"same":[91],"reliability":[92],"evaluation;":[94],"if":[95],"additional":[97],"goes":[99],"assessing":[102],"budget,":[103],"errors":[104],"evaluation":[106],"inevitable.":[108]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}