{"id":"https://openalex.org/W2060487963","doi":"https://doi.org/10.1145/1008992.1008997","title":"Scaling IR-system evaluation using term relevance sets","display_name":"Scaling IR-system evaluation using term relevance sets","publication_year":2004,"publication_date":"2004-07-25","ids":{"openalex":"https://openalex.org/W2060487963","doi":"https://doi.org/10.1145/1008992.1008997","mag":"2060487963"},"language":"en","primary_location":{"id":"doi:10.1145/1008992.1008997","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1008992.1008997","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th annual international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002303994","display_name":"Einat Amitay","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Einat Amitay","raw_affiliation_strings":["IBM Haifa Research Lab, Haifa, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Haifa Research Lab, Haifa, Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103324615","display_name":"David Carmel","orcid":"https://orcid.org/0000-0003-1161-7084"},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"David Carmel","raw_affiliation_strings":["IBM Haifa Research Lab, Haifa, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Haifa Research Lab, Haifa, Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071523173","display_name":"Ronny Lempel","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Ronny Lempel","raw_affiliation_strings":["IBM Haifa Research Lab, Haifa, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Haifa Research Lab, Haifa, Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010601302","display_name":"Aya Soffer","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Aya Soffer","raw_affiliation_strings":["IBM Haifa Research Lab, Haifa, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Haifa Research Lab, Haifa, Israel","institution_ids":["https://openalex.org/I4210167297"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210167297"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"10","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8196803331375122},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8108359575271606},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7758438587188721},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.740635335445404},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4444790482521057},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.44155165553092957},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.4251238703727722},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.4173426926136017},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.10709384083747864},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0828542709350586}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8196803331375122},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8108359575271606},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7758438587188721},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.740635335445404},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4444790482521057},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.44155165553092957},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.4251238703727722},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.4173426926136017},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10709384083747864},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0828542709350586},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1008992.1008997","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1008992.1008997","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th annual international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.5}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W55312212","https://openalex.org/W395909215","https://openalex.org/W1546195864","https://openalex.org/W1554394657","https://openalex.org/W1983595289","https://openalex.org/W1993467697","https://openalex.org/W1999991474","https://openalex.org/W2007687650","https://openalex.org/W2027752285","https://openalex.org/W2028513846","https://openalex.org/W2029341294","https://openalex.org/W2053049304","https://openalex.org/W2073722401","https://openalex.org/W2075893676","https://openalex.org/W2082158392","https://openalex.org/W2090805977","https://openalex.org/W2100240666","https://openalex.org/W2136309948","https://openalex.org/W2144200917","https://openalex.org/W2147164982","https://openalex.org/W2151005602","https://openalex.org/W2152565070","https://openalex.org/W2168469656","https://openalex.org/W2169557227","https://openalex.org/W4233760227","https://openalex.org/W4285719527","https://openalex.org/W6602266041","https://openalex.org/W6630516739","https://openalex.org/W6681574511"],"related_works":["https://openalex.org/W2085384747","https://openalex.org/W141820298","https://openalex.org/W2088166309","https://openalex.org/W1891216533","https://openalex.org/W4312133475","https://openalex.org/W2049584446","https://openalex.org/W4238976562","https://openalex.org/W2276587472","https://openalex.org/W2615795876","https://openalex.org/W2049612369"],"abstract_inverted_index":{"This":[0],"paper":[1],"describes":[2],"an":[3,14,103],"evaluation":[4],"method":[5,59,96,121],"based":[6],"on":[7,102,108],"Term":[8],"Relevance":[9],"Sets":[10],"Trels":[11,34],"that":[12,118],"measures":[13],"IR":[15],"system's":[16,100],"quality":[17],"by":[18,28,74],"examining":[19],"the":[20,23,77,92,119],"content":[21],"of":[22,36,39,54],"retrieved":[24],"results":[25],"rather":[26],"than":[27],"looking":[29],"for":[30,45],"pre-specified":[31],"relevant":[32,44],"pages.":[33],"consist":[35],"a":[37,46,52,99],"list":[38,53],"terms":[40],"believed":[41],"to":[42,76,85],"be":[43],"particular":[47],"query":[48],"as":[49,51,68,91],"well":[50],"irrelevant":[55],"terms.":[56],"The":[57],"proposed":[58,120],"does":[60],"not":[61,71],"involve":[62],"any":[63],"document":[64],"relevance":[65],"judgments,":[66],"and":[67],"such":[69,90],"is":[70,122],"adversely":[72],"affected":[73],"changes":[75],"underlying":[78],"collection.":[79],"Therefore,":[80],"it":[81],"can":[82,97],"better":[83],"scale":[84],"very":[86,123],"large,":[87],"dynamic":[88],"collections":[89,109],"Web.":[93],"Moreover,":[94],"this":[95],"evaluate":[98],"effectiveness":[101],"updatable":[104],"\"live\"":[105],"collection,":[106],"or":[107],"derived":[110],"from":[111],"different":[112],"data":[113],"sources.":[114],"Our":[115],"experiments":[116],"show":[117],"highly":[124],"correlated":[125],"with":[126],"official":[127],"TREC":[128],"measures.":[129]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":1},{"year":2012,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}