{"id":"https://openalex.org/W2106777568","doi":"https://doi.org/10.1145/1670564.1670567","title":"Report on the SIGIR 2009 workshop on the future of IR evaluation","display_name":"Report on the SIGIR 2009 workshop on the future of IR evaluation","publication_year":2009,"publication_date":"2009-12-14","ids":{"openalex":"https://openalex.org/W2106777568","doi":"https://doi.org/10.1145/1670564.1670567","mag":"2106777568"},"language":"en","primary_location":{"id":"doi:10.1145/1670564.1670567","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1670564.1670567","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044511901","display_name":"Jaap Kamps","orcid":"https://orcid.org/0000-0002-6614-0087"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jaap Kamps","raw_affiliation_strings":["University of Amsterdam, The Netherlands &amp; INEX","University of Amsterdam, The Netherlands & INEX#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam, The Netherlands &amp; INEX","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"University of Amsterdam, The Netherlands & INEX#TAB#","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045420626","display_name":"Shlomo Geva","orcid":"https://orcid.org/0000-0003-1340-2802"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shlomo Geva","raw_affiliation_strings":["QUT, Australia &amp; INEX"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"QUT, Australia &amp; INEX","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036058339","display_name":"C Peters","orcid":null},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Carol Peters","raw_affiliation_strings":["ISTI-CNR, Italy &amp; CLEF","ISTI-CNR, Italy & CLEF#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Italy &amp; CLEF","institution_ids":["https://openalex.org/I122991210"]},{"raw_affiliation_string":"ISTI-CNR, Italy & CLEF#TAB#","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Microsoft Research Asia, China &amp; NTCIR","Microsoft Research Asia, China & NTCIR#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia, China &amp; NTCIR","institution_ids":["https://openalex.org/I4210113369"]},{"raw_affiliation_string":"Microsoft Research Asia, China & NTCIR#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050903724","display_name":"Andrew Trotman","orcid":"https://orcid.org/0000-0003-1253-7123"},"institutions":[{"id":"https://openalex.org/I80281795","display_name":"University of Otago","ror":"https://ror.org/01jmxt844","country_code":"NZ","type":"education","lineage":["https://openalex.org/I80281795"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Andrew Trotman","raw_affiliation_strings":["University of Otago, New Zealand &amp; INEX"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Otago, New Zealand &amp; INEX","institution_ids":["https://openalex.org/I80281795"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028019543","display_name":"Ellen M. Voorhees","orcid":"https://orcid.org/0000-0002-5658-2308"},"institutions":[{"id":"https://openalex.org/I1321296531","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416","country_code":"US","type":"funder","lineage":["https://openalex.org/I1321296531","https://openalex.org/I1343035065"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ellen Voorhees","raw_affiliation_strings":["NIST, USA &amp; TREC/TAC","NIST, USA & TREC/TAC#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NIST, USA &amp; TREC/TAC","institution_ids":[]},{"raw_affiliation_string":"NIST, USA & TREC/TAC#TAB#","institution_ids":["https://openalex.org/I1321296531"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.5183,"has_fulltext":false,"cited_by_count":24,"citation_normalized_percentile":{"value":0.97777338,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"43","issue":"2","first_page":"13","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9775999784469604,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8853977918624878},{"id":"https://openalex.org/keywords/session","display_name":"Session (web analytics)","score":0.736946702003479},{"id":"https://openalex.org/keywords/breakout","display_name":"Breakout","score":0.6974431276321411},{"id":"https://openalex.org/keywords/library-science","display_name":"Library science","score":0.36235511302948},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3244176506996155},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.24602442979812622}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8853977918624878},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.736946702003479},{"id":"https://openalex.org/C2778091849","wikidata":"https://www.wikidata.org/wiki/Q4959649","display_name":"Breakout","level":2,"score":0.6974431276321411},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.36235511302948},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3244176506996155},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.24602442979812622},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/1670564.1670567","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1670564.1670567","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/84c1540e-a9b7-4f47-98d7-80939b1cb444","is_oa":false,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/report-on-the-sigir-2009-workshop-on-the-future-ir-evaluation(84c1540e-a9b7-4f47-98d7-80939b1cb444).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Kamps, J, Geva, S, Peters, C, Sakai, T, Trotman, A & Voorhees, E 2009, 'Report on the SIGIR 2009 Workshop on the Future IR Evaluation', SIGIR Forum, vol. 43, no. 2, pp. 13-23. < http://doi.acm.org/10.1145/1670564.1670567 >","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:dare.uva.nl:publications/84c1540e-a9b7-4f47-98d7-80939b1cb444","is_oa":false,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/report-on-the-sigir-2009-workshop-on-the-future-ir-evaluation(84c1540e-a9b7-4f47-98d7-80939b1cb444).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Kamps, J, Geva, S, Peters, C, Sakai, T, Trotman, A & Voorhees, E 2009, 'Report on the SIGIR 2009 Workshop on the Future IR Evaluation', SIGIR Forum, vol. 43, no. 2, pp. 13-23. < http://doi.acm.org/10.1145/1670564.1670567 >","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.44999998807907104,"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W209864582","https://openalex.org/W646687403","https://openalex.org/W2023166903","https://openalex.org/W2052201263","https://openalex.org/W2070702247","https://openalex.org/W2106731713","https://openalex.org/W2113557293","https://openalex.org/W2122070397","https://openalex.org/W2129978847","https://openalex.org/W2130955505","https://openalex.org/W2138107722","https://openalex.org/W2144353159","https://openalex.org/W2153292911","https://openalex.org/W2153636676","https://openalex.org/W2155513234","https://openalex.org/W2156415649","https://openalex.org/W2569387732","https://openalex.org/W2596051187","https://openalex.org/W2778795319","https://openalex.org/W2981103634","https://openalex.org/W4285719527","https://openalex.org/W6679353441","https://openalex.org/W6682573409","https://openalex.org/W6731279569"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2494799991","https://openalex.org/W2369586047","https://openalex.org/W2360720760","https://openalex.org/W2033715836","https://openalex.org/W2380756105","https://openalex.org/W2383313205","https://openalex.org/W2354342024","https://openalex.org/W2370950893","https://openalex.org/W2362842490"],"abstract_inverted_index":{"On":[0],"July":[1],"23,":[2],"2009":[3],"the":[4,8,45,52,55,59],"SIGIR":[5,18],"Workshop":[6],"on":[7],"Future":[9],"of":[10,17,24,44,54],"IR":[11],"Evaluation":[12],"was":[13],"held":[14],"as":[15],"part":[16],"in":[19],"Boston.":[20],"The":[21],"program":[22],"consisted":[23],"four":[25,36],"keynotes,":[26],"a":[27,40],"boaster":[28],"and":[29,39,57],"poster":[30],"session":[31],"with":[32],"20":[33],"accepted":[34],"papers,":[35],"breakout":[37,46],"groups,":[38],"final":[41],"panel":[42],"discussion":[43],"group":[47],"reports.":[48],"This":[49],"report":[50],"outlines":[51],"events":[53],"workshop":[56],"summarizes":[58],"major":[60],"outcomes.":[61]},"counts_by_year":[{"year":2021,"cited_by_count":3},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":5}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}