{"id":"https://openalex.org/W2041889759","doi":"https://doi.org/10.1145/2641383.2641388","title":"Report on the CIKM workshop on living labs for information retrieval evaluation","display_name":"Report on the CIKM workshop on living labs for information retrieval evaluation","publication_year":2014,"publication_date":"2014-06-26","ids":{"openalex":"https://openalex.org/W2041889759","doi":"https://doi.org/10.1145/2641383.2641388","mag":"2041889759"},"language":"en","primary_location":{"id":"doi:10.1145/2641383.2641388","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2641383.2641388","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://doras.dcu.ie/20108/","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059926999","display_name":"Krisztian Balog","orcid":"https://orcid.org/0000-0003-2762-721X"},"institutions":[{"id":"https://openalex.org/I92008406","display_name":"University of Stavanger","ror":"https://ror.org/02qte9q33","country_code":"NO","type":"education","lineage":["https://openalex.org/I92008406"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Krisztian Balog","raw_affiliation_strings":["University of Stavanger, Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Stavanger, Norway","institution_ids":["https://openalex.org/I92008406"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041921674","display_name":"David Elsweiler","orcid":"https://orcid.org/0000-0002-5791-0641"},"institutions":[{"id":"https://openalex.org/I60668342","display_name":"University of Regensburg","ror":"https://ror.org/01eezs655","country_code":"DE","type":"education","lineage":["https://openalex.org/I60668342"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"David Elsweiler","raw_affiliation_strings":["University of Regensburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Regensburg, Germany","institution_ids":["https://openalex.org/I60668342"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055639036","display_name":"Evangelos Kanoulas","orcid":"https://orcid.org/0000-0002-8312-0694"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH","US"],"is_corresponding":false,"raw_author_name":"Evangelos Kanoulas","raw_affiliation_strings":["Google Inc., Switzerland","Google Inc, Switzerland#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Inc., Switzerland","institution_ids":["https://openalex.org/I4210100430"]},{"raw_affiliation_string":"Google Inc, Switzerland#TAB#","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031680216","display_name":"Liadh Kelly","orcid":"https://orcid.org/0000-0003-1131-5238"},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Liadh Kelly","raw_affiliation_strings":["Dublin City University, Ireland","Dublin, City University, Ireland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dublin City University, Ireland","institution_ids":["https://openalex.org/I42934936"]},{"raw_affiliation_string":"Dublin, City University, Ireland","institution_ids":["https://openalex.org/I42934936"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049222552","display_name":"Mark D. Smucker","orcid":"https://orcid.org/0000-0003-4968-6405"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mark D. Smucker","raw_affiliation_strings":["University of Waterloo, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":6,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.8416,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.96232534,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"48","issue":"1","first_page":"21","last_page":"28"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9803000092506409,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9803000092506409,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14147","display_name":"Innovative Approaches in Technology and Social Development","score":0.974399983882904,"subfield":{"id":"https://openalex.org/subfields/1405","display_name":"Management of Technology and Innovation"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9699000120162964,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9176690578460693},{"id":"https://openalex.org/keywords/session","display_name":"Session (web analytics)","score":0.7846695780754089},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.7019904851913452},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.42331740260124207},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.36221498250961304},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3592785596847534}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9176690578460693},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.7846695780754089},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.7019904851913452},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.42331740260124207},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.36221498250961304},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3592785596847534},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2641383.2641388","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2641383.2641388","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},{"id":"pmh:oai:doras.dcu.ie:20108","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401511","display_name":"Dublin City University Open Access Institutional Repository (Dublin City University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I42934936","host_organization_name":"Dublin City University","host_organization_lineage":["https://openalex.org/I42934936"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Article"},{"id":"pmh:http://www.rian.ie/98274/","is_oa":true,"landing_page_url":"http://doras.dcu.ie/20108/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400033","display_name":"Arrow@dit (Dublin Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I115570527","host_organization_name":"Dublin Institute of Technology","host_organization_lineage":["https://openalex.org/I115570527"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Balog, Krisztian, Elsweiler, David, Kanoulas, Evangelos, Kelly, Liadh ORCID: 0000-0003-1131-5238 <https://orcid.org/0000-0003-1131-5238> and Smucker, Mark (2014) Report on the CIKM workshop on living labs for information retrieval evaluation. ACM SIGIR Forum, 48 (1). pp. 21-28. ISSN 0163-5840","raw_type":"Other"}],"best_oa_location":{"id":"pmh:http://www.rian.ie/98274/","is_oa":true,"landing_page_url":"http://doras.dcu.ie/20108/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400033","display_name":"Arrow@dit (Dublin Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I115570527","host_organization_name":"Dublin Institute of Technology","host_organization_lineage":["https://openalex.org/I115570527"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Balog, Krisztian, Elsweiler, David, Kanoulas, Evangelos, Kelly, Liadh ORCID: 0000-0003-1131-5238 <https://orcid.org/0000-0003-1131-5238> and Smucker, Mark (2014) Report on the CIKM workshop on living labs for information retrieval evaluation. ACM SIGIR Forum, 48 (1). pp. 21-28. ISSN 0163-5840","raw_type":"Other"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W1257366642","https://openalex.org/W2015770094","https://openalex.org/W2106731713","https://openalex.org/W2189233738","https://openalex.org/W3125695435","https://openalex.org/W6603887595"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W4296749040","https://openalex.org/W4230197055","https://openalex.org/W621808327","https://openalex.org/W644007644","https://openalex.org/W178231042","https://openalex.org/W2360028903","https://openalex.org/W4280543773","https://openalex.org/W3012257603","https://openalex.org/W3177475962"],"abstract_inverted_index":{"Evaluation":[0],"is":[1],"a":[2,15,27,74],"central":[3],"aspect":[4],"of":[5,62,82,87],"information":[6],"retrieval":[7],"(IR)":[8],"research.":[9],"In":[10],"the":[11,83,88,92],"past":[12],"few":[13],"years,":[14],"new":[16],"evaluation":[17,47],"methodology":[18],"known":[19],"as":[20,26],"living":[21],"labs":[22],"has":[23],"been":[24],"proposed":[25],"way":[28],"for":[29,45],"researchers":[30],"to":[31,34],"be":[32],"able":[33],"perform":[35],"in-situ":[36],"evaluation.":[37],"The":[38,59],"first":[39],"CIKM":[40],"workshop":[41,60,89],"on":[42,51],"Living":[43],"Labs":[44],"IR":[46],"(LL'13)":[48],"was":[49],"held":[50],"1st":[52],"November":[53],"2013":[54],"in":[55],"San":[56],"Francisco,":[57],"USA.":[58],"consisted":[61],"an":[63,80],"industrial":[64],"keynote,":[65],"four":[66],"oral":[67],"paper":[68],"presentations,":[69,72],"three":[70],"demo":[71],"and":[73,85,90],"discussion":[75],"session.":[76],"This":[77],"report":[78],"presents":[79],"overview":[81],"scope":[84],"contents":[86],"outlines":[91],"major":[93],"outcomes.":[94]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":3}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}