{"id":"https://openalex.org/W2740136032","doi":"https://doi.org/10.1145/3077136.3080644","title":"On the Reusability of \"Living Labs\" Test Collections","display_name":"On the Reusability of \"Living Labs\" Test Collections","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2740136032","doi":"https://doi.org/10.1145/3077136.3080644","mag":"2740136032"},"language":"en","primary_location":{"id":"doi:10.1145/3077136.3080644","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080644","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043679408","display_name":"Luchen Tan","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Luchen Tan","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080828705","display_name":"Gaurav Baruah","orcid":"https://orcid.org/0000-0002-4992-0524"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Gaurav Baruah","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5043679408"],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":1.7585,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.8691238,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"793","last_page":"796"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14147","display_name":"Innovative Approaches in Technology and Social Development","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1405","display_name":"Management of Technology and Innovation"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14147","display_name":"Innovative Approaches in Technology and Social Development","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1405","display_name":"Management of Technology and Innovation"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9510999917984009,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reusability","display_name":"Reusability","score":0.8731701374053955},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6408261656761169},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6225877404212952},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3404393792152405},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.32368898391723633},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.11487957835197449},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.05817693471908569},{"id":"https://openalex.org/keywords/ecology","display_name":"Ecology","score":0.05780375003814697}],"concepts":[{"id":"https://openalex.org/C137981799","wikidata":"https://www.wikidata.org/wiki/Q1369184","display_name":"Reusability","level":3,"score":0.8731701374053955},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6408261656761169},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6225877404212952},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3404393792152405},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.32368898391723633},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.11487957835197449},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.05817693471908569},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.05780375003814697}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3077136.3080644","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080644","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.5400000214576721}],"awards":[],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W18860330","https://openalex.org/W1554394657","https://openalex.org/W1798038286","https://openalex.org/W1982530130","https://openalex.org/W2053100920","https://openalex.org/W2069003154","https://openalex.org/W2075893676","https://openalex.org/W2102698119","https://openalex.org/W2124108136","https://openalex.org/W2135046706","https://openalex.org/W2156186232","https://openalex.org/W2271209846","https://openalex.org/W2338920854"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W4282938614","https://openalex.org/W2906929912","https://openalex.org/W2099755394","https://openalex.org/W2111589395","https://openalex.org/W2155486208","https://openalex.org/W2753086708","https://openalex.org/W1977071681","https://openalex.org/W2157191986","https://openalex.org/W2157615579"],"abstract_inverted_index":{"Information":[0],"retrieval":[1,31],"test":[2,26,57,122],"collections":[3,27,58,123],"are":[4],"typically":[5],"built":[6,124],"using":[7],"data":[8,85],"from":[9,60,87,125],"large-scale":[10],"evaluations":[11],"in":[12,47,128,141],"international":[13],"forums":[14],"such":[15,143],"as":[16],"TREC,":[17],"CLEF,":[18],"and":[19,94,130],"NTCIR.":[20],"Previous":[21],"validation":[22],"studies":[23],"on":[24,65],"pool-based":[25],"for":[28,138],"ad":[29],"hoc":[30],"have":[32],"examined":[33],"their":[34],"reusability":[35,55,120],"to":[36,102,110],"accurately":[37],"assess":[38],"the":[39,48,54,88,119,132,136],"effectiveness":[40],"of":[41,56,67,82,121],"systems":[42],"that":[43,96],"did":[44],"not":[45,71,100],"participate":[46],"original":[49],"evaluation.":[50],"To":[51],"our":[52],"knowledge,":[53],"derived":[59,86],"\"living":[61],"labs\"":[62],"evaluations,":[63],"based":[64],"logs":[66],"user":[68],"activity,":[69],"has":[70],"been":[72],"explored.":[73],"In":[74],"this":[75,106],"paper,":[76],"we":[77],"performed":[78],"a":[79],"\"leave-one-out\"":[80],"analysis":[81],"human":[83],"judgment":[84],"TREC":[89],"2016":[90],"Real-Time":[91],"Summarization":[92],"Track":[93],"show":[95],"those":[97],"judgments":[98],"do":[99],"appear":[101],"be":[103],"reusable.":[104],"While":[105],"finding":[107],"is":[108],"limited":[109],"one":[111],"specific":[112],"evaluation,":[113],"it":[114],"does":[115],"call":[116],"into":[117],"question":[118],"living":[126],"labs":[127],"general,":[129],"at":[131],"very":[133],"least":[134],"suggests":[135],"need":[137],"additional":[139],"work":[140],"validating":[142],"experimental":[144],"instruments.":[145]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2017,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}