{"id":"https://openalex.org/W2208056883","doi":"https://doi.org/10.1007/s10791-015-9276-9","title":"The strange case of reproducibility versus representativeness in contextual suggestion test collections","display_name":"The strange case of reproducibility versus representativeness in contextual suggestion test collections","publication_year":2015,"publication_date":"2015-12-27","ids":{"openalex":"https://openalex.org/W2208056883","doi":"https://doi.org/10.1007/s10791-015-9276-9","mag":"2208056883"},"language":"en","primary_location":{"id":"doi:10.1007/s10791-015-9276-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10791-015-9276-9","pdf_url":"https://link.springer.com/content/pdf/10.1007%2Fs10791-015-9276-9.pdf","source":{"id":"https://openalex.org/S79460864","display_name":"Information Retrieval","issn_l":"1386-4564","issn":["1386-4564","1573-7659"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Retrieval Journal","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007%2Fs10791-015-9276-9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046069889","display_name":"Thaer Samar","orcid":"https://orcid.org/0000-0002-9872-5258"},"institutions":[{"id":"https://openalex.org/I1341640284","display_name":"Centrum Wiskunde & Informatica","ror":"https://ror.org/00x7ekv49","country_code":"NL","type":"facility","lineage":["https://openalex.org/I1341640284","https://openalex.org/I2800991832","https://openalex.org/I4405262988"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Thaer Samar","raw_affiliation_strings":["Centrum Wiskunde & Informatica, Amsterdam, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centrum Wiskunde & Informatica, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I1341640284"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029757626","display_name":"Alejandro Bellog\u00edn","orcid":"https://orcid.org/0000-0001-6368-2510"},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Alejandro Bellog\u00edn","raw_affiliation_strings":["Universidad Aut\u00f3noma de Madrid, Madrid, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universidad Aut\u00f3noma de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I63634437"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060259737","display_name":"Arjen P. de Vries","orcid":"https://orcid.org/0000-0002-2888-4202"},"institutions":[{"id":"https://openalex.org/I1341640284","display_name":"Centrum Wiskunde & Informatica","ror":"https://ror.org/00x7ekv49","country_code":"NL","type":"facility","lineage":["https://openalex.org/I1341640284","https://openalex.org/I2800991832","https://openalex.org/I4405262988"]},{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Arjen P. de Vries","raw_affiliation_strings":["Centrum Wiskunde & Informatica, Amsterdam, The Netherlands","Delft University of Technology, Delft, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centrum Wiskunde & Informatica, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I1341640284"]},{"raw_affiliation_string":"Delft University of Technology, Delft, The Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5060259737"],"corresponding_institution_ids":["https://openalex.org/I1341640284","https://openalex.org/I98358874"],"apc_list":{"value":1520,"currency":"USD","value_usd":1520},"apc_paid":{"value":1520,"currency":"USD","value_usd":1520},"fwci":2.1971,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.91713107,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"19","issue":"3","first_page":"230","last_page":"255"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/representativeness-heuristic","display_name":"Representativeness heuristic","score":0.7796940803527832},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7554275989532471},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6839643716812134},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6718142032623291},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6674957275390625},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5806952714920044},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.5699052214622498},{"id":"https://openalex.org/keywords/web-modeling","display_name":"Web modeling","score":0.43000078201293945},{"id":"https://openalex.org/keywords/web-page","display_name":"Web page","score":0.4268742501735687},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.41595399379730225},{"id":"https://openalex.org/keywords/web-application","display_name":"Web application","score":0.4137565791606903},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.332850843667984},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08846735954284668}],"concepts":[{"id":"https://openalex.org/C37381756","wikidata":"https://www.wikidata.org/wiki/Q20203288","display_name":"Representativeness heuristic","level":2,"score":0.7796940803527832},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7554275989532471},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6839643716812134},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6718142032623291},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6674957275390625},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5806952714920044},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.5699052214622498},{"id":"https://openalex.org/C130436687","wikidata":"https://www.wikidata.org/wiki/Q7978591","display_name":"Web modeling","level":3,"score":0.43000078201293945},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.4268742501735687},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.41595399379730225},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.4137565791606903},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.332850843667984},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08846735954284668},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1007/s10791-015-9276-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10791-015-9276-9","pdf_url":"https://link.springer.com/content/pdf/10.1007%2Fs10791-015-9276-9.pdf","source":{"id":"https://openalex.org/S79460864","display_name":"Information Retrieval","issn_l":"1386-4564","issn":["1386-4564","1573-7659"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Retrieval Journal","raw_type":"journal-article"},{"id":"pmh:oai:www.tdx.cat:10486/674485","is_oa":false,"landing_page_url":"http://hdl.handle.net/10486/674485","pdf_url":null,"source":{"id":"https://openalex.org/S4306401111","display_name":"Tesis Doctorals en Xarxa (Consorci de Serveis Universitaris de Catalunya)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210090028","host_organization_name":"Consorci de Serveis Universitaris de Catalunya","host_organization_lineage":["https://openalex.org/I4210090028"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:cwi.nl:25193","is_oa":true,"landing_page_url":"https://ir.cwi.nl/pub/25193","pdf_url":"https://ir.cwi.nl/pub/25193/IRTR3007.pdf","source":{"id":"https://openalex.org/S7407055335","display_name":"Centrum Wiskunde & Informatica (CWI), the national research institute for mathematics and computer science in the Netherlands","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information Retrieval Journal vol. 19 no. 3, pp. 230-255","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1007/s10791-015-9276-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10791-015-9276-9","pdf_url":"https://link.springer.com/content/pdf/10.1007%2Fs10791-015-9276-9.pdf","source":{"id":"https://openalex.org/S79460864","display_name":"Information Retrieval","issn_l":"1386-4564","issn":["1386-4564","1573-7659"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Retrieval Journal","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.550000011920929}],"awards":[{"id":"https://openalex.org/G1563932804","display_name":null,"funder_award_id":"TIN2013-47090-C3-2","funder_id":"https://openalex.org/F4320322930","funder_display_name":"Ministerio de Ciencia e Innovaci\u00f3n"},{"id":"https://openalex.org/G2544847368","display_name":null,"funder_award_id":"640.005.001","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320322930","display_name":"Ministerio de Ciencia e Innovaci\u00f3n","ror":"https://ror.org/034900433"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2208056883.pdf","grobid_xml":"https://content.openalex.org/works/W2208056883.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W36361912","https://openalex.org/W269085259","https://openalex.org/W1560147776","https://openalex.org/W1598759141","https://openalex.org/W1878790362","https://openalex.org/W1987690005","https://openalex.org/W1988144039","https://openalex.org/W1989468977","https://openalex.org/W1994594731","https://openalex.org/W2006124775","https://openalex.org/W2014134732","https://openalex.org/W2035569891","https://openalex.org/W2077066542","https://openalex.org/W2090909419","https://openalex.org/W2094607109","https://openalex.org/W2104708548","https://openalex.org/W2116206254","https://openalex.org/W2124555297","https://openalex.org/W2125055259","https://openalex.org/W2150461528","https://openalex.org/W2151005602","https://openalex.org/W2155106456","https://openalex.org/W2166646729","https://openalex.org/W2185885129","https://openalex.org/W2296693058","https://openalex.org/W2398973937","https://openalex.org/W4205324931","https://openalex.org/W4245605774","https://openalex.org/W6601497724","https://openalex.org/W6686599164"],"related_works":["https://openalex.org/W2137912379","https://openalex.org/W2120222276","https://openalex.org/W3025289787","https://openalex.org/W143181582","https://openalex.org/W2404573736","https://openalex.org/W2036104219","https://openalex.org/W2167278502","https://openalex.org/W2124649262","https://openalex.org/W1986474863","https://openalex.org/W2389737178"],"abstract_inverted_index":{"The":[0,17,38],"most":[1,152],"common":[2],"approach":[3,160,347],"to":[4,32,48,161,194,299],"measuring":[5],"the":[6,45,54,59,66,69,72,75,115,130,143,154,166,198,204,217,225,237,241,253,256,261,273,277,283,291,301,315,340,343,360,370,382,394],"effectiveness":[7,207,373],"of":[8,41,65,121,139,153,175,180,200,220,240,255,276,314,318,321,393],"Information":[9],"Retrieval":[10],"systems":[11,28,131,155,169,211,214],"is":[12],"by":[13,98,103,171],"using":[14],"test":[15],"collections.":[16],"Contextual":[18],"Suggestion":[19],"(CS)":[20],"TREC":[21],"track":[22,43,323],"provides":[23],"an":[24,364],"evaluation":[25,120],"framework":[26],"for":[27,118,197,348],"that":[29,129,132,182,234,325],"recommend":[30],"items":[31],"users":[33],"given":[34],"their":[35,134],"geographical":[36],"context.":[37],"specific":[39],"nature":[40],"this":[42,164],"allows":[44],"participating":[46],"teams":[47],"identify":[49,232,270],"candidate":[50,350],"documents":[51,73,221,233,271,298,328,341,351],"either":[52,94],"from":[53,58,74,111,142,223,260,265,272,352,359],"Open":[55,76,95,144,209,226,242,262,278,326,361,375],"Web":[56,77,96,145,210,227,243,263,279,327,362,376],"or":[57,100,264],"ClueWeb12":[60,79,101,112,176,213,284,292,302,333,353,378],"collection,":[61],"a":[62,148,157,247],"static":[63],"version":[64],"web.":[67,395],"In":[68],"judging":[70],"pool,":[71],"and":[78,212,228,244,252,377],"collection":[80,285,354],"are":[81,177,308],"distinguished.":[82],"Hence,":[83],"each":[84],"system":[85],"submission":[86],"should":[87,113],"be":[88,114],"based":[89,355],"only":[90],"on":[91,137,173,186,356,389],"one":[92],"resource,":[93],"(identified":[97,102],"URLs)":[99],"ids).":[104],"To":[105],"achieve":[106,146,386],"reproducibility,":[107],"ranking":[108],"web":[109,188],"pages":[110],"preferred":[116],"method":[117],"scientific":[119],"CS":[122,322],"systems,":[123,379],"but":[124,286],"it":[125],"has":[126],"been":[127],"found":[128],"build":[133],"suggestion":[135],"algorithms":[136],"top":[138,174],"input":[140],"taken":[141,259],"consistently":[147],"higher":[149],"effectiveness.":[150],"Because":[151],"take":[156],"rather":[158],"similar":[159],"making":[162],"CSs,":[163],"raises":[165],"question":[167],"whether":[168],"built":[170],"researchers":[172],"still":[178],"representative":[179,391],"those":[181],"would":[183],"work":[184],"directly":[185],"industry-strength":[187],"search":[189],"engines.":[190],"Do":[191],"we":[192,231,269,337,385],"need":[193],"sacrifice":[195],"reproducibility":[196],"sake":[199],"representativeness?":[201],"We":[202,295],"study":[203],"difference":[205],"in":[206,282,290,342,372],"between":[208,236,249,374],"through":[215],"analyzing":[216],"relevance":[218,238,250,274,293,303,316],"assessments":[219,239,251,275,317],"identified":[222],"both":[224],"ClueWeb12.":[229,266],"Then,":[230],"overlap":[235],"ClueWeb12,":[245],"observing":[246],"dependency":[248],"source":[254],"document":[257],"being":[258],"After":[267],"that,":[268],"which":[280],"exist":[281,289],"do":[287],"not":[288],"assessments.":[294,304],"use":[296],"these":[297],"expand":[300],"Our":[305],"main":[306],"findings":[307],"twofold.":[309],"First,":[310],"our":[311,346],"empirical":[312],"analysis":[313],"2":[319],"years":[320],"shows":[324],"receive":[329],"better":[330],"ratings":[331],"than":[332],"documents,":[334],"especially":[335],"if":[336],"look":[338],"at":[339,381],"overlap.":[344],"Second,":[345],"selecting":[349],"information":[357],"obtained":[358],"makes":[363],"improvement":[365],"step":[366],"towards":[367],"partially":[368],"bridging":[369],"gap":[371],"while":[380],"same":[383],"time":[384],"reproducible":[387],"results":[388],"well-known":[390],"sample":[392]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}