{"id":"https://openalex.org/W4412394816","doi":"https://doi.org/10.1145/3726302.3730308","title":"Variations in Relevance Judgments and the Shelf Life of Test Collections","display_name":"Variations in Relevance Judgments and the Shelf Life of Test Collections","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412394816","doi":"https://doi.org/10.1145/3726302.3730308"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730308","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730308","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730308","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730308","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075770550","display_name":"Andrew Parry","orcid":"https://orcid.org/0000-0001-5446-8328"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Andrew Parry","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0001-5446-8328","affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068616222","display_name":"Maik Fr\u00f6be","orcid":"https://orcid.org/0000-0002-1003-981X"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Maik Fr\u00f6be","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"raw_orcid":"https://orcid.org/0000-0002-1003-981X","affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075371460","display_name":"Harrisen Scells","orcid":"https://orcid.org/0000-0001-9578-7157"},"institutions":[{"id":"https://openalex.org/I8087733","display_name":"University of T\u00fcbingen","ror":"https://ror.org/03a1kwz48","country_code":"DE","type":"education","lineage":["https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Harrisen Scells","raw_affiliation_strings":["University of T\u00fcbingen, Germany, Germany"],"raw_orcid":"https://orcid.org/0000-0001-9578-7157","affiliations":[{"raw_affiliation_string":"University of T\u00fcbingen, Germany, Germany","institution_ids":["https://openalex.org/I8087733"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020636685","display_name":"Ferdinand Schlatt","orcid":"https://orcid.org/0000-0002-6032-909X"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ferdinand Schlatt","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"raw_orcid":"https://orcid.org/0000-0002-6032-909X","affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079246354","display_name":"Guglielmo Faggioli","orcid":"https://orcid.org/0000-0002-5070-2049"},"institutions":[{"id":"https://openalex.org/I138689650","display_name":"University of Padua","ror":"https://ror.org/00240q980","country_code":"IT","type":"education","lineage":["https://openalex.org/I138689650"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Guglielmo Faggioli","raw_affiliation_strings":["University of Padua, Padua, Italy"],"raw_orcid":"https://orcid.org/0000-0002-5070-2049","affiliations":[{"raw_affiliation_string":"University of Padua, Padua, Italy","institution_ids":["https://openalex.org/I138689650"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077001732","display_name":"Saber Zerhoudi","orcid":"https://orcid.org/0000-0003-2259-0462"},"institutions":[{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Saber Zerhoudi","raw_affiliation_strings":["Universit\u00e4t Passau, Passau, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2259-0462","affiliations":[{"raw_affiliation_string":"Universit\u00e4t Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014199889","display_name":"Sean MacAvaney","orcid":"https://orcid.org/0000-0002-8914-2659"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sean MacAvaney","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-8914-2659","affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062016266","display_name":"Eugene Yang","orcid":"https://orcid.org/0000-0002-0051-1535"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eugene Yang","raw_affiliation_strings":["Johns Hopkins University, Baltimore, USA"],"raw_orcid":"https://orcid.org/0000-0002-0051-1535","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, USA","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3387","last_page":"3397"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9829999804496765,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6865158081054688},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6590019464492798},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5770243406295776},{"id":"https://openalex.org/keywords/off-the-shelf","display_name":"Off the shelf","score":0.43091005086898804},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.37481552362442017},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35114747285842896},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.09140247106552124},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.08907654881477356},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.059831976890563965}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6865158081054688},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6590019464492798},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5770243406295776},{"id":"https://openalex.org/C2982819384","wikidata":"https://www.wikidata.org/wiki/Q7078910","display_name":"Off the shelf","level":2,"score":0.43091005086898804},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.37481552362442017},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35114747285842896},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.09140247106552124},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.08907654881477356},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.059831976890563965},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3726302.3730308","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730308","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730308","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.gla.ac.uk:352744","is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/69236.html>,","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Conference Proceedings"},{"id":"pmh:oai:arXiv.org:2502.20937","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.20937","pdf_url":"https://arxiv.org/pdf/2502.20937","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730308","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730308","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730308","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Climate action","score":0.5199999809265137,"id":"https://metadata.un.org/sdg/13"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412394816.pdf","grobid_xml":"https://content.openalex.org/works/W4412394816.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W125362197","https://openalex.org/W1981065051","https://openalex.org/W2001142465","https://openalex.org/W2006996936","https://openalex.org/W2009571180","https://openalex.org/W2052989395","https://openalex.org/W2069870183","https://openalex.org/W2075893676","https://openalex.org/W2077324407","https://openalex.org/W2113640060","https://openalex.org/W2138680231","https://openalex.org/W2160892561","https://openalex.org/W2799108077","https://openalex.org/W2968287414","https://openalex.org/W3021397474","https://openalex.org/W3099700870","https://openalex.org/W3100107515","https://openalex.org/W3154280800","https://openalex.org/W3155375847","https://openalex.org/W3196248941","https://openalex.org/W4226391416","https://openalex.org/W4284664419","https://openalex.org/W4284691825","https://openalex.org/W4284704763","https://openalex.org/W4327644104","https://openalex.org/W4385571915","https://openalex.org/W4389469970","https://openalex.org/W4389523765","https://openalex.org/W4391285994","https://openalex.org/W4393096553","https://openalex.org/W4401381512"],"related_works":["https://openalex.org/W2085384747","https://openalex.org/W2088166309","https://openalex.org/W1891216533","https://openalex.org/W4312133475","https://openalex.org/W1967370444","https://openalex.org/W2150136235","https://openalex.org/W2085215424","https://openalex.org/W2088814244","https://openalex.org/W2157408137","https://openalex.org/W4312252109"],"abstract_inverted_index":{"The":[0],"fundamental":[1],"property":[2],"of":[3,36,48,117,129,174],"Cranfield-style":[4],"evaluations,":[5],"that":[6,80,146,157,180],"system":[7,70,152],"rankings":[8],"are":[9,42,86],"stable":[10],"even":[11],"when":[12],"assessors":[13],"disagree":[14],"on":[15,21],"individual":[16],"relevance":[17,127,165],"decisions,":[18],"was":[19],"validated":[20],"traditional":[22],"test":[23,38,84,104,181],"collections.":[24],"However,":[25,154],"the":[26,34,77,81,126,130,141,172],"paradigm":[27],"shift":[28],"towards":[29],"neural":[30,142],"retrieval":[31,143],"models":[32,159],"affected":[33],"characteristics":[35],"modern":[37,83],"collections,":[39],"e.g.,":[40],"documents":[41],"short,":[43],"judged":[44],"with":[45,162],"four":[46],"grades":[47],"relevance,":[49],"and":[50,124,167],"information":[51,98],"needs":[52],"have":[53,169],"no":[54],"descriptions":[55],"or":[56],"narratives.":[57],"Under":[58],"these":[59],"changes,":[60],"it":[61],"is":[62],"unclear":[63],"whether":[64],"assessor":[65,147],"disagreement":[66,148],"remains":[67],"negligible":[68],"for":[69,103],"comparisons.":[71],"We":[72,119,135],"investigate":[73],"this":[74],"aspect":[75],"under":[76],"additional":[78],"condition":[79],"few":[82],"collections":[85,105,182],"heavily":[87],"re-used.":[88],"Given":[89],"more":[90],"possible":[91],"query":[92],"interpretations":[93],"due":[94],"to":[95,113],"less":[96],"formalized":[97],"needs,":[99],"an":[100],"''expiration":[101],"date''":[102],"might":[106],"be":[107],"needed":[108],"if":[109],"top-effectiveness":[110],"requires":[111],"overfitting":[112],"a":[114,121],"single":[115],"interpretation":[116],"relevance.":[118],"run":[120],"reproducibility":[122],"study":[123],"re-annotate":[125],"judgments":[128],"2019~TREC":[131],"Deep":[132],"Learning":[133],"track.":[134],"can":[136,183],"reproduce":[137],"prior":[138],"work":[139],"in":[140],"setting,":[144],"showing":[145],"does":[149],"not":[150],"affect":[151],"rankings.":[153],"we":[155],"observe":[156],"some":[158,168],"substantially":[160],"degrade":[161],"our":[163],"new":[164],"judgments,":[166],"already":[170],"reached":[171],"effectiveness":[173],"humans":[175],"as":[176],"rankers,":[177],"providing":[178],"evidence":[179],"expire.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}