{"id":"https://openalex.org/W2094607109","doi":"https://doi.org/10.1145/2009916.2010057","title":"Quantifying test collection quality based on the consistency of relevance judgements","display_name":"Quantifying test collection quality based on the consistency of relevance judgements","publication_year":2011,"publication_date":"2011-07-24","ids":{"openalex":"https://openalex.org/W2094607109","doi":"https://doi.org/10.1145/2009916.2010057","mag":"2094607109"},"language":"en","primary_location":{"id":"doi:10.1145/2009916.2010057","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2009916.2010057","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://figshare.com/articles/conference_contribution/Quantifying_test_collection_quality_based_on_the_consistency_of_relevance_judgements/27379491","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014748363","display_name":"Andrew Turpin","orcid":"https://orcid.org/0000-0003-2559-8769"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Andrew Turpin","raw_affiliation_strings":["University of Melbourne, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Melbourne, Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072614956","display_name":"Mark Sanderson","orcid":"https://orcid.org/0000-0003-0487-9609"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mark Sanderson","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":80,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1063","last_page":"1072"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11492","display_name":"Academic integrity and plagiarism","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8519245386123657},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.7499697208404541},{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.7334010601043701},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7006348371505737},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6726173758506775},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6034764051437378},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4863350987434387},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.46486029028892517},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4441056251525879},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.35907262563705444},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.329096257686615},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.27260303497314453},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.0676291286945343}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8519245386123657},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7499697208404541},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.7334010601043701},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7006348371505737},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6726173758506775},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6034764051437378},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4863350987434387},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.46486029028892517},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4441056251525879},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35907262563705444},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.329096257686615},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27260303497314453},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0676291286945343},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/2009916.2010057","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2009916.2010057","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:researchbank.rmit.edu.au:rmit:13971","is_oa":false,"landing_page_url":"http://researchbank.rmit.edu.au/view/rmit:13971","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},{"id":"pmh:oai:researchbank.rmit.edu.au:rmit:14983","is_oa":false,"landing_page_url":"http://researchbank.rmit.edu.au/view/rmit:14983","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},{"id":"pmh:oai:alma.61RMIT_INST:11246625860001341","is_oa":false,"landing_page_url":"http://doi.org/10.1145/2009916.2010057","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:figshare.com:article/27379491","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/Quantifying_test_collection_quality_based_on_the_consistency_of_relevance_judgements/27379491","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference contribution"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/27379491","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/Quantifying_test_collection_quality_based_on_the_consistency_of_relevance_judgements/27379491","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference contribution"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6299999952316284}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W132882894","https://openalex.org/W191231183","https://openalex.org/W1480224957","https://openalex.org/W1588207697","https://openalex.org/W1964488075","https://openalex.org/W1976898639","https://openalex.org/W1987690005","https://openalex.org/W1990190154","https://openalex.org/W1990420106","https://openalex.org/W2008285050","https://openalex.org/W2009543464","https://openalex.org/W2023217403","https://openalex.org/W2037022968","https://openalex.org/W2037140704","https://openalex.org/W2052989395","https://openalex.org/W2057495142","https://openalex.org/W2062236692","https://openalex.org/W2075893676","https://openalex.org/W2105280231","https://openalex.org/W2120308175","https://openalex.org/W2130955505","https://openalex.org/W2160484851","https://openalex.org/W2621280964","https://openalex.org/W2781361141","https://openalex.org/W4205625510","https://openalex.org/W4230511558","https://openalex.org/W4296816479","https://openalex.org/W6628462790"],"related_works":["https://openalex.org/W2503032459","https://openalex.org/W3140222185","https://openalex.org/W2347551783","https://openalex.org/W3024185757","https://openalex.org/W2018097218","https://openalex.org/W1588207697","https://openalex.org/W4252055628","https://openalex.org/W2031812225","https://openalex.org/W2072440657","https://openalex.org/W2781814656"],"abstract_inverted_index":{"Relevance":[0],"assessments":[1],"are":[2,46,127,164],"a":[3,18,27,62,84,113,180],"key":[4],"component":[5],"for":[6],"test":[7,44,189],"collection-based":[8],"evaluation":[9],"of":[10,20,29,36,43,57,65,73,88,110,158,168,187],"information":[11],"retrieval":[12,162],"systems.":[13],"This":[14],"paper":[15],"reports":[16],"on":[17,95,134],"feature":[19],"such":[21],"collections":[22,45,69],"that":[23,100,123,161],"is":[24,75],"used":[25],"as":[26],"form":[28],"ground":[30],"truth":[31],"data":[32],"to":[33,49,77,119,147,152],"allow":[34],"analysis":[35],"human":[37],"assessment":[38,89,143],"error.":[39,90],"A":[40,155],"wide":[41],"range":[42],"retrospectively":[47],"examined":[48,188],"determine":[50],"how":[51],"accurately":[52],"assessors":[53,124],"judge":[54],"the":[55,68,96,108,121,131,159,176,185],"relevance":[56,149,169],"documents.":[58,138],"Our":[59],"results":[60],"demonstrate":[61,99],"high":[63,86],"level":[64,72,87,109],"inconsistency":[66,101],"across":[67,79,184],"studied.":[70],"The":[71],"irregularity":[74],"shown":[76],"vary":[78],"topics,":[80],"with":[81,105],"some":[82],"showing":[83],"very":[85],"We":[91],"investigate":[92],"possible":[93],"influences":[94],"error,":[97],"and":[98,173],"in":[102,112,175],"judging":[103],"increases":[104],"time.":[106],"While":[107],"detail":[111],"topic":[114],"specification":[115],"does":[116],"not":[117],"appear":[118,151],"influence":[120,182],"errors":[122],"make,":[125],"judgements":[126,150,170],"significantly":[128],"affected":[129],"by":[130],"decisions":[132],"made":[133],"previously":[135],"seen":[136],"similar":[137],"Assessors":[139],"also":[140],"display":[141],"an":[142],"inertia.":[144],"Alternate":[145],"approaches":[146],"generating":[148],"reduce":[153],"errors.":[154],"further":[156],"investigation":[157],"way":[160],"systems":[163],"ranked":[165],"using":[166],"sets":[167],"produced":[171],"early":[172],"late":[174],"judgement":[177],"process":[178],"reveals":[179],"consistent":[181],"measured":[183],"majority":[186],"collections.":[190]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":8},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":7},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":9},{"year":2015,"cited_by_count":7},{"year":2014,"cited_by_count":7},{"year":2013,"cited_by_count":6},{"year":2012,"cited_by_count":7}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}