{"id":"https://openalex.org/W2102500012","doi":"https://doi.org/10.1353/lib.0.0000","title":"Effects of Inconsistent Relevance Judgments on Information Retrieval Test Results: A Historical Perspective","display_name":"Effects of Inconsistent Relevance Judgments on Information Retrieval Test Results: A Historical Perspective","alternate_titles":[],"publication_year":2008,"publication_date":"2008-03-01","ids":{"openalex":"https://openalex.org/W2102500012","doi":"https://doi.org/10.1353/lib.0.0000","mag":"2102500012"},"language":"en","primary_location":{"id":"doi:10.1353/lib.0.0000","doi":"https://doi.org/10.1353/lib.0.0000","is_oa":false,"landing_page_url":"https://doi.org/10.1353/lib.0.0000","pdf_url":null,"source":{"id":"https://openalex.org/S9186059","display_name":"Library trends","issn_l":"0024-2594","issn":["0024-2594","1559-0682"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","jufo-1","norway-1","poland-20","russia-white-list-2"],"host_organization":"https://openalex.org/P4310315678","host_organization_name":"Johns Hopkins University Press","host_organization_lineage":["https://openalex.org/P4310315678","https://openalex.org/P4310316510"],"host_organization_lineage_names":["Johns Hopkins University Press","Johns Hopkins University"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Library Trends","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W2102500012"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ideals.illinois.edu/items/9535/bitstreams/35471/data.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059509085","display_name":"Tefko Sara\u010devi\u0107","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Tefko Saracevic","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5059509085"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.4891,"has_fulltext":true,"cited_by_count":58,"reviewed_by_count":0,"citation_normalized_percentile":{"value":0.96203185,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":86,"max":98},"biblio":{"volume":"56","issue":"4","first_page":"763","last_page":"783"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10712","display_name":"Library Science and Information Literacy","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13673","display_name":"Library Science and Information","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance-assessment","display_name":"relevance assessment","score":0.8650000095367432},{"id":"https://openalex.org/keywords/information-retrieval-evaluation","display_name":"information retrieval evaluation","score":0.9210000038146973},{"id":"https://openalex.org/keywords/test-validity","display_name":"test validity","score":0.6980000138282776},{"id":"https://openalex.org/keywords/user-relevance","display_name":"user relevance","score":0.5950000286102295},{"id":"https://openalex.org/keywords/gold-standard","display_name":"gold standard","score":0.9129999876022339},{"id":"https://openalex.org/keywords/retrieval-performance","display_name":"retrieval effectiveness","score":0.8080000281333923},{"id":"https://openalex.org/keywords/retrieval-failure","display_name":"retrieval failure","score":0.6460000276565552},{"id":"https://openalex.org/keywords/medlars","display_name":"MEDLARS","score":0.8799999952316284},{"id":"https://openalex.org/keywords/lancaster","display_name":"Lancaster","score":0.6460000276565552}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.9403636455535889},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6235237717628479},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6213861107826233},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5657528638839722},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5465377569198608},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5163137316703796},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.4295719265937805},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3220213055610657},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.12154361605644226},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09147065877914429},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.08597654104232788},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1353/lib.0.0000","doi":"https://doi.org/10.1353/lib.0.0000","is_oa":false,"landing_page_url":"https://doi.org/10.1353/lib.0.0000","pdf_url":null,"source":{"id":"https://openalex.org/S9186059","display_name":"Library trends","issn_l":"0024-2594","issn":["0024-2594","1559-0682"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","jufo-1","norway-1","poland-20","russia-white-list-2"],"host_organization":"https://openalex.org/P4310315678","host_organization_name":"Johns Hopkins University Press","host_organization_lineage":["https://openalex.org/P4310315678","https://openalex.org/P4310316510"],"host_organization_lineage_names":["Johns Hopkins University Press","Johns Hopkins University"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Library Trends","raw_type":"journal-article"},{"id":"pmh:oai:www.ideals.illinois.edu:2142/9492","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/9492","pdf_url":"https://www.ideals.illinois.edu/items/9535/bitstreams/35471/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"pmh:oai:muse.jhu.edu:/article/246678","doi":null,"is_oa":false,"landing_page_url":"https://muse.jhu.edu/article/246678","pdf_url":null,"source":{"id":"https://openalex.org/S4377196299","display_name":"Project Muse (Johns Hopkins University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I145311948","host_organization_name":"Johns Hopkins University","host_organization_lineage":["https://openalex.org/I145311948"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.224.8980","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.224.8980","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.ideals.uiuc.edu/bitstream/handle/2142/9492/56.4.saracevic.pdf?sequence=2","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.387.7868","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.387.7868","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://comminfo.rutgers.edu/~tefko/LibraryTrends2008.pdf","raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/9492","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/9492","pdf_url":"https://www.ideals.illinois.edu/items/9535/bitstreams/35471/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2102500012.pdf","grobid_xml":"https://content.openalex.org/works/W2102500012.grobid-xml"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W132056723","https://openalex.org/W180959710","https://openalex.org/W296674361","https://openalex.org/W1495821370","https://openalex.org/W1557757161","https://openalex.org/W1914011393","https://openalex.org/W1956559956","https://openalex.org/W1965690069","https://openalex.org/W1975730959","https://openalex.org/W1976898639","https://openalex.org/W1979818898","https://openalex.org/W1982149803","https://openalex.org/W1986866538","https://openalex.org/W1991546151","https://openalex.org/W2000255786","https://openalex.org/W2015720094","https://openalex.org/W2037140704","https://openalex.org/W2049342053","https://openalex.org/W2052989395","https://openalex.org/W2053528567","https://openalex.org/W2053584122","https://openalex.org/W2056973643","https://openalex.org/W2068912397","https://openalex.org/W2069820131","https://openalex.org/W2077324407","https://openalex.org/W2080745753","https://openalex.org/W2083605078","https://openalex.org/W2084097969","https://openalex.org/W2084675619","https://openalex.org/W2095396650","https://openalex.org/W2104061763","https://openalex.org/W2107125709","https://openalex.org/W2119652477","https://openalex.org/W2120308175","https://openalex.org/W2151012289","https://openalex.org/W2151664495","https://openalex.org/W2156849843","https://openalex.org/W2163484262","https://openalex.org/W2168469656","https://openalex.org/W2170707709","https://openalex.org/W2171695133","https://openalex.org/W2177637792","https://openalex.org/W2401023160","https://openalex.org/W2771255756","https://openalex.org/W2781361141","https://openalex.org/W2896335299","https://openalex.org/W4245894172","https://openalex.org/W4254168416","https://openalex.org/W4255321213"],"related_works":["https://openalex.org/W2094607109","https://openalex.org/W2151664495","https://openalex.org/W4394711204","https://openalex.org/W4300354420","https://openalex.org/W2077324407","https://openalex.org/W4382118803","https://openalex.org/W2120308175","https://openalex.org/W2052989395","https://openalex.org/W1980158193","https://openalex.org/W2000205710"],"abstract_inverted_index":{"The":[0,79],"main":[1],"objective":[2],"of":[3,71,77,81,98,107,113,115,140,147,152,176,180,189],"information":[4,11,13],"retrieval":[5,25,159],"(IR)":[6],"systems":[7,32,100],"is":[8,27,47,57,66,85,171],"to":[9,16,86],"retrieve":[10],"or":[12,38],"objects":[14],"relevant":[15],"user":[17,39,45],"requests":[18],"and":[19,61,96,101,132,155,167,182],"possible":[20],"needs.":[21],"In":[22],"IR":[23,31,99,108,117,153,169,190],"tests,":[24],"effectiveness":[26],"established":[28],"by":[29,64,143],"comparing":[30],"retrievals":[33],"(systems":[34],"relevance)":[35],"with":[36,68,137],"users\u2019":[37],"surrogates\u2019":[40],"assessments":[41],"(user":[42],"relevance),":[43],"where":[44],"relevance":[46,63,91,124,141],"treated":[48],"as":[49],"the":[50,88,93,104,111,116,187],"gold":[51],"standard":[52],"for":[53,158,164,168],"performance":[54],"evaluation.":[55,191],"Relevance":[56],"a":[58,69],"human":[59],"notion,":[60],"establishing":[62],"humans":[65],"fraught":[67],"number":[70],"problems\u2014inconsistency":[72],"in":[73,186],"judgment":[74],"being":[75],"one":[76,94],"them.":[78],"aim":[80],"this":[82],"critical":[83],"review":[84,130],"explore":[87],"relationship":[89],"between":[90],"on":[92,103,123,150],"hand":[95],"testing":[97,170],"procedures":[102],"other.":[105],"Critics":[106],"tests":[109,118,154],"raised":[110],"issue":[112],"validity":[114],"because":[119],"they":[120],"were":[121],"based":[122],"judgments":[125,142],"that":[126],"are":[127],"inconsistent.":[128],"This":[129],"traces":[131],"synthesizes":[133],"experimental":[134],"studies":[135,166],"dealing":[136],"(1)":[138],"inconsistency":[139,149],"people,":[144],"(2)":[145],"effects":[146],"such":[148],"results":[151],"(3)":[156],"reasons":[157],"failures.":[160],"A":[161],"historical":[162],"context":[163],"these":[165],"provided":[172],"including":[173],"an":[174],"assessment":[175],"Lancaster\u2019s":[177],"(1969)":[178],"evaluation":[179],"MEDLARS":[181],"its":[183],"unique":[184],"place":[185],"history":[188]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2102500012","counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":6},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":2},{"year":2012,"cited_by_count":2}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2016-06-24T00:00:00"}
