{"id":"https://openalex.org/W1588207697","doi":null,"title":"Relatively relevant: Assessor shift in document judgements","display_name":"Relatively relevant: Assessor shift in document judgements","alternate_titles":[],"publication_year":2010,"publication_date":"2010-01-01","ids":{"openalex":"https://openalex.org/W1588207697","mag":"1588207697"},"language":"en","primary_location":{"id":"pmh:oai:alma61RMIT.INST:11246405530001341","doi":null,"is_oa":false,"landing_page_url":"https://researchrepository.rmit.edu.au/esploro/outputs/conferenceProceeding/Relatively-relevant-Assessor-shift-in-document/9921857752801341","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W1588207697"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072614956","display_name":"Mark Sanderson","orcid":"https://orcid.org/0000-0003-0487-9609","observed_orcids":["https://orcid.org/0000-0003-0487-9609"]},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Mark Sanderson","raw_affiliation_strings":["RMIT-University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT-University","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810","observed_orcids":["https://orcid.org/0000-0001-9094-0810"]},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT-University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT-University","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014748363","display_name":"Andrew H. Turpin","orcid":"https://orcid.org/0000-0003-2559-8769","observed_orcids":["https://orcid.org/0000-0003-2559-8769"]},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Andrew Turpin","raw_affiliation_strings":["University of Melbourne"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Melbourne","institution_ids":["https://openalex.org/I165779595"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7405,"has_fulltext":false,"cited_by_count":17,"reviewed_by_count":0,"citation_normalized_percentile":{"value":0.87411598,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"60","last_page":"67"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance-assessment","display_name":"relevance assessment","score":0.7630000114440918},{"id":"https://openalex.org/keywords/information-retrieval-evaluation","display_name":"information retrieval evaluation","score":0.6380000114440918},{"id":"https://openalex.org/keywords/test-collections","display_name":"test collections","score":0.746999979019165},{"id":"https://openalex.org/keywords/document-similarity","display_name":"document similarity","score":0.6060000061988831}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8979779481887817},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.8871617317199707},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.8238099217414856},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6481764912605286},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5665026903152466},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.47070571780204773},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.46970078349113464},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4187760651111603},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18597596883773804},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"pmh:oai:alma61RMIT.INST:11246405530001341","doi":null,"is_oa":false,"landing_page_url":"https://researchrepository.rmit.edu.au/esploro/outputs/conferenceProceeding/Relatively-relevant-Assessor-shift-in-document/9921857752801341","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:researchbank.rmit.edu.au:rmit:13015","doi":null,"is_oa":false,"landing_page_url":"http://researchbank.rmit.edu.au/view/rmit:13015","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},{"id":"pmh:oai:figshare.com:article/27375045","doi":null,"is_oa":false,"landing_page_url":"https://figshare.com/articles/conference_contribution/Relatively_relevant_Assessor_shift_in_document_judgements/27375045","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference contribution"},{"id":"mag:1588207697","doi":null,"is_oa":false,"landing_page_url":"http://marksanderson.org/publications/my_papers/adcs2010.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306417788","display_name":"Australasian Document Computing Symposium","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Australasian Document Computing Symposium","raw_type":null},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.338.1011","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.338.1011","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.seg.rmit.edu.au/research/download.php?manuscript=437","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W132882894","https://openalex.org/W1553682320","https://openalex.org/W1965690069","https://openalex.org/W1982149803","https://openalex.org/W1987690005","https://openalex.org/W1990190154","https://openalex.org/W2009543464","https://openalex.org/W2023217403","https://openalex.org/W2037022968","https://openalex.org/W2037098674","https://openalex.org/W2037140704","https://openalex.org/W2045207820","https://openalex.org/W2052688978","https://openalex.org/W2052989395","https://openalex.org/W2053100920","https://openalex.org/W2053528567","https://openalex.org/W2056361398","https://openalex.org/W2062324828","https://openalex.org/W2075893676","https://openalex.org/W2096452841","https://openalex.org/W2099341654","https://openalex.org/W2120308175","https://openalex.org/W2141623655","https://openalex.org/W2742538188"],"related_works":["https://openalex.org/W2094607109","https://openalex.org/W2515377091","https://openalex.org/W1967160681","https://openalex.org/W2923651429","https://openalex.org/W7098577501","https://openalex.org/W6996193160","https://openalex.org/W2333065964","https://openalex.org/W1972821056","https://openalex.org/W1987016195","https://openalex.org/W2159048649"],"abstract_inverted_index":{"Abstract":[0],"The":[1],"evaluation":[2],"of":[3,14,45,58,68,97,160,171,194],"information":[4],"retrieval":[5,199],"systems":[6],"relies":[7],"on":[8,42,60,197,217],"relevance":[9,44,85,108,154,172],"judgements":[10,180],"\u2013":[11],"human":[12],"assessments":[13],"whether":[15],"a":[16,21,56,84,100,153,163,214],"document":[17,113,165],"is":[18,181],"relevant":[19,91,164,187],"to":[20,39,81,93,147,204,212],"specified":[22],"search":[23],"request.":[24],"In":[25],"the":[26,43,66,118,126,130,157,177,192],"past,":[27],"it":[28],"was":[29],"demonstrated":[30],"that":[31,65,76,107,138,142,176,209],"test":[32],"collection":[33],"assessors":[34,82,109],"disagree":[35],"with":[36],"each":[37,124,205],"other":[38,206],"some":[40],"extent":[41],"documents":[46,78,88,98],"and":[47,185,207],"can":[48],"be":[49,213],"inconsistent":[50],"in":[51,83,99,179],"themselves.":[52],"This":[53],"paper":[54],"describes":[55],"series":[57],"investigations":[59],"assessor":[61,70],"consistency,":[62],"which":[63],"demonstrate":[64],"inconsistency":[67,196],"an":[69],"varies":[71],"over":[72,167],"time.":[73,168],"We":[74,136,189],"show":[75],"when":[77,117,129],"are":[79,120,132,144,201],"presented":[80],"independent":[86],"order,":[87],"judged":[89,133],"as":[90],"appear":[92],"cluster.":[94],"Examining":[95],"pairs":[96,114,119,131],"sequence":[101],"ordered":[102],"by":[103],"timeof-judgement,":[104],"we":[105,174],"find":[106,175,208],"judge":[110],"highly":[111,184],"similar":[112],"more":[115],"consistently":[116],"seen":[121],"soon":[122],"after":[123],"other;":[125],"consistency":[127],"reduces":[128],"further":[134],"apart.":[135],"contend":[137],"our":[139],"analysis":[140],"shows":[141],"changes":[143,166],"not":[145],"due":[146],"random":[148],"error,":[149],"but":[150],"instead":[151],"reflect":[152],"shift,":[155],"whereby":[156],"assessor\u2019s":[158],"conception":[159],"what":[161],"constitutes":[162],"Studying":[169],"types":[170],"judgement":[173],"shift":[178],"greatest":[182],"between":[183],"partially":[186],"documents.":[188],"also":[190],"examine":[191],"impact":[193],"this":[195],"how":[198],"runs":[200],"ranked":[202],"relative":[203],"there":[210],"appears":[211],"noticeable":[215],"effect":[216],"such":[218],"rankings.":[219]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W1588207697","counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2013,"cited_by_count":2}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2016-06-24T00:00:00"}
