{"id":"https://openalex.org/W2515377091","doi":"https://doi.org/10.1145/2970398.2970431","title":"The Effect of Document Order and Topic Difficulty on Assessor Agreement","display_name":"The Effect of Document Order and Topic Difficulty on Assessor Agreement","publication_year":2016,"publication_date":"2016-09-09","ids":{"openalex":"https://openalex.org/W2515377091","doi":"https://doi.org/10.1145/2970398.2970431","mag":"2515377091"},"language":"en","primary_location":{"id":"doi:10.1145/2970398.2970431","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2970398.2970431","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008446102","display_name":"Tadele T. Damessie","orcid":null},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Tadele T. Damessie","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032537393","display_name":"Kalvero J\u00e4rvelin","orcid":null},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]},{"id":"https://openalex.org/I4210133110","display_name":"Tampere University","ror":null,"country_code":"FI","type":null,"lineage":["https://openalex.org/I4210133110"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Kalvero J\u00e4rvelin","raw_affiliation_strings":["University of Tampere, Tampere, Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Tampere, Tampere, Finland","institution_ids":["https://openalex.org/I166825849","https://openalex.org/I4210133110"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070937840","display_name":"J. Shane Culpepper","orcid":"https://orcid.org/0000-0002-1902-9087"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"J. Shane Culpepper","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.452,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.93590723,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"73","last_page":"76"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.7420485615730286},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.6059038043022156},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5982482433319092},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4353167712688446},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4134797155857086},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.17978256940841675},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.05823037028312683},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.0574265718460083}],"concepts":[{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.7420485615730286},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.6059038043022156},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5982482433319092},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4353167712688446},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4134797155857086},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.17978256940841675},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.05823037028312683},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0574265718460083},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2970398.2970431","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2970398.2970431","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:figshare.com:article/27397404","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/27397404","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1566355232","display_name":"Beyond keyword search for ranked document retrieval","funder_award_id":"DE140100275","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1593272667","https://openalex.org/W1976898639","https://openalex.org/W1980245224","https://openalex.org/W1991774506","https://openalex.org/W2008285050","https://openalex.org/W2035569891","https://openalex.org/W2044361482","https://openalex.org/W2061504941","https://openalex.org/W2066289827","https://openalex.org/W2068912397","https://openalex.org/W2101287943","https://openalex.org/W2120308175","https://openalex.org/W2141653139","https://openalex.org/W2918049428","https://openalex.org/W6760044250"],"related_works":["https://openalex.org/W2384888906","https://openalex.org/W2144190808","https://openalex.org/W2376314740","https://openalex.org/W2366644548","https://openalex.org/W2357241418","https://openalex.org/W2611614995","https://openalex.org/W2115485936","https://openalex.org/W2368651715","https://openalex.org/W2789919619","https://openalex.org/W38394648"],"abstract_inverted_index":{"Human":[0],"relevance":[1,19,67],"judgments":[2],"are":[3,108],"a":[4,34,75],"key":[5],"component":[6],"for":[7,33],"measuring":[8],"the":[9,43],"effectiveness":[10],"of":[11,36,95],"information":[12],"retrieval":[13],"systems":[14],"using":[15],"test":[16],"collections.":[17],"Since":[18],"is":[20,104,117],"not":[21],"an":[22],"absolute":[23],"concept,":[24],"human":[25],"assessors":[26],"can":[27],"disagree":[28],"on":[29,50,90,121,125],"particular":[30],"topic-document":[31],"pairs":[32],"variety":[35],"reasons.":[37],"In":[38,114],"this":[39],"work":[40],"we":[41],"investigate":[42],"effect":[44],"that":[45,101],"document":[46,70,111],"presentation":[47,55],"order":[48,68,85],"has":[49],"inter-rater":[51,91],"agreement,":[52],"comparing":[53],"two":[54],"ordering":[56],"approaches":[57],"similar":[58],"to":[59,86],"those":[60],"used":[61],"in":[62,84,110],"IR":[63],"evaluation":[64],"campaigns:":[65],"decreasing":[66],"and":[69,81],"identifier":[71,112],"order.":[72,113],"We":[73],"make":[74],"further":[76],"distinction":[77],"between":[78],"\"easy\"":[79],"topics":[80,83,123],"\"hard\"":[82],"explore":[87],"system":[88],"effects":[89],"agreement.":[92],"The":[93],"results":[94],"our":[96],"pilot":[97],"user":[98],"study":[99],"indicate":[100],"assessor":[102],"agreement":[103,120],"higher":[105,118],"when":[106],"documents":[107],"judged":[109],"addition,":[115],"there":[116],"overall":[119],"easy":[122],"than":[124],"hard":[126],"topics.":[127]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1}],"updated_date":"2026-06-22T08:00:12.763002","created_date":"2025-10-10T00:00:00"}