{"id":"https://openalex.org/W2896022497","doi":"https://doi.org/10.1145/3269206.3271750","title":"Presentation Ordering Effects On Assessor Agreement","display_name":"Presentation Ordering Effects On Assessor Agreement","publication_year":2018,"publication_date":"2018-10-17","ids":{"openalex":"https://openalex.org/W2896022497","doi":"https://doi.org/10.1145/3269206.3271750","mag":"2896022497"},"language":"en","primary_location":{"id":"doi:10.1145/3269206.3271750","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3269206.3271750","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://figshare.com/articles/conference_contribution/Presentation_Ordering_Effects_On_Assessor_Agreement/27582528","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008446102","display_name":"Tadele T. Damessie","orcid":null},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Tadele T. Damessie","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070937840","display_name":"J. Shane Culpepper","orcid":"https://orcid.org/0000-0002-1902-9087"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"J. Shane Culpepper","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100413748","display_name":"Jaewon Kim","orcid":"https://orcid.org/0000-0003-2717-1531"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Jaewon Kim","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I82951845"],"apc_list":null,"apc_paid":null,"fwci":0.8225,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.81867779,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"723","last_page":"732"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.902901291847229},{"id":"https://openalex.org/keywords/presentation","display_name":"Presentation (obstetrics)","score":0.8584928512573242},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.7909448742866516},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7505977153778076},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6830548048019409},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.5559344291687012},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5258631706237793},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.5188999772071838},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.5048492550849915},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36464300751686096},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18245679140090942},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.07035163044929504}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.902901291847229},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.8584928512573242},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7909448742866516},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7505977153778076},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6830548048019409},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5559344291687012},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5258631706237793},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.5188999772071838},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.5048492550849915},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36464300751686096},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18245679140090942},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.07035163044929504},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3269206.3271750","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3269206.3271750","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:alma.61RMIT_INST:11248137000001341","is_oa":false,"landing_page_url":"http://doi.org/10.1145/3269206.3271750","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:figshare.com:article/27582528","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/Presentation_Ordering_Effects_On_Assessor_Agreement/27582528","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/27582528","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/Presentation_Ordering_Effects_On_Assessor_Agreement/27582528","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"sustainable_development_goals":[{"score":0.6499999761581421,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G8319290515","display_name":null,"funder_award_id":"DP170102231, DP170102726, DP180102687","funder_id":"https://openalex.org/F4320337726","funder_display_name":"Centre of Excellence for Core to Crust Fluid Systems, Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320337726","display_name":"Centre of Excellence for Core to Crust Fluid Systems, Australian Research Council","ror":"https://ror.org/03nk9pp38"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W320171562","https://openalex.org/W1497983610","https://openalex.org/W1509991416","https://openalex.org/W1541075651","https://openalex.org/W1556861584","https://openalex.org/W1592933613","https://openalex.org/W1965690069","https://openalex.org/W1966758856","https://openalex.org/W1968667761","https://openalex.org/W1982149803","https://openalex.org/W1989220366","https://openalex.org/W1990420106","https://openalex.org/W1991774506","https://openalex.org/W2004737172","https://openalex.org/W2008285050","https://openalex.org/W2014581807","https://openalex.org/W2022995284","https://openalex.org/W2027359109","https://openalex.org/W2035569891","https://openalex.org/W2037140704","https://openalex.org/W2045334584","https://openalex.org/W2053528567","https://openalex.org/W2061504941","https://openalex.org/W2061935394","https://openalex.org/W2066451131","https://openalex.org/W2068130800","https://openalex.org/W2068912397","https://openalex.org/W2071446250","https://openalex.org/W2120308175","https://openalex.org/W2122912498","https://openalex.org/W2131915193","https://openalex.org/W2141653139","https://openalex.org/W2150136235","https://openalex.org/W2154200620","https://openalex.org/W2165062189","https://openalex.org/W2167431750","https://openalex.org/W2181793185","https://openalex.org/W2319586349","https://openalex.org/W2408365773","https://openalex.org/W2515377091","https://openalex.org/W2559342890","https://openalex.org/W2740069864","https://openalex.org/W2785937154","https://openalex.org/W3005284662","https://openalex.org/W3143291062","https://openalex.org/W4244611409"],"related_works":["https://openalex.org/W4387426029","https://openalex.org/W4254162896","https://openalex.org/W4388792380","https://openalex.org/W1477999932","https://openalex.org/W4386731653","https://openalex.org/W2376001620","https://openalex.org/W2032233321","https://openalex.org/W2925303117","https://openalex.org/W3121970507","https://openalex.org/W3167529338"],"abstract_inverted_index":{"Consistency":[0],"of":[1,11,30,49,122,180],"relevance":[2,19,55,123],"judgments":[3,35],"is":[4,36,60,171],"a":[5,37,91,119,159,178],"vital":[6],"issue":[7],"for":[8],"the":[9,47,70,110,125],"construction":[10],"test":[12,43],"collections":[13,25],"in":[14,72],"information":[15],"retrieval.":[16],"As":[17],"human":[18,54],"assessments":[20],"are":[21,75,147],"costly,":[22],"and":[23,104,141,166],"large":[24],"can":[26,66,114],"contain":[27],"many":[28],"documents":[29,74],"varying":[31],"relevance,":[32],"collecting":[33],"reliable":[34],"critical":[38],"component":[39],"to":[40,61,77,95,98,129],"building":[41],"reusable":[42],"collections.":[44],"We":[45],"explore":[46],"impact":[48,161],"document":[50,153],"presentation":[51,88,111,139,154],"order":[52,71,155],"on":[53,118,144,162],"assessments.":[56],"Our":[57,149],"primary":[58],"goal":[59],"determine":[62],"if":[63],"assessor":[64,145,163],"disagreement":[65],"be":[67],"minimized":[68],"through":[69],"which":[73],"presented":[76],"assessors.":[78],"To":[79],"achieve":[80],"this":[81],"goal,":[82],"we":[83],"compare":[84],"two":[85],"commonly":[86],"used":[87],"orderings":[89],"with":[90],"new":[92,169],"ordering":[93,140,170],"designed":[94],"aid":[96],"assessors":[97,113],"more":[99,115,172],"easily":[100],"discriminate":[101],"between":[102,138],"relevant":[103],"non-relevant":[105],"documents.":[106],"By":[107],"carefully":[108],"controlling":[109],"ordering,":[112],"quickly":[116],"converge":[117],"consistent":[120],"notion":[121],"during":[124],"assessment":[126],"exercise,":[127],"leading":[128],"higher":[130],"overall":[131],"judging":[132],"agreement.":[133],"In":[134],"addition,":[135],"important":[136],"interactions":[137],"topic":[142,182],"difficulty":[143],"agreement":[146,164],"highlighted.":[148],"findings":[150],"suggest":[151],"that":[152,167],"does":[156],"indeed":[157],"have":[158],"substantial":[160],",":[165],"our":[168],"robust":[173],"than":[174],"previous":[175],"approaches":[176],"across":[177],"variety":[179],"different":[181],"types.":[183]},"counts_by_year":[{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}