{"id":"https://openalex.org/W2740172188","doi":"https://doi.org/10.1145/3077136.3084154","title":"Relevance Judgments","display_name":"Relevance Judgments","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2740172188","doi":"https://doi.org/10.1145/3077136.3084154","mag":"2740172188"},"language":"en","primary_location":{"id":"doi:10.1145/3077136.3084154","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3084154","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015195541","display_name":"Ziying Yang","orcid":"https://orcid.org/0000-0001-7705-3280"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Ziying Yang","raw_affiliation_strings":["University of Melbourne, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Melbourne, Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5015195541"],"corresponding_institution_ids":["https://openalex.org/I165779595"],"apc_list":null,"apc_paid":null,"fwci":0.4315,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.60013631,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1373","last_page":"1373"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9879999756813049,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.915207028388977},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.7817693948745728},{"id":"https://openalex.org/keywords/viewpoints","display_name":"Viewpoints","score":0.622647762298584},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.558529257774353},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.5274134874343872},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.5119460821151733},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.4812217056751251},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.44524097442626953},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4288513660430908},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.39537855982780457},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21176493167877197},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1460561752319336},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14132583141326904},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.11122044920921326}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.915207028388977},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7817693948745728},{"id":"https://openalex.org/C2776035091","wikidata":"https://www.wikidata.org/wiki/Q7928819","display_name":"Viewpoints","level":2,"score":0.622647762298584},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.558529257774353},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.5274134874343872},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.5119460821151733},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.4812217056751251},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.44524097442626953},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4288513660430908},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.39537855982780457},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21176493167877197},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1460561752319336},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14132583141326904},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11122044920921326},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3077136.3084154","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3084154","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Gender equality","id":"https://metadata.un.org/sdg/5","score":0.7200000286102295}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W131852725","https://openalex.org/W1533179050","https://openalex.org/W1546195864","https://openalex.org/W1592933613","https://openalex.org/W1990420106","https://openalex.org/W2008285050","https://openalex.org/W2053528567","https://openalex.org/W2094607109","https://openalex.org/W2159048649","https://openalex.org/W2569307494","https://openalex.org/W3045153201"],"related_works":["https://openalex.org/W2385368906","https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2902924992","https://openalex.org/W648502197","https://openalex.org/W4205262062","https://openalex.org/W181477314","https://openalex.org/W2002629668"],"abstract_inverted_index":{"Conventionally,":[0],"relevance":[1,7,58,74,119,137,167],"judgments":[2,17,138],"were":[3],"assessed":[4,48],"using":[5,105],"ordinal":[6,106],"scales":[8],"such":[9,63],"as":[10,35,64,93],"binary":[11],"and":[12,27,37,67,120],"Sormunen":[13],"categories":[14],"[9].":[15],"Such":[16],"record":[18],"how":[19],"much":[20],"overlap":[21],"there":[22],"is":[23],"between":[24],"the":[25,28,71,130,144,162],"document":[26],"topic.":[29],"However":[30],"they":[31],"have":[32,111],"been":[33],"argued":[34],"unreliable":[36],"not":[38],"objective":[39],"[3,":[40],"5,":[41],"10]":[42],"because:":[43],"(1)":[44],"documents":[45,95,154],"are":[46,96,103],"usually":[47],"by":[49,77],"limited":[50],"numbers":[51],"of":[52,57,60,73,115,118,164],"experts,":[53],"with":[54,123,161],"different":[55],"viewpoints":[56],"because":[59],"individual":[61],"factors":[62],"gender,":[65],"age":[66],"background":[68],"[1];":[69],"(2)":[70],"distinctions":[72],"levels":[75],"expected":[76],"users":[78],"disparate":[79],"types":[80],"may":[81],"be":[82,159],"diverse":[83],"[7];":[84],"(3)":[85],"assessors'":[86],"examining":[87],"criteria":[88],"drift":[89],"in":[90],"varying":[91],"degrees":[92],"more":[94],"judged":[97,153],"[8];":[98],"(4)":[99],"many":[100],"judgment":[101],"ties":[102],"generated":[104,160],"scales.":[107],"In":[108],"order":[109],"to":[110,128,135],"a":[112,140,147],"better":[113],"understanding":[114],"users'":[116],"perceptions":[117],"collect":[121,136],"data":[122],"high":[124],"fidelity,":[125],"we":[126],"propose":[127],"use":[129],"Pairwise":[131],"Preference":[132],"technique":[133],"[2]":[134],"from":[139],"crowdsourcing":[141],"platform.":[142],"With":[143],"collected":[145],"judgments,":[146],"computed":[148],"rank":[149],"list":[150],"containing":[151],"all":[152],"for":[155],"each":[156],"topic":[157],"will":[158],"goal":[163],"having":[165],"fewer":[166],"ties.":[168]},"counts_by_year":[{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}