{"id":"https://openalex.org/W2903058329","doi":"https://doi.org/10.1145/3291992.3291995","title":"Pairwise Crowd Judgments","display_name":"Pairwise Crowd Judgments","publication_year":2018,"publication_date":"2018-12-04","ids":{"openalex":"https://openalex.org/W2903058329","doi":"https://doi.org/10.1145/3291992.3291995","mag":"2903058329"},"language":"en","primary_location":{"id":"doi:10.1145/3291992.3291995","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3291992.3291995","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd Australasian Document Computing Symposium","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015195541","display_name":"Ziying Yang","orcid":"https://orcid.org/0000-0001-7705-3280"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Ziying Yang","raw_affiliation_strings":["The University of Melbourne, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081861848","display_name":"Alistair Moffat","orcid":"https://orcid.org/0000-0002-6638-0232"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Alistair Moffat","raw_affiliation_strings":["The University of Melbourne, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014748363","display_name":"Andrew Turpin","orcid":"https://orcid.org/0000-0003-2559-8769"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Andrew Turpin","raw_affiliation_strings":["The University of Melbourne, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5015195541"],"corresponding_institution_ids":["https://openalex.org/I165779595"],"apc_list":null,"apc_paid":null,"fwci":1.9816,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.88833976,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9836000204086304,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9678999781608582,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8814027309417725},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.880622148513794},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6995068192481995},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6479672193527222},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5436827540397644},{"id":"https://openalex.org/keywords/nist","display_name":"NIST","score":0.44503462314605713},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.42058542370796204},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3802538514137268},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3581191301345825},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.25573647022247314},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.18620064854621887}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8814027309417725},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.880622148513794},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6995068192481995},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6479672193527222},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5436827540397644},{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.44503462314605713},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.42058542370796204},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3802538514137268},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3581191301345825},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25573647022247314},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.18620064854621887},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3291992.3291995","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3291992.3291995","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd Australasian Document Computing Symposium","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.6100000143051147}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W131852725","https://openalex.org/W1529341914","https://openalex.org/W1592933613","https://openalex.org/W1968927634","https://openalex.org/W1990420106","https://openalex.org/W2008285050","https://openalex.org/W2051752522","https://openalex.org/W2053528567","https://openalex.org/W2088901214","https://openalex.org/W2094607109","https://openalex.org/W2114386757","https://openalex.org/W2119207355","https://openalex.org/W2120308175","https://openalex.org/W2132232498","https://openalex.org/W2134366819","https://openalex.org/W2159048649","https://openalex.org/W2164545125","https://openalex.org/W2530530386","https://openalex.org/W2559342890","https://openalex.org/W2798782484","https://openalex.org/W6631869169","https://openalex.org/W6759030081"],"related_works":["https://openalex.org/W2158491338","https://openalex.org/W2807901368","https://openalex.org/W2133733652","https://openalex.org/W2072658171","https://openalex.org/W2029032313","https://openalex.org/W4315563560","https://openalex.org/W2553559590","https://openalex.org/W2171766007","https://openalex.org/W2772061936","https://openalex.org/W2143990275"],"abstract_inverted_index":{"Relevance":[0],"judgments":[1,22,81,95,157],"are":[2],"conventionally":[3],"formed":[4],"by":[5,15,36],"small":[6],"numbers":[7],"of":[8,43,115,135,161],"experts":[9],"using":[10,50],"ordinal":[11],"relevance":[12,19,48,56,90],"scales":[13],"defined":[14],"two":[16],"or":[17],"more":[18],"categories.":[20],"Such":[21],"often":[23],"contain":[24],"many":[25],"ties:":[26],"documents":[27,136],"in":[28],"the":[29,41,79,124,148,152,159],"same":[30],"category":[31],"that":[32,126],"cannot":[33],"be":[34],"separated":[35],"relevance.":[37],"Here":[38],"we":[39,146],"explore":[40],"use":[42],"crowd-sourcing":[44],"and":[45,55,62,72,92,104,113,138],"combined":[46],"three-way":[47],"assessments":[49],"pairwise":[51],"preference,":[52],"absolute":[53],"relevance,":[54],"ratio,":[57],"with":[58],"forced":[59],"choice":[60],"testing":[61],"embedded":[63],"quality":[64],"control":[65],"processes,":[66],"seeking":[67],"to":[68,73,150,155,164],"reduce":[69],"assessment":[70,139,143],"ties,":[71],"increase":[74],"judgment":[75,111],"consistency.":[76],"In":[77],"particular,":[78],"crowd-sourced":[80],"from":[82],"these":[83],"three":[84,98],"approaches":[85],"were":[86],"normalized":[87],"into":[88],"numeric":[89],"scores,":[91],"compared":[93],"against":[94],"arising":[96],"via":[97],"previous":[99],"techniques:":[100],"NIST":[101],"binary;":[102],"Sormunen;":[103],"magnitude":[105],"estimation.":[106],"The":[107],"relationship":[108],"between":[109,166],"generated":[110],"reliability":[112],"number":[114,134],"document":[116,130],"pairs":[117],"assessed":[118],"was":[119,123],"also":[120],"explored,":[121],"as":[122,129],"effect":[125],"factors":[127],"such":[128],"length,":[131],"topic":[132],"difficulty,":[133],"judged,":[137],"time,":[140],"have":[141],"on":[142],"reliability.":[144],"Lastly,":[145],"investigate":[147],"extent":[149],"which":[151],"methodology":[153],"used":[154],"collect":[156],"affects":[158],"ability":[160],"an":[162],"experiment":[163],"discriminate":[165],"IR":[167],"systems.":[168]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":3}],"updated_date":"2026-03-25T14:56:36.534964","created_date":"2025-10-10T00:00:00"}