{"id":"https://openalex.org/W1986805046","doi":"https://doi.org/10.1145/2484028.2484170","title":"A document rating system for preference judgements","display_name":"A document rating system for preference judgements","publication_year":2013,"publication_date":"2013-07-28","ids":{"openalex":"https://openalex.org/W1986805046","doi":"https://doi.org/10.1145/2484028.2484170","mag":"1986805046"},"language":"en","primary_location":{"id":"doi:10.1145/2484028.2484170","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2484028.2484170","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045011600","display_name":"Maryam Bashir","orcid":"https://orcid.org/0000-0001-6124-5317"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Maryam Bashir","raw_affiliation_strings":["Northeastern University, Boston, Massachusetts, USA"],"affiliations":[{"raw_affiliation_string":"Northeastern University, Boston, Massachusetts, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061171961","display_name":"Jesse Anderton","orcid":"https://orcid.org/0000-0003-0405-2441"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jesse Anderton","raw_affiliation_strings":["Northeastern University, Boston, Massachusetts, USA"],"affiliations":[{"raw_affiliation_string":"Northeastern University, Boston, Massachusetts, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100724762","display_name":"Jie Wu","orcid":"https://orcid.org/0000-0002-6995-1213"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jie Wu","raw_affiliation_strings":["Northeastern University, Boston, Massachusetts, USA"],"affiliations":[{"raw_affiliation_string":"Northeastern University, Boston, Massachusetts, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047199726","display_name":"Peter B. Golbus","orcid":null},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter B. Golbus","raw_affiliation_strings":["Northeastern University, Boston, Massachusetts, USA"],"affiliations":[{"raw_affiliation_string":"Northeastern University, Boston, Massachusetts, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056048295","display_name":"Virgil Pavlu","orcid":null},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Virgil Pavlu","raw_affiliation_strings":["Northeastern University, Boston, Massachusetts, USA"],"affiliations":[{"raw_affiliation_string":"Northeastern University, Boston, Massachusetts, USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107005206","display_name":"Javed A. Aslam","orcid":"https://orcid.org/0009-0006-5098-6594"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Javed A. Aslam","raw_affiliation_strings":["Northeastern University, Boston, Massachusetts, USA"],"affiliations":[{"raw_affiliation_string":"Northeastern University, Boston, Massachusetts, USA","institution_ids":["https://openalex.org/I12912129"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5045011600"],"corresponding_institution_ids":["https://openalex.org/I12912129"],"apc_list":null,"apc_paid":null,"fwci":8.63605116,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.92733174,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"909","last_page":"912"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.991100013256073,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.9378440380096436},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8317187428474426},{"id":"https://openalex.org/keywords/transitive-relation","display_name":"Transitive relation","score":0.8267775177955627},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.7805705070495605},{"id":"https://openalex.org/keywords/arbitrariness","display_name":"Arbitrariness","score":0.5302305817604065},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4995718002319336},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4933582842350006},{"id":"https://openalex.org/keywords/analogy","display_name":"Analogy","score":0.4884992837905884},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3581204414367676},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.34832555055618286},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3480337858200073},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.31839656829833984},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.07898133993148804},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.07151767611503601}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.9378440380096436},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8317187428474426},{"id":"https://openalex.org/C191399111","wikidata":"https://www.wikidata.org/wiki/Q64861","display_name":"Transitive relation","level":2,"score":0.8267775177955627},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.7805705070495605},{"id":"https://openalex.org/C2777451423","wikidata":"https://www.wikidata.org/wiki/Q629962","display_name":"Arbitrariness","level":2,"score":0.5302305817604065},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4995718002319336},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4933582842350006},{"id":"https://openalex.org/C521332185","wikidata":"https://www.wikidata.org/wiki/Q185816","display_name":"Analogy","level":2,"score":0.4884992837905884},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3581204414367676},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.34832555055618286},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3480337858200073},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.31839656829833984},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.07898133993148804},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.07151767611503601},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2484028.2484170","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2484028.2484170","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W1556861584","https://openalex.org/W1938221098","https://openalex.org/W1989344419","https://openalex.org/W2047221353","https://openalex.org/W2072112759","https://openalex.org/W2098865355","https://openalex.org/W2142927605","https://openalex.org/W2162185655","https://openalex.org/W2164545125"],"related_works":["https://openalex.org/W2370653978","https://openalex.org/W2376033856","https://openalex.org/W2110868472","https://openalex.org/W2372540915","https://openalex.org/W3140422530","https://openalex.org/W1947281443","https://openalex.org/W2767187055","https://openalex.org/W4205262062","https://openalex.org/W2002629668","https://openalex.org/W181477314"],"abstract_inverted_index":{"High":[0],"quality":[1],"relevance":[2,17,80,112,144],"judgments":[3,18,30,117,138],"are":[4,19,31,64],"essential":[5],"for":[6,52,146],"the":[7,40,61,86,98,107,123],"evaluation":[8],"of":[9,15,42,88,100,109,135],"information":[10],"retrieval":[11],"systems.":[12],"Traditional":[13],"methods":[14],"collecting":[16,22],"based":[20],"on":[21],"binary":[23],"or":[24],"graded":[25],"nominal":[26],"judgments,":[27],"but":[28],"such":[29,35],"limited":[32],"by":[33,118],"factors":[34],"as":[36],"inter-assessor":[37],"disagreement":[38],"and":[39],"arbitrariness":[41],"grades.":[43],"Previous":[44],"research":[45],"has":[46],"shown":[47],"that":[48,85,90],"it":[49,67],"is":[50,68,84,95],"easier":[51],"assessors":[53,141],"to":[54,72,77,92,120,130,142],"make":[55],"pairwise":[56,115,136],"preference":[57,116,137],"judgments.":[58],"However,":[59],"unless":[60],"preferences":[62],"collected":[63],"largely":[65],"transitive,":[66],"not":[69],"clear":[70],"how":[71,129],"combine":[73,131],"them":[74],"in":[75,97],"order":[76],"obtain":[78],"document":[79,111],"scores.":[81],"Another":[82],"difficulty":[83],"number":[87,99,134],"pairs":[89],"need":[91],"be":[93],"assessed":[94],"quadratic":[96],"documents.":[101],"In":[102],"this":[103],"work,":[104],"we":[105],"consider":[106],"problem":[108],"inferring":[110],"scores":[113,145],"from":[114,139],"analogy":[119],"tournaments":[121],"using":[122],"Elo":[124],"rating":[125],"system.":[126],"We":[127],"show":[128],"a":[132],"linear":[133],"multiple":[140],"compute":[143],"every":[147],"document.":[148]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}