{"id":"https://openalex.org/W2950051380","doi":"https://doi.org/10.48550/arxiv.1405.7192","title":"The PeerRank Method for Peer Assessment","display_name":"The PeerRank Method for Peer Assessment","publication_year":2014,"publication_date":"2014-05-28","ids":{"openalex":"https://openalex.org/W2950051380","doi":"https://doi.org/10.48550/arxiv.1405.7192","mag":"2950051380"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1405.7192","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1405.7192","pdf_url":"https://arxiv.org/pdf/1405.7192","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1405.7192","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072902302","display_name":"Toby Walsh","orcid":"https://orcid.org/0000-0003-2998-8668","observed_orcids":["https://orcid.org/0000-0003-2998-8668"]},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]},{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Walsh, Toby","raw_affiliation_strings":["NICTA and UNSW, Sydney, Australia#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NICTA and UNSW, Sydney, Australia#TAB#","institution_ids":["https://openalex.org/I31746571","https://openalex.org/I42894916"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5072902302"],"corresponding_institution_ids":["https://openalex.org/I31746571","https://openalex.org/I42894916"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":37,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10991","display_name":"Game Theory and Voting Systems","score":0.7041000127792358,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10991","display_name":"Game Theory and Voting Systems","score":0.7041000127792358,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.041600000113248825,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10927","display_name":"Access Control and Trust","score":0.030899999663233757,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-assessment","display_name":"peer assessment","score":0.8569999933242798},{"id":"https://openalex.org/keywords/pagerank","display_name":"PageRank","score":0.5770000219345093},{"id":"https://openalex.org/keywords/grade-prediction","display_name":"grade prediction","score":0.609000027179718},{"id":"https://openalex.org/keywords/incentive-compatibility","display_name":"incentive compatibility","score":0.7760000228881836},{"id":"https://openalex.org/keywords/fixed-point-equations","display_name":"fixed point equations","score":0.8169999718666077},{"id":"https://openalex.org/keywords/axiomatic-properties","display_name":"axiomatic properties","score":0.7279999852180481},{"id":"https://openalex.org/keywords/multi-agent-systems","display_name":"multi-agent systems","score":0.5989999771118164}],"concepts":[{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.6743039488792419},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6132680177688599},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5385422110557556},{"id":"https://openalex.org/C167729594","wikidata":"https://www.wikidata.org/wiki/Q17736","display_name":"Axiom","level":2,"score":0.5313872694969177},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3435038924217224},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.294524222612381},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"pmh:oai:arXiv.org:1405.7192","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1405.7192","pdf_url":"https://arxiv.org/pdf/1405.7192","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2950051380","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1405.7192.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.644.6439","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.644.6439","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.cse.unsw.edu.au/~tw/wecai2014.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.748.7445","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.748.7445","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://arxiv.org/pdf/1405.7192.pdf","raw_type":"text"},{"id":"doi:10.48550/arxiv.1405.7192","doi":"https://doi.org/10.48550/arxiv.1405.7192","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1405.7192","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1405.7192","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1405.7192","pdf_url":"https://arxiv.org/pdf/1405.7192","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2950051380.pdf","grobid_xml":"https://content.openalex.org/works/W2950051380.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W1558752712","https://openalex.org/W1854214752","https://openalex.org/W2006490781","https://openalex.org/W2091152488","https://openalex.org/W2093375346","https://openalex.org/W2120810064","https://openalex.org/W2122588206","https://openalex.org/W2278055525"],"related_works":["https://openalex.org/W2920988802","https://openalex.org/W6922608902","https://openalex.org/W2949835649","https://openalex.org/W3174330101","https://openalex.org/W7123340088","https://openalex.org/W2070635790","https://openalex.org/W3168821299","https://openalex.org/W4302148109","https://openalex.org/W2166164009","https://openalex.org/W2978979629"],"abstract_inverted_index":{"We":[0,109],"propose":[1],"the":[2,17,21,24,27,41,47,50,66,73,76,86,93,103,115,140,152,168],"PeerRank":[3,42,54,94,116],"method":[4,43,55,95,105,117],"for":[5,12,60,106],"peer":[6,173],"assessment.":[7],"This":[8],"constructs":[9],"a":[10,33,97,158],"grade":[11,28,39,63,155],"an":[13,30,58,69],"agent":[14,31,70],"based":[15],"on":[16,72,85,142],"grades":[18,45,48,67,74,82,87],"proposed":[19],"by":[20,46,96,157],"agents":[22,61],"evaluating":[23],"agent.":[25,52],"Since":[26],"of":[29,35,49,68,75,88,114,123,160,171],"is":[32],"measure":[34],"their":[36],"ability":[37],"to":[38,62,102],"correctly,":[40],"weights":[44],"grading":[51,77],"The":[53],"also":[56],"provides":[57],"incentive":[59],"correctly.":[64],"As":[65],"depend":[71,84],"agents,":[78,90],"and":[79,129,138],"as":[80],"these":[81],"themselves":[83],"other":[89],"we":[91],"define":[92],"fixed":[98],"point":[99],"equation":[100],"similar":[101],"PageRank":[104],"ranking":[107],"web-pages.":[108],"identify":[110],"some":[111,132,143],"formal":[112],"properties":[113],"(for":[118],"example,":[119],"it":[120],"satisfies":[121],"axioms":[122],"unanimity,":[124],"no":[125,127],"dummy,":[126],"discrimination":[128],"symmetry),":[130],"discuss":[131],"examples,":[133],"compare":[134],"with":[135],"related":[136],"work":[137],"evaluate":[139],"performance":[141],"synthetic":[144],"data.":[145],"Our":[146],"results":[147],"show":[148],"considerable":[149],"promise,":[150],"reducing":[151],"error":[153],"in":[154,164],"predictions":[156],"factor":[159],"2":[161],"or":[162],"more":[163],"many":[165],"cases":[166],"over":[167],"natural":[169],"baseline":[170],"averaging":[172],"grades.":[174]},"counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":9},{"year":2015,"cited_by_count":6}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2019-06-27T00:00:00"}