{"id":"https://openalex.org/W4383604699","doi":"https://doi.org/10.48550/arxiv.2307.02762","title":"PRD: Peer Rank and Discussion Improve Large Language Model based Evaluations","display_name":"PRD: Peer Rank and Discussion Improve Large Language Model based Evaluations","publication_year":2023,"publication_date":"2023-07-06","ids":{"openalex":"https://openalex.org/W4383604699","doi":"https://doi.org/10.48550/arxiv.2307.02762"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2307.02762","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.02762","pdf_url":"https://arxiv.org/pdf/2307.02762","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2307.02762","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015964613","display_name":"Ruosen Li","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Ruosen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108335356","display_name":"Teerth Patel","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Patel, Teerth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5059181012","display_name":"Xinya Du","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Xinya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6801999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6801999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1525000035762787,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.07119999825954437,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.9039999842643738},{"id":"https://openalex.org/keywords/llm-based-evaluation","display_name":"LLM-based evaluation","score":0.6110000014305115},{"id":"https://openalex.org/keywords/peer-discussion","display_name":"peer discussion","score":0.9810000061988831},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"pairwise comparison","score":0.6899999976158142},{"id":"https://openalex.org/keywords/self-enhancement-bias","display_name":"self-enhancement bias","score":0.7929999828338623},{"id":"https://openalex.org/keywords/position-bias","display_name":"position bias","score":0.9900000095367432},{"id":"https://openalex.org/keywords/reference-free-evaluation","display_name":"reference-free evaluation","score":0.6970000267028809},{"id":"https://openalex.org/keywords/open-ended-question-answering","display_name":"open-ended question answering","score":0.878000020980835},{"id":"https://openalex.org/keywords/model-ranking","display_name":"model ranking","score":0.6679999828338623}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7887998223304749},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7830034494400024},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.7147692441940308},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6523106098175049},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6335172057151794},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6310009360313416},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.4636951684951782},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.38962364196777344},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36630478501319885},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3637771010398865},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33166366815567017},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1352798044681549},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07579922676086426},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2307.02762","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.02762","pdf_url":"https://arxiv.org/pdf/2307.02762","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2307.02762","doi":"https://doi.org/10.48550/arxiv.2307.02762","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2307.02762","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2307.02762","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.02762","pdf_url":"https://arxiv.org/pdf/2307.02762","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391591722","https://openalex.org/W4403582514","https://openalex.org/W4391376184","https://openalex.org/W7117645573","https://openalex.org/W4402670841","https://openalex.org/W7171368255","https://openalex.org/W7127581972","https://openalex.org/W4393247764","https://openalex.org/W4415885255","https://openalex.org/W4411113256"],"abstract_inverted_index":{"Nowadays,":[0],"the":[1,38,43,83,100,145,182],"quality":[2],"of":[3,30,49,114,123,147,179],"responses":[4],"generated":[5],"by":[6],"different":[7],"modern":[8],"large":[9],"language":[10],"models":[11,180,198],"(LLMs)":[12],"is":[13,189],"hard":[14,201],"to":[15,92,135,139,195,202],"evaluate":[16],"and":[17,23,53,74,80,118,125,137,165],"compare":[18,203],"automatically.":[19],"Recent":[20],"studies":[21],"suggest":[22],"predominantly":[24],"use":[25,37],"LLMs":[26,134],"for":[27,204],"reference-free":[28],"evaluation":[29],"open-ended":[31],"question":[32],"answering.":[33],"More":[34],"specifically,":[35],"they":[36],"recognized":[39],"\"strongest\"":[40],"LLM":[41],"as":[42,66],"evaluator,":[44],"which":[45],"conducts":[46],"pairwise":[47,112],"comparisons":[48],"candidate":[50],"models'":[51],"answers":[52],"provides":[54,193],"a":[55,120,141,175],"ranking":[56,122],"score.":[57],"However,":[58],"this":[59],"intuitive":[60],"method":[61],"has":[62],"multiple":[63],"problems,":[64],"such":[65],"bringing":[67],"in":[68],"self-enhancement":[69],"(favoring":[70],"its":[71],"own":[72],"answers)":[73],"positional":[75],"bias.":[76],"We":[77,150,157],"draw":[78],"insights":[79],"lessons":[81],"from":[82],"educational":[84],"domain":[85],"(Cho":[86],"&":[87],"MacArthur,":[88],"2011;":[89],"Walsh,":[90],"2014)":[91],"improve":[93],"LLM-based":[94],"evaluations.":[95],"Specifically,":[96],"we":[97,131],"propose":[98],"(1)":[99],"peer":[101,110,127],"rank":[102],"(PR)":[103],"algorithm":[104],"that":[105,159,199],"takes":[106],"into":[107],"account":[108],"each":[109,186],"LLM's":[111],"preferences":[113,146],"all":[115],"answer":[116],"pairs,":[117],"outputs":[119],"final":[121],"models;":[124],"(2)":[126],"discussion":[128],"(PD),":[129],"where":[130,185],"prompt":[132],"two":[133,148,154],"discuss":[136],"try":[138],"reach":[140],"mutual":[142],"agreement":[143],"on":[144,153],"answers.":[149],"conduct":[151],"experiments":[152],"benchmark":[155],"datasets.":[156],"find":[158],"our":[160],"approaches":[161],"achieve":[162],"higher":[163],"accuracy":[164],"align":[166],"better":[167],"with":[168],"human":[169],"judgments.":[170],"Interestingly,":[171],"PR":[172],"can":[173],"induce":[174],"relatively":[176],"accurate":[177],"self-ranking":[178],"under":[181],"anonymous":[183],"setting,":[184],"model's":[187],"name":[188],"unrevealed.":[190],"Our":[191],"work":[192],"space":[194],"explore":[196],"evaluating":[197],"are":[200],"humans.":[205]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2023-07-08T00:00:00"}