{"id":"https://openalex.org/W7155192603","doi":"https://doi.org/10.48550/arxiv.2604.19502","title":"Beyond Rating: A Comprehensive Evaluation and Benchmark for AI Reviews","display_name":"Beyond Rating: A Comprehensive Evaluation and Benchmark for AI Reviews","publication_year":2026,"publication_date":"2026-04-21","ids":{"openalex":"https://openalex.org/W7155192603","doi":"https://doi.org/10.48550/arxiv.2604.19502"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.19502","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19502","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.19502","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134264139","display_name":"Bowen Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134310556","display_name":"Haochen Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Haochen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134334727","display_name":"Yuxin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yuxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134272989","display_name":"Jie Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Jie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134325312","display_name":"Xinchi Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Yining","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134353671","display_name":"Xuanjing Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xinchi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134318251","display_name":"Yining Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Xuanjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134237947","display_name":"Xipeng Qiu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Xipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.19370000064373016,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.11240000277757645,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5760999917984009},{"id":"https://openalex.org/keywords/argumentative","display_name":"Argumentative","score":0.5458999872207642},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.49889999628067017},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.36039999127388},{"id":"https://openalex.org/keywords/subject-matter-expert","display_name":"Subject-matter expert","score":0.34130001068115234},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.3296999931335449}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7017999887466431},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5760999917984009},{"id":"https://openalex.org/C2781306805","wikidata":"https://www.wikidata.org/wiki/Q4789761","display_name":"Argumentative","level":2,"score":0.5458999872207642},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5320000052452087},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.49889999628067017},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4214000105857849},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41679999232292175},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.36039999127388},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.34130001068115234},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.3296999931335449},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.30140000581741333},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2919999957084656},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.29100000858306885},{"id":"https://openalex.org/C114073186","wikidata":"https://www.wikidata.org/wiki/Q2631895","display_name":"Automated planning and scheduling","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.2619999945163727}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.19502","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19502","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.19502","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19502","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"rapid":[1],"adoption":[2],"of":[3,36,96,126],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"has":[8],"spurred":[9],"interest":[10],"in":[11,40],"automated":[12,149],"peer":[13],"review;":[14],"however,":[15],"progress":[16],"is":[17,144],"currently":[18],"stifled":[19],"by":[20],"benchmarks":[21],"that":[22,33,63,110,136],"treat":[23],"reviewing":[24],"primarily":[25],"as":[26],"a":[27,37,49,59,83,93,145,152],"rating":[28,131],"prediction":[29],"task.":[30],"We":[31],"argue":[32],"the":[34,124],"utility":[35],"review":[38],"lies":[39],"its":[41],"textual":[42],"justification--its":[43],"arguments,":[44],"questions,":[45],"and":[46,78,91],"critique--rather":[47],"than":[48],"scalar":[50],"score.":[51],"To":[52],"address":[53],"this,":[54],"we":[55,81],"introduce":[56,92],"Beyond":[57],"Rating,":[58],"holistic":[60],"evaluation":[61],"framework":[62],"assesses":[64],"AI":[65,138],"reviewers":[66],"across":[67],"five":[68],"dimensions:":[69],"Content":[70],"Faithfulness,":[71],"Argumentative":[72],"Alignment,":[73],"Focus":[74],"Consistency,":[75],"Question":[76],"Constructiveness,":[77],"AI-Likelihood.":[79],"Notably,":[80],"propose":[82],"Max-Recall":[84],"strategy":[85],"to":[86,103,116],"accommodate":[87],"valid":[88],"expert":[89],"disagreement":[90],"curated":[94],"dataset":[95],"paper":[97],"with":[98,130,141],"high-confidence":[99],"reviews,":[100],"rigorously":[101],"filtered":[102],"remove":[104],"procedural":[105],"noise.":[106],"Extensive":[107],"experiments":[108],"demonstrate":[109],"while":[111],"traditional":[112],"n-gram":[113],"metrics":[114],"fail":[115],"reflect":[117],"human":[118,142],"preferences,":[119],"our":[120],"proposed":[121],"text-centric":[122],"metrics--particularly":[123],"recall":[125],"weakness":[127],"arguments--correlate":[128],"strongly":[129],"accuracy.":[132],"These":[133],"findings":[134],"establish":[135],"aligning":[137],"critique":[139],"focus":[140],"experts":[143],"prerequisite":[146],"for":[147,155],"reliable":[148],"scoring,":[150],"offering":[151],"robust":[153],"standard":[154],"future":[156],"research.":[157]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-23T00:00:00"}