{"id":"https://openalex.org/W7204199737","doi":"https://doi.org/10.48550/arxiv.2608.21374","title":"LitReview Arena: Evaluating Literature Review Agents with Battle-Style Peer Review Platform","display_name":"LitReview Arena: Evaluating Literature Review Agents with Battle-Style Peer Review Platform","publication_year":2026,"publication_date":"2026-07-01","ids":{"openalex":"https://openalex.org/W7204199737","doi":"https://doi.org/10.48550/arxiv.2608.21374"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.21374","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.21374","pdf_url":"https://arxiv.org/pdf/2608.21374","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.21374","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101184074","display_name":"Ruotong Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhao, Ruotong","raw_affiliation_strings":["Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148149344","display_name":"Zhiyu Zoey Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhiyu","raw_affiliation_strings":["Zhongguancun Academy , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhongguancun Academy , Beijing , China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128096959","display_name":"Xurui Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Xurui","raw_affiliation_strings":["Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148103415","display_name":"Haidong Xue","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue, Haidong","raw_affiliation_strings":["Zhongguancun Institute of AI , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhongguancun Institute of AI , Beijing , China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148080421","display_name":"Dong Liang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang, Dong","raw_affiliation_strings":["Huazhong University of Science and Technology , Wuhan , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huazhong University of Science and Technology , Wuhan , China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148073604","display_name":"Jigao Fu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Jigao","raw_affiliation_strings":["Zhongguancun Institute of AI , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhongguancun Institute of AI , Beijing , China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148139160","display_name":"Wu YanBiao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"YanBiao, Wu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029414400","display_name":"Yuanyi Zhen","orcid":"https://orcid.org/0000-0003-2712-1198","observed_orcids":["https://orcid.org/0000-0003-2712-1198"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhen, Yuanyi","raw_affiliation_strings":["Zhongguancun Academy , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhongguancun Academy , Beijing , China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062365263","display_name":"Fengli Xu","orcid":"https://orcid.org/0000-0002-5720-4026","observed_orcids":["https://orcid.org/0000-0002-5720-4026"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu, Fengli","raw_affiliation_strings":["Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148858536","display_name":"Yong Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Yong","raw_affiliation_strings":["Tsinghua University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University , Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.29789999127388,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.29789999127388,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.13339999318122864,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12648","display_name":"Academic Writing and Publishing","score":0.07639999687671661,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/expert-opinion","display_name":"expert opinion","score":0.5669999718666077},{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.7649999856948853},{"id":"https://openalex.org/keywords/agentic-llms","display_name":"agentic LLMs","score":0.703000009059906},{"id":"https://openalex.org/keywords/expert-calibration","display_name":"expert calibration","score":0.5569999814033508},{"id":"https://openalex.org/keywords/evidence-synthesis","display_name":"evidence synthesis","score":0.33500000834465027}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6705999970436096},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6000000238418579},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5141000151634216},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5031999945640564},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.4359000027179718},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.4334000051021576},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.38989999890327454},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38440001010894775},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3693999946117401},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.3000999987125397},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.29899999499320984},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2978000044822693},{"id":"https://openalex.org/C72161134","wikidata":"https://www.wikidata.org/wiki/Q5421219","display_name":"Expert elicitation","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2858000099658966},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2639999985694885},{"id":"https://openalex.org/C2781083858","wikidata":"https://www.wikidata.org/wiki/Q17327049","display_name":"Scientific literature","level":2,"score":0.2614000141620636},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.25690001249313354}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.21374","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.21374","pdf_url":"https://arxiv.org/pdf/2608.21374","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.21374","doi":"https://doi.org/10.48550/arxiv.2608.21374","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.21374","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.21374","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.21374","pdf_url":"https://arxiv.org/pdf/2608.21374","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204199737.pdf","grobid_xml":"https://content.openalex.org/works/W7204199737.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7140197738","https://openalex.org/W4403006832","https://openalex.org/W4416034483","https://openalex.org/W7162804100","https://openalex.org/W4416018011","https://openalex.org/W7212253519","https://openalex.org/W7155065644","https://openalex.org/W7154729717","https://openalex.org/W7214341427","https://openalex.org/W4405713902"],"abstract_inverted_index":{"Literature":[0],"reviews":[1,12],"are":[2,55,127,170],"essential":[3],"to":[4,42,57,160,164],"scientific":[5],"progress,":[6],"but":[7],"rigorously":[8],"evaluating":[9],"automatically":[10],"generated":[11],"remains":[13],"difficult":[14],"because":[15],"many":[16],"aspects":[17],"of":[18,95],"research":[19,144],"utility":[20],"depend":[21],"on":[22,101,136],"expert":[23,77],"judgment":[24],"rather":[25],"than":[26],"reference-overlap":[27],"metrics.":[28],"We":[29,120],"introduce":[30],"LitReview":[31],"Arena,":[32],"a":[33,38],"battle-style":[34],"evaluation":[35],"platform":[36],"with":[37,48,130],"structured":[39],"protocol":[40],"tailored":[41],"literature":[43],"review":[44],"quality:":[45],"domain":[46],"experts":[47,132],"AI":[49],"paper-writing":[50],"experience":[51],"compare":[52],"anonymized":[53],"drafts,":[54],"matched":[56],"topics":[58],"within":[59],"their":[60],"expertise,":[61],"and":[62,84,143,168],"provide":[63,152],"dimension-wise":[64,82],"outcomes":[65],"over":[66,118],"five":[67,81],"literature-review-specific":[68],"criteria.":[69],"From":[70],"this":[71],"protocol,":[72],"we":[73,151],"collect":[74],"approximately":[75],"3k":[76],"judgments,":[78],"each":[79],"containing":[80],"outcomes,":[83],"show":[85],"that":[86,123],"even":[87],"the":[88,147],"strongest":[89],"current":[90],"systems":[91],"win":[92],"only":[93],"23.0%":[94],"decisive":[96],"matches":[97],"against":[98],"human":[99,131],"drafts":[100],"overall":[102],"utility,":[103],"while":[104],"agentic":[105],"LLMs":[106],"such":[107,139],"as":[108,140],"Sonar":[109],"Deep":[110],"Research":[111],"substantially":[112,128],"outperform":[113],"base":[114],"language":[115],"models":[116],"by":[117],"60%.":[119],"further":[121],"find":[122],"existing":[124],"LLM-as-a-judge":[125],"methods":[126],"misaligned":[129],"(Spearman's":[133],"rho=0.467),":[134],"especially":[135],"synthesis-heavy":[137],"criteria":[138],"paper":[141],"structure":[142],"suggestions.":[145],"Using":[146],"collected":[148],"preference":[149],"data,":[150],"an":[153],"expert-calibrated":[154],"evaluator,":[155],"LitJudge,":[156],"which":[157],"improves":[158],"alignment":[159],"Spearman's":[161],"rho=0.78,":[162],"comparable":[163],"inter-expert":[165],"consistency;":[166],"code":[167],"data":[169],"publicly":[171],"available":[172],"at":[173],"https://github.com/VanellopeAsher/LitReview-Arena.":[174]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-08-26T00:00:00"}