{"id":"https://openalex.org/W7138840507","doi":"https://doi.org/10.48550/arxiv.2603.17588","title":"From Isolated Scoring to Collaborative Ranking: A Comparison-Native Framework for LLM-Based Paper Evaluation","display_name":"From Isolated Scoring to Collaborative Ranking: A Comparison-Native Framework for LLM-Based Paper Evaluation","publication_year":2026,"publication_date":"2026-03-18","ids":{"openalex":"https://openalex.org/W7138840507","doi":"https://doi.org/10.48550/arxiv.2603.17588"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.17588","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.17588","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.17588","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129849984","display_name":"Pujun Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Pujun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130211273","display_name":"Jiacheng Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Jiacheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130072917","display_name":"Jinquan Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Jinquan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130193645","display_name":"Chenyang Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Chenyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000341481","display_name":"Guoxiu He","orcid":"https://orcid.org/0000-0002-1419-7495"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Guoxiu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129911805","display_name":"Jiawei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130100391","display_name":"Yong Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Yong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101278308","display_name":"Tianrui Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Tianrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130182605","display_name":"Wei Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2953999936580658,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2953999936580658,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10102","display_name":"scientometrics and bibliometrics research","score":0.08699999749660492,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.05209999904036522,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.7473000288009644},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.7379000186920166},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6414999961853027},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.614300012588501},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.5016999840736389},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.475600004196167},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4228000044822693},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.39719998836517334}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7473000288009644},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.7379000186920166},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.667900025844574},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6650999784469604},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6414999961853027},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6355999708175659},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.614300012588501},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.5016999840736389},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.475600004196167},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4228000044822693},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.39719998836517334},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3917999863624573},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3862999975681305},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.32749998569488525},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.31700000166893005},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.30399999022483826},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.28369998931884766},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.2685000002384186},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.26260000467300415},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.2549999952316284},{"id":"https://openalex.org/C3020001037","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assessment","level":3,"score":0.2549999952316284},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.25440001487731934}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.17588","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.17588","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.17588","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.17588","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.7477567791938782}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,32],"(LLMs)":[3],"are":[4,37],"currently":[5],"applied":[6],"to":[7,16,39,62,93,166],"scientific":[8],"paper":[9,18,57,102,131],"evaluation":[10,30,58],"by":[11],"assigning":[12],"an":[13,151],"absolute":[14,35],"score":[15,22],"each":[17],"independently.":[19],"However,":[20],"since":[21],"scales":[23],"vary":[24],"across":[25],"conferences,":[26],"time":[27],"periods,":[28],"and":[29,82,100,116,133],"criteria,":[31],"trained":[33],"on":[34],"scores":[36],"prone":[38],"fitting":[40],"narrow,":[41],"context-specific":[42],"rules":[43],"rather":[44],"than":[45],"developing":[46],"robust":[47,164],"scholarly":[48],"judgment.":[49],"To":[50],"overcome":[51],"this":[52],"limitation,":[53],"we":[54,67],"propose":[55,87],"shifting":[56],"from":[59,104],"isolated":[60],"scoring":[61],"collaborative":[63],"ranking.":[64,143],"In":[65],"particular,":[66],"design":[68],"a":[69,88,105,139],"$\\textbf{C}$omparison-$\\textbf{N}$ative":[70],"framework":[71,149],"for":[72],"$\\textbf{P}$aper":[73],"$\\textbf{E}$valuation":[74],"($\\textbf{CNPE}$),":[75],"integrating":[76],"comparison":[77],"into":[78,138],"both":[79],"data":[80],"construction":[81],"model":[83,125],"learning.":[84],"We":[85,107],"first":[86],"graph-based":[89],"similarity":[90],"ranking":[91],"algorithm":[92],"facilitate":[94],"the":[95,124,158],"sampling":[96],"of":[97,155],"more":[98],"informative":[99],"discriminative":[101],"pairs":[103,132],"collection.":[106],"then":[108],"enhance":[109],"relative":[110,141,153],"quality":[111,142],"judgment":[112],"through":[113],"supervised":[114],"fine-tuning":[115],"reinforcement":[117],"learning":[118],"with":[119],"comparison-based":[120],"rewards.":[121],"At":[122],"inference,":[123],"performs":[126],"pairwise":[127],"comparisons":[128],"over":[129,157],"sampled":[130],"aggregates":[134],"these":[135],"preference":[136],"signals":[137],"global":[140],"Experimental":[144],"results":[145],"demonstrate":[146],"that":[147],"our":[148],"achieves":[150],"average":[152],"improvement":[154],"21.8%":[156],"strong":[159],"baseline":[160],"DeepReview-14B,":[161],"while":[162],"exhibiting":[163],"generalization":[165],"five":[167],"previously":[168],"unseen":[169],"datasets.":[170],"Our":[171],"code":[172],"is":[173],"available":[174],"at":[175],"https://github.com/ECNU-Text-Computing/ComparisonReview.":[176]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-20T00:00:00"}