{"id":"https://openalex.org/W4412888671","doi":"https://doi.org/10.18653/v1/2025.findings-acl.301","title":"Judge as A Judge: Improving the Evaluation of Retrieval-Augmented Generation through the Judge-Consistency of Large Language Models","display_name":"Judge as A Judge: Improving the Evaluation of Retrieval-Augmented Generation through the Judge-Consistency of Large Language Models","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888671","doi":"https://doi.org/10.18653/v1/2025.findings-acl.301"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.301","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.301","pdf_url":"https://aclanthology.org/2025.findings-acl.301.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.301.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027064196","display_name":"Shuliang Liu","orcid":"https://orcid.org/0000-0002-9341-3116"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shuliang Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083202487","display_name":"Xinze Li","orcid":"https://orcid.org/0009-0005-0035-6802"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xinze Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070196864","display_name":"Zhenghao Liu","orcid":"https://orcid.org/0000-0001-8067-1477"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhenghao Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104248364","display_name":"Yukun Yan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yukun Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111968626","display_name":"Cheng Yang","orcid":"https://orcid.org/0009-0003-5050-7387"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008361378","display_name":"Zheni Zeng","orcid":"https://orcid.org/0000-0001-6901-5292"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheni Zeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100320723","display_name":"Zhiyuan Liu","orcid":"https://orcid.org/0000-0002-7709-2543"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhiyuan Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046448314","display_name":"Maosong Sun","orcid":"https://orcid.org/0000-0002-6011-6115"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maosong Sun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5113657466","display_name":"Yu Ge","orcid":"https://orcid.org/0000-0001-7153-5035"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ge Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5788","last_page":"5807"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.8039648532867432},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7501620650291443},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6017608642578125},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5862950086593628},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49111148715019226}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.8039648532867432},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7501620650291443},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6017608642578125},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5862950086593628},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49111148715019226}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.301","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.301","pdf_url":"https://aclanthology.org/2025.findings-acl.301.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.301","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.301","pdf_url":"https://aclanthology.org/2025.findings-acl.301.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5699999928474426}],"awards":[{"id":"https://openalex.org/G2602313477","display_name":null,"funder_award_id":"62206042","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3078953789","display_name":null,"funder_award_id":"202403","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4466500611","display_name":null,"funder_award_id":"62137001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320323086","display_name":"Natural Science Foundation of Liaoning Province","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888671.pdf","grobid_xml":"https://content.openalex.org/works/W4412888671.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Retrieval-Augmented":[0],"Generation":[1],"(RAG)":[2],"has":[3],"proven":[4],"its":[5,180],"effectiveness":[6],"in":[7,185],"alleviating":[8],"hallucinations":[9],"for":[10,75,106,119],"Large":[11],"Language":[12],"Models":[13],"(LLMs).However,":[14],"existing":[15],"automated":[16],"evaluation":[17,47],"metrics":[18],"cannot":[19],"fairly":[20],"evaluate":[21,96],"the":[22,35,54,61,93,101,141,152,173,231,234,241],"outputs":[23],"generated":[24,133],"by":[25,134,172],"RAG":[26,57,76,121,125],"models":[27,33,122,126],"during":[28],"training":[29],"and":[30,99,103,127,187,196,203,222],"evaluation.LLM-based":[31],"judgment":[32,90],"provide":[34,115],"potential":[36],"to":[37,46,50,67,70,81,95,179,182],"produce":[38],"high-quality":[39],"judgments,":[40,98],"but":[41],"they":[42,228],"are":[43,145,151,166,224],"highly":[44],"sensitive":[45],"prompts,":[48],"leading":[49],"inconsistencies":[51],"when":[52],"judging":[53],"output":[55],"of":[56,89,155,164,169,233,243],"models.This":[58],"paper":[59],"introduces":[60],"Judge-Consistency":[62],"(ConsJudge)":[63],"method,":[64],"which":[65],"aims":[66],"enhance":[68],"LLMs":[69,80],"generate":[71,82],"more":[72,116,225],"accurate":[73,117],"evaluations":[74],"models.Specifically,":[77],"ConsJudge":[78,112,135],"prompts":[79],"different":[83],"judgments":[84,105,118,132],"based":[85],"on":[86],"various":[87,124],"combinations":[88],"dimensions,":[91],"utilizes":[92],"judge-consistency":[94],"these":[97],"selects":[100],"chosen":[102],"rejected":[104],"DPO":[107],"training.Our":[108],"experiments":[109],"show":[110],"that":[111,131,177],"can":[113],"effectively":[114],"optimizing":[120],"across":[123],"datasets.Further":[128],"analysis":[129],"reveals":[130],"have":[136],"a":[137,167],"high":[138],"agreement":[139],"with":[140],"superior":[142],"LLM.All":[143],"codes":[144],"available":[146],"at":[147],"https://github.com/OpenBMB/ConsJudge.Case":[148],"1:":[149],"What":[150],"virulence":[153,162,235],"factors":[154,163],"anthrax?Ground":[156],"Truth:":[157],"Bacillus":[158,174],"anthracis.Choice":[159,190],"A:":[160],"The":[161],"anthrax":[165],"group":[168],"proteins":[170],"produced":[171],"anthracis":[175],"bacterium":[176],"contribute":[178],"ability":[181],"cause":[183],"disease":[184],"humans":[186],"animals.Choice":[188],"B:":[189],"C:":[191],"lethal":[192,200],"factor,":[193],"edema":[194],"factor":[195,202],"antiphagocytic":[197],"factor.Choice":[198],"D:":[199],"factor,antiphagocytic":[201],"other":[204],"factors.Raw":[205],"Metric":[206],"(ROUGE-L):":[207],"Choice":[208,211,214,217],"B(0.667)":[209],">":[210,213],"A(0.143)":[212],"C(0)":[215],"=":[216],"D(0)":[218],"Zero-Shot:":[219],"...choice":[220],"C":[221],"D":[223],"complete":[226],"as":[227],"specifically":[229],"mention":[230],"names":[232],"factor...":[236],"Best":[237],"answer":[238],":C.SFT:...":[239],"from":[240],"perspective":[242]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-16T13:24:37.021932","created_date":"2025-10-10T00:00:00"}