{"id":"https://openalex.org/W7168354759","doi":"https://doi.org/10.1145/3805712.3809940","title":"RubricRAG: Towards Interpretable and Reliable LLM Evaluation via Domain Knowledge Retrieval for Rubric Generation","display_name":"RubricRAG: Towards Interpretable and Reliable LLM Evaluation via Domain Knowledge Retrieval for Rubric Generation","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7168354759","doi":"https://doi.org/10.1145/3805712.3809940"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3809940","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809940","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3809940","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140882850","display_name":"Kaustubh Dhole","orcid":"https://orcid.org/0009-0006-6907-2530"},"institutions":[{"id":"https://openalex.org/I150468666","display_name":"Emory University","ror":"https://ror.org/03czfpz43","country_code":"US","type":"education","lineage":["https://openalex.org/I150468666"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kaustubh Dhole","raw_affiliation_strings":["Department of Computer Science, Emory University, Atlanta, Georgia, USA"],"raw_orcid":"https://orcid.org/0009-0006-6907-2530","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Emory University, Atlanta, Georgia, USA","institution_ids":["https://openalex.org/I150468666"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5140844773","display_name":"Eugene Agichtein","orcid":"https://orcid.org/0000-0002-3148-5448"},"institutions":[{"id":"https://openalex.org/I150468666","display_name":"Emory University","ror":"https://ror.org/03czfpz43","country_code":"US","type":"education","lineage":["https://openalex.org/I150468666"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eugene Agichtein","raw_affiliation_strings":["Department of Computer Science, Emory University, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0002-3148-5448","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Emory University, Atlanta, GA, USA","institution_ids":["https://openalex.org/I150468666"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I150468666"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3681","last_page":"3687"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.679099977016449},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5386000275611877},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"Domain knowledge","score":0.3919999897480011},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.29739999771118164},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.2904999852180481}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.679099977016449},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6754000186920166},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5386000275611877},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5001000165939331},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.3919999897480011},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39149999618530273},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3361999988555908},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2904999852180481},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2881999909877777},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26989999413490295},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C115925183","wikidata":"https://www.wikidata.org/wiki/Q1412694","display_name":"Knowledge-based systems","level":2,"score":0.2524000108242035},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.2524000108242035}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805712.3809940","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809940","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805712.3809940","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809940","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2970641574","https://openalex.org/W2979826702","https://openalex.org/W4245724339","https://openalex.org/W4389519598","https://openalex.org/W4400484844","https://openalex.org/W4401042808","https://openalex.org/W4401330297","https://openalex.org/W4411119051","https://openalex.org/W4411119262","https://openalex.org/W4411120242","https://openalex.org/W4412889928","https://openalex.org/W4412945636","https://openalex.org/W4415795891","https://openalex.org/W4416033719","https://openalex.org/W4416034300","https://openalex.org/W7140126704"],"related_works":[],"abstract_inverted_index":{"LLMs":[0,71,100],"are":[1],"increasingly":[2],"evaluated":[3],"or":[4,16,28],"trained":[5],"using":[6],"automated":[7],"graders":[8],"such":[9,53],"as":[10],"LLM-as-judges,":[11],"but":[12],"their":[13],"scalar":[14],"scores":[15,40],"preferences":[17],"often":[18],"provide":[19,126],"little":[20],"explanation":[21],"of":[22,139],"why":[23],"an":[24],"answer":[25],"is":[26,59],"good":[27],"bad.":[29],"Rubric-based":[30],"evaluation":[31],"offers":[32],"a":[33,115,149],"more":[34,45],"interpretable":[35,156],"alternative":[36],"by":[37],"decomposing":[38],"final":[39],"into":[41],"explicit":[42],"criteria,":[43],"enabling":[44],"consistent":[46],"and":[47,61,82,90,93,136,155],"controllable":[48],"evaluation.":[49,85,158],"However,":[50],"manually":[51],"creating":[52],"rubrics":[54,76,81,102,121],"requires":[55],"extensive":[56],"domain":[57,127],"knowledge,":[58],"labor-intensive,":[60],"difficult":[62],"to":[63,106,125,133],"scale.":[64],"In":[65,109],"this":[66],"work,":[67],"we":[68,96,112],"study":[69],"whether":[70],"can":[72],"generate":[73],"useful":[74],"instance-specific":[75],"that":[77,98,110,119],"align":[78],"with":[79,103],"human-authored":[80,134],"improve":[83],"response":[84],"Across":[86],"two":[87],"rubric":[88],"benchmarks":[89],"multiple":[91],"prompting":[92],"post-training":[94],"strategies,":[95],"find":[97],"off-the-shelf":[99],"produce":[101],"limited":[104],"alignment":[105],"human-written":[107],"ones.":[108],"regard,":[111],"propose":[113],"RubricRAG,":[114],"simple":[116],"inference-time":[117],"strategy":[118],"retrieves":[120],"from":[122],"related":[123],"queries":[124],"knowledge.":[128],"RubricRAG":[129],"improves":[130],"both":[131],"similarity":[132],"rubrics,":[135,147],"the":[137,145],"accuracy":[138],"downstream":[140],"LLM":[141],"judges":[142],"based":[143],"on":[144],"generated":[146],"highlighting":[148],"promising":[150],"path":[151],"toward":[152],"automated,":[153],"scalable,":[154],"rubric-based":[157]},"counts_by_year":[],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2026-07-16T00:00:00"}