{"id":"https://openalex.org/W4416958338","doi":"https://doi.org/10.1145/3767695.3769523","title":"BREV-RAG: Beyond Relevance-based EValuation of RAG systems: A SIGIR-AP 2025 Workshop Proposal","display_name":"BREV-RAG: Beyond Relevance-based EValuation of RAG systems: A SIGIR-AP 2025 Workshop Proposal","publication_year":2025,"publication_date":"2025-12-03","ids":{"openalex":"https://openalex.org/W4416958338","doi":"https://doi.org/10.1145/3767695.3769523"},"language":null,"primary_location":{"id":"doi:10.1145/3767695.3769523","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3767695.3769523","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University/Naver Corporation, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6720-963X","affiliations":[{"raw_affiliation_string":"Waseda University/Naver Corporation, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091105421","display_name":"Sijie Tao","orcid":"https://orcid.org/0000-0002-6751-5303"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Sijie Tao","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6751-5303","affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010558184","display_name":"Zhicheng Dou","orcid":"https://orcid.org/0000-0002-9781-948X"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhicheng Dou","raw_affiliation_strings":["Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9781-948X","affiliations":[{"raw_affiliation_string":"Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100395785","display_name":"Junjie Wang","orcid":"https://orcid.org/0000-0001-9869-7085"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junjie Wang","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9869-7085","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020364411","display_name":"Haoxiang Shi","orcid":"https://orcid.org/0009-0002-9204-0351"},"institutions":[{"id":"https://openalex.org/I55654194","display_name":"Inner Mongolia University of Technology","ror":"https://ror.org/05564e019","country_code":"CN","type":"education","lineage":["https://openalex.org/I55654194"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoxiang Shi","raw_affiliation_strings":["Inner Mongolia University of Technology, Ordos, China"],"raw_orcid":"https://orcid.org/0009-0002-9204-0351","affiliations":[{"raw_affiliation_string":"Inner Mongolia University of Technology, Ordos, China","institution_ids":["https://openalex.org/I55654194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007524571","display_name":"Nuo Chen","orcid":"https://orcid.org/0000-0001-8600-8203"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Nuo Chen","raw_affiliation_strings":["The Hong Kong Polytechnic University, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-8600-8203","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University, Hong Kong, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004113066","display_name":"Atsushi Keyaki","orcid":"https://orcid.org/0000-0001-6495-117X"},"institutions":[{"id":"https://openalex.org/I111428342","display_name":"Hitotsubashi University","ror":"https://ror.org/04jqj7p05","country_code":"JP","type":"education","lineage":["https://openalex.org/I111428342"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Atsushi Keyaki","raw_affiliation_strings":["Hitotsubashi University, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0001-6495-117X","affiliations":[{"raw_affiliation_string":"Hitotsubashi University, Tokyo, Japan","institution_ids":["https://openalex.org/I111428342"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5023595778"],"corresponding_institution_ids":["https://openalex.org/I150744194"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.19524658,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"459","last_page":"460"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.692300021648407,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.692300021648407,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12268","display_name":"Deception detection and forensic psychology","score":0.03999999910593033,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.01979999989271164,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/plan","display_name":"Plan (archaeology)","score":0.612500011920929},{"id":"https://openalex.org/keywords/theme","display_name":"Theme (computing)","score":0.5990999937057495},{"id":"https://openalex.org/keywords/breakout","display_name":"Breakout","score":0.519599974155426},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5189999938011169},{"id":"https://openalex.org/keywords/publication","display_name":"Publication","score":0.4092999994754791}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6983000040054321},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.612500011920929},{"id":"https://openalex.org/C33566652","wikidata":"https://www.wikidata.org/wiki/Q1065927","display_name":"Theme (computing)","level":2,"score":0.5990999937057495},{"id":"https://openalex.org/C2778091849","wikidata":"https://www.wikidata.org/wiki/Q4959649","display_name":"Breakout","level":2,"score":0.519599974155426},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5189999938011169},{"id":"https://openalex.org/C41458344","wikidata":"https://www.wikidata.org/wiki/Q732577","display_name":"Publication","level":2,"score":0.4092999994754791},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3691999912261963},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3361000120639801},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3122999966144562},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.28279998898506165},{"id":"https://openalex.org/C59656382","wikidata":"https://www.wikidata.org/wiki/Q191536","display_name":"Conjunction (astronomy)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2630999982357025}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3767695.3769523","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3767695.3769523","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"propose":[1],"a":[2,88,150],"new":[3],"SIGIR-AP":[4],"workshop,":[5],"which":[6],"we":[7,134],"call":[8],"BREV-RAG":[9],"(Beyond":[10],"Relevance-based":[11],"EValuation":[12],"of":[13,58,72,95],"RAG":[14,17,86,118],"systems).":[15],"While":[16],"evaluation":[18,30,90,110,129],"primarily":[19],"considers":[20],"relevance,":[21],"correctness,":[22],"and":[23,40,53,120,123,147],"groundedness,":[24],"there":[25],"probably":[26],"are":[27],"many":[28],"other":[29],"axes":[31],"that":[32,64,112],"need":[33],"to":[34,126,136,153],"be":[35,99],"considered":[36],"for":[37,41,101,116],"the":[38,46,56,81,128,138,142],"user":[39],"downstream":[42],"tasks.":[43],"For":[44],"example,":[45],"upcoming":[47],"NTCIR-19":[48],"R2C2":[49],"(RAG":[50],"Responses:":[51],"Confident":[52],"Correct?)":[54],"addresses":[55],"problem":[57],"returning":[59],"an":[60,109],"answer":[61,67],"confidence":[62],"score":[63],"aligns":[65],"with":[66,87],"accuracy.":[68],"The":[69,92],"first":[70],"half":[71,94],"this":[73,96],"workshop":[74,97,144,151],"will":[75,98,107,148],"feature":[76],"refereed":[77],"paper":[78],"presentations,":[79],"where":[80,104],"central":[82],"theme":[83],"is":[84],"evaluating":[85],"novel":[89],"axis.":[91],"second":[93],"used":[100],"breakout":[102],"discussions,":[103],"each":[105],"group":[106],"choose":[108],"axis":[111],"they":[113],"deem":[114],"important":[115],"future":[117],"evaluations,":[119],"discuss":[121],"possible":[122],"feasible":[124],"approaches":[125],"implementing":[127],"scheme.":[130],"As":[131],"tangible":[132],"output,":[133],"plan":[135],"publish":[137],"accepted":[139],"papers":[140],"in":[141],"CEUR":[143],"proceedings":[145],"series,":[146],"submit":[149],"summary":[152],"SIGIR":[154],"Forum":[155],"(June":[156],"2026":[157],"issue).":[158]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-12-03T00:00:00"}