{"id":"https://openalex.org/W4416035856","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.236","title":"GRADE: Generating multi-hop QA and fine-gRAined Difficulty matrix for RAG Evaluation","display_name":"GRADE: Generating multi-hop QA and fine-gRAined Difficulty matrix for RAG Evaluation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416035856","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.236"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.236","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.236","pdf_url":"https://aclanthology.org/2025.findings-emnlp.236.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-emnlp.236.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120308926","display_name":"Jeongsoo Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jeongsoo Lee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120308927","display_name":"Daeyong Kwon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Daeyong Kwon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5120308928","display_name":"Kyohoon Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kyohoon Jin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.28969582,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4405","last_page":"4424"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10901","display_name":"Advanced Data Compression Techniques","score":0.04859999939799309,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10901","display_name":"Advanced Data Compression Techniques","score":0.04859999939799309,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11034","display_name":"Digital Filter Design and Implementation","score":0.04659999907016754,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10057","display_name":"Face and Expression Recognition","score":0.04650000110268593,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/matrix","display_name":"Matrix (chemical analysis)","score":0.40310001373291016},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.2842999994754791},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.25220000743865967},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.25189998745918274}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5335000157356262},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.44209998846054077},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4115000069141388},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.40310001373291016},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.33570000529289246},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.28450000286102295},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2842999994754791},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.25220000743865967},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.25189998745918274},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.2386000007390976}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-emnlp.236","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.236","pdf_url":"https://aclanthology.org/2025.findings-emnlp.236.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.236","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.236","pdf_url":"https://aclanthology.org/2025.findings-emnlp.236.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416035856.pdf","grobid_xml":"https://content.openalex.org/works/W4416035856.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Retrieval-Augmented":[0],"Generation":[1],"(RAG)":[2],"systems":[3],"are":[4],"widely":[5],"adopted":[6],"in":[7,23,158],"knowledgeintensive":[8],"NLP":[9],"tasks,":[10],"but":[11],"current":[12],"evaluations":[13],"often":[14],"overlook":[15,27],"the":[16,32,63,74],"structural":[17],"complexity":[18],"and":[19,37,69,76,94,109,123,147,154],"multi-step":[20],"reasoning":[21,38,59,157],"required":[22],"real-world":[24,159],"scenarios.These":[25],"benchmarks":[26],"key":[28],"factors":[29],"such":[30],"as":[31],"interaction":[33],"between":[34,73],"retrieval":[35],"difficulty":[36,53,118,135],"depth.To":[39],"address":[40],"this":[41],"gap,":[42],"we":[43],"propose":[44],"GRADE,":[45],"a":[46,81,116,149],"novel":[47],"evaluation":[48],"framework":[49,114],"that":[50,120,128],"models":[51],"task":[52],"along":[54],"two":[55],"orthogonal":[56],"dimensions:":[57],"(1)":[58],"depth,":[60],"defined":[61],"by":[62,90],"number":[64],"of":[65,144],"inference":[66],"steps":[67],"(hops),":[68],"(2)":[70],"semantic":[71,98],"distance":[72],"query":[75],"its":[77],"supporting":[78],"evidence.We":[79],"construct":[80],"synthetic":[82],"multi-hop":[83],"QA":[84],"dataset":[85],"from":[86],"factual":[87],"news":[88],"articles":[89],"extracting":[91],"knowledge":[92],"graphs":[93],"augmenting":[95],"them":[96],"through":[97],"clustering":[99],"to":[100,106,112],"recover":[101],"missing":[102],"links,":[103],"allowing":[104],"us":[105],"generate":[107],"diverse":[108],"difficulty-controlled":[110],"queries.Central":[111],"our":[113,134],"is":[115],"2D":[117],"matrix":[119],"combines":[121],"generator-side":[122],"retriever-side":[124],"difficulty.Extensive":[125],"experiments":[126],"show":[127],"error":[129],"rates":[130],"strongly":[131],"correlate":[132],"with":[133],"measures,":[136],"validating":[137],"their":[138],"diagnostic":[139],"utility.GRADE":[140],"enables":[141],"fine-grained":[142],"analysis":[143],"RAG":[145],"performance":[146],"provides":[148],"scalable":[150],"foundation":[151],"for":[152],"evaluating":[153],"improving":[155],"multihop":[156],"applications.":[160]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-08T00:00:00"}