{"id":"https://openalex.org/W7116799455","doi":"https://doi.org/10.1007/978-3-032-13109-6_3","title":"Towards Dynamically Generated KGQA Benchmark Datasets for Memorization-Resistant Evaluations","display_name":"Towards Dynamically Generated KGQA Benchmark Datasets for Memorization-Resistant Evaluations","publication_year":2025,"publication_date":"2025-12-22","ids":{"openalex":"https://openalex.org/W7116799455","doi":"https://doi.org/10.1007/978-3-032-13109-6_3"},"language":"en","primary_location":{"id":"doi:10.1007/978-3-032-13109-6_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-13109-6_3","pdf_url":null,"source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1007/978-3-032-13109-6_3","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120999975","display_name":"Aleksandr Gashkov","orcid":null},"institutions":[{"id":"https://openalex.org/I147765834","display_name":"Leipzig University of Applied Sciences","ror":"https://ror.org/03xgcq477","country_code":"DE","type":"education","lineage":["https://openalex.org/I147765834"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Aleksandr Gashkov","raw_affiliation_strings":["Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany"],"raw_orcid":"https://orcid.org/0000-0001-6894-2094","affiliations":[{"raw_affiliation_string":"Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany","institution_ids":["https://openalex.org/I147765834"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Aleksandr Perevalov","orcid":"https://orcid.org/0000-0002-0877-7063"},"institutions":[{"id":"https://openalex.org/I147765834","display_name":"Leipzig University of Applied Sciences","ror":"https://ror.org/03xgcq477","country_code":"DE","type":"education","lineage":["https://openalex.org/I147765834"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Aleksandr Perevalov","raw_affiliation_strings":["Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany"],"raw_orcid":"https://orcid.org/0000-0002-0877-7063","affiliations":[{"raw_affiliation_string":"Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany","institution_ids":["https://openalex.org/I147765834"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023792502","display_name":"Maria Nikolaevna Eltsova","orcid":null},"institutions":[{"id":"https://openalex.org/I147765834","display_name":"Leipzig University of Applied Sciences","ror":"https://ror.org/03xgcq477","country_code":"DE","type":"education","lineage":["https://openalex.org/I147765834"]},{"id":"https://openalex.org/I41121874","display_name":"Heilbronn University","ror":"https://ror.org/04g5gcg95","country_code":"DE","type":"education","lineage":["https://openalex.org/I41121874"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Maria Eltsova","raw_affiliation_strings":["CBZ M\u00fcnchen GmbH, Heilbronn, Germany","Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany"],"raw_orcid":"https://orcid.org/0000-0003-3792-8518","affiliations":[{"raw_affiliation_string":"CBZ M\u00fcnchen GmbH, Heilbronn, Germany","institution_ids":["https://openalex.org/I41121874"]},{"raw_affiliation_string":"Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany","institution_ids":["https://openalex.org/I147765834"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5120957164","display_name":"Andreas Both","orcid":null},"institutions":[{"id":"https://openalex.org/I147765834","display_name":"Leipzig University of Applied Sciences","ror":"https://ror.org/03xgcq477","country_code":"DE","type":"education","lineage":["https://openalex.org/I147765834"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Andreas Both","raw_affiliation_strings":["Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany"],"raw_orcid":"https://orcid.org/0000-0001-7116-9338","affiliations":[{"raw_affiliation_string":"Web & Software Engineering (WSE) Research Group, Leipzig University of Applied Sciences (HTWK Leipzig), Leipzig, Germany","institution_ids":["https://openalex.org/I147765834"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5120999975"],"corresponding_institution_ids":["https://openalex.org/I147765834"],"apc_list":{"value":5000,"currency":"EUR","value_usd":5392},"apc_paid":{"value":5000,"currency":"EUR","value_usd":5392},"fwci":6.5449,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.97270407,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"25","last_page":"40"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.5238999724388123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.5238999724388123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.36410000920295715,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.012199999764561653,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8672000169754028},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8531000018119812},{"id":"https://openalex.org/keywords/sparql","display_name":"SPARQL","score":0.7878999710083008},{"id":"https://openalex.org/keywords/memorization","display_name":"Memorization","score":0.6100000143051147},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5782999992370605},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.531000018119812},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.384799987077713}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8964999914169312},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8672000169754028},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8531000018119812},{"id":"https://openalex.org/C41009113","wikidata":"https://www.wikidata.org/wiki/Q54871","display_name":"SPARQL","level":4,"score":0.7878999710083008},{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.6100000143051147},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5782999992370605},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.531000018119812},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5199000239372253},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44119998812675476},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.39820000529289246},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.384799987077713},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.3756999969482422},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.3653999865055084},{"id":"https://openalex.org/C2987255567","wikidata":"https://www.wikidata.org/wiki/Q33002955","display_name":"Knowledge graph","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3596999943256378},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.2996000051498413},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2793999910354614},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.26840001344680786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/978-3-032-13109-6_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-13109-6_3","pdf_url":null,"source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"}],"best_oa_location":{"id":"doi:10.1007/978-3-032-13109-6_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-13109-6_3","pdf_url":null,"source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W150699991","https://openalex.org/W2803728898","https://openalex.org/W2885274892","https://openalex.org/W2963900171","https://openalex.org/W2980401255","https://openalex.org/W3008811720","https://openalex.org/W3030844777","https://openalex.org/W3164228909","https://openalex.org/W3165041216","https://openalex.org/W3176757281","https://openalex.org/W3194934287","https://openalex.org/W4220844664","https://openalex.org/W4285192944","https://openalex.org/W4287027440","https://openalex.org/W4381329270","https://openalex.org/W4386834841","https://openalex.org/W4387171349","https://openalex.org/W4399665841","https://openalex.org/W4399699439","https://openalex.org/W4399699770","https://openalex.org/W4400487882","https://openalex.org/W4403615738","https://openalex.org/W4404025604","https://openalex.org/W4407737208","https://openalex.org/W4409569681","https://openalex.org/W4411119670","https://openalex.org/W4412377805","https://openalex.org/W4415080825"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"demonstrate":[4],"impressive":[5],"capabilities":[6,26],"in":[7,79,105],"Question":[8,85],"Answering":[9,86],"tasks,":[10],"yet":[11],"previous":[12],"research":[13],"suggests":[14],"current":[15],"approaches":[16],"fail":[17],"to":[18,28,70],"accurately":[19],"reflect":[20],"their":[21],"real":[22],"SPARQL":[23,106],"query":[24],"generation":[25],"due":[27],"memorization":[29,147],"effects":[30,41],"caused":[31],"by":[32,52,101],"integrating":[33],"benchmark":[34,129,154],"datasets":[35,74,91,95,130],"into":[36],"the":[37,47,50,80,115,123,144,151],"training":[38],"data.":[39],"These":[40],"artificially":[42],"inflate":[43],"perceived":[44],"performance,":[45],"i.e.,":[46],"quality":[48],"of":[49,82,146,153],"showing":[51],"LLMs":[53],"cannot":[54],"actually":[55],"be":[56,133],"achieved":[57],"on":[58,93],"previously":[59],"unseen":[60],"queries.":[61],"This":[62],"paper":[63],"presents":[64],"DynBench":[65],"\u2013":[66,96,100],"a":[67],"novel":[68],"approach":[69,125,141],"creating":[71],"high-quality":[72],"benchmarking":[73],"that":[75,122,131],"address":[76],"this":[77],"challenge":[78],"field":[81],"Knowledge":[83],"Graph":[84],"(KGQA).":[87],"We":[88],"develop":[89],"new":[90,128],"based":[92],"two":[94],"QALD-9-Plus":[97],"and":[98,113],"LC-QuAD":[99],"systematically":[102],"replacing":[103],"entities":[104],"queries":[107],"with":[108],"alternatives":[109],"retrieved":[110],"from":[111],"Wikidata":[112],"within":[114],"corresponding":[116],"natural-language":[117],"questions.":[118],"Our":[119],"findings":[120],"confirm":[121],"proposed":[124],"successfully":[126],"creates":[127],"can":[132],"used":[134],"for":[135,156],"evaluating":[136],"KGQA":[137,158],"systems.":[138],"Therefore,":[139],"our":[140],"drastically":[142],"reduces":[143],"risk":[145],"effects,":[148],"thereby":[149],"increasing":[150],"trustworthiness":[152],"results":[155],"LLM-based":[157],"approaches.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-06-19T17:40:00.097472","created_date":"2025-12-22T00:00:00"}