{"id":"https://openalex.org/W7138278709","doi":"https://doi.org/10.1609/aaai.v40i37.40397","title":"From Chaos to Clarity: A Knowledge Graph-Driven Audit Dataset Generation Framework for LLM Unlearning","display_name":"From Chaos to Clarity: A Knowledge Graph-Driven Audit Dataset Generation Framework for LLM Unlearning","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138278709","doi":"https://doi.org/10.1609/aaai.v40i37.40397"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i37.40397","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i37.40397","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v40i37.40397","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129733634","display_name":"Weipeng Jiang","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weipeng Jiang","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129673069","display_name":"Juan Zhai","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juan Zhai","raw_affiliation_strings":["University of Massachusetts Amherst"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109689136","display_name":"Shiqing Ma","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shiqing Ma","raw_affiliation_strings":["University of Massachusetts Amherst"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129665430","display_name":"Ziyan Lei","orcid":null},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ziyan Lei","raw_affiliation_strings":["University of Bristol"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bristol","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129655218","display_name":"Xiaofei Xie","orcid":null},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xiaofei Xie","raw_affiliation_strings":["Singapore Management University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Singapore Management University","institution_ids":["https://openalex.org/I79891267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129749179","display_name":"Yige Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yige Wang","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5129718373","display_name":"Chao Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Shen","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.20648473,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"37","first_page":"31337","last_page":"31345"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3303999900817871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3303999900817871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.058400001376867294,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.046300001442432404,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.6704000234603882},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.6621000170707703},{"id":"https://openalex.org/keywords/audit-trail","display_name":"Audit trail","score":0.5198000073432922},{"id":"https://openalex.org/keywords/unstructured-data","display_name":"Unstructured data","score":0.5037999749183655},{"id":"https://openalex.org/keywords/clarity","display_name":"CLARITY","score":0.4717000126838684},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.3643999993801117},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.36250001192092896},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.35569998621940613}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7649999856948853},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.6704000234603882},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.6621000170707703},{"id":"https://openalex.org/C80958533","wikidata":"https://www.wikidata.org/wiki/Q1047174","display_name":"Audit trail","level":3,"score":0.5198000073432922},{"id":"https://openalex.org/C2781252014","wikidata":"https://www.wikidata.org/wiki/Q1141900","display_name":"Unstructured data","level":3,"score":0.5037999749183655},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.4717000126838684},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4212000072002411},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4113999903202057},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3643999993801117},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.36250001192092896},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.35569998621940613},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.310699999332428},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2976999878883362},{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.2842999994754791},{"id":"https://openalex.org/C152550464","wikidata":"https://www.wikidata.org/wiki/Q3355715","display_name":"Performance audit","level":5,"score":0.2799000144004822},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27810001373291016},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C120567893","wikidata":"https://www.wikidata.org/wiki/Q1582085","display_name":"Knowledge extraction","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25940001010894775},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2572999894618988},{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.25429999828338623},{"id":"https://openalex.org/C115925183","wikidata":"https://www.wikidata.org/wiki/Q1412694","display_name":"Knowledge-based systems","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v40i37.40397","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i37.40397","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/40397","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/40397","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i37.40397","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i37.40397","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.44263678789138794,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recently":[0],"LLMs":[1],"have":[2],"faced":[3],"increasing":[4],"demands":[5],"to":[6,61,82,94,107,151,191,198],"selectively":[7],"remove":[8],"specific":[9],"information":[10],"through":[11],"Machine":[12],"Unlearning.":[13],"While":[14],"evaluating":[15],"unlearning":[16,85],"effectiveness":[17],"is":[18],"crucial,":[19],"existing":[20],"benchmarks":[21],"suffer":[22],"from":[23,30,57,87,189,196],"fundamental":[24],"limitations":[25],"in":[26],"audit":[27,39,110,132,161],"dataset":[28,111,133],"generation":[29,56,112,134,141],"unstructured":[31,58,122],"corpora.":[32],"We":[33],"identify":[34],"two":[35],"critical":[36],"challenges:":[37],"ensuring":[38],"adequacy":[40,146],"and":[41,47,65,73,116,139,147,159,165,193],"handling":[42],"knowledge":[43,105,126,174,181],"redundancy":[44,70,118,182],"between":[45,71],"forget":[46,72],"retain":[48,74],"datasets.":[49],"Current":[50],"approaches":[51],"rely":[52],"on":[53],"ad-hoc":[54,131],"question":[55],"text,":[59],"leading":[60],"unpredictable":[62],"coverage":[63,115],"gaps":[64],"evaluation":[66],"blind":[67],"spots.":[68],"Knowledge":[69],"corpora":[75,123],"further":[76],"obscures":[77],"evaluation,":[78],"making":[79],"it":[80,128],"difficult":[81],"distinguish":[83],"genuine":[84],"failures":[86],"legitimately":[88],"retained":[89],"knowledge.":[90],"To":[91],"bring":[92],"clarity":[93],"this":[95],"challenge,":[96],"we":[97,155],"propose":[98],"LUCID,":[99],"an":[100],"automated":[101,140],"framework":[102],"that":[103,143,180],"leverages":[104],"graphs":[106],"achieve":[108],"comprehensive":[109],"with":[113],"fine-grained":[114],"systematic":[117],"elimination.":[119],"By":[120],"converting":[121],"into":[124,136],"structured":[125],"representations,":[127],"transforms":[129],"the":[130,152,201],"process":[135],"a":[137],"transparent":[138],"pipeline":[142],"ensures":[144],"both":[145],"non-redundancy.":[148],"Applying":[149],"LUCID":[150],"MUSE":[153],"benchmark,":[154],"generated":[156],"over":[157],"69,000":[158],"111,000":[160],"cases":[162],"for":[163,205],"News":[164],"Books":[166],"datasets":[167],"respectively,":[168],"identifying":[169],"thousands":[170],"of":[171,203],"previously":[172],"undetected":[173],"memorization":[175],"instances.":[176],"Our":[177],"analysis":[178],"reveals":[179],"significantly":[183],"skews":[184],"metrics,":[185],"artificially":[186],"inflating":[187],"ROUGE":[188],"19.7%":[190],"26.1%":[192],"Entailment":[194],"Scores":[195],"32.4%":[197],"35.2%,":[199],"highlighting":[200],"necessity":[202],"deduplication":[204],"accurate":[206],"assessment.":[207]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-03-18T00:00:00"}