{"id":"https://openalex.org/W2098908907","doi":"https://doi.org/10.1162/08997660460733994","title":"A Machine Learning Method for Extracting Symbolic Knowledge from Recurrent Neural Networks","display_name":"A Machine Learning Method for Extracting Symbolic Knowledge from Recurrent Neural Networks","publication_year":2004,"publication_date":"2004-01-01","ids":{"openalex":"https://openalex.org/W2098908907","doi":"https://doi.org/10.1162/08997660460733994","mag":"2098908907","pmid":"https://pubmed.ncbi.nlm.nih.gov/15006023"},"language":"en","primary_location":{"id":"doi:10.1162/08997660460733994","is_oa":false,"landing_page_url":"https://doi.org/10.1162/08997660460733994","pdf_url":null,"source":{"id":"https://openalex.org/S207023548","display_name":"Neural Computation","issn_l":"0899-7667","issn":["0899-7667","1530-888X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Computation","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029294908","display_name":"Amir Taghizadeh Vahed","orcid":null},"institutions":[{"id":"https://openalex.org/I869660684","display_name":"University of the Western Cape","ror":"https://ror.org/00h2vm590","country_code":"ZA","type":"education","lineage":["https://openalex.org/I869660684"]}],"countries":["ZA"],"is_corresponding":true,"raw_author_name":"A. Vahed","raw_affiliation_strings":["Department of Computer Science, University of the Western Cape, 7535 Bellville, South Africa,"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of the Western Cape, 7535 Bellville, South Africa,","institution_ids":["https://openalex.org/I869660684"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030608087","display_name":"Christian W. Omlin","orcid":"https://orcid.org/0000-0003-0299-171X"},"institutions":[{"id":"https://openalex.org/I869660684","display_name":"University of the Western Cape","ror":"https://ror.org/00h2vm590","country_code":"ZA","type":"education","lineage":["https://openalex.org/I869660684"]}],"countries":["ZA"],"is_corresponding":false,"raw_author_name":"C. W. Omlin","raw_affiliation_strings":["Department of Computer Science, University of the Western Cape, 7535 Bellville, South Africa,"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of the Western Cape, 7535 Bellville, South Africa,","institution_ids":["https://openalex.org/I869660684"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5029294908"],"corresponding_institution_ids":["https://openalex.org/I869660684"],"apc_list":null,"apc_paid":null,"fwci":2.2485,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.89395728,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"16","issue":"1","first_page":"59","last_page":"71"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7061108946800232},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6934731006622314},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.6436253190040588},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.561424970626831},{"id":"https://openalex.org/keywords/finite-state-machine","display_name":"Finite-state machine","score":0.5567342638969421},{"id":"https://openalex.org/keywords/limit","display_name":"Limit (mathematics)","score":0.4761231541633606},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.47582414746284485},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4563404619693756},{"id":"https://openalex.org/keywords/automaton","display_name":"Automaton","score":0.4310305118560791},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39502888917922974},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.38162076473236084},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.2940959930419922},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.202642023563385}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7061108946800232},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6934731006622314},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.6436253190040588},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.561424970626831},{"id":"https://openalex.org/C167822520","wikidata":"https://www.wikidata.org/wiki/Q176452","display_name":"Finite-state machine","level":2,"score":0.5567342638969421},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.4761231541633606},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.47582414746284485},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4563404619693756},{"id":"https://openalex.org/C112505250","wikidata":"https://www.wikidata.org/wiki/Q787116","display_name":"Automaton","level":2,"score":0.4310305118560791},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39502888917922974},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.38162076473236084},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2940959930419922},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.202642023563385},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":2,"locations":[{"id":"doi:10.1162/08997660460733994","is_oa":false,"landing_page_url":"https://doi.org/10.1162/08997660460733994","pdf_url":null,"source":{"id":"https://openalex.org/S207023548","display_name":"Neural Computation","issn_l":"0899-7667","issn":["0899-7667","1530-888X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Computation","raw_type":"journal-article"},{"id":"pmid:15006023","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/15006023","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural computation","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4399999976158142,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W60533568","https://openalex.org/W1538240903","https://openalex.org/W1679913846","https://openalex.org/W1957841678","https://openalex.org/W1982370770","https://openalex.org/W1984666354","https://openalex.org/W1990813072","https://openalex.org/W1998937898","https://openalex.org/W2010315176","https://openalex.org/W2016589492","https://openalex.org/W2024169057","https://openalex.org/W2053987251","https://openalex.org/W2063046703","https://openalex.org/W2071906671","https://openalex.org/W2082345008","https://openalex.org/W2107878631","https://openalex.org/W2110485445","https://openalex.org/W2121553911","https://openalex.org/W2141703670","https://openalex.org/W2142148616","https://openalex.org/W2143096623","https://openalex.org/W2144612220","https://openalex.org/W2153827974","https://openalex.org/W2156960699","https://openalex.org/W2161017408","https://openalex.org/W2165301751","https://openalex.org/W2167615167"],"related_works":["https://openalex.org/W2280422768","https://openalex.org/W4242459863","https://openalex.org/W127357700","https://openalex.org/W2724597727","https://openalex.org/W1480528870","https://openalex.org/W2508541108","https://openalex.org/W4243191745","https://openalex.org/W2315419748","https://openalex.org/W1029445510","https://openalex.org/W2072177288"],"abstract_inverted_index":{"Neural":[0],"networks":[1,24,74,92],"do":[2],"not":[3,38,165,175],"readily":[4,39],"provide":[5],"an":[6],"explanation":[7],"of":[8,17,49,66,107,117,131,142,145,170,199,215],"the":[9,32,64,96,129,140,143,161,167,178,197,209,213,216],"knowledge":[10,33,53,67,89,163],"stored":[11,34],"in":[12,35,46,54,68,159],"their":[13,18,36],"weights":[14,37],"as":[15],"part":[16],"information":[19],"processing.":[20],"Until":[21],"recently,":[22],"neural":[23,59,73],"were":[25],"considered":[26],"to":[27,76,87,102,111,124,156,192,211],"be":[28],"black":[29],"boxes,":[30],"with":[31],"accessible.":[40],"Since":[41],"then,":[42],"research":[43],"has":[44,122,208],"resulted":[45],"a":[47,119,171,186,200],"number":[48,130],"algorithms":[50],"for":[51,177],"extracting":[52],"symbolic":[55,69,189],"form":[56,70,135],"from":[57,71,90,196],"trained":[58,75,172,201],"networks.":[60],"This":[61],"article":[62],"addresses":[63],"extraction":[65],"recurrent":[72,147],"behave":[77],"like":[78],"deterministic":[79],"finite-state":[80],"automata":[81],"(DFAs).":[82],"To":[83],"date,":[84],"methods":[85],"used":[86],"extract":[88],"such":[91,118],"have":[93],"relied":[94],"on":[95],"hypothesis":[97],"that":[98,105,126,133],"networks'":[99],"states":[100,109],"tend":[101],"cluster":[103,120],"and":[104],"clusters":[106,132],"network":[108],"correspond":[110],"DFA":[112],"states.":[113],"The":[114,181],"computational":[115],"complexity":[116],"analysis":[121],"led":[123],"heuristics":[125],"either":[127],"limit":[128,139],"may":[134,154,164],"during":[136],"training":[137,179],"or":[138],"exploration":[141],"space":[144],"hidden":[146],"state":[148],"neurons.":[149],"These":[150],"limitations,":[151],"while":[152],"necessary,":[153],"lead":[155],"decreased":[157],"fidelity,":[158],"which":[160],"extracted":[162,217],"model":[166],"true":[168],"behavior":[169],"network,":[173],"perhaps":[174],"even":[176],"set.":[180],"method":[182,207],"proposed":[183],"here":[184],"uses":[185],"polynomial":[187],"time,":[188],"learning":[190],"algorithm":[191],"infer":[193],"DFAs":[194],"solely":[195],"observation":[198],"network's":[202],"input-output":[203],"behavior.":[204],"Thus,":[205],"this":[206],"potential":[210],"increase":[212],"fidelity":[214],"knowledge.":[218]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2014,"cited_by_count":2},{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}