{"id":"https://openalex.org/W4392341695","doi":"https://doi.org/10.48550/arxiv.2402.16726","title":"Towards Empirical Interpretation of Internal Circuits and Properties in Grokked Transformers on Modular Polynomials","display_name":"Towards Empirical Interpretation of Internal Circuits and Properties in Grokked Transformers on Modular Polynomials","publication_year":2024,"publication_date":"2024-02-26","ids":{"openalex":"https://openalex.org/W4392341695","doi":"https://doi.org/10.48550/arxiv.2402.16726"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.16726","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.16726","pdf_url":"https://arxiv.org/pdf/2402.16726","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.16726","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064007269","display_name":"Hiroki Furuta","orcid":"https://orcid.org/0009-0002-7209-810X"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Furuta, Hiroki","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012454552","display_name":"Gouki Minegishi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Minegishi, Gouki","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063925941","display_name":"Yusuke Iwasawa","orcid":"https://orcid.org/0000-0002-1321-2622"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Iwasawa, Yusuke","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5074059447","display_name":"Yutaka Matsuo","orcid":"https://orcid.org/0000-0002-2070-4393"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matsuo, Yutaka","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5064007269"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12002","display_name":"Computability, Logic, AI Algorithms","score":0.9244999885559082,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12002","display_name":"Computability, Logic, AI Algorithms","score":0.9244999885559082,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.7788369655609131},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.6732101440429688},{"id":"https://openalex.org/keywords/modular-arithmetic","display_name":"Modular arithmetic","score":0.5912598371505737},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5845308303833008},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.444930762052536},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3915322721004486},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.17634513974189758},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.17017295956611633},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.15678051114082336},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.05115357041358948}],"concepts":[{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.7788369655609131},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.6732101440429688},{"id":"https://openalex.org/C32049820","wikidata":"https://www.wikidata.org/wiki/Q319400","display_name":"Modular arithmetic","level":3,"score":0.5912598371505737},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5845308303833008},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.444930762052536},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3915322721004486},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.17634513974189758},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.17017295956611633},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.15678051114082336},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.05115357041358948}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.16726","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.16726","pdf_url":"https://arxiv.org/pdf/2402.16726","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.16726","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.16726","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.16726","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.16726","pdf_url":"https://arxiv.org/pdf/2402.16726","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4246298538","https://openalex.org/W2255754718","https://openalex.org/W377643703","https://openalex.org/W590541131","https://openalex.org/W4206809139","https://openalex.org/W3138611917","https://openalex.org/W4294326371","https://openalex.org/W4386489432","https://openalex.org/W2587849783","https://openalex.org/W2073239372"],"abstract_inverted_index":{"Grokking":[0,30],"has":[1,34],"been":[2,35],"actively":[3],"explored":[4],"to":[5,26,37,60,207,215,224],"reveal":[6],"the":[7,19,51,56,69,163,166,189,194,197,233,251],"mystery":[8],"of":[9,151,165,253],"delayed":[10],"generalization":[11],"and":[12,16,41,65,107,133,143,236],"identifying":[13],"interpretable":[14],"representations":[15,150],"algorithms":[17],"inside":[18],"grokked":[20,98,152,199],"models":[21,99,153,191,198],"is":[22,59],"a":[23,78],"suggestive":[24],"hint":[25],"understanding":[27],"its":[28,42],"mechanism.":[29],"on":[31,71,123,188],"modular":[32,54,73,86,125,138,155],"addition":[33],"known":[36],"implement":[38],"Fourier":[39,93,131,140,144,167],"representation":[40,94],"calculation":[43],"circuits":[44],"with":[45,91,110,200],"trigonometric":[46],"identities":[47],"in":[48,53,162,170,179],"Transformers.":[49],"Considering":[50],"periodicity":[52],"arithmetic,":[55,139,172],"natural":[57],"question":[58],"what":[61],"extent":[62],"these":[63],"explanations":[64],"interpretations":[66],"hold":[67],"for":[68,137,157,231],"grokking":[70,228],"other":[72],"operations":[74,87,112],"beyond":[75],"addition.":[76],"For":[77],"closer":[79],"look,":[80],"we":[81,116],"first":[82],"hypothesize":[83],"that":[84,193],"any":[85],"can":[88,203],"be":[89,204],"characterized":[90],"distinctive":[92,148],"or":[95],"internal":[96,149,254],"circuits,":[97],"obtain":[100],"common":[101],"features":[102],"transferable":[103],"among":[104,196],"similar":[105,111],"operations,":[106],"mixing":[108],"datasets":[109],"promotes":[113],"grokking.":[114],"Then,":[115],"extensively":[117],"examine":[118],"them":[119],"by":[120],"learning":[121],"Transformers":[122],"complex":[124],"arithmetic":[126,214],"tasks,":[127],"including":[128],"polynomials.":[129,182],"Our":[130],"analysis":[132],"novel":[134],"progress":[135],"measure":[136],"Frequency":[141],"Density":[142],"Coefficient":[145],"Ratio,":[146],"characterize":[147],"per":[154],"operation;":[156],"instance,":[158],"polynomials":[159],"often":[160],"result":[161],"superposition":[164],"components":[168],"seen":[169],"elementary":[171,213],"but":[173],"clear":[174],"patterns":[175],"do":[176],"not":[177,242],"emerge":[178],"challenging":[180],"non-factorizable":[181],"In":[183],"contrast,":[184],"our":[185],"ablation":[186],"study":[187],"pre-grokked":[190],"reveals":[192],"transferability":[195],"each":[201],"operation":[202],"only":[205],"limited":[206],"specific":[208],"combinations,":[209],"such":[210],"as":[211],"from":[212],"linear":[216],"expressions.":[217],"Moreover,":[218],"some":[219],"multi-task":[220],"mixtures":[221],"may":[222,241],"lead":[223],"co-grokking":[225],"--":[226,235],"where":[227],"simultaneously":[229],"happens":[230],"all":[232],"tasks":[234],"accelerate":[237],"generalization,":[238],"while":[239],"others":[240],"find":[243],"optimal":[244],"solutions.":[245],"We":[246],"provide":[247],"empirical":[248],"steps":[249],"towards":[250],"interpretability":[252],"circuits.":[255]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-02-09T09:26:11.010843","created_date":"2024-03-05T00:00:00"}