{"id":"https://openalex.org/W4409670972","doi":"https://doi.org/10.1063/5.0256873","title":"Graph-aware isomorphic attention for adaptive dynamics in transformers","display_name":"Graph-aware isomorphic attention for adaptive dynamics in transformers","publication_year":2025,"publication_date":"2025-04-22","ids":{"openalex":"https://openalex.org/W4409670972","doi":"https://doi.org/10.1063/5.0256873"},"language":"en","primary_location":{"id":"doi:10.1063/5.0256873","doi":"https://doi.org/10.1063/5.0256873","is_oa":true,"landing_page_url":"https://doi.org/10.1063/5.0256873","pdf_url":null,"source":{"id":"https://openalex.org/S4387287287","display_name":"APL Machine Learning","issn_l":"2770-9019","issn":["2770-9019"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-1"],"host_organization":"https://openalex.org/P4310320257","host_organization_name":"American Institute of Physics","host_organization_lineage":["https://openalex.org/P4310320257"],"host_organization_lineage_names":["American Institute of Physics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"APL Machine Learning","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["arxiv","crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1063/5.0256873","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011504360","display_name":"Markus J. Buehler","orcid":"https://orcid.org/0000-0002-4173-9659","observed_orcids":["https://orcid.org/0000-0002-4173-9659"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Markus J. Buehler","raw_affiliation_strings":["Laboratory for Atomistic and Molecular Mechanics (LAMM) MIT , Cambridge, Massachusetts 02139,"],"raw_orcid":"https://orcid.org/0000-0002-4173-9659","affiliations":[{"raw_affiliation_string":"Laboratory for Atomistic and Molecular Mechanics (LAMM) MIT , Cambridge, Massachusetts 02139,","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5011504360"],"corresponding_institution_ids":["https://openalex.org/I63966007"],"apc_list":null,"apc_paid":null,"fwci":3.5857,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.93702947,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"3","issue":"2","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.7699999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.7699999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13099999725818634,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.015200000256299973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/graph-isomorphism-network","display_name":"graph isomorphism network","score":0.7400000095367432},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.5180000066757202},{"id":"https://openalex.org/keywords/relational-reasoning","display_name":"relational reasoning","score":0.8989999890327454},{"id":"https://openalex.org/keywords/fine-tuning-pre-trained-models","display_name":"pre-trained model fine-tuning","score":0.746999979019165},{"id":"https://openalex.org/keywords/graph-neural-networks","display_name":"graph neural networks","score":0.6679999828338623},{"id":"https://openalex.org/keywords/category-theory","display_name":"category theory","score":0.7020000219345093},{"id":"https://openalex.org/keywords/hierarchical-representation-learning","display_name":"hierarchical feature learning","score":0.9340000152587891},{"id":"https://openalex.org/keywords/generalization-gap","display_name":"generalization gap","score":0.7950000166893005}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5087712407112122},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.48068007826805115},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.37864935398101807}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1063/5.0256873","doi":"https://doi.org/10.1063/5.0256873","is_oa":true,"landing_page_url":"https://doi.org/10.1063/5.0256873","pdf_url":null,"source":{"id":"https://openalex.org/S4387287287","display_name":"APL Machine Learning","issn_l":"2770-9019","issn":["2770-9019"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-1"],"host_organization":"https://openalex.org/P4310320257","host_organization_name":"American Institute of Physics","host_organization_lineage":["https://openalex.org/P4310320257"],"host_organization_lineage_names":["American Institute of Physics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"APL Machine Learning","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2501.02393","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.02393","pdf_url":"https://arxiv.org/pdf/2501.02393","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dspace.mit.edu:1721.1/174466","doi":null,"is_oa":true,"landing_page_url":"https://dspace.mit.edu/handle/1721.1/174466","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"AIP Publishing","raw_type":"Article"},{"id":"pmh:oai:doaj.org/article:c46209075a2b4011b3aa964783503aa8","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/c46209075a2b4011b3aa964783503aa8","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"APL Machine Learning, Vol 3, Iss 2, Pp 026108-026108-24 (2025)","raw_type":"article"},{"id":"doi:10.48550/arxiv.2501.02393","doi":"https://doi.org/10.48550/arxiv.2501.02393","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2501.02393","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1063/5.0256873","doi":"https://doi.org/10.1063/5.0256873","is_oa":true,"landing_page_url":"https://doi.org/10.1063/5.0256873","pdf_url":null,"source":{"id":"https://openalex.org/S4387287287","display_name":"APL Machine Learning","issn_l":"2770-9019","issn":["2770-9019"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-1"],"host_organization":"https://openalex.org/P4310320257","host_organization_name":"American Institute of Physics","host_organization_lineage":["https://openalex.org/P4310320257"],"host_organization_lineage_names":["American Institute of Physics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"APL Machine Learning","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309369","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409670972.pdf","grobid_xml":"https://content.openalex.org/works/W4409670972.grobid-xml"},"referenced_works_count":69,"referenced_works":["https://openalex.org/W1686946872","https://openalex.org/W1983193888","https://openalex.org/W1996454832","https://openalex.org/W2038000659","https://openalex.org/W2058788751","https://openalex.org/W2064675550","https://openalex.org/W2086238435","https://openalex.org/W2127811140","https://openalex.org/W2128084896","https://openalex.org/W2133564696","https://openalex.org/W2166712553","https://openalex.org/W2167839676","https://openalex.org/W2333333255","https://openalex.org/W2626778328","https://openalex.org/W2755092149","https://openalex.org/W2766453196","https://openalex.org/W2785885194","https://openalex.org/W2794367096","https://openalex.org/W2894175828","https://openalex.org/W2901995873","https://openalex.org/W2947403094","https://openalex.org/W2952409190","https://openalex.org/W3030163527","https://openalex.org/W3092584515","https://openalex.org/W3104591796","https://openalex.org/W3164045210","https://openalex.org/W3168547639","https://openalex.org/W3168867926","https://openalex.org/W4210738256","https://openalex.org/W4226119281","https://openalex.org/W4226275767","https://openalex.org/W4310273071","https://openalex.org/W4310460724","https://openalex.org/W4313555834","https://openalex.org/W4315706348","https://openalex.org/W4320713584","https://openalex.org/W4366815931","https://openalex.org/W4386839891","https://openalex.org/W4387561528","https://openalex.org/W4387934665","https://openalex.org/W4388979610","https://openalex.org/W4389267611","https://openalex.org/W4390824394","https://openalex.org/W4392012692","https://openalex.org/W4392181665","https://openalex.org/W4392886466","https://openalex.org/W4395484023","https://openalex.org/W4398777692","https://openalex.org/W4401306886","https://openalex.org/W4402269933","https://openalex.org/W4402672092","https://openalex.org/W4403564469","https://openalex.org/W4403753254","https://openalex.org/W4404353289","https://openalex.org/W4404719247","https://openalex.org/W4404782921","https://openalex.org/W4405090274","https://openalex.org/W4405433019","https://openalex.org/W4405599065","https://openalex.org/W4405710765","https://openalex.org/W4410374120","https://openalex.org/W4414589084","https://openalex.org/W4415795994","https://openalex.org/W4415797144","https://openalex.org/W4415797334","https://openalex.org/W4415915064","https://openalex.org/W7133192525","https://openalex.org/W7133224126","https://openalex.org/W7133231369"],"related_works":["https://openalex.org/W7208816960","https://openalex.org/W4305008697","https://openalex.org/W4389518412","https://openalex.org/W4391590633","https://openalex.org/W6929468500","https://openalex.org/W4391948449","https://openalex.org/W7205011756","https://openalex.org/W4221163422","https://openalex.org/W4407759301","https://openalex.org/W4416260998"],"abstract_inverted_index":{"We":[0,95,125],"present":[1],"an":[2],"approach":[3,71,107],"for":[4,217,224,241],"modifying":[5],"transformer":[6,184],"architectures":[7,185],"by":[8,86],"integrating":[9],"graph-aware":[10,48,100,123,165],"relational":[11,68,139,219],"reasoning":[12,140,220],"into":[13],"the":[14,28,38,65,73,97,110,128,239],"attention":[15,32,40,101,166,170,193],"mechanism,":[16],"merging":[17],"concepts":[18],"from":[19,134],"graph":[20,34,44,55,143,181],"neural":[21],"networks":[22],"and":[23,33,46,80,91,154,175,183,186,228,236,243],"language":[24,234],"modeling.":[25],"Building":[26],"on":[27],"inherent":[29],"connection":[30],"between":[31],"theory,":[35],"we":[36,212],"reformulate":[37],"transformer\u2019s":[39],"mechanism":[41],"as":[42,84,208],"a":[43,87,105,196],"operation":[45],"propose":[47],"isomorphic":[49],"attention.":[50],"This":[51],"method":[52],"leverages":[53,131],"advanced":[54],"modeling":[56,146,245],"strategies,":[57],"including":[58],"Graph":[59],"Isomorphism":[60],"Networks":[61],"(GINs),":[62],"to":[63,76,102,137],"enrich":[64],"representation":[66,148],"of":[67,99,112],"structures.":[69],"Our":[70,161],"improves":[72],"model\u2019s":[74],"ability":[75],"capture":[77],"complex":[78],"dependencies":[79],"generalize":[81],"across":[82,158],"tasks,":[83],"evidenced":[85],"reduced":[88],"generalization":[89],"gap":[90],"improved":[92],"learning":[93,149],"performance.":[94,177],"expand":[96],"concept":[98],"introduce":[103],"sparse-GIN-attention,":[104],"fine-tuning":[106],"that":[108,127,150,164],"enhances":[109],"adaptability":[111],"pre-trained":[113],"foundational":[114,225],"models":[115],"with":[116,122,141,221],"minimal":[117],"computational":[118],"overhead,":[119],"endowing":[120],"them":[121],"capabilities.":[124],"show":[126],"sparse-GIN-attention":[129],"framework":[130],"compositional":[132],"principles":[133],"category":[135],"theory":[136,182],"align":[138],"sparsified":[142],"structures":[144,190],"while":[145],"hierarchical":[147,209],"bridges":[151],"local":[152],"interactions":[153],"global":[155],"task":[156],"objectives":[157],"diverse":[159],"domains.":[160],"results":[162],"demonstrate":[163],"mechanisms":[167],"outperform":[168],"traditional":[169,192],"in":[171,230],"both":[172],"training":[173],"efficiency":[174],"validation":[176],"These":[178],"insights":[179],"bridge":[180],"uncover":[187],"latent":[188],"graph-like":[189],"within":[191],"mechanisms,":[194],"offering":[195],"new":[197],"lens":[198],"through":[199],"which":[200],"transformers":[201,207],"can":[202],"be":[203],"optimized.":[204],"By":[205],"evolving":[206],"GIN":[210],"models,":[211],"reveal":[213],"their":[214],"implicit":[215],"capacity":[216],"graph-level":[218],"profound":[222],"implications":[223],"model":[226],"development":[227],"applications":[229],"bioinformatics,":[231],"materials":[232],"science,":[233],"modeling,":[235],"beyond,":[237],"setting":[238],"stage":[240],"interpretable":[242],"generalizable":[244],"strategies.":[246]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":8}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2025-04-23T00:00:00"}