{"id":"https://openalex.org/W4366835585","doi":"https://doi.org/10.48550/arxiv.2304.10933","title":"Self-Attention in Colors: Another Take on Encoding Graph Structure in Transformers","display_name":"Self-Attention in Colors: Another Take on Encoding Graph Structure in Transformers","publication_year":2023,"publication_date":"2023-04-21","ids":{"openalex":"https://openalex.org/W4366835585","doi":"https://doi.org/10.48550/arxiv.2304.10933"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2304.10933","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10933","pdf_url":"https://arxiv.org/pdf/2304.10933","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2304.10933","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050069872","display_name":"Romain Menegaux","orcid":"https://orcid.org/0000-0001-8814-4911"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Menegaux, Romain","raw_affiliation_strings":["Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)"],"affiliations":[{"raw_affiliation_string":"Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012412991","display_name":"Emmanuel Jehanno","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jehanno, Emmanuel","raw_affiliation_strings":["Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)"],"affiliations":[{"raw_affiliation_string":"Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010575148","display_name":"Margot Selosse","orcid":"https://orcid.org/0000-0003-1422-688X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Selosse, Margot","raw_affiliation_strings":["Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)"],"affiliations":[{"raw_affiliation_string":"Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062817741","display_name":"Julien Mairal","orcid":"https://orcid.org/0000-0001-6991-2110"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mairal, Julien","raw_affiliation_strings":["Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)"],"affiliations":[{"raw_affiliation_string":"Thoth - Apprentissage de mod\u00e8les \u00e0 partir de donn\u00e9es massives (France)","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5050069872"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10211","display_name":"Computational Drug Discovery Methods","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.979200005531311,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6628350019454956},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.5489615797996521},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.5327727198600769},{"id":"https://openalex.org/keywords/chromatic-scale","display_name":"Chromatic scale","score":0.4952940046787262},{"id":"https://openalex.org/keywords/lattice-graph","display_name":"Lattice graph","score":0.48732790350914},{"id":"https://openalex.org/keywords/attention-network","display_name":"Attention network","score":0.45535707473754883},{"id":"https://openalex.org/keywords/topology","display_name":"Topology (electrical circuits)","score":0.43953120708465576},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4168981909751892},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.32457733154296875},{"id":"https://openalex.org/keywords/line-graph","display_name":"Line graph","score":0.2890268564224243},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2501147389411926},{"id":"https://openalex.org/keywords/voltage-graph","display_name":"Voltage graph","score":0.22558048367500305},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.19880637526512146},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.14250725507736206}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6628350019454956},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.5489615797996521},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.5327727198600769},{"id":"https://openalex.org/C196956537","wikidata":"https://www.wikidata.org/wiki/Q202021","display_name":"Chromatic scale","level":2,"score":0.4952940046787262},{"id":"https://openalex.org/C145580432","wikidata":"https://www.wikidata.org/wiki/Q6497118","display_name":"Lattice graph","level":5,"score":0.48732790350914},{"id":"https://openalex.org/C2993807640","wikidata":"https://www.wikidata.org/wiki/Q103709453","display_name":"Attention network","level":2,"score":0.45535707473754883},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.43953120708465576},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4168981909751892},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.32457733154296875},{"id":"https://openalex.org/C203776342","wikidata":"https://www.wikidata.org/wiki/Q1378376","display_name":"Line graph","level":3,"score":0.2890268564224243},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2501147389411926},{"id":"https://openalex.org/C22149727","wikidata":"https://www.wikidata.org/wiki/Q7940747","display_name":"Voltage graph","level":4,"score":0.22558048367500305},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19880637526512146},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.14250725507736206},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2304.10933","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10933","pdf_url":"https://arxiv.org/pdf/2304.10933","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2304.10933","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2304.10933","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2304.10933","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10933","pdf_url":"https://arxiv.org/pdf/2304.10933","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3104333717","display_name":null,"funder_award_id":"MIAI@Grenoble Alpes","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320326256","display_name":"Grand \u00c9quipement National De Calcul Intensif","ror":"https://ror.org/0010d1q40"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366835585.pdf","grobid_xml":"https://content.openalex.org/works/W4366835585.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2181667894","https://openalex.org/W2105223880","https://openalex.org/W2379048343","https://openalex.org/W4256403421","https://openalex.org/W3178681420","https://openalex.org/W4389995241","https://openalex.org/W4200094402","https://openalex.org/W4320149722","https://openalex.org/W3213655484","https://openalex.org/W2366758840"],"abstract_inverted_index":{"We":[0,27,77],"introduce":[1],"a":[2,31,72,79,119],"novel":[3],"self-attention":[4],"mechanism,":[5],"which":[6,12,39],"we":[7],"call":[8],"CSA":[9,29],"(Chromatic":[10,36],"Self-Attention),":[11],"extends":[13],"the":[14,24,50,67,113],"notion":[15],"of":[16],"attention":[17,20],"scores":[18],"to":[19,96],"_filters_,":[21],"independently":[22],"modulating":[23],"feature":[25],"channels.":[26],"showcase":[28],"in":[30,104],"fully-attentional":[32],"graph":[33,42,60,124],"Transformer":[34],"CGT":[35],"Graph":[37],"Transformer)":[38],"integrates":[40],"both":[41,88],"structural":[43,89],"information":[44],"and":[45,90,93,126],"edge":[46,69],"features,":[47],"completely":[48],"bypassing":[49],"need":[51],"for":[52,122],"local":[53],"message-passing":[54],"components.":[55],"Our":[56,107],"method":[57],"flexibly":[58],"encodes":[59,87],"structure":[61,125],"through":[62],"node-node":[63],"interactions,":[64],"by":[65],"enriching":[66],"original":[68],"features":[70],"with":[71],"relative":[73],"positional":[74,91],"encoding":[75,123],"scheme.":[76],"propose":[78],"new":[80],"scheme":[81],"based":[82],"on":[83,112],"random":[84],"walks":[85],"that":[86],"information,":[92,100],"show":[94],"how":[95],"incorporate":[97],"higher-order":[98,128],"topological":[99],"such":[101],"as":[102],"rings":[103],"molecular":[105],"graphs.":[106],"approach":[108],"achieves":[109],"state-of-the-art":[110],"results":[111],"ZINC":[114],"benchmark":[115],"dataset,":[116],"while":[117],"providing":[118],"flexible":[120],"framework":[121],"incorporating":[127],"topology.":[129]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}