{"id":"https://openalex.org/W7163361645","doi":"https://doi.org/10.48550/arxiv.2606.03712","title":"When Graph Tokens Sink: A Mechanistic Analysis of Graph Language Models","display_name":"When Graph Tokens Sink: A Mechanistic Analysis of Graph Language Models","publication_year":2026,"publication_date":"2026-06-02","ids":{"openalex":"https://openalex.org/W7163361645","doi":"https://doi.org/10.48550/arxiv.2606.03712"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.03712","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03712","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.03712","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137755023","display_name":"Ding Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ding","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120645437","display_name":"Runtao Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Runtao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137795447","display_name":"Wenqing Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Wenqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005784479","display_name":"Rizal Fathony","orcid":"https://orcid.org/0000-0003-1538-9090"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fathony, Rizal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021146490","display_name":"Bayan Bruss","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bruss, Bayan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137780626","display_name":"Chirag Agarwal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agarwal, Chirag","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.8115000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.8115000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.04969999939203262,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.029500000178813934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/graph-property","display_name":"Graph property","score":0.5148000121116638},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.4830000102519989},{"id":"https://openalex.org/keywords/voltage-graph","display_name":"Voltage graph","score":0.43810001015663147},{"id":"https://openalex.org/keywords/null-graph","display_name":"Null graph","score":0.43220001459121704},{"id":"https://openalex.org/keywords/line-graph","display_name":"Line graph","score":0.4052000045776367},{"id":"https://openalex.org/keywords/complement-graph","display_name":"Complement graph","score":0.3862999975681305}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6396999955177307},{"id":"https://openalex.org/C64339825","wikidata":"https://www.wikidata.org/wiki/Q722659","display_name":"Graph property","level":5,"score":0.5148000121116638},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.5141000151634216},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.4830000102519989},{"id":"https://openalex.org/C22149727","wikidata":"https://www.wikidata.org/wiki/Q7940747","display_name":"Voltage graph","level":4,"score":0.43810001015663147},{"id":"https://openalex.org/C17169500","wikidata":"https://www.wikidata.org/wiki/Q3033506","display_name":"Null graph","level":5,"score":0.43220001459121704},{"id":"https://openalex.org/C203776342","wikidata":"https://www.wikidata.org/wiki/Q1378376","display_name":"Line graph","level":3,"score":0.4052000045776367},{"id":"https://openalex.org/C168291704","wikidata":"https://www.wikidata.org/wiki/Q902252","display_name":"Complement graph","level":5,"score":0.3862999975681305},{"id":"https://openalex.org/C18819970","wikidata":"https://www.wikidata.org/wiki/Q3035340","display_name":"Butterfly graph","level":5,"score":0.3257000148296356},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C2993807640","wikidata":"https://www.wikidata.org/wiki/Q103709453","display_name":"Attention network","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.26100000739097595}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.03712","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03712","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.03712","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03712","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.4303871989250183,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Graph":[0,99],"Language":[1,12],"Models":[2,13],"(GLMs)":[3],"have":[4],"become":[5],"a":[6,116,218,227],"promising":[7],"direction":[8],"for":[9,190],"adapting":[10],"Large":[11],"(LLMs)":[14],"to":[15,32,95,238],"graph":[16,21,27,36,50,54,61,71,88,96,144,155,178,203],"learning":[17],"tasks.":[18],"By":[19],"transforming":[20],"topology":[22],"and":[23,38,52,122,152,172,232,245],"node":[24],"information":[25,72,97],"into":[26,205],"tokens,":[28],"GLMs":[29,91,201],"allow":[30],"LLMs":[31,46,69],"jointly":[33],"process":[34,70],"structured":[35],"inputs":[37],"textual":[39],"instructions.":[40],"Yet,":[41],"it":[42],"remains":[43],"unclear":[44],"how":[45,68],"internally":[47],"interpret":[48],"these":[49,137,195],"tokens":[51,55,89,101,138,157,180,189],"whether":[53],"act":[56],"as":[57,104],"meaningful":[58],"carriers":[59,142],"of":[60,87,119,143],"structure.":[62],"In":[63],"this":[64,130],"work,":[65],"we":[66,175],"analyze":[67],"through":[73],"graph-token":[74,127,212,242],"behavior":[75],"in":[76,90,150,240],"representative":[77],"GLM":[78],"architectures.":[79],"Findings.":[80],"We":[81],"find":[82],"that":[83,136,177,198],"the":[84,140,162,183,206,210],"internal":[85,222],"saliency":[86,132,231],"is":[92],"not":[93,134,159,182,215],"equivalent":[94],"utilization.":[98],"sink":[100,156,179],"consistently":[102],"emerge":[103],"activation-level":[105,131,230],"outliers:":[106],"they":[107,225],"can":[108],"be":[109],"identified":[110],"by":[111],"massive":[112],"activation":[113],"values":[114],"along":[115],"small":[117],"set":[118],"hidden-state":[120],"dimensions":[121],"are":[123,139,181],"biased":[124],"toward":[125],"early":[126],"positions.":[128],"However,":[129],"does":[133],"imply":[135],"main":[141],"information.":[145],"Unlike":[146],"classical":[147],"attention":[148,164],"sinks":[149],"language":[151],"vision-language":[153],"models,":[154],"do":[158,214],"necessarily":[160],"attract":[161],"largest":[163],"weights":[165],"from":[166],"query":[167],"tokens.":[168],"Through":[169],"pruning,":[170],"repositioning,":[171],"swapping":[173],"interventions,":[174],"show":[176],"most":[184],"important":[185],"semantic":[186],"or":[187],"structural":[188],"downstream":[191],"prediction.":[192],"Implications.":[193],"Together,":[194],"results":[196],"suggest":[197],"after":[199],"current":[200],"map":[202],"structure":[204],"LLM":[207],"token":[208],"space,":[209],"resulting":[211],"representations":[213],"naturally":[216],"form":[217],"fully":[219],"usable":[220],"topology-aware":[221],"representation;":[223],"instead,":[224],"exhibit":[226],"decoupling":[228,236],"between":[229],"graph-semantic":[233],"utility.":[234],"This":[235],"points":[237],"limitations":[239],"existing":[241],"construction,":[243],"placement,":[244],"alignment":[246],"mechanisms.":[247]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-04T00:00:00"}