{"id":"https://openalex.org/W4407098117","doi":"https://doi.org/10.1109/tpami.2025.3538221","title":"Pre-Training a Graph Recurrent Network for Text Understanding","display_name":"Pre-Training a Graph Recurrent Network for Text Understanding","publication_year":2025,"publication_date":"2025-02-03","ids":{"openalex":"https://openalex.org/W4407098117","doi":"https://doi.org/10.1109/tpami.2025.3538221","pmid":"https://pubmed.ncbi.nlm.nih.gov/40031392"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3538221","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3538221","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044326908","display_name":"Yile Wang","orcid":"https://orcid.org/0000-0001-8705-9598"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["CN","ES"],"is_corresponding":true,"raw_author_name":"Yile Wang","raw_affiliation_strings":["College of Computer Science and Software Engineering, Shenzhen University, Shenzhen, China","College of Computer Science and Software Engineering, Shenzhen University, china"],"raw_orcid":"https://orcid.org/0000-0001-8705-9598","affiliations":[{"raw_affiliation_string":"College of Computer Science and Software Engineering, Shenzhen University, Shenzhen, China","institution_ids":["https://openalex.org/I180726961"]},{"raw_affiliation_string":"College of Computer Science and Software Engineering, Shenzhen University, china","institution_ids":["https://openalex.org/I180726961","https://openalex.org/I97750245"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112427338","display_name":"Linyi Yang","orcid":"https://orcid.org/0000-0002-5928-148X"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]},{"id":"https://openalex.org/I3133055985","display_name":"Westlake University","ror":"https://ror.org/05hfa4n20","country_code":"CN","type":"education","lineage":["https://openalex.org/I3133055985"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linyi Yang","raw_affiliation_strings":["School of Engineering, Westlake University, Hangzhou, China","Southern University of Science and Technology, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Engineering, Westlake University, Hangzhou, China","institution_ids":["https://openalex.org/I3133055985"]},{"raw_affiliation_string":"Southern University of Science and Technology, China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012543243","display_name":"Zhiyang Teng","orcid":"https://orcid.org/0000-0001-6757-6897"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhiyang Teng","raw_affiliation_strings":["ByteDance, Singapore","ByteDance"],"raw_orcid":"https://orcid.org/0000-0001-6757-6897","affiliations":[{"raw_affiliation_string":"ByteDance, Singapore","institution_ids":[]},{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100701572","display_name":"Ming Zhou","orcid":"https://orcid.org/0000-0002-2551-2964"},"institutions":[{"id":"https://openalex.org/I4210110097","display_name":"Langdon Hospital","ror":"https://ror.org/01wx7k245","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I4210110097","https://openalex.org/I4210156727"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ming Zhou","raw_affiliation_strings":["Langboat Ltd., Beijing, China","Langboat Ltd"],"raw_orcid":"https://orcid.org/0000-0002-2551-2964","affiliations":[{"raw_affiliation_string":"Langboat Ltd., Beijing, China","institution_ids":["https://openalex.org/I4210110097"]},{"raw_affiliation_string":"Langboat Ltd","institution_ids":["https://openalex.org/I4210110097"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100333729","display_name":"Yue Zhang","orcid":"https://orcid.org/0000-0002-5214-2268"},"institutions":[{"id":"https://openalex.org/I3133055985","display_name":"Westlake University","ror":"https://ror.org/05hfa4n20","country_code":"CN","type":"education","lineage":["https://openalex.org/I3133055985"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yue Zhang","raw_affiliation_strings":["School of Engineering, Westlake University, Hangzhou, China","School of Engineering, Westlake University, China"],"raw_orcid":"https://orcid.org/0000-0002-5214-2268","affiliations":[{"raw_affiliation_string":"School of Engineering, Westlake University, Hangzhou, China","institution_ids":["https://openalex.org/I3133055985"]},{"raw_affiliation_string":"School of Engineering, Westlake University, China","institution_ids":["https://openalex.org/I3133055985"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5044326908"],"corresponding_institution_ids":["https://openalex.org/I180726961","https://openalex.org/I97750245"],"apc_list":null,"apc_paid":null,"fwci":2.1733,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.87068193,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"47","issue":"5","first_page":"4061","last_page":"4074"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9745000004768372,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9508000016212463,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7260177135467529},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6501545310020447},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5342971682548523},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.5109450221061707},{"id":"https://openalex.org/keywords/graph-theory","display_name":"Graph theory","score":0.4158879816532135},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.40696418285369873},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3815248906612396},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3422994613647461},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.1966848075389862},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11821934580802917},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.07169729471206665}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7260177135467529},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6501545310020447},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5342971682548523},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.5109450221061707},{"id":"https://openalex.org/C88230418","wikidata":"https://www.wikidata.org/wiki/Q131476","display_name":"Graph theory","level":2,"score":0.4158879816532135},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40696418285369873},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3815248906612396},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3422994613647461},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.1966848075389862},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11821934580802917},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.07169729471206665},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2025.3538221","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3538221","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40031392","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40031392","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6200000047683716}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":93,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W2064675550","https://openalex.org/W2096707493","https://openalex.org/W2507974895","https://openalex.org/W2600702321","https://openalex.org/W2888128175","https://openalex.org/W2888482885","https://openalex.org/W2891553865","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W2933138175","https://openalex.org/W2952179106","https://openalex.org/W2962739339","https://openalex.org/W2962784628","https://openalex.org/W2963250244","https://openalex.org/W2963323070","https://openalex.org/W2963374482","https://openalex.org/W2963497309","https://openalex.org/W2963653811","https://openalex.org/W2963748441","https://openalex.org/W2963809228","https://openalex.org/W2970120757","https://openalex.org/W2970641574","https://openalex.org/W2972324944","https://openalex.org/W2979826702","https://openalex.org/W2988217457","https://openalex.org/W3021636956","https://openalex.org/W3022104542","https://openalex.org/W3034296505","https://openalex.org/W3035451444","https://openalex.org/W3035529900","https://openalex.org/W3085139254","https://openalex.org/W3096609285","https://openalex.org/W3104136798","https://openalex.org/W3104162037","https://openalex.org/W3114651185","https://openalex.org/W3156636935","https://openalex.org/W3162090017","https://openalex.org/W3170719901","https://openalex.org/W3173529047","https://openalex.org/W3173581459","https://openalex.org/W3175049034","https://openalex.org/W3177828909","https://openalex.org/W3198659451","https://openalex.org/W4206136559","https://openalex.org/W4225102708","https://openalex.org/W4244502008","https://openalex.org/W4253028564","https://openalex.org/W4385245566","https://openalex.org/W4389169833","https://openalex.org/W4389524555","https://openalex.org/W6631435780","https://openalex.org/W6632455782","https://openalex.org/W6720006811","https://openalex.org/W6726873649","https://openalex.org/W6727099177","https://openalex.org/W6737778391","https://openalex.org/W6745537798","https://openalex.org/W6753143414","https://openalex.org/W6753640285","https://openalex.org/W6757423643","https://openalex.org/W6757585730","https://openalex.org/W6762392948","https://openalex.org/W6763701032","https://openalex.org/W6766673545","https://openalex.org/W6768021236","https://openalex.org/W6768851824","https://openalex.org/W6769627184","https://openalex.org/W6771626834","https://openalex.org/W6771917389","https://openalex.org/W6774242638","https://openalex.org/W6776048684","https://openalex.org/W6776684981","https://openalex.org/W6778883912","https://openalex.org/W6779068807","https://openalex.org/W6779163297","https://openalex.org/W6779902714","https://openalex.org/W6780218876","https://openalex.org/W6780226713","https://openalex.org/W6781533629","https://openalex.org/W6788135285","https://openalex.org/W6789317445","https://openalex.org/W6789463137","https://openalex.org/W6789642857","https://openalex.org/W6791776128","https://openalex.org/W6796417832","https://openalex.org/W6802852670","https://openalex.org/W6803444062","https://openalex.org/W6811129797","https://openalex.org/W6850625674","https://openalex.org/W6854866820","https://openalex.org/W6859298233","https://openalex.org/W6863493712"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W4366179611","https://openalex.org/W2996078371"],"abstract_inverted_index":{"Transformer-based":[0,55],"pre-trained":[1,155,189],"models":[2,56,156,190],"have":[3,51,57,63],"gained":[4],"much":[5],"advance":[6],"in":[7,20,187],"recent":[8],"years,":[9],"Transformer":[10,32,38],"architecture":[11],"also":[12,52,158],"becomes":[13],"one":[14],"of":[15,84,181],"the":[16,28,69,167,185],"most":[17],"important":[18],"backbones":[19],"natural":[21],"language":[22,114],"processing.":[23],"Recent":[24],"works":[25],"show":[26],"that":[27,166,180],"attention":[29,71],"mechanism":[30],"inside":[31],"may":[33],"not":[34],"be":[35],"necessary,":[36],"and":[37,47,142,176],"alternatives":[39],"such":[40,88,191],"as":[41,89,192],"convolutional":[42],"neural":[43],"networks,":[44],"multi-layer":[45],"perceptron,":[46],"state":[48],"space":[49],"model":[50,115,148,172],"been":[53],"investigated.":[54],"two":[58],"main":[59],"limitations:":[60],"First,":[61],"they":[62,80],"quadratic":[64],"time":[65,111],"complexity":[66,112],"due":[67],"to":[68,75,91,153,179],"full":[70],"mechanism,":[72],"which":[73,95,117,183],"leads":[74],"high":[76],"computational":[77],"costs.":[78],"Second,":[79],"rely":[81],"on":[82],"representation":[83,133,193],"a":[85,105,119,131],"special":[86],"token":[87],"[CLS]":[90],"encode":[92],"entire":[93],"text,":[94],"limits":[96],"its":[97],"sentence-level":[98,132],"expressiveness.":[99],"In":[100],"this":[101],"paper,":[102],"we":[103,164],"consider":[104],"graph":[106,120],"recurrent":[107],"network":[108],"with":[109,125,130],"linear":[110],"for":[113,122],"pre-training,":[116],"builds":[118],"structure":[121],"each":[123],"sequence":[124],"local":[126],"token-level":[127],"communications,":[128],"together":[129],"detached":[134],"from":[135],"other":[136],"normal":[137],"tokens.":[138],"On":[139],"both":[140],"English":[141],"Chinese":[143],"text":[144],"understanding":[145],"tasks,":[146],"our":[147,171],"can":[149],"achieve":[150],"comparable":[151],"performance":[152],"existing":[154,188],"while":[157],"achieving":[159],"higher":[160],"inference":[161],"efficiency.":[162],"Furthermore,":[163],"discovered":[165],"representations":[168],"generated":[169],"by":[170],"are":[173],"more":[174],"diverse":[175],"uniform":[177],"compared":[178],"Transformer,":[182],"alleviates":[184],"problems":[186],"degradation.":[194]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-12-27T23:08:20.325037","created_date":"2025-10-10T00:00:00"}