{"id":"https://openalex.org/W4410345129","doi":"https://doi.org/10.1145/3735653","title":"U-Net Encapsulated Transformer for Reducing Dimensionality in Training Large Language Models","display_name":"U-Net Encapsulated Transformer for Reducing Dimensionality in Training Large Language Models","publication_year":2025,"publication_date":"2025-05-13","ids":{"openalex":"https://openalex.org/W4410345129","doi":"https://doi.org/10.1145/3735653"},"language":"en","primary_location":{"id":"doi:10.1145/3735653","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3735653","pdf_url":null,"source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100256169","display_name":"Marvin John Ignacio","orcid":"https://orcid.org/0000-0003-2145-934X"},"institutions":[{"id":"https://openalex.org/I28777354","display_name":"Sejong University","ror":"https://ror.org/00aft1q37","country_code":"KR","type":"education","lineage":["https://openalex.org/I28777354"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Marvin John Ignacio","raw_affiliation_strings":["Department of Computer Engineering, Sejong University, Seoul, the Republic of Korea","Sejong University, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0003-2145-934X","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Sejong University, Seoul, the Republic of Korea","institution_ids":["https://openalex.org/I28777354"]},{"raw_affiliation_string":"Sejong University, Republic of Korea","institution_ids":["https://openalex.org/I28777354"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064727471","display_name":"Yong-Guk Kim","orcid":"https://orcid.org/0000-0003-4645-1395"},"institutions":[{"id":"https://openalex.org/I28777354","display_name":"Sejong University","ror":"https://ror.org/00aft1q37","country_code":"KR","type":"education","lineage":["https://openalex.org/I28777354"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yong-Guk Kim","raw_affiliation_strings":["Department of Computer Engineering, Sejong University, Seoul, the Republic of Korea","Sejong University, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0003-4645-1395","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Sejong University, Seoul, the Republic of Korea","institution_ids":["https://openalex.org/I28777354"]},{"raw_affiliation_string":"Sejong University, Republic of Korea","institution_ids":["https://openalex.org/I28777354"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103935927","display_name":"Hulin Jin","orcid":"https://orcid.org/0000-0002-6896-2498"},"institutions":[{"id":"https://openalex.org/I143868143","display_name":"Anhui University","ror":"https://ror.org/05th6yx34","country_code":"CN","type":"education","lineage":["https://openalex.org/I143868143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hulin Jin","raw_affiliation_strings":["School of Computer Science and Technology, Anhui University, Hefei, China","Anhui University, China"],"raw_orcid":"https://orcid.org/0000-0002-6896-2498","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Anhui University, Hefei, China","institution_ids":["https://openalex.org/I143868143"]},{"raw_affiliation_string":"Anhui University, China","institution_ids":["https://openalex.org/I143868143"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111256746","display_name":"Seunghee Yu","orcid":"https://orcid.org/0009-0001-6249-9156"},"institutions":[{"id":"https://openalex.org/I28777354","display_name":"Sejong University","ror":"https://ror.org/00aft1q37","country_code":"KR","type":"education","lineage":["https://openalex.org/I28777354"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Seunghee Yu","raw_affiliation_strings":["Department of Business Administration, Sejong University, Seoul, the Republic of Korea","Sejong University, Republic of Korea"],"raw_orcid":"https://orcid.org/0009-0001-6249-9156","affiliations":[{"raw_affiliation_string":"Department of Business Administration, Sejong University, Seoul, the Republic of Korea","institution_ids":["https://openalex.org/I28777354"]},{"raw_affiliation_string":"Sejong University, Republic of Korea","institution_ids":["https://openalex.org/I28777354"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.396,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.83548323,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"17","issue":"3","first_page":"1","last_page":"32"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8885393142700195},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7855395078659058},{"id":"https://openalex.org/keywords/curse-of-dimensionality","display_name":"Curse of dimensionality","score":0.6703598499298096},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5176056623458862},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.49118396639823914},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47229769825935364},{"id":"https://openalex.org/keywords/net","display_name":"Net (polyhedron)","score":0.4140082597732544},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.38617485761642456},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.35723790526390076},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.11657071113586426},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.06981104612350464}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8885393142700195},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7855395078659058},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.6703598499298096},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5176056623458862},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.49118396639823914},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47229769825935364},{"id":"https://openalex.org/C14166107","wikidata":"https://www.wikidata.org/wiki/Q253829","display_name":"Net (polyhedron)","level":2,"score":0.4140082597732544},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38617485761642456},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35723790526390076},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.11657071113586426},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.06981104612350464},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3735653","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3735653","pdf_url":null,"source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5396177265","display_name":null,"funder_award_id":"2020R1A6A1A03038540","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"}],"funders":[{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W2064675550","https://openalex.org/W2157331557","https://openalex.org/W2890894339","https://openalex.org/W2963122961","https://openalex.org/W2998617917","https://openalex.org/W3115532187","https://openalex.org/W3177265267","https://openalex.org/W4239019441","https://openalex.org/W4394993979","https://openalex.org/W6893007900"],"related_works":["https://openalex.org/W4288365749","https://openalex.org/W2936497627","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W3098382480","https://openalex.org/W4287598411","https://openalex.org/W3100913109","https://openalex.org/W3198458223","https://openalex.org/W3126642501","https://openalex.org/W2964413124"],"abstract_inverted_index":{"Training":[0],"language":[1,81,127,156,176],"models":[2,82,100,128],"from":[3],"scratch":[4],"presents":[5],"a":[6,50,89,106],"critical":[7],"challenge":[8],"in":[9,129,141],"Natural":[10],"Language":[11,23],"Processing":[12],"(NLP),":[13],"primarily":[14],"due":[15],"to":[16,53,71,98,173,175],"the":[17,55,62,78,135,138,163],"computational":[18],"demands":[19],"of":[20,58,80,92,101,110,137,165],"pre-trained":[21],"Large":[22],"Models,":[24],"which":[25,67],"are":[26],"predominantly":[27],"trained":[28],"on":[29,44],"English":[30],"corpora":[31],"using":[32],"extensive":[33],"resources.":[34],"While":[35],"offering":[36],"viable":[37],"solutions,":[38],"existing":[39],"alternatives":[40],"still":[41],"rely":[42],"heavily":[43],"high-performance":[45],"hardware.":[46],"This":[47,119,159],"work":[48,160],"introduces":[49],"different":[51],"approach":[52],"reducing":[54],"algorithmic":[56],"complexity":[57],"Transformer-based":[59],"architectures":[60],"through":[61],"U-Net":[63],"Encapsulated":[64],"Transformer":[65,111],"(UET),":[66],"applies":[68],"dimensionality":[69],"reduction":[70],"token":[72],"embeddings.":[73],"The":[74],"UET":[75,139],"architecture":[76,140],"enables":[77],"development":[79],"with":[83,105,170],"significantly":[84],"reduced":[85],"parameter":[86],"sizes":[87],"for":[88,125,155],"given":[90],"set":[91],"hyperparameters.":[93],"Alternatively,":[94],"it":[95],"allows":[96],"researchers":[97,169],"design":[99],"comparable":[102],"size":[103],"but":[104],"substantially":[107],"greater":[108],"number":[109],"blocks,":[112],"enhancing":[113],"model":[114,157,177],"depth":[115],"and":[116],"potential":[117,136],"capacity.":[118],"study":[120],"also":[121],"outlines":[122],"practical":[123],"methodologies":[124],"training":[126],"resource-constrained":[130,146],"environments.":[131],"Experimental":[132],"results":[133],"illustrate":[134],"achieving":[142],"reasonable":[143],"performance":[144],"under":[145],"conditions,":[147],"highlighting":[148],"its":[149],"promise":[150],"as":[151],"an":[152],"accessible":[153],"alternative":[154],"development.":[158,178],"could":[161],"broaden":[162],"accessibility":[164],"NLP":[166],"research,":[167],"empowering":[168],"hardware":[171],"constraints":[172],"contribute":[174]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}