{"id":"https://openalex.org/W4389633786","doi":"https://doi.org/10.1109/access.2023.3341512","title":"How Does a Transformer Learn Compression? An Attention Study on Huffman and LZ4","display_name":"How Does a Transformer Learn Compression? An Attention Study on Huffman and LZ4","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389633786","doi":"https://doi.org/10.1109/access.2023.3341512"},"language":"en","primary_location":{"id":"doi:10.1109/access.2023.3341512","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3341512","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10353923.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10353923.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009604313","display_name":"Beomseok Seo","orcid":"https://orcid.org/0009-0008-3376-9330"},"institutions":[{"id":"https://openalex.org/I94588446","display_name":"Hongik University","ror":"https://ror.org/00egdv862","country_code":"KR","type":"education","lineage":["https://openalex.org/I94588446"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Beomseok Seo","raw_affiliation_strings":["Department of Electronic and Electrical Engineering, Hongik University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0009-0008-3376-9330","affiliations":[{"raw_affiliation_string":"Department of Electronic and Electrical Engineering, Hongik University, Seoul, South Korea","institution_ids":["https://openalex.org/I94588446"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049196468","display_name":"Albert No","orcid":"https://orcid.org/0000-0002-6346-4182"},"institutions":[{"id":"https://openalex.org/I94588446","display_name":"Hongik University","ror":"https://ror.org/00egdv862","country_code":"KR","type":"education","lineage":["https://openalex.org/I94588446"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Albert No","raw_affiliation_strings":["Department of Electronic and Electrical Engineering, Hongik University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0002-6346-4182","affiliations":[{"raw_affiliation_string":"Department of Electronic and Electrical Engineering, Hongik University, Seoul, South Korea","institution_ids":["https://openalex.org/I94588446"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I94588446"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.3173,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.64760513,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"11","issue":null,"first_page":"140559","last_page":"140568"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/huffman-coding","display_name":"Huffman coding","score":0.7406291365623474},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6182050108909607},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5297155380249023},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.40437787771224976},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2787880301475525},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.1785459816455841},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.10720750689506531},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08251339197158813}],"concepts":[{"id":"https://openalex.org/C46900642","wikidata":"https://www.wikidata.org/wiki/Q2647","display_name":"Huffman coding","level":3,"score":0.7406291365623474},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6182050108909607},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5297155380249023},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.40437787771224976},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2787880301475525},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.1785459816455841},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.10720750689506531},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08251339197158813}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2023.3341512","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3341512","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10353923.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:ace7842737334f43b8174d5323da6829","is_oa":true,"landing_page_url":"https://doaj.org/article/ace7842737334f43b8174d5323da6829","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 11, Pp 140559-140568 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2023.3341512","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3341512","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10353923.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.7799999713897705,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G3363376426","display_name":null,"funder_award_id":"IITP-2022-RS-2022-00156295","funder_id":"https://openalex.org/F4320328359","funder_display_name":"Ministry of Science and ICT, South Korea"},{"id":"https://openalex.org/G4700831490","display_name":null,"funder_award_id":"2022-","funder_id":"https://openalex.org/F4320335489","funder_display_name":"Institute for Information and Communications Technology Promotion"},{"id":"https://openalex.org/G8962085891","display_name":null,"funder_award_id":"IITP-2022-RS-2022-00156295","funder_id":"https://openalex.org/F4320324891","funder_display_name":"Iran Telecommunication Research Center"}],"funders":[{"id":"https://openalex.org/F4320324891","display_name":"Iran Telecommunication Research Center","ror":"https://ror.org/01a3g2z22"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"},{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389633786.pdf","grobid_xml":"https://content.openalex.org/works/W4389633786.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1902237438","https://openalex.org/W1990653637","https://openalex.org/W2060108852","https://openalex.org/W2107745473","https://openalex.org/W2122962290","https://openalex.org/W2892009249","https://openalex.org/W2896457183","https://openalex.org/W2951025380","https://openalex.org/W2952028139","https://openalex.org/W2970726176","https://openalex.org/W2972498556","https://openalex.org/W2977944219","https://openalex.org/W2995744795","https://openalex.org/W3035281110","https://openalex.org/W3173247797","https://openalex.org/W4288103164","https://openalex.org/W4313447594","https://openalex.org/W4322718191","https://openalex.org/W4384643949","https://openalex.org/W4385245566","https://openalex.org/W4387818804","https://openalex.org/W6679434410","https://openalex.org/W6739901393","https://openalex.org/W6755207826","https://openalex.org/W6764072591","https://openalex.org/W6767868144","https://openalex.org/W6768061436","https://openalex.org/W6771550025","https://openalex.org/W6848036820","https://openalex.org/W6850625674","https://openalex.org/W6854424554"],"related_works":["https://openalex.org/W2011446102","https://openalex.org/W1979511889","https://openalex.org/W3154960554","https://openalex.org/W2158701093","https://openalex.org/W2352725890","https://openalex.org/W3119430538","https://openalex.org/W4391249354","https://openalex.org/W2787192521","https://openalex.org/W2756410301","https://openalex.org/W3028580730"],"abstract_inverted_index":{"Transformers":[0,17,62],"have":[1],"excelled":[2],"in":[3,151,204],"natural":[4],"language":[5],"processing":[6,182],"and":[7,45,68,166,186,193,208],"vision":[8],"domains.":[9],"This":[10,79,198],"leads":[11],"to":[12,20,85,117,127,212],"the":[13,92,98,110,119,123,137,144,160,170,175,180,187,191,201],"intriguing":[14],"proposition:":[15],"can":[16,63],"be":[18],"adapted":[19],"a":[21,37,133],"more":[22,171],"generalized":[23],"framework,":[24],"such":[25],"as":[26,49,220],"understanding":[27],"general":[28,54],"finite":[29,55,88],"state":[30,56,89],"machines?":[31],"To":[32],"explore":[33],"this,":[34],"we":[35],"trained":[36],"Transformer":[38],"network":[39],"on":[40,114],"compression":[41,105,206],"algorithms":[42,207],"like":[43],"Huffman":[44,108],"LZ4,":[46],"viewing":[47],"them":[48],"stepping":[50],"stones":[51],"towards":[52],"mastering":[53],"machines.":[57,90],"Our":[58],"analysis":[59],"indicates":[60],"that":[61,217],"adeptly":[64],"internalize":[65],"these":[66,104],"methods":[67],"replicate":[69],"essential":[70],"states,":[71],"echoing":[72],"human-like":[73],"interpretation":[74],"via":[75],"their":[76,83],"attention":[77,93,139,176,223],"mechanisms.":[78],"provides":[80],"evidence":[81],"of":[82,183],"capability":[84],"decipher":[86],"practical":[87],"Examining":[91],"maps":[94,157,177],"offers":[95],"insights":[96],"into":[97],"Transformer\u2019s":[99,181,202],"methodology":[100],"when":[101],"engaging":[102],"with":[103,132],"techniques.":[106],"In":[107],"encodings,":[109],"encoder":[111],"predominantly":[112],"focuses":[113],"input":[115,153,164,184,192,214],"statistics":[116],"define":[118],"present":[120],"state,":[121],"which":[122],"decoder":[124],"subsequently":[125],"leverages":[126],"produce":[128],"output":[129,167,195],"bits.":[130,168],"Remarkably,":[131],"2nd-order":[134],"Markov":[135],"input,":[136],"encoder\u2019s":[138],"is":[140],"prominently":[141],"directed":[142],"at":[143],"previous":[145],"two":[146],"symbols,":[147],"underscoring":[148],"its":[149,209,218],"proficiency":[150,203],"summarizing":[152],"statistics.":[154],"The":[155],"cross-attention":[156],"further":[158],"elucidate":[159],"exact":[161],"association":[162],"between":[163,190],"symbols":[165],"For":[169],"complex":[172],"LZ4":[173],"compression,":[174],"vividly":[178],"portray":[179],"sequences":[185],"close":[188],"linkage":[189],"resulting":[194],"bit":[196],"stream.":[197],"study":[199],"underscores":[200],"comprehending":[205],"keen":[210],"ability":[211],"grasp":[213],"statistics,":[215],"implying":[216],"mechanisms,":[219],"illustrated":[221],"by":[222],"maps,":[224],"provide":[225],"profound":[226],"interpretability.":[227]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}