{"id":"https://openalex.org/W7214192558","doi":"https://doi.org/10.48550/arxiv.2609.27248","title":"Repurposing Pre-trained LLMs as High Fidelity Continuous Text Autoencoders","display_name":"Repurposing Pre-trained LLMs as High Fidelity Continuous Text Autoencoders","publication_year":2026,"publication_date":"2026-09-23","ids":{"openalex":"https://openalex.org/W7214192558","doi":"https://doi.org/10.48550/arxiv.2609.27248"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.27248","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.27248","pdf_url":"https://arxiv.org/pdf/2609.27248","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.27248","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061607107","display_name":"Arkanath Pathak","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I204250578","display_name":"University of California, Irvine","ror":"https://ror.org/04gyf1771","country_code":"US","type":"education","lineage":["https://openalex.org/I204250578"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pathak, Arkanath","raw_affiliation_strings":["University of California , Irvine"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Irvine","institution_ids":["https://openalex.org/I204250578"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015496512","display_name":"Unnat Jain","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I204250578","display_name":"University of California, Irvine","ror":"https://ror.org/04gyf1771","country_code":"US","type":"education","lineage":["https://openalex.org/I204250578"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jain, Unnat","raw_affiliation_strings":["University of California , Irvine"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Irvine","institution_ids":["https://openalex.org/I204250578"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5104361813","display_name":"Alexander C. Berg","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I204250578","display_name":"University of California, Irvine","ror":"https://ror.org/04gyf1771","country_code":"US","type":"education","lineage":["https://openalex.org/I204250578"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Berg, Alexander C.","raw_affiliation_strings":["University of California , Irvine"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Irvine","institution_ids":["https://openalex.org/I204250578"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I204250578"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9814000129699707,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9814000129699707,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.007600000128149986,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.006399999838322401,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pre-trained-language-models","display_name":"pre-trained language models","score":0.7129999995231628},{"id":"https://openalex.org/keywords/lora-adaptation","display_name":"LoRA adaptation","score":0.7760000228881836},{"id":"https://openalex.org/keywords/kl-regularization","display_name":"KL regularization","score":0.8450000286102295},{"id":"https://openalex.org/keywords/latent-diffusion-models","display_name":"latent diffusion models","score":0.6930000185966492},{"id":"https://openalex.org/keywords/image-captioning","display_name":"image captioning","score":0.9679999947547913},{"id":"https://openalex.org/keywords/textual-reconstruction","display_name":"textual reconstruction","score":0.4970000088214874},{"id":"https://openalex.org/keywords/parameter-efficient-fine-tuning","display_name":"parameter-efficient fine-tuning","score":0.6510000228881836}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8047000169754028},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.621999979019165},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.5598000288009644},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.5591999888420105},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.5429999828338623},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.49720001220703125},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.4869999885559082},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.47040000557899475},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.4514999985694885},{"id":"https://openalex.org/C171686336","wikidata":"https://www.wikidata.org/wiki/Q3532085","display_name":"Topic model","level":2,"score":0.45100000500679016},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4325999915599823},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.4239000082015991},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3944000005722046},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38749998807907104},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3781999945640564},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.37049999833106995},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.3375000059604645},{"id":"https://openalex.org/C52723943","wikidata":"https://www.wikidata.org/wiki/Q1127410","display_name":"Serialization","level":2,"score":0.3273000121116638},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.326200008392334},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.3215000033378601},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.3138999938964844},{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.2962999939918518},{"id":"https://openalex.org/C2778689934","wikidata":"https://www.wikidata.org/wiki/Q1313396","display_name":"Headline","level":2,"score":0.2946000099182129},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.2831999957561493},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.27619999647140503},{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.2761000096797943},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.2718999981880188},{"id":"https://openalex.org/C519536355","wikidata":"https://www.wikidata.org/wiki/Q21021151","display_name":"Repurposing","level":2,"score":0.2718000113964081},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2651999890804291},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.26030001044273376},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C203357204","wikidata":"https://www.wikidata.org/wiki/Q1089605","display_name":"Chunking (psychology)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.25360000133514404}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.27248","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.27248","pdf_url":"https://arxiv.org/pdf/2609.27248","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.27248","doi":"https://doi.org/10.48550/arxiv.2609.27248","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.27248","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.27248","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.27248","pdf_url":"https://arxiv.org/pdf/2609.27248","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214192558.pdf","grobid_xml":"https://content.openalex.org/works/W7214192558.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4416035791","https://openalex.org/W7172411991","https://openalex.org/W2947054491","https://openalex.org/W3005116366","https://openalex.org/W4415928588","https://openalex.org/W7160596537","https://openalex.org/W7223405065","https://openalex.org/W7133493128","https://openalex.org/W7163149319","https://openalex.org/W7133234250"],"abstract_inverted_index":{"Next-token":[0],"prediction":[1],"has":[2],"enabled":[3],"highly":[4],"fluent":[5],"autoregressive":[6],"language":[7,57],"models,":[8],"but":[9],"it":[10],"represents":[11],"global":[12],"structure":[13],"only":[14],"indirectly":[15],"through":[16],"sequential":[17],"factorization.":[18],"In":[19],"contrast,":[20],"high-fidelity":[21],"autoencoders":[22],"have":[23],"become":[24],"a":[25,42,50,54,60,73,128,148],"standard":[26],"primitive":[27],"in":[28,138],"image":[29,135],"generation,":[30],"enabling":[31],"generative":[32,90],"models":[33,173],"to":[34,109,117],"operate":[35],"over":[36],"continuous":[37,45,61,150],"latent":[38,75,142,151],"spaces;":[39],"text":[40,62,146],"lacks":[41],"comparably":[43],"faithful":[44],"representation.":[46],"We":[47,122],"propose":[48],"LLMAE,":[49],"method":[51],"for":[52,159],"repurposing":[53],"pretrained":[55],"decoder-only":[56],"model":[58,131],"as":[59,72],"autoencoder":[63],"by":[64,126],"exposing":[65],"the":[66,89,93,139,165,168],"activations":[67],"of":[68,92,119,167],"an":[69,156],"intermediate":[70],"layer":[71],"fixed-length":[74,149],"bottleneck.":[76],"LLMAE":[77,105,141],"achieves":[78],"this":[79],"interface":[80],"with":[81,112],"structured":[82],"attention":[83],"masks":[84],"and":[85,102,172],"LoRA":[86],"adaptation,":[87],"leveraging":[88],"prior":[91],"original":[94,169],"LLM.":[95,170],"Across":[96],"two":[97],"backbones":[98],"(270M":[99],"Gemma":[100],"3":[101],"0.5B":[103],"Qwen2.5),":[104],"reconstructs":[106],"sequences":[107],"up":[108,116],"1024":[110],"tokens":[111],"high":[113],"accuracy,":[114],"reproducing":[115],"97%":[118],"documents":[120],"verbatim.":[121],"demonstrate":[123],"downstream":[124,160],"utility":[125],"training":[127],"lightweight":[129],"diffusion":[130],"that":[132],"generates":[133],"detailed":[134],"captions":[136],"directly":[137],"frozen":[140],"space.":[143],"By":[144],"mapping":[145],"into":[147],"space,":[152],"our":[153],"approach":[154],"provides":[155],"effective":[157],"substrate":[158],"adaptation":[161],"while":[162],"benefiting":[163],"from":[164],"fluency":[166],"Code":[171],"are":[174],"available":[175],"at":[176],"https://github.com/arkanath/LLMAE.":[177]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-25T00:00:00"}