{"id":"https://openalex.org/W4416620459","doi":"https://doi.org/10.48550/arxiv.2510.20535","title":"ARC-Encoder: learning compressed text representations for large language models","display_name":"ARC-Encoder: learning compressed text representations for large language models","publication_year":2025,"publication_date":"2025-10-23","ids":{"openalex":"https://openalex.org/W4416620459","doi":"https://doi.org/10.48550/arxiv.2510.20535"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.20535","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.20535","pdf_url":"https://arxiv.org/pdf/2510.20535","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.20535","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120663902","display_name":"Hippolyte Pilchen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pilchen, Hippolyte","raw_affiliation_strings":["Kyutai , Paris , France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyutai , Paris , France","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069316249","display_name":"\u00c9douard Grave","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Grave, Edouard","raw_affiliation_strings":["Kyutai , Paris , France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyutai , Paris , France","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5113908330","display_name":"Patrick P\u00e9rez","orcid":"https://orcid.org/0000-0002-8124-1206","observed_orcids":["https://orcid.org/0000-0002-8124-1206"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"P\u00e9rez, Patrick","raw_affiliation_strings":["Kyutai , Paris , France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyutai , Paris , France","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context-compression","display_name":"context compression","score":0.9440000057220459},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8029999732971191},{"id":"https://openalex.org/keywords/encoder-decoder-architecture","display_name":"encoder-decoder architecture","score":0.492000013589859},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.6420000195503235},{"id":"https://openalex.org/keywords/context-window-extension","display_name":"context window extension","score":0.9750000238418579},{"id":"https://openalex.org/keywords/inference-efficiency","display_name":"inference efficiency","score":0.7260000109672546},{"id":"https://openalex.org/keywords/token-embeddings","display_name":"token embeddings","score":0.3659999966621399},{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.7870000004768372},{"id":"https://openalex.org/keywords/chain-of-thought-reasoning","display_name":"chain-of-thought reasoning","score":0.9589999914169312}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8414999842643738},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6974999904632568},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6359999775886536},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6172000169754028},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5985999703407288},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5626000165939331},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5583999752998352},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4702000021934509},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4618000090122223},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.44690001010894775},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.43799999356269836},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.4056999981403351},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4016000032424927},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4000000059604645},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39879998564720154},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.3671000003814697},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.35350000858306885},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.34779998660087585},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.3301999866962433},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.31769999861717224},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.3082999885082245},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.2831999957561493},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.27619999647140503},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.2639999985694885},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.20535","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.20535","pdf_url":"https://arxiv.org/pdf/2510.20535","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.20535","doi":"https://doi.org/10.48550/arxiv.2510.20535","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.20535","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.20535","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.20535","pdf_url":"https://arxiv.org/pdf/2510.20535","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416620459.pdf","grobid_xml":"https://content.openalex.org/works/W4416620459.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W2124509324","https://openalex.org/W2612431505","https://openalex.org/W2946088473","https://openalex.org/W2951365061","https://openalex.org/W2951534261","https://openalex.org/W2952862139","https://openalex.org/W2963748441","https://openalex.org/W2972522091","https://openalex.org/W2989743967","https://openalex.org/W3003928769","https://openalex.org/W3030163527","https://openalex.org/W3085139254","https://openalex.org/W3169369929","https://openalex.org/W3175218683","https://openalex.org/W3176503701","https://openalex.org/W4301243929","https://openalex.org/W4382603041","https://openalex.org/W4384392961","https://openalex.org/W4385573898","https://openalex.org/W4385890381","https://openalex.org/W4387561528","https://openalex.org/W4389519019","https://openalex.org/W4389519226","https://openalex.org/W4389524473","https://openalex.org/W4390722419","https://openalex.org/W4399115160","https://openalex.org/W4401306886","https://openalex.org/W4402671910","https://openalex.org/W4407124183","https://openalex.org/W4412887989","https://openalex.org/W4412944941","https://openalex.org/W4415335368","https://openalex.org/W4415795887","https://openalex.org/W4416120571","https://openalex.org/W4417114725","https://openalex.org/W4417300457","https://openalex.org/W6910722336","https://openalex.org/W6947991961","https://openalex.org/W7133221612"],"related_works":["https://openalex.org/W7164000695","https://openalex.org/W7141214090","https://openalex.org/W7090139269","https://openalex.org/W4409348132","https://openalex.org/W4416184858","https://openalex.org/W4389524473","https://openalex.org/W4389524288","https://openalex.org/W4404609179","https://openalex.org/W7221816807","https://openalex.org/W7204538597"],"abstract_inverted_index":{"Recent":[0],"techniques":[1,20],"such":[2],"as":[3],"retrieval-augmented":[4],"generation":[5],"or":[6,35],"chain-of-thought":[7],"reasoning":[8],"have":[9],"led":[10,92],"to":[11,93,129,164,172],"longer":[12],"contexts":[13],"and":[14,84,136,183,205],"increased":[15],"inference":[16],"costs.":[17],"Context":[18],"compression":[19],"can":[21,41,161],"reduce":[22],"these":[23],"costs,":[24],"but":[25],"the":[26,32,63,88,94],"most":[27],"effective":[28],"approaches":[29],"require":[30],"fine-tuning":[31,203],"target":[33],"model":[34],"even":[36],"modifying":[37],"its":[38,43],"architecture.":[39],"This":[40,178],"degrade":[42],"general":[44],"abilities":[45],"when":[46],"not":[47],"used":[48],"for":[49,87,186],"this":[50],"specific":[51],"purpose.":[52],"Here":[53],"we":[54,76,156],"explore":[55],"an":[56,59,97],"alternative":[57],"approach:":[58],"encoder":[60,171],"that":[61,141,158,189],"compresses":[62],"context":[64,130],"into":[65],"continuous":[66,108],"representations":[67,109],"which":[68,104],"replace":[69],"token":[70],"embeddings":[71],"in":[72],"decoder":[73,176],"LLMs.":[74,177,194],"First,":[75],"perform":[77],"a":[78,119,169,181,197],"systematic":[79],"study":[80],"of":[81,96,121],"training":[82,198],"strategies":[83],"architecture":[85],"choices":[86],"encoder.":[89],"Our":[90],"findings":[91],"design":[95],"Adaptable":[98],"text":[99,113],"Representations":[100],"Compressor,":[101],"named":[102],"ARC-Encoder,":[103],"outputs":[105],"$x$-times":[106],"fewer":[107],"(typically":[110],"$x\\!\\in\\!\\{4,8\\}$)":[111],"than":[112],"tokens.":[114],"We":[115,195],"evaluate":[116],"ARC-Encoder":[117,142,180],"across":[118,174],"variety":[120],"LLM":[122],"usage":[123],"scenarios,":[124],"ranging":[125],"from":[126],"in-context":[127],"learning":[128],"window":[131],"extension,":[132],"on":[133,146],"both":[134],"instruct":[135],"base":[137],"decoders.":[138],"Results":[139],"show":[140],"achieves":[143],"state-of-the-art":[144],"performance":[145],"several":[147],"benchmarks":[148],"while":[149],"improving":[150],"computational":[151],"efficiency":[152],"at":[153,200,210],"inference.":[154],"Finally,":[155],"demonstrate":[157],"our":[159],"models":[160,207],"be":[162],"adapted":[163],"multiple":[165,193],"decoders":[166],"simultaneously,":[167],"allowing":[168],"single":[170],"generalize":[173],"different":[175],"makes":[179],"flexible":[182],"efficient":[184],"solution":[185],"portable":[187],"encoders":[188],"work":[190],"seamlessly":[191],"with":[192],"release":[196],"code":[199],"https://github.com/kyutai-labs/ARC-Encoder":[201],",":[202],"dataset":[204],"pretrained":[206],"are":[208],"available":[209],"https://huggingface.co/collections/kyutai/arc-encoders-68ee18787301407d60a57047":[211],".":[212]},"counts_by_year":[{"year":2026,"cited_by_count":6}],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2025-10-25T00:00:00"}