{"id":"https://openalex.org/W4412945523","doi":"https://doi.org/10.18653/v1/2025.acl-long.366","title":"How to Train Long-Context Language Models (Effectively)","display_name":"How to Train Long-Context Language Models (Effectively)","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412945523","doi":"https://doi.org/10.18653/v1/2025.acl-long.366"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.366","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.366","pdf_url":"https://aclanthology.org/2025.acl-long.366.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.366.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033296587","display_name":"Tianyu Gao","orcid":"https://orcid.org/0000-0002-1860-7252"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tianyu Gao","raw_affiliation_strings":["Princeton Language and Intelligence , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton Language and Intelligence , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003423990","display_name":"Alexander Wettig","orcid":null},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander Wettig","raw_affiliation_strings":["Princeton Language and Intelligence , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton Language and Intelligence , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111465448","display_name":"H. W. Yen","orcid":null},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Howard Yen","raw_affiliation_strings":["Princeton Language and Intelligence , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton Language and Intelligence , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051064208","display_name":"Danqi Chen","orcid":"https://orcid.org/0000-0002-6226-6838"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danqi Chen","raw_affiliation_strings":["Princeton Language and Intelligence , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton Language and Intelligence , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I20089843"],"apc_list":null,"apc_paid":null,"fwci":6.6921,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.97010214,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"7376","last_page":"7399"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9787999987602234,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9787999987602234,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9650999903678894,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7451245784759521},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5406410694122314},{"id":"https://openalex.org/keywords/context-model","display_name":"Context model","score":0.41199660301208496},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3526453971862793},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2795253396034241},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.08521753549575806}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7451245784759521},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5406410694122314},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.41199660301208496},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3526453971862793},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2795253396034241},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.08521753549575806},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.366","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.366","pdf_url":"https://aclanthology.org/2025.acl-long.366.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.366","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.366","pdf_url":"https://aclanthology.org/2025.acl-long.366.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5537390954","display_name":"CAREER: Building Next-Generation Language Models Based on Retrieval","funder_award_id":"2239290","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6671297155","display_name":null,"funder_award_id":"CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412945523.pdf","grobid_xml":"https://content.openalex.org/works/W4412945523.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2349222429","https://openalex.org/W2116230991","https://openalex.org/W3117430770","https://openalex.org/W2046058552","https://openalex.org/W2590751808","https://openalex.org/W1972377868","https://openalex.org/W2132709506","https://openalex.org/W2098730537","https://openalex.org/W2186895195","https://openalex.org/W2748557612"],"abstract_inverted_index":{"We":[0],"study":[1],"continued":[2,72],"training":[3,111],"and":[4,46,78,92,145],"supervised":[5],"fine-tuning":[6],"(SFT)":[7],"of":[8,17,30,42,97,161,168,190,195],"a":[9,22,39,113,159],"language":[10],"model":[11,28],"(LM)":[12],"to":[13,26,66,104,186],"make":[14],"effective":[15],"use":[16,38],"long-context":[18,43,56,121,135,152,169,179],"information.We":[19],"first":[20],"establish":[21],"reliable":[23],"evaluation":[24,118],"protocol":[25],"guide":[27],"development-instead":[29],"perplexity":[31],"or":[32],"simple":[33],"needle-in-a-haystack":[34],"(NIAH)":[35],"tests,":[36],"we":[37,47,62],"broad":[40],"set":[41],"downstream":[44],"tasks,":[45],"evaluate":[48],"models":[49,157],"after":[50],"SFT":[51],"as":[52,84,175],"this":[53],"better":[54],"reveals":[55],"abilities.Supported":[57],"by":[58],"our":[59],"robust":[60],"evaluations,":[61],"run":[63],"thorough":[64],"experiments":[65],"decide":[67],"the":[68,74,117,166,191],"data":[69],"mix":[70],"for":[71,124],"pre-training,":[73],"instruction":[75,129],"tuning":[76],"dataset,":[77],"many":[79,176],"other":[80],"design":[81],"choices":[82],"such":[83],"position":[85],"extrapolation.We":[86],"find":[87],"that":[88],"(1)":[89],"code":[90],"repositories":[91],"books":[93],"are":[94],"excellent":[95],"sources":[96],"long":[98],"data,":[99],"but":[100],"it":[101],"is":[102,141],"crucial":[103],"combine":[105],"them":[106],"with":[107,112],"high-quality":[108],"short-context":[109],"data;(2)":[110],"sequence":[114],"length":[115,119,160],"beyond":[116],"boosts":[120],"performance;":[122],"(3)":[123],"SFT,":[125],"using":[126,172],"only":[127,173],"short":[128],"datasets":[130],"yields":[131],"strong":[132],"performance":[133,153],"on":[134,147,165],"tasks.Our":[136],"final":[137],"model,":[138],"ProLong-8B,":[139],"which":[140],"initialized":[142],"from":[143],"Llama-3":[144],"trained":[146],"40B":[148],"tokens,":[149,188],"demonstrates":[150],"state-ofthe-art":[151],"among":[154],"similarly":[155],"sized":[156],"at":[158],"128K.Pro-Long":[162],"outperforms":[163],"Llama-3.1-8B-Instruct":[164],"majority":[167],"tasks":[170],"despite":[171],"5%":[174],"tokens":[177],"during":[178],"training.Additionally,":[180],"ProLong":[181],"can":[182],"effectively":[183],"process":[184],"up":[185],"512K":[187],"one":[189],"longest":[192],"context":[193],"windows":[194],"publicly":[196],"available":[197],"LMs.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}