{"id":"https://openalex.org/W4412886870","doi":"https://doi.org/10.18653/v1/2025.acl-long.62","title":"Untie the Knots: An Efficient Data Augmentation Strategy for Long-Context Pre-Training in Language Models","display_name":"Untie the Knots: An Efficient Data Augmentation Strategy for Long-Context Pre-Training in Language Models","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412886870","doi":"https://doi.org/10.18653/v1/2025.acl-long.62"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.62","pdf_url":"https://aclanthology.org/2025.acl-long.62.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.62.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006589025","display_name":"Junfeng Tian","orcid":"https://orcid.org/0000-0003-1759-3255"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Junfeng Tian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101155301","display_name":"Da Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Da Zheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037659355","display_name":"Yang Chen","orcid":"https://orcid.org/0000-0001-7269-7933"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113746153","display_name":"Rui Wang","orcid":"https://orcid.org/0009-0008-0763-4280"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rui Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039034006","display_name":"Colin Zhang","orcid":"https://orcid.org/0009-0009-2411-9640"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Colin Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5103145743","display_name":"Debing Zhang","orcid":"https://orcid.org/0000-0003-4048-0531"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Debing Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.673,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.83272103,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1223","last_page":"1242"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9679999947547913,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.958899974822998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7794903516769409},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5786595344543457},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.47056958079338074},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4623611569404602},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.4301450252532959},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.36485159397125244},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.20445001125335693}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7794903516769409},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5786595344543457},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.47056958079338074},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4623611569404602},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.4301450252532959},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.36485159397125244},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.20445001125335693},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.62","pdf_url":"https://aclanthology.org/2025.acl-long.62.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.62","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.62","pdf_url":"https://aclanthology.org/2025.acl-long.62.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412886870.pdf","grobid_xml":"https://content.openalex.org/works/W4412886870.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W4412817058","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,19,148,179],"(LLM)":[3],"have":[4],"focused":[5],"on":[6,40,147,155,167],"expanding":[7],"the":[8,28,35,44,57,67,81,85,92,95,125,141],"context":[9,171],"window":[10],"in":[11,134],"order":[12],"to":[13,20,72,76,109,131],"incorporate":[14],"more":[15],"information":[16,133],"effectively.However,":[17],"training":[18,46,142],"handle":[21],"long":[22,103,135],"contexts":[23],"poses":[24],"significant":[25],"challenges.These":[26],"include":[27],"scarcity":[29],"of":[30,37,83,102],"high-quality":[31],"natural":[32],"long-context":[33,78,176],"data,":[34],"potential":[36],"performance":[38,127],"degradation":[39],"short-context":[41],"tasks,":[42],"and":[43,97,113,151,164,180],"reduced":[45],"efficiency":[47],"associated":[48],"with":[49,149],"attention":[50],"mechanisms.In":[51],"this":[52],"paper,":[53],"we":[54,90],"introduce":[55],"Untie":[56],"Knots":[58],"(UtK),":[59],"a":[60,99],"novel":[61],"data":[62,87,181],"augmentation":[63],"strategy":[64],"employed":[65],"during":[66],"continue":[68],"pre-training":[69],"phase,":[70],"designed":[71],"efficiently":[73],"enable":[74],"LLMs":[75,105],"gain":[77],"capabilities":[79],"without":[80],"need":[82],"modifying":[84],"existing":[86],"mixture.In":[88],"particular,":[89],"chunk":[91],"documents,":[93],"shuffle":[94],"chunks,":[96],"create":[98],"knotted":[100],"structure":[101],"texts;":[104],"are":[106,184],"then":[107],"trained":[108,154,178],"untie":[110],"these":[111],"knots":[112],"identify":[114],"relevant":[115,132],"segments":[116],"within":[117],"seemingly":[118],"chaotic":[119],"token":[120],"sequences.This":[121],"approach":[122],"substantially":[123],"enhances":[124],"model's":[126],"by":[128],"accurately":[129],"attending":[130],"contexts,":[136],"while":[137],"also":[138],"greatly":[139],"improving":[140],"efficiency.We":[143],"conduct":[144],"extensive":[145],"experiments":[146],"7B":[150],"72B":[152],"parameters,":[153],"20":[156],"billion":[157],"tokens,":[158],"demonstrating":[159],"that":[160],"UtK":[161],"achieves":[162],"75%":[163],"84.5%":[165],"accuracy":[166],"RULER":[168],"at":[169],"128K":[170],"length,":[172],"significantly":[173],"outperforming":[174],"other":[175],"strategies.The":[177],"processing":[182],"code":[183],"open-sourced":[185],"for":[186],"further":[187],"research.https://github.com/rgtjf/Untie-the-Knots":[188]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}