{"id":"https://openalex.org/W4412887757","doi":"https://doi.org/10.18653/v1/2025.findings-acl.1040","title":"FRAME: Boosting LLMs with A Four-Quadrant Multi-Stage Pretraining Strategy","display_name":"FRAME: Boosting LLMs with A Four-Quadrant Multi-Stage Pretraining Strategy","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412887757","doi":"https://doi.org/10.18653/v1/2025.findings-acl.1040"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.1040","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.1040","pdf_url":"https://aclanthology.org/2025.findings-acl.1040.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.1040.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xuemiao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuemiao Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091614124","display_name":"Feiyu Duan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feiyu Duan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100307712","display_name":"Liangyu Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu Liangyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102195851","display_name":"Yongwei Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yongwei Zhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Sirui Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sirui Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081667345","display_name":"Rongxiang Weng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rongxiang Weng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100695184","display_name":"Jingang Wang","orcid":"https://orcid.org/0000-0002-7889-1881"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jingang Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5114353815","display_name":"Xunliang Cai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xunliang Cai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.1420381,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"20278","last_page":"20297"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.7960854172706604},{"id":"https://openalex.org/keywords/quadrant","display_name":"Quadrant (abdomen)","score":0.5718333721160889},{"id":"https://openalex.org/keywords/stage","display_name":"Stage (stratigraphy)","score":0.47232168912887573},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4513194262981415},{"id":"https://openalex.org/keywords/frame","display_name":"Frame (networking)","score":0.4196723401546478},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3018365502357483},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.19171857833862305},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.13397279381752014},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.07809394598007202},{"id":"https://openalex.org/keywords/surgery","display_name":"Surgery","score":0.06176033616065979}],"concepts":[{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.7960854172706604},{"id":"https://openalex.org/C2780639617","wikidata":"https://www.wikidata.org/wiki/Q6516972","display_name":"Quadrant (abdomen)","level":2,"score":0.5718333721160889},{"id":"https://openalex.org/C146357865","wikidata":"https://www.wikidata.org/wiki/Q1123245","display_name":"Stage (stratigraphy)","level":2,"score":0.47232168912887573},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4513194262981415},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.4196723401546478},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3018365502357483},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.19171857833862305},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.13397279381752014},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.07809394598007202},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.06176033616065979},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.1040","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.1040","pdf_url":"https://aclanthology.org/2025.findings-acl.1040.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.1040","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.1040","pdf_url":"https://aclanthology.org/2025.findings-acl.1040.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412887757.pdf","grobid_xml":"https://content.openalex.org/works/W4412887757.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2125652721","https://openalex.org/W1540371141","https://openalex.org/W2124235242","https://openalex.org/W1549363203","https://openalex.org/W2147697413","https://openalex.org/W2154063878","https://openalex.org/W4231274751","https://openalex.org/W2556012038","https://openalex.org/W1489772951","https://openalex.org/W2070465345"],"abstract_inverted_index":{"Large":[0],"language":[1,8],"models":[2],"(LLMs)":[3],"have":[4],"significantly":[5,114],"advanced":[6],"human":[7],"understanding":[9],"and":[10,16,38,94,116,124,139],"generation,":[11],"with":[12],"pretraining":[13,23,63],"data":[14,36,88,102,120],"quality":[15],"organization":[17],"being":[18],"crucial":[19],"to":[20,68,112],"their":[21],"performance.Multi-stage":[22],"is":[24,76],"a":[25,130],"promising":[26],"approach,":[27],"but":[28],"existing":[29],"methods":[30],"often":[31],"lack":[32],"quantitative":[33],"criteria":[34],"for":[35,141],"partitioning":[37,119],"instead":[39],"rely":[40],"on":[41,84,97],"intuitive":[42],"heuristics.In":[43],"this":[44],"paper,":[45],"we":[46],"propose":[47],"the":[48,57,62,110,142],"novel":[49],"Four-quadRAnt":[50],"Multistage":[51],"prEtraining":[52],"strategy":[53],"(FRAME),":[54],"guided":[55],"by":[56,90,104],"established":[58],"principle":[59,75],"of":[60],"organizing":[61,126],"process":[64],"into":[65,121],"four":[66,73,122],"stages":[67],"achieve":[69],"significant":[70],"loss":[71,111],"reductions":[72],"times.This":[74],"grounded":[77],"in":[78],"two":[79],"key":[80],"findings:":[81],"first,":[82],"training":[83,96],"high":[85,105],"Perplexity":[86],"(PPL)":[87],"followed":[89,103],"low":[91,98],"PPL":[92,99],"data,":[93,107],"second,":[95],"difference":[100],"(PD)":[101],"PD":[106],"both":[108],"causing":[109],"drop":[113],"twice":[115],"performance":[117],"enhancements.By":[118],"quadrants":[123],"strategically":[125],"them,":[127],"FRAME":[128],"achieves":[129],"remarkable":[131],"16.8%":[132],"average":[133],"improvement":[134],"over":[135],"random":[136],"across":[137],"MMLU":[138],"CMMLU":[140],"3B":[143],"model,":[144],"effectively":[145],"boosting":[146],"LLM":[147],"performance.":[148]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}