{"id":"https://openalex.org/W4412888761","doi":"https://doi.org/10.18653/v1/2025.findings-acl.111","title":"Drop Dropout on Single Epoch Language Model Pretraining","display_name":"Drop Dropout on Single Epoch Language Model Pretraining","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888761","doi":"https://doi.org/10.18653/v1/2025.findings-acl.111"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.111","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.111","pdf_url":"https://aclanthology.org/2025.findings-acl.111.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.111.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103146684","display_name":"Houjun Liu","orcid":"https://orcid.org/0000-0002-2156-5903"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Houjun Liu","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023322733","display_name":"John Bauer","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Bauer","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046006076","display_name":"Christopher D. Manning","orcid":"https://orcid.org/0000-0001-6155-649X"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher D Manning","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2157","last_page":"2166"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.8833000063896179,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.8833000063896179,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.8695999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8454999923706055,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dropout","display_name":"Dropout (neural networks)","score":0.7094889283180237},{"id":"https://openalex.org/keywords/drop-out","display_name":"Drop out","score":0.6271846294403076},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5542722940444946},{"id":"https://openalex.org/keywords/drop","display_name":"Drop (telecommunication)","score":0.5494151711463928},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.41089341044425964},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31067684292793274},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.11900606751441956},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.10755303502082825},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.10140261054039001}],"concepts":[{"id":"https://openalex.org/C2776145597","wikidata":"https://www.wikidata.org/wiki/Q25339462","display_name":"Dropout (neural networks)","level":2,"score":0.7094889283180237},{"id":"https://openalex.org/C2984949393","wikidata":"https://www.wikidata.org/wiki/Q1260241","display_name":"Drop out","level":2,"score":0.6271846294403076},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5542722940444946},{"id":"https://openalex.org/C2781345722","wikidata":"https://www.wikidata.org/wiki/Q5308388","display_name":"Drop (telecommunication)","level":2,"score":0.5494151711463928},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.41089341044425964},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31067684292793274},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.11900606751441956},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.10755303502082825},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.10140261054039001},{"id":"https://openalex.org/C4249254","wikidata":"https://www.wikidata.org/wiki/Q3044431","display_name":"Demographic economics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.111","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.111","pdf_url":"https://aclanthology.org/2025.findings-acl.111.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.111","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.111","pdf_url":"https://aclanthology.org/2025.findings-acl.111.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309949","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888761.pdf","grobid_xml":"https://content.openalex.org/works/W4412888761.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2783917878","https://openalex.org/W2058575413","https://openalex.org/W3161635404","https://openalex.org/W2062676549","https://openalex.org/W3094219133","https://openalex.org/W2114414302","https://openalex.org/W1480756546","https://openalex.org/W3205604289","https://openalex.org/W4312919775","https://openalex.org/W4386836482"],"abstract_inverted_index":{"Originally,":[0],"dropout":[1,37,55,98,117,131,151],"was":[2],"seen":[3],"as":[4],"a":[5],"breakthrough":[6],"regularization":[7],"technique":[8],"that":[9,81,106,127],"reduced":[10],"overfitting":[11],"and":[12,67,71,92,125,140],"improved":[13],"performance":[14,83,113],"in":[15,56,60,84,135,142],"almost":[16],"all":[17],"applications":[18],"of":[19,54,63,77],"deep":[20],"learning":[21],"by":[22],"reducing":[23],"overfitting.Yet,":[24],"single-epoch":[25,61,153],"pretraining":[26,62],"tasks":[27],"common":[28],"to":[29,36,149],"modern":[30],"LLMs":[31],"yield":[32],"minimal":[33],"overfitting,":[34],"leading":[35],"not":[38,100],"being":[39],"used":[40],"for":[41],"large":[42],"LLMs.Nevertheless,":[43],"no":[44,116],"thorough":[45],"empirical":[46],"investigation":[47],"has":[48],"been":[49],"done":[50],"on":[51],"the":[52,107,122],"role":[53],"LM":[57],"pretraining.Through":[58],"experiments":[59],"both":[64],"masked":[65],"(BERT)":[66],"autoregressive":[68],"(Pythia":[69],"160M":[70],"1.4B)":[72],"LMs":[73],"with":[74],"varying":[75],"levels":[76],"dropout,":[78],"we":[79,147],"find":[80,105,126],"downstream":[82],"language":[85],"modeling,":[86],"morpho-syntax":[87],"(BLiMP),":[88],"question":[89],"answering":[90],"(SQuAD),":[91],"natural-language":[93],"inference":[94],"(MNLI)":[95],"improves":[96],"when":[97],"is":[99],"applied":[101],"during":[102,152],"pretraining.We":[103],"additionally":[104],"recentlyintroduced":[108],"\"early":[109],"dropout\"":[110],"also":[111],"degrades":[112],"over":[114],"applying":[115],"at":[118],"all.We":[119],"further":[120],"investigate":[121],"models'":[123],"editability,":[124],"models":[128],"trained":[129],"without":[130],"are":[132],"more":[133],"successful":[134],"gradient-based":[136],"model":[137,144],"editing":[138,145],"(MEND)":[139],"equivalent":[141],"representationbased":[143],"(ReFT).Therefore,":[146],"advocate":[148],"drop":[150],"pretraining.":[154]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}