{"id":"https://openalex.org/W2963121782","doi":"https://doi.org/10.18653/v1/p19-1439","title":"Can You Tell Me How to Get Past Sesame Street? Sentence-Level Pretraining Beyond Language Modeling","display_name":"Can You Tell Me How to Get Past Sesame Street? Sentence-Level Pretraining Beyond Language Modeling","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2963121782","doi":"https://doi.org/10.18653/v1/p19-1439","mag":"2963121782"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1439","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1439","pdf_url":"https://www.aclweb.org/anthology/P19-1439.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1439.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103982993","display_name":"Alex Wang","orcid":"https://orcid.org/0000-0003-0353-2937"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Wang","raw_affiliation_strings":["New York University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University,","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076891199","display_name":"Jan H\u016fla","orcid":"https://orcid.org/0000-0001-7639-864X"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jan Hula","raw_affiliation_strings":["Johns Hopkins University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018805044","display_name":"Patrick Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Patrick Xia","raw_affiliation_strings":["Johns Hopkins University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040448607","display_name":"Raghavendra Pappagari","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Raghavendra Pappagari","raw_affiliation_strings":["Johns Hopkins University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003082850","display_name":"R. Thomas McCoy","orcid":"https://orcid.org/0000-0001-9383-4079"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"R. Thomas McCoy","raw_affiliation_strings":["Johns Hopkins University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042032005","display_name":"Roma Patel","orcid":"https://orcid.org/0000-0003-2075-2560"},"institutions":[{"id":"https://openalex.org/I175594653","display_name":"John Brown University","ror":"https://ror.org/02ct41q97","country_code":"US","type":"education","lineage":["https://openalex.org/I175594653"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Roma Patel","raw_affiliation_strings":["Brown University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University,","institution_ids":["https://openalex.org/I175594653"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071542832","display_name":"Najoung Kim","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Najoung Kim","raw_affiliation_strings":["Johns Hopkins University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082431329","display_name":"Ian Tenney","orcid":"https://orcid.org/0000-0002-6225-9283"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ian Tenney","raw_affiliation_strings":["Google AI Language,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google AI Language,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101673889","display_name":"Yinghui Huang","orcid":"https://orcid.org/0000-0003-0607-1507"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yinghui Huang","raw_affiliation_strings":["IBM,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM,","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022436932","display_name":"Katherin Yu","orcid":null},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Katherin Yu","raw_affiliation_strings":["Facebook,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook,","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112540008","display_name":"Shuning Jin","orcid":null},"institutions":[{"id":"https://openalex.org/I4210115145","display_name":"University of Minnesota, Duluth","ror":"https://ror.org/01hy4qx27","country_code":"US","type":"education","lineage":["https://openalex.org/I4210115145"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shuning Jin","raw_affiliation_strings":["University of Minnesota Duluth,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Minnesota Duluth,","institution_ids":["https://openalex.org/I4210115145"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115595070","display_name":"Berlin Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I118020396","display_name":"Swarthmore College","ror":"https://ror.org/012dg8a96","country_code":"US","type":"education","lineage":["https://openalex.org/I118020396"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Berlin Chen","raw_affiliation_strings":["Swarthmore College"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Swarthmore College","institution_ids":["https://openalex.org/I118020396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075825791","display_name":"Benjamin Van Durme","orcid":"https://orcid.org/0000-0003-4328-4288"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benjamin Van Durme","raw_affiliation_strings":["Johns Hopkins University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069316249","display_name":"\u00c9douard Grave","orcid":null},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Edouard Grave","raw_affiliation_strings":["Facebook,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook,","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053850863","display_name":"Ellie Pavlick","orcid":"https://orcid.org/0000-0002-7155-5420"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I175594653","display_name":"John Brown University","ror":"https://ror.org/02ct41q97","country_code":"US","type":"education","lineage":["https://openalex.org/I175594653"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ellie Pavlick","raw_affiliation_strings":["Brown University,","Google AI Language,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University,","institution_ids":["https://openalex.org/I175594653"]},{"raw_affiliation_string":"Google AI Language,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112713734","display_name":"Samuel R. Bowman","orcid":null},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samuel R. Bowman","raw_affiliation_strings":["New York University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University,","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":16,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":10.7018,"has_fulltext":true,"cited_by_count":91,"citation_normalized_percentile":{"value":0.98566692,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"4465","last_page":"4476"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chen","display_name":"Chen","score":0.7551721334457397},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.4778953790664673},{"id":"https://openalex.org/keywords/computational-linguistics","display_name":"Computational linguistics","score":0.47124797105789185},{"id":"https://openalex.org/keywords/art-history","display_name":"Art history","score":0.3768376410007477},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.36169344186782837},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3211330771446228},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.3094831109046936},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.29153838753700256},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.06916278600692749}],"concepts":[{"id":"https://openalex.org/C2776085556","wikidata":"https://www.wikidata.org/wiki/Q183361","display_name":"Chen","level":2,"score":0.7551721334457397},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.4778953790664673},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.47124797105789185},{"id":"https://openalex.org/C52119013","wikidata":"https://www.wikidata.org/wiki/Q50637","display_name":"Art history","level":1,"score":0.3768376410007477},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.36169344186782837},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3211330771446228},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.3094831109046936},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.29153838753700256},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.06916278600692749},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1439","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1439","pdf_url":"https://www.aclweb.org/anthology/P19-1439.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1439","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1439","pdf_url":"https://www.aclweb.org/anthology/P19-1439.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8700000047683716}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2963121782.pdf","grobid_xml":"https://content.openalex.org/works/W2963121782.grobid-xml"},"referenced_works_count":72,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W1486649854","https://openalex.org/W1599016936","https://openalex.org/W1632114991","https://openalex.org/W1861492603","https://openalex.org/W2064675550","https://openalex.org/W2130158090","https://openalex.org/W2158899491","https://openalex.org/W2159636675","https://openalex.org/W2250473257","https://openalex.org/W2250861254","https://openalex.org/W2251939518","https://openalex.org/W2257408573","https://openalex.org/W2525778437","https://openalex.org/W2548036585","https://openalex.org/W2551396370","https://openalex.org/W2592170186","https://openalex.org/W2610858497","https://openalex.org/W2611669587","https://openalex.org/W2761988601","https://openalex.org/W2771275742","https://openalex.org/W2785523195","https://openalex.org/W2793978524","https://openalex.org/W2794365787","https://openalex.org/W2798284509","https://openalex.org/W2823875838","https://openalex.org/W2888329843","https://openalex.org/W2893141505","https://openalex.org/W2896457183","https://openalex.org/W2898700502","https://openalex.org/W2907849599","https://openalex.org/W2911300548","https://openalex.org/W2911681509","https://openalex.org/W2923014074","https://openalex.org/W2946417913","https://openalex.org/W2951714314","https://openalex.org/W2952230511","https://openalex.org/W2952729433","https://openalex.org/W2962739339","https://openalex.org/W2963026768","https://openalex.org/W2963090765","https://openalex.org/W2963149412","https://openalex.org/W2963241825","https://openalex.org/W2963264012","https://openalex.org/W2963310665","https://openalex.org/W2963323070","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963494889","https://openalex.org/W2963691697","https://openalex.org/W2963748441","https://openalex.org/W2963756346","https://openalex.org/W2963804993","https://openalex.org/W2963842982","https://openalex.org/W2963846996","https://openalex.org/W2963854351","https://openalex.org/W2963918774","https://openalex.org/W2964303773","https://openalex.org/W2964331819","https://openalex.org/W2970352191","https://openalex.org/W2978670439","https://openalex.org/W3104033643","https://openalex.org/W4241499611","https://openalex.org/W4288351520","https://openalex.org/W4289490673","https://openalex.org/W4297823153","https://openalex.org/W4298422451","https://openalex.org/W4300822525","https://openalex.org/W4302343710","https://openalex.org/W4322588812","https://openalex.org/W4322614701","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2370310922","https://openalex.org/W2386975637","https://openalex.org/W2891436901","https://openalex.org/W2087355708","https://openalex.org/W2388837937","https://openalex.org/W2385339070","https://openalex.org/W2390434941","https://openalex.org/W2385344059","https://openalex.org/W2351657747"],"abstract_inverted_index":{"Alex":[0],"Wang,":[1],"Jan":[2],"Hula,":[3],"Patrick":[4],"Xia,":[5],"Raghavendra":[6],"Pappagari,":[7],"R.":[8,33],"Thomas":[9],"McCoy,":[10],"Roma":[11],"Patel,":[12],"Najoung":[13],"Kim,":[14],"Ian":[15],"Tenney,":[16],"Yinghui":[17],"Huang,":[18],"Katherin":[19],"Yu,":[20],"Shuning":[21],"Jin,":[22],"Berlin":[23],"Chen,":[24],"Benjamin":[25],"Van":[26],"Durme,":[27],"Edouard":[28],"Grave,":[29],"Ellie":[30],"Pavlick,":[31],"Samuel":[32],"Bowman.":[34],"Proceedings":[35],"of":[36,41],"the":[37,42],"57th":[38],"Annual":[39],"Meeting":[40],"Association":[43],"for":[44],"Computational":[45],"Linguistics.":[46],"2019.":[47]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":21},{"year":2020,"cited_by_count":30},{"year":2019,"cited_by_count":4}],"updated_date":"2026-06-18T10:00:31.954636","created_date":"2025-10-10T00:00:00"}