{"id":"https://openalex.org/W3131648208","doi":"https://doi.org/10.1145/3561970","title":"A Primer on Contrastive Pretraining in Language Processing: Methods, Lessons Learned, and Perspectives","display_name":"A Primer on Contrastive Pretraining in Language Processing: Methods, Lessons Learned, and Perspectives","publication_year":2022,"publication_date":"2022-09-07","ids":{"openalex":"https://openalex.org/W3131648208","doi":"https://doi.org/10.1145/3561970","mag":"3131648208"},"language":"en","primary_location":{"id":"doi:10.1145/3561970","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3561970","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2102.12982","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089802125","display_name":"Nils Rethmeier","orcid":"https://orcid.org/0000-0002-4496-7307"},"institutions":[{"id":"https://openalex.org/I33256026","display_name":"German Research Centre for Artificial Intelligence","ror":"https://ror.org/01ayc5b57","country_code":"DE","type":"funder","lineage":["https://openalex.org/I33256026"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Nils Rethmeier","raw_affiliation_strings":["German Research Center for AI, Berlin, Germany, University of Copenhagen, Denmark, Berlin, Germany","German Research Centre for Artificial Intelligence"],"raw_orcid":"https://orcid.org/0000-0002-4496-7307","affiliations":[{"raw_affiliation_string":"German Research Center for AI, Berlin, Germany, University of Copenhagen, Denmark, Berlin, Germany","institution_ids":[]},{"raw_affiliation_string":"German Research Centre for Artificial Intelligence","institution_ids":["https://openalex.org/I33256026"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018976680","display_name":"Isabelle Augenstein","orcid":"https://orcid.org/0000-0003-1562-7909"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Isabelle Augenstein","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark","University of Copenhagen"],"raw_orcid":"https://orcid.org/0000-0003-1562-7909","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]},{"raw_affiliation_string":"University of Copenhagen","institution_ids":["https://openalex.org/I124055696"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5089802125"],"corresponding_institution_ids":["https://openalex.org/I33256026"],"apc_list":null,"apc_paid":null,"fwci":0.3047,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.51767732,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"55","issue":"10","first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7497371435165405},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6968779563903809},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6947872042655945},{"id":"https://openalex.org/keywords/contrast","display_name":"Contrast (vision)","score":0.5475165247917175},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.441377192735672},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.43207883834838867},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.41561025381088257},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.32763469219207764}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7497371435165405},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6968779563903809},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6947872042655945},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.5475165247917175},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.441377192735672},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.43207883834838867},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.41561025381088257},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.32763469219207764},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/3561970","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3561970","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2102.12982","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.12982","pdf_url":"https://arxiv.org/pdf/2102.12982","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3131648208","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2102.12982.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/574d698c-e595-4e5d-860f-7dce4461f2f0","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/574d698c-e595-4e5d-860f-7dce4461f2f0","pdf_url":null,"source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Rethmeier , N & Augenstein , I 2023 , ' A Primer on Contrastive Pretraining in Language Processing : Methods, Lessons Learned, and Perspectives ' , ACM Computing Surveys , vol. 55 , no. 10 , 203 . https://doi.org/10.1145/3561970","raw_type":"article"},{"id":"doi:10.48550/arxiv.2102.12982","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2102.12982","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2102.12982","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.12982","pdf_url":"https://arxiv.org/pdf/2102.12982","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.6200000047683716,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3131648208.pdf","grobid_xml":"https://content.openalex.org/works/W3131648208.grobid-xml"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W1614298861","https://openalex.org/W2153579005","https://openalex.org/W2161914416","https://openalex.org/W2887997457","https://openalex.org/W2890487780","https://openalex.org/W2911379778","https://openalex.org/W2951585248","https://openalex.org/W2963644595","https://openalex.org/W2965373594","https://openalex.org/W2971141904","https://openalex.org/W2971246534","https://openalex.org/W2981852735","https://openalex.org/W2990947836","https://openalex.org/W3011927872","https://openalex.org/W3026732421","https://openalex.org/W3033406728","https://openalex.org/W3034381157","https://openalex.org/W3034985160","https://openalex.org/W3035027743","https://openalex.org/W3035060554","https://openalex.org/W3035324702","https://openalex.org/W3035524453","https://openalex.org/W3035542229","https://openalex.org/W3090732401","https://openalex.org/W3092435621","https://openalex.org/W3093274308","https://openalex.org/W3098656025","https://openalex.org/W3100345210","https://openalex.org/W3100859887","https://openalex.org/W3101066076","https://openalex.org/W3104453885","https://openalex.org/W3106061119","https://openalex.org/W3109684201","https://openalex.org/W3114632476","https://openalex.org/W3115295967","https://openalex.org/W3116921291","https://openalex.org/W3120793869","https://openalex.org/W3126337491","https://openalex.org/W3131870090","https://openalex.org/W3132311534","https://openalex.org/W3133441681","https://openalex.org/W3135367836","https://openalex.org/W3166677907","https://openalex.org/W3171975879","https://openalex.org/W3173783447","https://openalex.org/W3173813266","https://openalex.org/W3199446690","https://openalex.org/W3201980333","https://openalex.org/W4385567522"],"related_works":["https://openalex.org/W3115295967","https://openalex.org/W2963341956","https://openalex.org/W3131870090","https://openalex.org/W3035524453","https://openalex.org/W3033406728","https://openalex.org/W2842511635","https://openalex.org/W2963310665","https://openalex.org/W3012039748","https://openalex.org/W3123226376","https://openalex.org/W3137857706","https://openalex.org/W3153294582","https://openalex.org/W3197032408","https://openalex.org/W3021531920","https://openalex.org/W3088043193","https://openalex.org/W3206020652","https://openalex.org/W2811250445","https://openalex.org/W3101065397","https://openalex.org/W3040640815","https://openalex.org/W2886197491","https://openalex.org/W2278264165"],"abstract_inverted_index":{"Modern":[0],"natural":[1],"language":[2,13,117],"processing":[3],"(NLP)":[4],"methods":[5,25,108],"employ":[6],"self-supervised":[7,38,102],"pretraining":[8,24,47,107,123,163],"objectives":[9,40],"such":[10],"as":[11,57],"masked":[12],"modeling":[14],"to":[15,50,82,115,120,149,158,165],"boost":[16],"the":[17,71,88],"performance":[18],"of":[19,54,73],"various":[20],"downstream":[21],"tasks.":[22,128],"These":[23],"are":[26,113],"frequently":[27],"extended":[28],"with":[29,135],"recurrence,":[30],"adversarial,":[31],"or":[32,60],"linguistic":[33],"property":[34],"masking.":[35],"Recently,":[36],"contrastive":[37,78,84,105,132,156,161],"training":[39],"have":[41],"enabled":[42],"successes":[43,167],"in":[44,168],"image":[45,169],"representation":[46,170],"by":[48,90,144],"learning":[49,133],"contrast":[51],"input-input":[52,77],"pairs":[53],"augmented":[55],"images":[56],"either":[58],"similar":[59],"dissimilar.":[61],"In":[62,96],"NLP":[63,106,127,157,162],"however,":[64],"a":[65,74],"single":[66],"token":[67],"augmentation":[68],"can":[69],"invert":[70],"meaning":[72],"sentence":[75],"during":[76],"learning,":[79,122],"which":[80],"led":[81],"input-output":[83],"approaches":[85],"that":[86],"avoid":[87],"issue":[89],"instead":[91],"contrasting":[92],"over":[93],"input-label":[94],"pairs.":[95],"this":[97],"primer,":[98],"we":[99,147],"summarize":[100],"recent":[101,166],"and":[103,109,125,141,152],"supervised":[104],"describe":[110],"where":[111],"they":[112],"used":[114],"improve":[116],"modeling,":[118],"zero":[119],"few-shot":[121],"data-efficiency,":[124],"specific":[126],"We":[129],"overview":[130],"key":[131],"concepts":[134],"lessons":[136],"learned":[137],"from":[138],"prior":[139],"research":[140],"structure":[142],"works":[143],"applications.":[145],"Finally,":[146],"point":[148],"open":[150],"challenges":[151],"future":[153],"directions":[154],"for":[155],"encourage":[159],"bringing":[160],"closer":[164],"pretraining.":[171]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":1}],"updated_date":"2026-05-08T15:41:06.802602","created_date":"2022-07-25T00:00:00"}