{"id":"https://openalex.org/W3216797294","doi":null,"title":"Controlling Conditional Language Models with Distributional Policy Gradients","display_name":"Controlling Conditional Language Models with Distributional Policy Gradients","publication_year":2021,"publication_date":"2021-12-01","ids":{"openalex":"https://openalex.org/W3216797294","mag":"3216797294"},"language":"en","primary_location":{"id":"mag:3216797294","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2112.00791","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2112.00791","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002211679","display_name":"Tomasz Korbak","orcid":"https://orcid.org/0000-0002-6258-2013","observed_orcids":["https://orcid.org/0000-0002-6258-2013"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tomasz Korbak","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-6258-2013","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040371566","display_name":"Hady Elsahar","orcid":"https://orcid.org/0000-0001-9508-6721","observed_orcids":["https://orcid.org/0000-0001-9508-6721"]},"institutions":[{"id":"https://openalex.org/I60922564","display_name":"Naver (South Korea)","ror":"https://ror.org/04nzrnx83","country_code":"KR","type":"company","lineage":["https://openalex.org/I60922564"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hady Elsahar","raw_affiliation_strings":["NAVER CORPORATION"],"raw_orcid":"https://orcid.org/0000-0001-9508-6721","affiliations":[{"raw_affiliation_string":"NAVER CORPORATION","institution_ids":["https://openalex.org/I60922564"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007916691","display_name":"Germ\u00e1n Kruszewski","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I60922564","display_name":"Naver (South Korea)","ror":"https://ror.org/04nzrnx83","country_code":"KR","type":"company","lineage":["https://openalex.org/I60922564"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Germ\u00e1n Kruszewski","raw_affiliation_strings":["NAVER CORPORATION"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NAVER CORPORATION","institution_ids":["https://openalex.org/I60922564"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059468860","display_name":"Marc Dymetman","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I60922564","display_name":"Naver (South Korea)","ror":"https://ror.org/04nzrnx83","country_code":"KR","type":"company","lineage":["https://openalex.org/I60922564"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Marc Dymetman","raw_affiliation_strings":["NAVER CORPORATION"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NAVER CORPORATION","institution_ids":["https://openalex.org/I60922564"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8910999894142151,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8910999894142151,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.05620000138878822,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.015699999406933784,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/energy-based-models","display_name":"energy-based models","score":0.8410000205039978},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.5019999742507935},{"id":"https://openalex.org/keywords/catastrophic-forgetting","display_name":"catastrophic forgetting","score":0.7129999995231628},{"id":"https://openalex.org/keywords/abstractive-summarization","display_name":"abstractive summarization","score":0.9319999814033508},{"id":"https://openalex.org/keywords/automatic-code-generation","display_name":"automatic code generation","score":0.8349999785423279},{"id":"https://openalex.org/keywords/t5","display_name":"T5","score":0.6420000195503235},{"id":"https://openalex.org/keywords/gpt-neo","display_name":"GPT-Neo","score":0.9850000143051147}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9022084474563599},{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.7865011692047119},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7262318730354309},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.7135113477706909},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7020383477210999},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5992004871368408},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5869194269180298},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5746022462844849},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.508209228515625},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.48470473289489746},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.44972893595695496},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.42052847146987915},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35807526111602783},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3216797294","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2112.00791","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3216797294","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2112.00791","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1951216520","https://openalex.org/W1994573369","https://openalex.org/W2055309977","https://openalex.org/W2116064496","https://openalex.org/W2119717200","https://openalex.org/W2143861926","https://openalex.org/W2154652894","https://openalex.org/W2155027007","https://openalex.org/W2444132761","https://openalex.org/W2533695286","https://openalex.org/W2741986794","https://openalex.org/W2751448157","https://openalex.org/W2797269287","https://openalex.org/W2798363023","https://openalex.org/W2949918260","https://openalex.org/W2963206148","https://openalex.org/W2963929190","https://openalex.org/W2964121744","https://openalex.org/W2972427545","https://openalex.org/W2980282514","https://openalex.org/W2993398598","https://openalex.org/W2996287690","https://openalex.org/W3021338988","https://openalex.org/W3030163527","https://openalex.org/W3082274269","https://openalex.org/W3118781290","https://openalex.org/W3124120397","https://openalex.org/W3126675481","https://openalex.org/W3135367836","https://openalex.org/W3153193605","https://openalex.org/W3195140297","https://openalex.org/W3212496002","https://openalex.org/W7097033042","https://openalex.org/W7151510318"],"related_works":["https://openalex.org/W4310638782","https://openalex.org/W2973049837","https://openalex.org/W7220998593","https://openalex.org/W3124120397","https://openalex.org/W4391591062","https://openalex.org/W4281690218","https://openalex.org/W7167593278","https://openalex.org/W3158172841","https://openalex.org/W7221569776","https://openalex.org/W2967853777"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"is":[2,109],"shifting":[3],"towards":[4,165],"general-purpose":[5],"pretrained":[6,162],"generative":[7,71],"models,":[8],"trained":[9],"in":[10,51,57,171,180],"a":[11,26,74],"self-supervised":[12],"manner":[13],"on":[14,66,136],"large":[15,27],"amounts":[16],"of":[17,29,45],"data,":[18],"which":[19],"can":[20],"then":[21],"be":[22],"applied":[23],"to":[24,33,42,68,73,85,111,125],"solve":[25,86],"number":[28],"tasks.":[30],"However,":[31],"due":[32],"their":[34],"generic":[35],"training":[36],"methodology,":[37],"these":[38,99,161],"models":[39,72,95,163],"often":[40],"fail":[41],"meet":[43],"some":[44],"the":[46],"downstream":[47],"requirements":[48,92],"(e.g.":[49],"hallucination":[50],"abstractive":[52],"summarization":[53,144],"or":[54],"wrong":[55],"format":[56],"automatic":[58],"code":[59,148],"generation).":[60],"This":[61],"raises":[62],"an":[63],"important":[64],"question":[65],"how":[67],"adapt":[69],"pre-trained":[70],"new":[75],"task":[76],"without":[77],"destroying":[78],"its":[79],"capabilities.":[80],"Recent":[81],"work":[82],"has":[83],"suggested":[84],"this":[87,107,119,123],"problem":[88],"by":[89,115,128],"representing":[90],"task-specific":[91],"through":[93],"energy-based":[94],"(EBMs)":[96],"and":[97,147,169],"approximating":[98],"EBMs":[100],"using":[101,157],"distributional":[102],"policy":[103],"gradients":[104],"(DPG).":[105],"Unfortunately,":[106],"approach":[108,124],"limited":[110],"unconditional":[112,116],"distributions,":[113],"represented":[114],"EBMs.":[117],"In":[118],"paper,":[120],"we":[121],"extend":[122],"conditional":[126],"tasks":[127],"proposing":[129],"Conditional":[130],"DPG":[131],"(CDPG).":[132],"We":[133],"evaluate":[134],"CDPG":[135,158],"three":[137],"different":[138],"control":[139,167],"objectives":[140,168],"across":[141],"two":[142],"tasks:":[143],"with":[145,150,173],"T5":[146],"generation":[149],"GPT-Neo.":[151],"Our":[152],"results":[153],"show":[154],"that":[155],"fine-tuning":[156],"robustly":[159],"moves":[160],"closer":[164],"meeting":[166],"--":[170,176],"contrast":[172],"baseline":[174],"approaches":[175],"does":[177],"not":[178],"result":[179],"catastrophic":[181],"forgetting.":[182]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2021-12-06T00:00:00"}