{"id":"https://openalex.org/W2995404354","doi":null,"title":"Neural Text Generation With Unlikelihood Training","display_name":"Neural Text Generation With Unlikelihood Training","publication_year":2020,"publication_date":"2020-04-30","ids":{"openalex":"https://openalex.org/W2995404354","mag":"2995404354"},"language":"en","primary_location":{"id":"mag:2995404354","doi":null,"is_oa":false,"landing_page_url":"https://www.openreview.net/pdf?id=SJeYe0NtvH","pdf_url":null,"source":{"id":"https://openalex.org/S4306419637","display_name":"International Conference on Learning Representations","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Learning Representations","raw_type":null},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019030424","display_name":"Sean Welleck","orcid":"https://orcid.org/0009-0003-1744-0096","observed_orcids":["https://orcid.org/0009-0003-1744-0096"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sean Welleck","raw_affiliation_strings":["New York University, New York, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University, New York, United States","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103110668","display_name":"Ilia Kulikov","orcid":"https://orcid.org/0000-0001-5989-9733","observed_orcids":["https://orcid.org/0000-0001-5989-9733"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ilia Kulikov","raw_affiliation_strings":["New York University, New York, United States"],"raw_orcid":"https://orcid.org/0000-0001-5989-9733","affiliations":[{"raw_affiliation_string":"New York University, New York, United States","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108644922","display_name":"Stephen Roller","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Stephen Roller","raw_affiliation_strings":["Meta (Israel), Tel Aviv, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta (Israel), Tel Aviv, Israel","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061347220","display_name":"Emily Dinan","orcid":"https://orcid.org/0000-0003-0624-6311","observed_orcids":["https://orcid.org/0000-0003-0624-6311"]},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Emily Dinan","raw_affiliation_strings":["Meta (Israel), Tel Aviv, Israel"],"raw_orcid":"https://orcid.org/0000-0003-0624-6311","affiliations":[{"raw_affiliation_string":"Meta (Israel), Tel Aviv, Israel","institution_ids":["https://openalex.org/I2252078561"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091175785","display_name":"Kyunghyun Cho","orcid":"https://orcid.org/0000-0003-1669-3211","observed_orcids":["https://orcid.org/0000-0003-1669-3211"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kyunghyun Cho","raw_affiliation_strings":["New York University, New York, United States"],"raw_orcid":"https://orcid.org/0000-0003-1669-3211","affiliations":[{"raw_affiliation_string":"New York University, New York, United States","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076635608","display_name":"Jason Weston","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2252078561","display_name":"Meta (Israel)","ror":"https://ror.org/02388em19","country_code":"IL","type":"company","lineage":["https://openalex.org/I2252078561","https://openalex.org/I4210114444"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Jason Weston","raw_affiliation_strings":["Meta (Israel), Tel Aviv, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta (Israel), Tel Aviv, Israel","institution_ids":["https://openalex.org/I2252078561"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.3979,"has_fulltext":false,"cited_by_count":106,"citation_normalized_percentile":{"value":0.98290528,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.864300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.864300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10419999808073044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.01549999974668026,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/neural-text-generation","display_name":"neural text generation","score":0.9760000109672546},{"id":"https://openalex.org/keywords/beam-search","display_name":"beam search","score":0.621999979019165},{"id":"https://openalex.org/keywords/nucleus-sampling","display_name":"nucleus sampling","score":0.8870000243186951},{"id":"https://openalex.org/keywords/top-k-sampling","display_name":"top-k sampling","score":0.8690000176429749},{"id":"https://openalex.org/keywords/beam-blocking","display_name":"beam blocking","score":0.9440000057220459},{"id":"https://openalex.org/keywords/perplexity","display_name":"perplexity","score":0.8640000224113464}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7563868761062622},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.7555122971534729},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.688856303691864},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.566491425037384},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5491299629211426},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4798126220703125},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4753270447254181},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.46666616201400757},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.410575807094574},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3327009379863739},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.14735519886016846},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2995404354","doi":null,"is_oa":false,"landing_page_url":"https://www.openreview.net/pdf?id=SJeYe0NtvH","pdf_url":null,"source":{"id":"https://openalex.org/S4306419637","display_name":"International Conference on Learning Representations","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Learning Representations","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W1931877416","https://openalex.org/W2008652694","https://openalex.org/W2158349948","https://openalex.org/W2525332836","https://openalex.org/W2557436004","https://openalex.org/W2574872930","https://openalex.org/W2612675303","https://openalex.org/W2626778328","https://openalex.org/W2770553825","https://openalex.org/W2788277448","https://openalex.org/W2798363023","https://openalex.org/W2885421725","https://openalex.org/W2894175714","https://openalex.org/W2898658996","https://openalex.org/W2909379646","https://openalex.org/W2913443447","https://openalex.org/W2916772188","https://openalex.org/W2933138175","https://openalex.org/W2963096510","https://openalex.org/W2963463964","https://openalex.org/W2964268978","https://openalex.org/W2971883198","https://openalex.org/W2996287690","https://openalex.org/W3035239386"],"related_works":["https://openalex.org/W3120982476","https://openalex.org/W2938704169","https://openalex.org/W7133224479","https://openalex.org/W3092970162","https://openalex.org/W3137269947","https://openalex.org/W2792795990","https://openalex.org/W7117858983","https://openalex.org/W4387428011","https://openalex.org/W3087324061","https://openalex.org/W3128772897"],"abstract_inverted_index":{"Neural":[0],"text":[1,135],"generation":[2],"is":[3,13,75],"a":[4,80,103,173],"key":[5],"tool":[6],"in":[7,43,79],"natural":[8],"language":[9],"applications,":[10],"but":[11],"it":[12],"well":[14],"known":[15],"there":[16],"are":[17,63],"major":[18],"problems":[19],"at":[20,76],"its":[21],"core.":[22],"In":[23,65],"particular,":[24],"standard":[25,143,155],"likelihood":[26,72],"training":[27,99,129],"and":[28,33,46,91,125],"decoding":[29,163],"leads":[30],"to":[31,87,112,149,176],"dull":[32,134],"repetitive":[34],"outputs.":[35],"While":[36],"some":[37],"post-hoc":[38],"fixes":[39],"have":[40],"been":[41],"proposed,":[42],"particular":[44],"top-k":[45],"nucleus":[47,166],"sampling,":[48],"they":[49],"do":[50],"not":[51],"address":[52],"the":[53,56,61,71,97,118,160],"fact":[54],"that":[55,70,82,122],"token-level":[57],"probabilities":[58],"predicted":[59],"by":[60,117],"model":[62,81],"poor.":[64],"this":[66],"paper":[67],"we":[68],"show":[69,121],"objective":[73],"itself":[74],"fault,":[77],"resulting":[78],"assigns":[83],"too":[84],"much":[85],"probability":[86,116],"sequences":[88],"containing":[89],"repeats":[90],"frequent":[92],"words,":[93],"unlike":[94],"those":[95],"from":[96],"human":[98,150],"distribution.":[100],"We":[101,120],"propose":[102],"new":[104],"objective,":[105],"unlikelihood":[106,128],"training,":[107],"which":[108],"forces":[109],"unlikely":[110],"generations":[111,141],"be":[113],"assigned":[114],"lower":[115],"model.":[119],"both":[123],"token":[124],"sequence":[126],"level":[127],"give":[130],"less":[131,133],"repetitive,":[132],"while":[136],"maintaining":[137],"perplexity,":[138],"giving":[139],"superior":[140],"using":[142],"greedy":[144],"or":[145,168],"beam":[146,156,169],"search.":[147],"According":[148],"evaluations,":[151],"our":[152],"approach":[153],"with":[154],"search":[157],"also":[158],"outperforms":[159],"currently":[161],"popular":[162],"methods":[164],"of":[165],"sampling":[167],"blocking,":[170],"thus":[171],"providing":[172],"strong":[174],"alternative":[175],"existing":[177],"techniques.":[178]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":70},{"year":2020,"cited_by_count":28},{"year":2019,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2019-12-26T00:00:00"}