{"id":"https://openalex.org/W3201808024","doi":"https://doi.org/10.48550/arxiv.2110.02802","title":"Self-conditioning pre-trained language models","display_name":"Self-conditioning pre-trained language models","publication_year":2021,"publication_date":"2021-09-30","ids":{"openalex":"https://openalex.org/W3201808024","doi":"https://doi.org/10.48550/arxiv.2110.02802","mag":"3201808024"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2110.02802","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.02802","pdf_url":"https://arxiv.org/pdf/2110.02802","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2110.02802","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046208400","display_name":"Xavier Suau","orcid":"https://orcid.org/0000-0002-4052-7048","observed_orcids":["https://orcid.org/0000-0002-4052-7048"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Suau, Xavier","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043365442","display_name":"Luca Zappella","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zappella, Luca","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5071541814","display_name":"Nicholas Apostoloff","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Apostoloff, Nicholas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9154999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9154999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04520000144839287,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.008299999870359898,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-conditioning","display_name":"self-conditioning","score":0.7720000147819519},{"id":"https://openalex.org/keywords/pre-trained-language-models","display_name":"pre-trained language models","score":0.8889999985694885},{"id":"https://openalex.org/keywords/text-generation","display_name":"text generation","score":0.7919999957084656},{"id":"https://openalex.org/keywords/product-of-experts","display_name":"product of experts","score":0.9259999990463257},{"id":"https://openalex.org/keywords/gender-parity","display_name":"gender parity","score":0.8690000176429749},{"id":"https://openalex.org/keywords/generative-mechanisms","display_name":"generative mechanisms","score":0.515999972820282}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.8339900970458984},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7792885303497314},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6943891048431396},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6112602949142456},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5556509494781494},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5362343788146973},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5288230776786804},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.46080896258354187},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40772393345832825},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2199118733406067},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2110.02802","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.02802","pdf_url":"https://arxiv.org/pdf/2110.02802","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3201808024","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2110.02802","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2110.02802","doi":"https://doi.org/10.48550/arxiv.2110.02802","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2110.02802","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2110.02802","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.02802","pdf_url":"https://arxiv.org/pdf/2110.02802","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3201808024.pdf","grobid_xml":"https://content.openalex.org/works/W3201808024.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2079182758","https://openalex.org/W2132339004","https://openalex.org/W2557449848","https://openalex.org/W2606347107","https://openalex.org/W2610018085","https://openalex.org/W2735642330","https://openalex.org/W2784070736","https://openalex.org/W2805486818","https://openalex.org/W2889056219","https://openalex.org/W2901107321","https://openalex.org/W2908854766","https://openalex.org/W2922565841","https://openalex.org/W2931212643","https://openalex.org/W2938704169","https://openalex.org/W2951539960","https://openalex.org/W2962776659","https://openalex.org/W2964204621","https://openalex.org/W2973049837","https://openalex.org/W2980282514","https://openalex.org/W2993398598","https://openalex.org/W3017701505","https://openalex.org/W3023621444","https://openalex.org/W3030163527","https://openalex.org/W3039288825","https://openalex.org/W3042266831","https://openalex.org/W3170344956"],"related_works":["https://openalex.org/W3101294880","https://openalex.org/W2993398598","https://openalex.org/W3024936740","https://openalex.org/W4307653761","https://openalex.org/W3033434706","https://openalex.org/W3138217237","https://openalex.org/W2967853777","https://openalex.org/W7083028635","https://openalex.org/W4322716724","https://openalex.org/W3187134297"],"abstract_inverted_index":{"In":[0],"this":[1,118,218],"paper":[2,219],"we":[3,29,128,149],"aim":[4],"to":[5,63,69,76,99,121,157,190,203,208],"investigate":[6],"the":[7,21,51,82,88,115,163,226],"mechanisms":[8,228],"that":[9,34,87,130,151,185],"guide":[10],"text":[11,55,101],"generation":[12,56,102],"with":[13,111,179],"pre-trained":[14],"Transformer-based":[15],"Language":[16],"Models":[17],"(TLMs).":[18],"Grounded":[19],"on":[20,57,144],"Product":[22],"of":[23,90,95,117,165,229],"Experts":[24],"formulation":[25],"by":[26],"Hinton":[27],"(1999),":[28],"describe":[30,61],"a":[31,91,109,195,204,221],"generative":[32,227],"mechanism":[33],"exploits":[35],"expert":[36,65],"units":[37,44,66,96,107],"which":[38],"naturally":[39],"exist":[40],"in":[41,50,74,81,108,162,217,224],"TLMs.":[42,230],"Such":[43],"are":[45,220],"responsible":[46],"for":[47,135,171],"detecting":[48],"concepts":[49],"input":[52],"and":[53,67,167,181,183,211],"conditioning":[54,136],"such":[58],"concepts.":[59,147],"We":[60,85,175],"how":[62,68,125],"identify":[64],"activate":[70],"them":[71],"during":[72],"inference":[73],"order":[75],"induce":[77],"any":[78],"desired":[79],"concept":[80],"generated":[83],"output.":[84],"find":[86],"activation":[89],"surprisingly":[92],"small":[93],"amount":[94],"is":[97,120,133,188,201],"sufficient":[98],"steer":[100],"(as":[103],"little":[104],"as":[105],"3":[106],"model":[110],"345M":[112],"parameters).":[113],"While":[114],"objective":[116],"work":[119],"learn":[122],"more":[123],"about":[124],"TLMs":[126,166],"work,":[127],"show":[129,150,184],"our":[131,152,177,186],"method":[132,153,178,200],"effective":[134],"without":[137],"fine-tuning":[138],"or":[139],"using":[140],"extra":[141],"parameters,":[142],"even":[143],"fine-grained":[145],"homograph":[146],"Additionally,":[148],"can":[154],"be":[155],"used":[156],"correct":[158],"gender":[159,169,192],"bias":[160],"present":[161],"output":[164],"achieves":[168],"parity":[170,193],"all":[172],"evaluated":[173],"contexts.":[174],"compare":[176],"FUDGE":[180],"PPLM-BoW,":[182],"approach":[187],"able":[189],"achieve":[191],"at":[194],"lower":[196],"perplexity.":[197],"The":[198,215],"proposed":[199],"accessible":[202],"wide":[205],"audience":[206],"thanks":[207],"its":[209],"simplicity":[210],"minimal":[212],"compute":[213],"needs.":[214],"findings":[216],"step":[222],"forward":[223],"understanding":[225]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2021-10-11T00:00:00"}