{"id":"https://openalex.org/W3132426319","doi":null,"title":"On Efficient Training, Controllability and Compositional Generalization of Insertion-based Language Generators.","display_name":"On Efficient Training, Controllability and Compositional Generalization of Insertion-based Language Generators.","publication_year":2021,"publication_date":"2021-02-12","ids":{"openalex":"https://openalex.org/W3132426319","mag":"3132426319"},"language":"en","primary_location":{"id":"mag:3132426319","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2102.11008","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://export.arxiv.org/pdf/2102.11008","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028846454","display_name":"Sidi Lu","orcid":"https://orcid.org/0000-0001-9846-7570","observed_orcids":["https://orcid.org/0000-0001-9846-7570"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sidi Lu","raw_affiliation_strings":["University of California-Los Angeles"],"raw_orcid":"https://orcid.org/0000-0001-9846-7570","affiliations":[{"raw_affiliation_string":"University of California-Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030248499","display_name":"Nanyun Peng","orcid":"https://orcid.org/0000-0002-8509-6595","observed_orcids":["https://orcid.org/0000-0002-8509-6595"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nanyun Peng","raw_affiliation_strings":["University of California-Los Angeles"],"raw_orcid":"https://orcid.org/0000-0002-8509-6595","affiliations":[{"raw_affiliation_string":"University of California-Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I161318765"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7979000210762024,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7979000210762024,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1720999926328659,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.00839999970048666,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/efficient-training","display_name":"efficient training","score":0.3889999985694885},{"id":"https://openalex.org/keywords/compositional-generalization","display_name":"compositional generalization","score":0.8989999890327454},{"id":"https://openalex.org/keywords/story-generation","display_name":"story generation","score":0.8069999814033508},{"id":"https://openalex.org/keywords/long-text-generation","display_name":"long-form text generation","score":0.5770000219345093},{"id":"https://openalex.org/keywords/transformer-decoder","display_name":"transformer decoder","score":0.5109999775886536}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7856966257095337},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5435280203819275},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5430367588996887},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5336597561836243},{"id":"https://openalex.org/C48209547","wikidata":"https://www.wikidata.org/wiki/Q1331104","display_name":"Controllability","level":2,"score":0.5218878388404846},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5039307475090027},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4945325553417206},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4604618549346924},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.45000624656677246},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.4423539638519287},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.4377056658267975},{"id":"https://openalex.org/C2985684807","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Text generation","level":2,"score":0.4306762218475342},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37745752930641174},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3380393385887146},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12183308601379395},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3132426319","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2102.11008","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3132426319","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2102.11008","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W1514535095","https://openalex.org/W1895577753","https://openalex.org/W2194775991","https://openalex.org/W2561715562","https://openalex.org/W2581637843","https://openalex.org/W2592808142","https://openalex.org/W2613904329","https://openalex.org/W2626778328","https://openalex.org/W2920538220","https://openalex.org/W2950813464","https://openalex.org/W2952436057","https://openalex.org/W2963797754","https://openalex.org/W2963993699","https://openalex.org/W2964110616","https://openalex.org/W2964308564","https://openalex.org/W2981461746","https://openalex.org/W2985694911","https://openalex.org/W3001434439","https://openalex.org/W3030163527","https://openalex.org/W3034899654","https://openalex.org/W3035540807","https://openalex.org/W3037550066","https://openalex.org/W3099872554","https://openalex.org/W3167303745"],"related_works":["https://openalex.org/W7133252207","https://openalex.org/W4417092522","https://openalex.org/W2920538220","https://openalex.org/W7164156227","https://openalex.org/W7163134970","https://openalex.org/W4226520927","https://openalex.org/W2985694911","https://openalex.org/W2999668723","https://openalex.org/W3138728419","https://openalex.org/W3096160024"],"abstract_inverted_index":{"Auto-regressive":[0],"language":[1,14],"models":[2],"with":[3,29,51,135],"the":[4,21,44,59,73,82,90,130,150,162],"left-to-right":[5,23],"generation":[6,19,145,160],"order":[7],"have":[8],"been":[9],"a":[10,30,37,47,78,136],"predominant":[11],"paradigm":[12,24],"for":[13],"generation.":[15],"Recently,":[16],"out-of-order":[17],"text":[18],"beyond":[20],"traditional":[22,125],"has":[25],"attracted":[26],"extensive":[27],"attention,":[28],"notable":[31],"variation":[32],"of":[33,62,72,81,152],"insertion-based":[34,74,115],"generation,":[35],"where":[36],"model":[38,91,117],"is":[39,66],"used":[40],"to":[41,95,164],"gradually":[42],"extend":[43],"context":[45,138],"into":[46],"complete":[48],"sentence":[49],"purely":[50],"insertion":[52,56],"operations.":[53],"However,":[54],"since":[55],"operations":[57],"disturb":[58],"position":[60],"information":[61],"each":[63,70],"token,":[64],"it":[65],"often":[67],"believed":[68],"that":[69,118,134],"step":[71],"likelihood":[75],"estimation":[76],"requires":[77],"bi-directional":[79,137],"\\textit{re-encoding}":[80],"whole":[83],"generated":[84],"sequence.":[85],"This":[86],"computational":[87,158],"overhead":[88],"prohibits":[89],"from":[92],"scaling":[93],"up":[94],"generate":[96],"long,":[97],"diverse":[98],"texts":[99],"such":[100],"as":[101,122,124,133],"stories,":[102],"news":[103],"articles,":[104],"and":[105,146,169],"reports.":[106],"To":[107],"address":[108],"this":[109],"issue,":[110],"we":[111],"propose":[112],"InsNet,":[113],"an":[114],"sequence":[116],"can":[119],"be":[120],"trained":[121],"efficiently":[123],"transformer":[126],"decoders":[127],"while":[128],"maintaining":[129],"same":[131],"performance":[132],"encoder.":[139],"We":[140],"evaluate":[141],"InsNet":[142,153],"on":[143],"story":[144],"CleVR-CoGENT":[147],"captioning,":[148],"showing":[149],"advantages":[151],"in":[154],"several":[155],"dimensions,":[156],"including":[157],"costs,":[159],"quality,":[161],"ability":[163],"perfectly":[165],"incorporate":[166],"lexical":[167],"controls,":[168],"better":[170],"compositional":[171],"generalization.":[172]},"counts_by_year":[{"year":2022,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2021-03-01T00:00:00"}