{"id":"https://openalex.org/W4402474365","doi":"https://doi.org/10.1109/taslp.2024.3451977","title":"Improving Non-Autoregressive Translation Quality With Pretrained Language Model, Embedding Distillation and Upsampling Strategy for CTC","display_name":"Improving Non-Autoregressive Translation Quality With Pretrained Language Model, Embedding Distillation and Upsampling Strategy for CTC","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402474365","doi":"https://doi.org/10.1109/taslp.2024.3451977"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2024.3451977","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3451977","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054647497","display_name":"Shen-Sian Syu","orcid":"https://orcid.org/0009-0002-5307-3572"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Shen-sian Syu","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0002-5307-3572","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101212430","display_name":"Juncheng Xie","orcid":null},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Juncheng Xie","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0009-0002-6292-0585","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040508737","display_name":"Hung-yi Lee","orcid":"https://orcid.org/0000-0002-9654-5747"},"institutions":[{"id":"https://openalex.org/I16733864","display_name":"National Taiwan University","ror":"https://ror.org/05bqach95","country_code":"TW","type":"education","lineage":["https://openalex.org/I16733864"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Hung-yi Lee","raw_affiliation_strings":["National Taiwan University, Taipei, Taiwan"],"raw_orcid":"https://orcid.org/0000-0002-9654-5747","affiliations":[{"raw_affiliation_string":"National Taiwan University, Taipei, Taiwan","institution_ids":["https://openalex.org/I16733864"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I16733864"],"apc_list":{"value":2495,"currency":"USD","value_usd":2495},"apc_paid":null,"fwci":0.4647,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.71498618,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"32","issue":null,"first_page":"4121","last_page":"4133"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.980400025844574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.980400025844574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/upsampling","display_name":"Upsampling","score":0.8863424062728882},{"id":"https://openalex.org/keywords/autoregressive-model","display_name":"Autoregressive model","score":0.7271020412445068},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.6248230934143066},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.586190938949585},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.5604396462440491},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.49889421463012695},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.41755998134613037},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3809017539024353},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36987394094467163},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.2327060103416443},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.1971849799156189},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1882379949092865},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.12723475694656372}],"concepts":[{"id":"https://openalex.org/C110384440","wikidata":"https://www.wikidata.org/wiki/Q1143270","display_name":"Upsampling","level":3,"score":0.8863424062728882},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.7271020412445068},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.6248230934143066},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.586190938949585},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.5604396462440491},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.49889421463012695},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.41755998134613037},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3809017539024353},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36987394094467163},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2327060103416443},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.1971849799156189},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1882379949092865},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.12723475694656372},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2024.3451977","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taslp.2024.3451977","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W179875071","https://openalex.org/W2127141656","https://openalex.org/W2143612262","https://openalex.org/W2222512263","https://openalex.org/W2257408573","https://openalex.org/W2892213699","https://openalex.org/W2933138175","https://openalex.org/W2962915948","https://openalex.org/W2996843693","https://openalex.org/W3035289598","https://openalex.org/W4221164185","https://openalex.org/W4377079846","https://openalex.org/W4382317864","https://openalex.org/W4385245566","https://openalex.org/W6679434410","https://openalex.org/W6679855610","https://openalex.org/W6680804558","https://openalex.org/W6719020539","https://openalex.org/W6725857009","https://openalex.org/W6739901393","https://openalex.org/W6746208923","https://openalex.org/W6748305982","https://openalex.org/W6755207826","https://openalex.org/W6757383703","https://openalex.org/W6760017253","https://openalex.org/W6761895683","https://openalex.org/W6762471145","https://openalex.org/W6766240984","https://openalex.org/W6766557392","https://openalex.org/W6767737316","https://openalex.org/W6769243733","https://openalex.org/W6769627184","https://openalex.org/W6769692749","https://openalex.org/W6769921281","https://openalex.org/W6771713106","https://openalex.org/W6772534010","https://openalex.org/W6773419789","https://openalex.org/W6774835902","https://openalex.org/W6776353054","https://openalex.org/W6779584667","https://openalex.org/W6782268706","https://openalex.org/W6784577980","https://openalex.org/W6784698631","https://openalex.org/W6785248963","https://openalex.org/W6787459375","https://openalex.org/W6790947600","https://openalex.org/W6797256274","https://openalex.org/W6801296924","https://openalex.org/W6802318714","https://openalex.org/W6804648141","https://openalex.org/W6809666657","https://openalex.org/W6838282907","https://openalex.org/W6838407971","https://openalex.org/W6843218176","https://openalex.org/W6845526127","https://openalex.org/W6855210224","https://openalex.org/W6855261489","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W2062399876","https://openalex.org/W2607795551","https://openalex.org/W3155117723","https://openalex.org/W1991429770","https://openalex.org/W1983892167","https://openalex.org/W2281134365","https://openalex.org/W4310746709","https://openalex.org/W4385574037","https://openalex.org/W4386075645","https://openalex.org/W4306309518"],"abstract_inverted_index":{"Non-autoregressive":[0],"approaches,":[1],"especially":[2],"those":[3],"that":[4],"generate":[5],"output":[6],"in":[7,16,33,71],"a":[8,30,48,68,77,208,215],"one-pass":[9],"forward":[10],"manner,":[11],"have":[12],"shown":[13],"great":[14],"potential":[15],"improving":[17],"the":[18,55,90,104,123,133,136,155,169,174,190,224],"inference":[19,72],"speed":[20,217],"of":[21,50,58,109,125,135,201,219],"translation":[22,34,56,62],"models.":[23,100,127,160],"However,":[24],"these":[25],"approaches":[26],"often":[27],"suffer":[28],"from":[29],"significant":[31],"drop":[32],"quality":[35,57],"compared":[36,167,222],"to":[37,53,96,120,168,223],"autoregressive":[38,138,171,225],"models":[39,64,205],"(AT).":[40],"To":[41],"tackle":[42],"this":[43,45],"challenge,":[44],"paper":[46],"introduces":[47],"series":[49],"innovative":[51],"techniques":[52],"enhance":[54,122],"non-autoregressive":[59],"neural":[60],"machine":[61],"(NAT)":[63],"while":[65],"still":[66],"maintaining":[67],"substantial":[69],"acceleration":[70],"speed.":[73],"Specifically,":[74],"we":[75,102],"propose":[76],"method":[78,119],"called":[79],"CTCPMLM,":[80],"which":[81],"involves":[82],"fine-tuning":[83],"Pretrained":[84],"Multilingual":[85],"Language":[86],"Models":[87],"(PMLMs)":[88],"with":[89],"Connectionist":[91],"Temporal":[92],"Classification":[93],"(CTC)":[94],"loss":[95],"effectively":[97],"train":[98],"NAT":[99,126,159],"Additionally,":[101,211],"adopt":[103],"MASK":[105],"insertion":[106],"scheme":[107],"instead":[108],"token":[110],"duplication":[111],"for":[112],"up-sampling":[113],"and":[114,149,177,206],"present":[115],"an":[116,197],"embedding":[117],"distillation":[118,184],"further":[121],"performance":[124,134],"In":[128],"our":[129,162,194,212],"experiments,":[130],"CTCPMLM":[131,153],"surpasses":[132],"baseline":[137,170],"model":[139,163,172,195,213],"(Transformerbase)":[140],"on":[141,173,189],"various":[142],"datasets,":[143,180],"including":[144],"WMT'14":[145],"DE$\\leftrightarrow$EN,":[146],"WMT'16":[147,178],"RO$\\leftrightarrow$EN,":[148],"IWSLT'14":[150,175,191],"DE$\\leftrightarrow$EN.":[151],"Moreover,":[152],"represents":[154],"current":[156],"state-of-the-art":[157],"among":[158],"Notably,":[161],"achieves":[164,196],"superior":[165],"results":[166],"En$\\leftrightarrow$De":[176],"En$\\leftrightarrow$Ro":[179],"even":[181],"without":[182],"using":[183],"data":[185],"during":[186],"training.":[187],"Particularly,":[188],"DE$\\rightarrow$EN":[192],"dataset,":[193],"impressive":[198],"BLEU":[199],"score":[200],"39.93,":[202],"surpassing":[203],"AT":[204],"establishing":[207],"new":[209],"state-of-the-art.":[210],"exhibits":[214],"remarkable":[216],"improvement":[218],"16.35":[220],"times":[221],"model.":[226]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2024-09-13T00:00:00"}