{"id":"https://openalex.org/W7019341752","doi":null,"title":"Fine-Tuning Neural Codec Language Models from Feedback with Reinforcement Learning","display_name":"Fine-Tuning Neural Codec Language Models from Feedback with Reinforcement Learning","publication_year":2024,"publication_date":"2024-03-19","ids":{"openalex":"https://openalex.org/W7019341752"},"language":"en","primary_location":{"id":"pmh:oai:amslaurea.cib.unibo.it:31722","doi":null,"is_oa":true,"landing_page_url":"http://amslaurea.unibo.it/31722/1/tesi_pratesi.pdf","pdf_url":"http://amslaurea.unibo.it/31722/1/tesi_pratesi.pdf","source":{"id":"https://openalex.org/S4377196908","display_name":"AMS Laurea Institutional Research Repository (University of Bologna)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I9360294","host_organization_name":"University of Bologna","host_organization_lineage":["https://openalex.org/I9360294"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/masterThesis"},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://amslaurea.unibo.it/31722/1/tesi_pratesi.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5150828230","display_name":"Lorenzo Pratesi","orcid":"https://orcid.org/0009-0003-2083-1506","observed_orcids":["https://orcid.org/0009-0003-2083-1506"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Pratesi, Lorenzo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5150828230"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speech-synthesis","display_name":"speech synthesis","score":0.7409999966621399},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.9639999866485596},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.8870000243186951},{"id":"https://openalex.org/keywords/libritts","display_name":"LibriTTS","score":0.9599999785423279},{"id":"https://openalex.org/keywords/intelligibility","display_name":"intelligibility","score":0.8820000290870667},{"id":"https://openalex.org/keywords/naturalness","display_name":"naturalness","score":0.8550000190734863},{"id":"https://openalex.org/keywords/speaker-similarity","display_name":"speaker similarity","score":0.9319999814033508},{"id":"https://openalex.org/keywords/reward-shaping","display_name":"reward shaping","score":0.43700000643730164},{"id":"https://openalex.org/keywords/zero-shot-text-to-speech","display_name":"zero-shot text-to-speech","score":0.6980000138282776}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8008000254631042},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6412000060081482},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5677000284194946},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5403000116348267},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5382000207901001},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4120999872684479},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4016000032424927},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3813999891281128},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.37630000710487366},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.36649999022483826},{"id":"https://openalex.org/C60048801","wikidata":"https://www.wikidata.org/wiki/Q1433889","display_name":"Intelligibility (philosophy)","level":2,"score":0.3652999997138977},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34630000591278076},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.3456000089645386},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3228999972343445},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.321399986743927},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.3203999996185303},{"id":"https://openalex.org/C161765866","wikidata":"https://www.wikidata.org/wiki/Q184748","display_name":"Codec","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.28600001335144043},{"id":"https://openalex.org/C13895895","wikidata":"https://www.wikidata.org/wiki/Q3270773","display_name":"Speech coding","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C504749915","wikidata":"https://www.wikidata.org/wiki/Q9010971","display_name":"Speech technology","level":3,"score":0.2759999930858612},{"id":"https://openalex.org/C14999030","wikidata":"https://www.wikidata.org/wiki/Q16346","display_name":"Speech synthesis","level":2,"score":0.26579999923706055}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:amslaurea.cib.unibo.it:31722","doi":null,"is_oa":true,"landing_page_url":"http://amslaurea.unibo.it/31722/1/tesi_pratesi.pdf","pdf_url":"http://amslaurea.unibo.it/31722/1/tesi_pratesi.pdf","source":{"id":"https://openalex.org/S4377196908","display_name":"AMS Laurea Institutional Research Repository (University of Bologna)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I9360294","host_organization_name":"University of Bologna","host_organization_lineage":["https://openalex.org/I9360294"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/masterThesis"}],"best_oa_location":{"id":"pmh:oai:amslaurea.cib.unibo.it:31722","doi":null,"is_oa":true,"landing_page_url":"http://amslaurea.unibo.it/31722/1/tesi_pratesi.pdf","pdf_url":"http://amslaurea.unibo.it/31722/1/tesi_pratesi.pdf","source":{"id":"https://openalex.org/S4377196908","display_name":"AMS Laurea Institutional Research Repository (University of Bologna)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I9360294","host_organization_name":"University of Bologna","host_organization_lineage":["https://openalex.org/I9360294"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/masterThesis"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7019341752.pdf","grobid_xml":"https://content.openalex.org/works/W7019341752.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W1508913136","https://openalex.org/W1591740026","https://openalex.org/W1598796236","https://openalex.org/W1757796397","https://openalex.org/W1959608418","https://openalex.org/W1965555277","https://openalex.org/W2102737569","https://openalex.org/W2107726111","https://openalex.org/W2626778328","https://openalex.org/W2736601468","https://openalex.org/W2767052532","https://openalex.org/W2948211236","https://openalex.org/W2963341956","https://openalex.org/W2963609956","https://openalex.org/W2964121744","https://openalex.org/W2964243274","https://openalex.org/W2972359262","https://openalex.org/W2973379954","https://openalex.org/W2974231335","https://openalex.org/W2982399380","https://openalex.org/W3030163527","https://openalex.org/W3129009457","https://openalex.org/W3133702157","https://openalex.org/W3148330722","https://openalex.org/W3152733922","https://openalex.org/W3169320628","https://openalex.org/W3174758275","https://openalex.org/W3178321840","https://openalex.org/W3205878676","https://openalex.org/W4200300291","https://openalex.org/W4224035735","https://openalex.org/W4226112058","https://openalex.org/W4297841605","https://openalex.org/W4297841766","https://openalex.org/W4297964528","https://openalex.org/W4307323391","https://openalex.org/W4313679638","https://openalex.org/W4319586753","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4387799863","https://openalex.org/W4390215524","https://openalex.org/W4392903006","https://openalex.org/W6903974388","https://openalex.org/W6959978187","https://openalex.org/W7106552862","https://openalex.org/W7133208539"],"related_works":["https://openalex.org/W4411119187","https://openalex.org/W7155092298","https://openalex.org/W4408351949","https://openalex.org/W7166162710","https://openalex.org/W4415797742","https://openalex.org/W4313679638","https://openalex.org/W4415433683","https://openalex.org/W4415536855","https://openalex.org/W4415433194","https://openalex.org/W4393969253"],"abstract_inverted_index":{"Neural":[0],"codec":[1,101],"language":[2,15,52,80,102],"models":[3,38,81,103],"(NCLMs)":[4],"are":[5],"speech":[6,48,64,230],"synthesizers":[7],"that":[8,208],"address":[9],"the":[10,68,120,130,134,143,154,159,175,213,216,229,234],"TTS":[11,37],"task":[12,17,54],"as":[13,23,50],"a":[14,51,181,219],"modeling":[16,53],"rather":[18],"than":[19],"continous":[20],"signal":[21],"regression":[22],"in":[24,55,158,252],"previous":[25,34],"work.":[26,254],"They":[27],"showed":[28],"an":[29],"impressive":[30],"generalization":[31],"capability,":[32],"surpassing":[33],"state-of-the-art":[35],"zero-shot":[36],"by":[39,124,153],"means":[40],"of":[41,78,122,145,156,183,196,215,241],"speaker":[42,200],"similarity":[43,201],"and":[44,62,97,108,149,202,245],"naturalness.":[45],"Although":[46],"addressing":[47],"synthesis":[49,106],"part":[56],"allows":[57],"to":[58,76,118,128,133,141,166,192,211,223,227,233,249],"train":[59],"on":[60,188],"large":[61,79],"diverse":[63],"data":[65],"crawled":[66],"from":[67,105,112,169],"Internet,":[69],"it":[70,191],"also":[71,226],"brings":[72],"some":[73],"issues":[74,121],"common":[75],"those":[77],"(LLMs)":[82],"for":[83],"text":[84,160],"generation.":[85],"While":[86],"LLMs":[87,140],"may":[88],"generate":[89],"outputs":[90],"with":[91,171,185],"made":[92],"up":[93,222],"facts":[94],"or":[95],"biased":[96],"toxic":[98,147],"contents,":[99],"neural":[100],"suffers":[104],"robustness":[107],"expressivness.":[109],"Reinforcement":[110],"Learning":[111],"Human":[113],"Feedback":[114],"(RLHF)":[115],"has":[116],"emerged":[117],"tackle":[119],"LLMs,":[123],"using":[125],"human":[126],"feedback":[127,170],"align":[129],"generated":[131,146],"responses":[132],"user":[135],"preferences.":[136],"Using":[137],"RLHF":[138,157,176],"helped":[139,210],"reduce":[142],"amount":[144],"content":[148],"false":[150],"facts.":[151],"Motivated":[152],"success":[155],"generation":[161],"domain,":[162],"this":[163],"work":[164],"proposes":[165],"fine-tune":[167],"NCLMs":[168],"reinforcement":[172],"learning":[173],"following":[174],"training":[177],"pipeline.":[178],"We":[179],"conduct":[180],"series":[182],"experiments":[184],"VALL-E":[186],"pretrained":[187],"LibriTTS,":[189],"fine-tuning":[190,209],"optimize":[193],"different":[194],"kind":[195],"feedback:":[197],"intelligibility,":[198],"naturalness,":[199],"waveform":[203],"duration.":[204],"Our":[205],"results":[206],"show":[207],"increase":[212],"intelligibility":[214],"model,":[217],"showing":[218],"WER":[220],"reduction":[221],"20.954\\\\%,":[224],"but":[225],"change":[228],"duration":[231],"according":[232],"reward":[235],"signal.":[236],"Finally,":[237],"we":[238],"delineate":[239],"limitations":[240],"our":[242],"experimental":[243],"setup":[244],"propose":[246],"practical":[247],"mitigations,":[248],"be":[250],"explored":[251],"future":[253]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}