{"id":"https://openalex.org/W4403791207","doi":"https://doi.org/10.1145/3664647.3681182","title":"ListenFormer: Responsive Listening Head Generation with Non-autoregressive Transformers","display_name":"ListenFormer: Responsive Listening Head Generation with Non-autoregressive Transformers","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403791207","doi":"https://doi.org/10.1145/3664647.3681182"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681182","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664647.3681182","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3664647.3681182?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3664647.3681182?download=true","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045241725","display_name":"Miao Liu","orcid":"https://orcid.org/0000-0002-2039-2051"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Miao Liu","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2039-2051","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100378497","display_name":"Jing Wang","orcid":"https://orcid.org/0000-0002-3653-9951"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Wang","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3653-9951","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056495776","display_name":"Xinyuan Qian","orcid":"https://orcid.org/0000-0002-9511-6713"},"institutions":[{"id":"https://openalex.org/I92403157","display_name":"University of Science and Technology Beijing","ror":"https://ror.org/02egmk993","country_code":"CN","type":"education","lineage":["https://openalex.org/I92403157"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinyuan Qian","raw_affiliation_strings":["University of Science and Technology Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9511-6713","affiliations":[{"raw_affiliation_string":"University of Science and Technology Beijing, Beijing, China","institution_ids":["https://openalex.org/I92403157"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032690182","display_name":"Haizhou Li","orcid":"https://orcid.org/0000-0001-9158-9401"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Haizhou Li","raw_affiliation_strings":["Chinese University of Hong Kong (Shenzhen) & National University of Singapore, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0001-9158-9401","affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong (Shenzhen) & National University of Singapore, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924","https://openalex.org/I165932596"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"7094","last_page":"7103"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9807999730110168,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/active-listening","display_name":"Active listening","score":0.714656412601471},{"id":"https://openalex.org/keywords/autoregressive-model","display_name":"Autoregressive model","score":0.689205527305603},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5680927038192749},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5430270433425903},{"id":"https://openalex.org/keywords/head","display_name":"Head (geology)","score":0.5260070562362671},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3497348427772522},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.24228441715240479},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.18312564492225647},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.146986186504364},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.1049218475818634},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.09757250547409058},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.0904420018196106},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.07706069946289062},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.056576818227767944}],"concepts":[{"id":"https://openalex.org/C177291462","wikidata":"https://www.wikidata.org/wiki/Q423038","display_name":"Active listening","level":2,"score":0.714656412601471},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.689205527305603},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5680927038192749},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5430270433425903},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.5260070562362671},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3497348427772522},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.24228441715240479},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.18312564492225647},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.146986186504364},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.1049218475818634},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.09757250547409058},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0904420018196106},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.07706069946289062},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.056576818227767944},{"id":"https://openalex.org/C114793014","wikidata":"https://www.wikidata.org/wiki/Q52109","display_name":"Geomorphology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3681182","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664647.3681182","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3664647.3681182?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3664647.3681182","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664647.3681182","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3664647.3681182?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.41999998688697815}],"awards":[{"id":"https://openalex.org/G6471985988","display_name":"\u57fa\u4e8e\u5f20\u91cf\u6df1\u5ea6\u5b66\u4e60\u548c\u6ce8\u610f\u529b\u673a\u5236\u7684\u65e0\u53c2\u8003\u58f0\u97f3\u8d28\u91cf\u8bc4\u4ef7\u65b9\u6cd5\u7814\u7a76","funder_award_id":"62071039","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403791207.pdf","grobid_xml":"https://content.openalex.org/works/W4403791207.grobid-xml"},"referenced_works_count":45,"referenced_works":["https://openalex.org/W1530404542","https://openalex.org/W2008208299","https://openalex.org/W2025567128","https://openalex.org/W2122515591","https://openalex.org/W2133665775","https://openalex.org/W2150610356","https://openalex.org/W2150671164","https://openalex.org/W2161813860","https://openalex.org/W2550143307","https://openalex.org/W2769666294","https://openalex.org/W2772447065","https://openalex.org/W2914911817","https://openalex.org/W2964449965","https://openalex.org/W2969985801","https://openalex.org/W3099284785","https://openalex.org/W3162293946","https://openalex.org/W3170936177","https://openalex.org/W3174763799","https://openalex.org/W3186090335","https://openalex.org/W3204680331","https://openalex.org/W4200630629","https://openalex.org/W4223499953","https://openalex.org/W4224932526","https://openalex.org/W4225760288","https://openalex.org/W4225959162","https://openalex.org/W4250335329","https://openalex.org/W4281758439","https://openalex.org/W4283710679","https://openalex.org/W4304087182","https://openalex.org/W4304099282","https://openalex.org/W4310379947","https://openalex.org/W4312309978","https://openalex.org/W4312444931","https://openalex.org/W4319300274","https://openalex.org/W4319300466","https://openalex.org/W4327852215","https://openalex.org/W4385490328","https://openalex.org/W4385801060","https://openalex.org/W4386076250","https://openalex.org/W4387693486","https://openalex.org/W4387967994","https://openalex.org/W4387969668","https://openalex.org/W4388979610","https://openalex.org/W4390154864","https://openalex.org/W4390872769"],"related_works":["https://openalex.org/W2317723112","https://openalex.org/W2475724061","https://openalex.org/W2773393136","https://openalex.org/W2174706483","https://openalex.org/W2997121352","https://openalex.org/W419536403","https://openalex.org/W2506280730","https://openalex.org/W4237969969","https://openalex.org/W1594297642","https://openalex.org/W2366328218"],"abstract_inverted_index":{"As":[0],"one":[1],"of":[2,50,78,95,140,195],"the":[3,47,76,90,105,118,123,126,138,177,192],"crucial":[4],"elements":[5],"in":[6,56,117,198],"human-robot":[7],"interaction,":[8],"responsive":[9],"listening":[10,24],"head":[11,25],"generation":[12,48],"has":[13],"attracted":[14],"considerable":[15],"attention":[16,133],"from":[17,62,125],"researchers.":[18],"It":[19,99],"aims":[20],"to":[21,74],"generate":[22],"a":[23,36,149,157,187],"video":[26,32],"based":[27],"on":[28,181],"speaker's":[29],"audio":[30,141],"and":[31,65,114,142,169,183,205],"as":[33,35],"well":[34],"reference":[37],"listener":[38],"image.":[39],"However,":[40],"existing":[41,178],"methods":[42,180],"exhibit":[43],"two":[44],"limitations:":[45],"1)":[46],"capability":[49,94],"their":[51],"models":[52],"is":[53,160,210],"limited,":[54],"resulting":[55],"generated":[57],"videos":[58],"that":[59,88,174],"are":[60],"far":[61],"real":[63],"ones,":[64],"2)":[66],"they":[67],"mostly":[68],"employ":[69],"autoregressive":[70],"generative":[71],"models,":[72],"unable":[73],"mitigate":[75],"risk":[77],"error":[79],"accumulation.":[80],"To":[81,120],"tackle":[82],"these":[83],"issues,":[84],"we":[85,129],"propose":[86],"Listenformer":[87,175],"leverages":[89],"powerful":[91],"temporal":[92,112],"modeling":[93],"transformers":[96],"for":[97,145,162],"generation.":[98],"can":[100],"perform":[101],"non-autoregressive":[102],"prediction":[103],"with":[104,156],"proposed":[106,161],"two-stage":[107],"training":[108],"method,":[109],"simultaneously":[110],"achieving":[111],"continuity":[113],"overall":[115],"consistency":[116],"outputs.":[119],"fully":[121],"utilize":[122],"information":[124],"speaker":[127],"inputs,":[128],"designed":[130],"an":[131],"audio-motion":[132],"fusion":[134],"module,":[135],"which":[136],"improves":[137],"correlation":[139],"motion":[143],"features":[144],"accurate":[146],"response.":[147],"Additionally,":[148],"novel":[150],"decoding":[151],"method":[152,197],"called":[153],"sliding":[154],"window":[155],"large":[158],"shift":[159],"Listenformer,":[163],"demonstrating":[164],"both":[165],"excellent":[166],"computational":[167],"efficiency":[168],"effectiveness.":[170],"Extensive":[171],"experiments":[172],"show":[173],"outperforms":[176],"state-of-the-art":[179],"ViCo":[182],"L2L":[184],"datasets.":[185],"And":[186],"perceptual":[188],"user":[189],"study":[190],"demonstrates":[191],"comprehensive":[193],"performance":[194],"our":[196],"generating":[199],"diversity,":[200],"identity":[201],"preserving,":[202],"speaker-listener":[203],"synchronization,":[204],"attitude":[206],"matching.":[207],"Our":[208],"code":[209],"available":[211],"at":[212],"https://liushenme.github.io/ListenFormer.github.io/.":[213]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}