{"id":"https://openalex.org/W4390873634","doi":"https://doi.org/10.1109/iccv51070.2023.00925","title":"Can Language Models Learn to Listen?","display_name":"Can Language Models Learn to Listen?","publication_year":2023,"publication_date":"2023-10-01","ids":{"openalex":"https://openalex.org/W4390873634","doi":"https://doi.org/10.1109/iccv51070.2023.00925"},"language":"en","primary_location":{"id":"doi:10.1109/iccv51070.2023.00925","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51070.2023.00925","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043135509","display_name":"Evonne Ng","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evonne Ng","raw_affiliation_strings":["University of California Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020117260","display_name":"Sanjay Subramanian","orcid":"https://orcid.org/0009-0001-1399-9920"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sanjay Subramanian","raw_affiliation_strings":["University of California Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004921249","display_name":"Dan Klein","orcid":"https://orcid.org/0000-0002-8881-1902"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Klein","raw_affiliation_strings":["University of California Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031491881","display_name":"Angjoo Kanazawa","orcid":"https://orcid.org/0000-0003-2592-8430"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Angjoo Kanazawa","raw_affiliation_strings":["University of California Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029105520","display_name":"Trevor Darrell","orcid":"https://orcid.org/0000-0001-5453-8533"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Trevor Darrell","raw_affiliation_strings":["University of California Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5009491631","display_name":"Shiry Ginosar","orcid":"https://orcid.org/0000-0002-7362-1401"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shiry Ginosar","raw_affiliation_strings":["University of California Berkeley"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California Berkeley","institution_ids":["https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I95457486"],"apc_list":null,"apc_paid":null,"fwci":4.5735,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.96076796,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"10049","last_page":"10059"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11431","display_name":"Action Observation and Synchronization","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.749197781085968},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5003881454467773},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4075998067855835},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.38495391607284546}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.749197781085968},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5003881454467773},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4075998067855835},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.38495391607284546}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iccv51070.2023.00925","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51070.2023.00925","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7599999904632568,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":61,"referenced_works":["https://openalex.org/W1921786749","https://openalex.org/W1971199318","https://openalex.org/W1990723800","https://openalex.org/W2008208299","https://openalex.org/W2009266235","https://openalex.org/W2017107803","https://openalex.org/W2107037917","https://openalex.org/W2150610356","https://openalex.org/W2237250383","https://openalex.org/W2334778303","https://openalex.org/W2740440254","https://openalex.org/W2747610795","https://openalex.org/W2752796333","https://openalex.org/W2769666294","https://openalex.org/W2772447065","https://openalex.org/W2787512630","https://openalex.org/W2798338993","https://openalex.org/W2806379360","https://openalex.org/W2911776943","https://openalex.org/W2962701817","https://openalex.org/W2962795401","https://openalex.org/W2963799213","https://openalex.org/W2970641574","https://openalex.org/W2981133004","https://openalex.org/W3002310794","https://openalex.org/W3015783745","https://openalex.org/W3118898753","https://openalex.org/W3134307371","https://openalex.org/W3157070662","https://openalex.org/W3180794345","https://openalex.org/W3198482860","https://openalex.org/W4224308101","https://openalex.org/W4225760288","https://openalex.org/W4226405181","https://openalex.org/W4292779060","https://openalex.org/W4294255555","https://openalex.org/W4297981470","https://openalex.org/W4298186886","https://openalex.org/W4301206121","https://openalex.org/W4311000453","https://openalex.org/W4312309978","https://openalex.org/W4312590328","https://openalex.org/W4312635677","https://openalex.org/W4313145975","https://openalex.org/W4318348359","https://openalex.org/W4319165750","https://openalex.org/W4385245566","https://openalex.org/W4385801342","https://openalex.org/W4386076288","https://openalex.org/W6739901393","https://openalex.org/W6748883595","https://openalex.org/W6765779288","https://openalex.org/W6772578931","https://openalex.org/W6778883912","https://openalex.org/W6791141078","https://openalex.org/W6810081322","https://openalex.org/W6810089409","https://openalex.org/W6844223692","https://openalex.org/W6847363464","https://openalex.org/W6849713201","https://openalex.org/W6849968169"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"We":[0,103],"present":[1],"a":[2,10,36,39,41,49,54,71,83,99,121],"framework":[3],"for":[4],"generating":[5],"appropriate":[6],"facial":[7,45],"responses":[8,96],"from":[9,101],"listener":[11,44,95,108],"in":[12,91],"dyadic":[13],"social":[14],"interactions":[15],"based":[16],"on":[17,88],"the":[18,26,60,80,130],"speaker\u2019s":[19,27],"words.":[20],"Given":[21],"an":[22],"input":[23],"transcription":[24],"of":[25,38,43,82,114,139],"words":[28],"with":[29,79],"their":[30],"timestamps,":[31],"our":[32,77,106,126],"approach":[33],"autoregressively":[34],"predicts":[35],"response":[37],"listener:":[40],"sequence":[42],"gestures,":[46],"quantized":[47,61],"using":[48],"VQ-VAE.":[50],"Since":[51],"gesture":[52],"is":[53,110],"language":[55,67,74,84,115],"component,":[56],"we":[57,128],"propose":[58],"treating":[59],"atomic":[62],"motion":[63,109],"elements":[64],"as":[65],"additional":[66],"token":[68],"inputs":[69],"to":[70,133],"transformer-based":[72],"large":[73],"model.":[75],"Initializing":[76],"transformer":[78,100],"weights":[81],"model":[85],"pre-trained":[86],"only":[87],"text":[89],"results":[90],"significantly":[92],"higher":[93],"quality":[94],"than":[97],"training":[98],"scratch.":[102],"show":[104],"that":[105],"generated":[107],"fluent":[111],"and":[112,120,136],"reflective":[113],"semantics":[116],"through":[117],"quantitative":[118],"metrics":[119],"qualitative":[122],"user":[123],"study.":[124],"In":[125],"evaluation,":[127],"analyze":[129],"model\u2019s":[131],"ability":[132],"utilize":[134],"temporal":[135],"semantic":[137],"aspects":[138],"spoken":[140],"text.":[141]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":8}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}