{"id":"https://openalex.org/W4391021513","doi":"https://doi.org/10.1109/asru57964.2023.10389619","title":"Audio-Visual Neural Syntax Acquisition","display_name":"Audio-Visual Neural Syntax Acquisition","publication_year":2023,"publication_date":"2023-12-16","ids":{"openalex":"https://openalex.org/W4391021513","doi":"https://doi.org/10.1109/asru57964.2023.10389619"},"language":"en","primary_location":{"id":"doi:10.1109/asru57964.2023.10389619","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru57964.2023.10389619","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010825170","display_name":"Cheng-I Lai","orcid":"https://orcid.org/0000-0002-2343-8596"},"institutions":[{"id":"https://openalex.org/I4210109586","display_name":"Moscow Institute of Thermal Technology","ror":"https://ror.org/021es5e59","country_code":"RU","type":"facility","lineage":["https://openalex.org/I4210109586"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Cheng-I Jeff Lai","raw_affiliation_strings":["MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT","institution_ids":["https://openalex.org/I4210109586"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037519114","display_name":"Freda Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Freda Shi","raw_affiliation_strings":["TTIC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TTIC","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075735963","display_name":"Puyuan Peng","orcid":null},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Puyuan Peng","raw_affiliation_strings":["UT Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UT Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100693798","display_name":"Yoon Kim","orcid":"https://orcid.org/0000-0002-4837-8411"},"institutions":[{"id":"https://openalex.org/I4210109586","display_name":"Moscow Institute of Thermal Technology","ror":"https://ror.org/021es5e59","country_code":"RU","type":"facility","lineage":["https://openalex.org/I4210109586"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Yoon Kim","raw_affiliation_strings":["MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT","institution_ids":["https://openalex.org/I4210109586"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081022650","display_name":"Kevin Gimpel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kevin Gimpel","raw_affiliation_strings":["TTIC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TTIC","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112248869","display_name":"Shiyu Chang","orcid":null},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shiyu Chang","raw_affiliation_strings":["UC Santa Barbara"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UC Santa Barbara","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058729228","display_name":"Yung-Sung Chuang","orcid":"https://orcid.org/0000-0002-1723-5063"},"institutions":[{"id":"https://openalex.org/I4210109586","display_name":"Moscow Institute of Thermal Technology","ror":"https://ror.org/021es5e59","country_code":"RU","type":"facility","lineage":["https://openalex.org/I4210109586"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Yung-Sung Chuang","raw_affiliation_strings":["MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT","institution_ids":["https://openalex.org/I4210109586"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045247662","display_name":"Saurabhchand Bhati","orcid":"https://orcid.org/0000-0001-6477-3895"},"institutions":[{"id":"https://openalex.org/I4210109586","display_name":"Moscow Institute of Thermal Technology","ror":"https://ror.org/021es5e59","country_code":"RU","type":"facility","lineage":["https://openalex.org/I4210109586"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Saurabhchand Bhati","raw_affiliation_strings":["MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT","institution_ids":["https://openalex.org/I4210109586"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100763986","display_name":"David Cox","orcid":"https://orcid.org/0000-0002-2189-9743"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Cox","raw_affiliation_strings":["MIT-IBM Watson AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT-IBM Watson AI Lab","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004717608","display_name":"David Harwath","orcid":"https://orcid.org/0000-0003-0206-0253"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Harwath","raw_affiliation_strings":["UT Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UT Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100354733","display_name":"Yang Zhang","orcid":"https://orcid.org/0000-0002-8540-1254"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang Zhang","raw_affiliation_strings":["MIT-IBM Watson AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT-IBM Watson AI Lab","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015602781","display_name":"Karen Livescu","orcid":"https://orcid.org/0000-0003-4962-946X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karen Livescu","raw_affiliation_strings":["TTIC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TTIC","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112758056","display_name":"James Glass","orcid":"https://orcid.org/0000-0002-3097-360X"},"institutions":[{"id":"https://openalex.org/I4210109586","display_name":"Moscow Institute of Thermal Technology","ror":"https://ror.org/021es5e59","country_code":"RU","type":"facility","lineage":["https://openalex.org/I4210109586"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"James Glass","raw_affiliation_strings":["MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT","institution_ids":["https://openalex.org/I4210109586"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":13,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2246,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.5428214,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.830445408821106},{"id":"https://openalex.org/keywords/phrase","display_name":"Phrase","score":0.7031546831130981},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.695222020149231},{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.6377704739570618},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5934861898422241},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5575124025344849},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.5474059581756592},{"id":"https://openalex.org/keywords/phrase-structure-rules","display_name":"Phrase structure rules","score":0.5038754343986511},{"id":"https://openalex.org/keywords/grammar","display_name":"Grammar","score":0.48455873131752014},{"id":"https://openalex.org/keywords/german","display_name":"German","score":0.45803311467170715},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.4521810710430145},{"id":"https://openalex.org/keywords/active-listening","display_name":"Active listening","score":0.4441341161727905},{"id":"https://openalex.org/keywords/audio-mining","display_name":"Audio mining","score":0.42363476753234863},{"id":"https://openalex.org/keywords/speech-processing","display_name":"Speech processing","score":0.29966628551483154},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.25872233510017395},{"id":"https://openalex.org/keywords/acoustic-model","display_name":"Acoustic model","score":0.18902835249900818},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.1499570608139038},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.10210490226745605},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09532439708709717}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.830445408821106},{"id":"https://openalex.org/C2776224158","wikidata":"https://www.wikidata.org/wiki/Q187931","display_name":"Phrase","level":2,"score":0.7031546831130981},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.695222020149231},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.6377704739570618},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5934861898422241},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5575124025344849},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.5474059581756592},{"id":"https://openalex.org/C80877019","wikidata":"https://www.wikidata.org/wiki/Q7188074","display_name":"Phrase structure rules","level":3,"score":0.5038754343986511},{"id":"https://openalex.org/C26022165","wikidata":"https://www.wikidata.org/wiki/Q8091","display_name":"Grammar","level":2,"score":0.48455873131752014},{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.45803311467170715},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.4521810710430145},{"id":"https://openalex.org/C177291462","wikidata":"https://www.wikidata.org/wiki/Q423038","display_name":"Active listening","level":2,"score":0.4441341161727905},{"id":"https://openalex.org/C157968479","wikidata":"https://www.wikidata.org/wiki/Q3079876","display_name":"Audio mining","level":4,"score":0.42363476753234863},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.29966628551483154},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.25872233510017395},{"id":"https://openalex.org/C155635449","wikidata":"https://www.wikidata.org/wiki/Q4674699","display_name":"Acoustic model","level":3,"score":0.18902835249900818},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.1499570608139038},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.10210490226745605},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09532439708709717},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/asru57964.2023.10389619","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru57964.2023.10389619","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8299999833106995}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W1548450879","https://openalex.org/W1778492285","https://openalex.org/W1861492603","https://openalex.org/W2025482506","https://openalex.org/W2057007397","https://openalex.org/W2069859485","https://openalex.org/W2080090262","https://openalex.org/W2087165009","https://openalex.org/W2114347655","https://openalex.org/W2119717200","https://openalex.org/W2129882630","https://openalex.org/W2153568660","https://openalex.org/W2188924248","https://openalex.org/W2468716020","https://openalex.org/W2477749706","https://openalex.org/W2483390977","https://openalex.org/W2747874407","https://openalex.org/W2796738181","https://openalex.org/W2899663614","https://openalex.org/W2932376173","https://openalex.org/W2937176889","https://openalex.org/W2949399644","https://openalex.org/W2949579048","https://openalex.org/W2954930777","https://openalex.org/W2962753610","https://openalex.org/W2963084773","https://openalex.org/W2963521945","https://openalex.org/W2963909453","https://openalex.org/W2972885185","https://openalex.org/W2973230496","https://openalex.org/W2990883660","https://openalex.org/W2995680346","https://openalex.org/W3098643042","https://openalex.org/W3100776995","https://openalex.org/W3103427938","https://openalex.org/W3106150792","https://openalex.org/W3171691518","https://openalex.org/W3174311593","https://openalex.org/W3197974236","https://openalex.org/W3198702270","https://openalex.org/W3198782837","https://openalex.org/W3209059054","https://openalex.org/W4206003764","https://openalex.org/W4224875474","https://openalex.org/W4285064783","https://openalex.org/W4295308567","https://openalex.org/W4313182775","https://openalex.org/W4372259823","https://openalex.org/W4385823277","https://openalex.org/W6632788291","https://openalex.org/W6744839764","https://openalex.org/W6755977528","https://openalex.org/W6770596778","https://openalex.org/W7062406522"],"related_works":["https://openalex.org/W2077104824","https://openalex.org/W2168778844","https://openalex.org/W2166781037","https://openalex.org/W2129100979","https://openalex.org/W2208029369","https://openalex.org/W2804356946","https://openalex.org/W2032911973","https://openalex.org/W2108311530","https://openalex.org/W1913586201","https://openalex.org/W2093652469"],"abstract_inverted_index":{"We":[0,34],"study":[1],"phrase":[2,26,44,75],"structure":[3,27,45],"induction":[4],"from":[5,101],"visually-grounded":[6],"speech.":[7],"The":[8],"core":[9],"idea":[10],"is":[11],"to":[12,48,58,72,80,111],"first":[13],"segment":[14],"the":[15,29,36,70,113,116],"speech":[16,102],"waveform":[17],"into":[18],"sequences":[19],"of":[20],"word":[21],"segments,":[22],"and":[23,50,65,90,103,107],"subsequently":[24],"induce":[25],"using":[28],"inferred":[30],"segment-level":[31],"continuous":[32],"representations.":[33],"present":[35,108],"Audio-Visual":[37],"Neural":[38],"Syntax":[39],"Learner":[40],"(AV-NSL)":[41],"that":[42,77],"learns":[43],"by":[46,83],"listening":[47],"audio":[49],"looking":[51],"at":[52],"images,":[53],"without":[54],"ever":[55],"being":[56],"exposed":[57],"text.":[59],"By":[60],"training":[61],"on":[62],"paired":[63],"images":[64],"spoken":[66],"captions,":[67],"AV-NSL":[68],"exhibits":[69],"capability":[71],"infer":[73],"meaningful":[74],"structures":[76],"are":[78],"comparable":[79],"those":[81],"derived":[82],"naturally-supervised":[84],"text":[85],"parsers,":[86],"for":[87],"both":[88],"English":[89],"German.":[91],"Our":[92],"findings":[93],"extend":[94],"prior":[95],"work":[96],"in":[97],"unsupervised":[98],"language":[99],"acquisition":[100],"grounded":[104],"grammar":[105],"induction,":[106],"one":[109],"approach":[110],"bridge":[112],"gap":[114],"between":[115],"two":[117],"topics.":[118]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}