{"id":"https://openalex.org/W4404088558","doi":"https://doi.org/10.48550/arxiv.2410.15316","title":"Ichigo: Mixed-Modal Early-Fusion Realtime Voice Assistant","display_name":"Ichigo: Mixed-Modal Early-Fusion Realtime Voice Assistant","publication_year":2024,"publication_date":"2024-10-20","ids":{"openalex":"https://openalex.org/W4404088558","doi":"https://doi.org/10.48550/arxiv.2410.15316"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.15316","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.15316","pdf_url":"https://arxiv.org/pdf/2410.15316","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.15316","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114534344","display_name":"Alan Dao","orcid":null},"institutions":[{"id":"https://openalex.org/I1292318990","display_name":"Nuance Communications (Austria)","ror":"https://ror.org/04ggpbw68","country_code":"AT","type":"company","lineage":["https://openalex.org/I1292318990","https://openalex.org/I4210125787"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Dao, Alan","raw_affiliation_strings":["Gia Tuan Dao"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Gia Tuan Dao","institution_ids":["https://openalex.org/I1292318990"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114534345","display_name":"Dinh Bach Vu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vu, Dinh Bach","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5114534346","display_name":"Huy Hoang Ha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ha, Huy Hoang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9671000242233276,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9448999762535095,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.7155026197433472},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.5534654259681702},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5502984523773193},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4094022810459137},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.14453023672103882},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.07273545861244202},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.0483645498752594}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7155026197433472},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.5534654259681702},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5502984523773193},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4094022810459137},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.14453023672103882},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.07273545861244202},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0483645498752594},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.15316","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.15316","pdf_url":"https://arxiv.org/pdf/2410.15316","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.15316","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.15316","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.15316","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.15316","pdf_url":"https://arxiv.org/pdf/2410.15316","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404088558.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"revolutionized":[5],"natural":[6],"language":[7,113],"processing,":[8],"but":[9,150],"their":[10],"application":[11],"to":[12,18,119,131,159,162],"speech-based":[13],"tasks":[14],"remains":[15],"challenging":[16],"due":[17],"the":[19,76,145],"complexities":[20],"of":[21,39,127,147],"integrating":[22],"audio":[23],"and":[24,41,54,63,71,94,115],"text":[25,64],"modalities.":[26,65],"This":[27,66],"paper":[28],"introduces":[29],"Ichigo,":[30],"a":[31,44,56,83,97,125,153],"mixed-modal":[32],"model":[33],"that":[34],"seamlessly":[35],"processes":[36],"interleaved":[37],"sequences":[38],"speech":[40,50,62,91,106,112],"text.":[42],"Utilizing":[43],"tokenized":[45],"early-fusion":[46],"approach,":[47],"Ichigo":[48,101,123],"quantizes":[49],"into":[51],"discrete":[52],"tokens":[53],"employs":[55],"uniform":[57],"transformer-based":[58],"architecture":[59],"for":[60,78,155],"both":[61],"method":[67],"enables":[68],"joint":[69],"reasoning":[70],"generation":[72],"across":[73],"modalities":[74],"without":[75],"need":[77],"separate":[79],"adapters.":[80],"We":[81],"present":[82],"comprehensive":[84],"training":[85],"methodology,":[86],"including":[87],"pre-training":[88],"on":[89,96,105],"multilingual":[90],"recognition":[92],"datasets":[93],"fine-tuning":[95],"curated":[98],"instruction":[99],"dataset.":[100],"demonstrates":[102],"state-of-the-art":[103],"performance":[104],"question-answering":[107],"benchmarks,":[108],"outperforming":[109],"existing":[110],"open-source":[111,163],"models":[114],"achieving":[116],"comparable":[117],"results":[118],"cascaded":[120],"systems.":[121],"Notably,":[122],"exhibits":[124],"latency":[126],"just":[128],"111":[129],"ms":[130],"first":[132],"token":[133],"generation,":[134],"significantly":[135],"lower":[136],"than":[137],"current":[138],"models.":[139,165],"Our":[140],"approach":[141],"not":[142],"only":[143],"advances":[144],"field":[146],"multimodal":[148],"AI":[149],"also":[151],"provides":[152],"framework":[154],"smaller":[156],"research":[157],"teams":[158],"contribute":[160],"effectively":[161],"speech-language":[164]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}