{"id":"https://openalex.org/W7164009274","doi":"https://doi.org/10.48550/arxiv.2606.07577","title":"OmniMem: Perturbation-aware Memory Compression for Streaming Audio-Visual LLMs","display_name":"OmniMem: Perturbation-aware Memory Compression for Streaming Audio-Visual LLMs","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7164009274","doi":"https://doi.org/10.48550/arxiv.2606.07577"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.07577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.07577","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138218725","display_name":"Guangzhi Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Guangzhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138201238","display_name":"Yixuan Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yixuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138263634","display_name":"Yudong Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yudong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138239665","display_name":"Chao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Chao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.37070000171661377,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.37070000171661377,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.13120000064373016,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.11729999631643295,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/compression","display_name":"Compression (physics)","score":0.6327999830245972},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6240000128746033},{"id":"https://openalex.org/keywords/data-compression","display_name":"Data compression","score":0.5705000162124634},{"id":"https://openalex.org/keywords/memory-management","display_name":"Memory management","score":0.511900007724762},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.4934000074863434},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4729999899864197},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.3937999904155731}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7792999744415283},{"id":"https://openalex.org/C180016635","wikidata":"https://www.wikidata.org/wiki/Q2712821","display_name":"Compression (physics)","level":2,"score":0.6327999830245972},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6240000128746033},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.5705000162124634},{"id":"https://openalex.org/C176649486","wikidata":"https://www.wikidata.org/wiki/Q2308807","display_name":"Memory management","level":3,"score":0.511900007724762},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4934000074863434},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4729999899864197},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3937999904155731},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.36649999022483826},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3199000060558319},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.313400000333786},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31209999322891235},{"id":"https://openalex.org/C13481523","wikidata":"https://www.wikidata.org/wiki/Q412438","display_name":"Image compression","level":4,"score":0.29589998722076416},{"id":"https://openalex.org/C98986596","wikidata":"https://www.wikidata.org/wiki/Q1143031","display_name":"Semiconductor memory","level":2,"score":0.28369998931884766},{"id":"https://openalex.org/C2985957978","wikidata":"https://www.wikidata.org/wiki/Q492","display_name":"Human memory","level":3,"score":0.28119999170303345},{"id":"https://openalex.org/C82687282","wikidata":"https://www.wikidata.org/wiki/Q66221","display_name":"Auxiliary memory","level":2,"score":0.2802000045776367},{"id":"https://openalex.org/C2778192920","wikidata":"https://www.wikidata.org/wiki/Q16874989","display_name":"Signal compression","level":4,"score":0.2646999955177307},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.25189998745918274}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.07577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.07577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Audio-visual":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"hold":[5],"strong":[6,134],"promise":[7],"for":[8,39],"long-form":[9],"video":[10,24],"understanding,":[11],"yet":[12],"their":[13],"long-video":[14],"inference":[15],"is":[16],"fundamentally":[17],"limited":[18],"by":[19,138],"the":[20,66,71,107,143],"linear":[21],"growth":[22],"of":[23],"tokens":[25,49],"and":[26,62,78,121,126],"key-value":[27],"(KV)":[28],"caches.":[29],"We":[30],"present":[31],"OmniMem,":[32],"a":[33,53],"memory-efficient":[34],"streaming":[35],"framework":[36],"designed":[37],"specifically":[38],"audio-visual":[40],"LLMs.":[41],"Unlike":[42],"existing":[43],"compression":[44,95,136],"methods":[45],"that":[46,58,129],"treat":[47],"all":[48],"uniformly,":[50],"OmniMem":[51,74,130],"introduces":[52],"modality-aware":[54],"memory":[55,84,88,145],"allocation":[56],"strategy":[57],"separately":[59],"manages":[60],"visual":[61],"audio":[63],"contexts,":[64],"addressing":[65],"severe":[67],"token":[68],"imbalance":[69],"between":[70],"two":[72],"modalities.":[73],"further":[75],"preserves":[76],"informative":[77],"non-redundant":[79],"KV":[80],"states":[81],"through":[82],"perturbation-aware":[83],"selection,":[85],"enabling":[86],"compact":[87],"without":[89],"sacrificing":[90],"long-range":[91],"understanding.":[92],"To":[93],"strengthen":[94],"under":[96,142],"realistic":[97],"deployment":[98],"constraints,":[99],"we":[100],"also":[101],"explore":[102],"budget-aware":[103],"fine-tuning,":[104],"which":[105],"encourages":[106],"model":[108],"to":[109],"consolidate":[110],"useful":[111],"information":[112],"into":[113],"retained":[114],"memory.":[115],"Experiments":[116],"on":[117],"VideoMME":[118],"Long,":[119],"LVBench,":[120],"LVOmniBench":[122],"with":[123,147],"video-SALMONN":[124],"2+":[125],"Qwen-2.5-Omni":[127],"show":[128],"consistently":[131],"improves":[132],"over":[133],"training-free":[135],"baselines":[137],"2-4%":[139],"absolute":[140],"accuracy":[141],"same":[144],"budgets,":[146],"an":[148],"additional":[149],"1-2%":[150],"gain":[151],"after":[152],"fine-tuning.":[153]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-10T00:00:00"}