{"id":"https://openalex.org/W4308222497","doi":"https://doi.org/10.1145/3536221.3556630","title":"Make Acoustic and Visual Cues Matter: CH-SIMS v2.0 Dataset and AV-Mixup Consistent Module","display_name":"Make Acoustic and Visual Cues Matter: CH-SIMS v2.0 Dataset and AV-Mixup Consistent Module","publication_year":2022,"publication_date":"2022-11-04","ids":{"openalex":"https://openalex.org/W4308222497","doi":"https://doi.org/10.1145/3536221.3556630"},"language":"en","primary_location":{"id":"doi:10.1145/3536221.3556630","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3536221.3556630","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3536221.3556630","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2022 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3536221.3556630","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075349120","display_name":"Yihe Liu","orcid":"https://orcid.org/0000-0001-7487-2213"},"institutions":[{"id":"https://openalex.org/I34155123","display_name":"Hebei University of Science and Technology","ror":"https://ror.org/05h3pkk68","country_code":"CN","type":"education","lineage":["https://openalex.org/I34155123"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yihe Liu","raw_affiliation_strings":["Hebei University of Science and Technology, China"],"raw_orcid":"https://orcid.org/0000-0001-7487-2213","affiliations":[{"raw_affiliation_string":"Hebei University of Science and Technology, China","institution_ids":["https://openalex.org/I34155123"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101845160","display_name":"Ziqi Yuan","orcid":"https://orcid.org/0000-0003-2397-2163"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziqi Yuan","raw_affiliation_strings":["Tsinghua University, China"],"raw_orcid":"https://orcid.org/0000-0003-2397-2163","affiliations":[{"raw_affiliation_string":"Tsinghua University, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013271085","display_name":"Huisheng Mao","orcid":"https://orcid.org/0000-0002-6321-4239"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huisheng Mao","raw_affiliation_strings":["Tsinghua University, China"],"raw_orcid":"https://orcid.org/0000-0002-6321-4239","affiliations":[{"raw_affiliation_string":"Tsinghua University, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033915643","display_name":"Zhiyun Liang","orcid":"https://orcid.org/0000-0002-1959-7949"},"institutions":[{"id":"https://openalex.org/I52158045","display_name":"China Agricultural University","ror":"https://ror.org/04v3ywz14","country_code":"CN","type":"education","lineage":["https://openalex.org/I52158045"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyun Liang","raw_affiliation_strings":["China Agricultural University, China"],"raw_orcid":"https://orcid.org/0000-0002-1959-7949","affiliations":[{"raw_affiliation_string":"China Agricultural University, China","institution_ids":["https://openalex.org/I52158045"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012506420","display_name":"Wanqiuyue Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wanqiuyue Yang","raw_affiliation_strings":["Beijing University of Post and Telecommunications, China"],"raw_orcid":"https://orcid.org/0000-0002-2109-6684","affiliations":[{"raw_affiliation_string":"Beijing University of Post and Telecommunications, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088936558","display_name":"Yuanzhe Qiu","orcid":"https://orcid.org/0000-0003-0870-8807"},"institutions":[{"id":"https://openalex.org/I34155123","display_name":"Hebei University of Science and Technology","ror":"https://ror.org/05h3pkk68","country_code":"CN","type":"education","lineage":["https://openalex.org/I34155123"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanzhe Qiu","raw_affiliation_strings":["Hebei University of Science and Technology, China"],"raw_orcid":"https://orcid.org/0000-0003-0870-8807","affiliations":[{"raw_affiliation_string":"Hebei University of Science and Technology, China","institution_ids":["https://openalex.org/I34155123"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014238437","display_name":"Tie Cheng","orcid":"https://orcid.org/0000-0001-8791-9957"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tie Cheng","raw_affiliation_strings":["Beijing University of Post and Telecommunications, China"],"raw_orcid":"https://orcid.org/0000-0001-8791-9957","affiliations":[{"raw_affiliation_string":"Beijing University of Post and Telecommunications, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101809383","display_name":"Xiaoteng Li","orcid":null},"institutions":[{"id":"https://openalex.org/I34155123","display_name":"Hebei University of Science and Technology","ror":"https://ror.org/05h3pkk68","country_code":"CN","type":"education","lineage":["https://openalex.org/I34155123"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoteng Li","raw_affiliation_strings":["Hebei University of Science and Technology, Shijiazhuang,China, China"],"raw_orcid":"https://orcid.org/0000-0003-2878-9675","affiliations":[{"raw_affiliation_string":"Hebei University of Science and Technology, Shijiazhuang,China, China","institution_ids":["https://openalex.org/I34155123"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086201515","display_name":"Hua Xu","orcid":"https://orcid.org/0000-0002-7401-307X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hua Xu","raw_affiliation_strings":["Tsinghua University, China"],"raw_orcid":"https://orcid.org/0000-0002-7401-307X","affiliations":[{"raw_affiliation_string":"Tsinghua University, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101943964","display_name":"Kai Gao","orcid":"https://orcid.org/0000-0002-6920-850X"},"institutions":[{"id":"https://openalex.org/I34155123","display_name":"Hebei University of Science and Technology","ror":"https://ror.org/05h3pkk68","country_code":"CN","type":"education","lineage":["https://openalex.org/I34155123"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Gao","raw_affiliation_strings":["Hebei University of Science and Technology, China"],"raw_orcid":"https://orcid.org/0000-0002-6920-850X","affiliations":[{"raw_affiliation_string":"Hebei University of Science and Technology, China","institution_ids":["https://openalex.org/I34155123"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5075349120"],"corresponding_institution_ids":["https://openalex.org/I34155123"],"apc_list":null,"apc_paid":null,"fwci":6.1041,"has_fulltext":true,"cited_by_count":48,"citation_normalized_percentile":{"value":0.96831075,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"247","last_page":"258"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11795","display_name":"Humor Studies and Applications","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7109899520874023},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6869269609451294},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6495460271835327},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.6260560750961304},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"Sentiment analysis","score":0.5304056406021118},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5270338654518127},{"id":"https://openalex.org/keywords/sensory-cue","display_name":"Sensory cue","score":0.42494627833366394},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.369823694229126},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3494863510131836},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.34271180629730225}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7109899520874023},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6869269609451294},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6495460271835327},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.6260560750961304},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.5304056406021118},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5270338654518127},{"id":"https://openalex.org/C111370547","wikidata":"https://www.wikidata.org/wiki/Q7451120","display_name":"Sensory cue","level":2,"score":0.42494627833366394},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.369823694229126},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3494863510131836},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.34271180629730225},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3536221.3556630","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3536221.3556630","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3536221.3556630","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2022 International Conference on Multimodal Interaction","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3536221.3556630","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3536221.3556630","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3536221.3556630","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2022 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4675302116","display_name":null,"funder_award_id":"62173195","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5101386881","display_name":null,"funder_award_id":"F2022208006","funder_id":"https://openalex.org/F4320322163","funder_display_name":"Natural Science Foundation of Hebei Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322163","display_name":"Natural Science Foundation of Hebei Province","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4308222497.pdf","grobid_xml":"https://content.openalex.org/works/W4308222497.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W128638292","https://openalex.org/W1520861770","https://openalex.org/W2064675550","https://openalex.org/W2085662862","https://openalex.org/W2122563357","https://openalex.org/W2239141610","https://openalex.org/W2341528187","https://openalex.org/W2510170536","https://openalex.org/W2556418146","https://openalex.org/W2787581402","https://openalex.org/W2883409523","https://openalex.org/W2924126491","https://openalex.org/W2963104701","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964216663","https://openalex.org/W2964346351","https://openalex.org/W3034266838","https://openalex.org/W3034849760","https://openalex.org/W3035542229","https://openalex.org/W3037611961","https://openalex.org/W3100921325","https://openalex.org/W3101998545","https://openalex.org/W3103167052","https://openalex.org/W3159769479","https://openalex.org/W3174311454","https://openalex.org/W3206008172","https://openalex.org/W3209458476","https://openalex.org/W3214432797","https://openalex.org/W4221155339","https://openalex.org/W4241307704","https://openalex.org/W4287798527","https://openalex.org/W4300833946","https://openalex.org/W4301104990","https://openalex.org/W4307823382","https://openalex.org/W4312639100"],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W203959209","https://openalex.org/W2004831463","https://openalex.org/W2110287964","https://openalex.org/W2167701463"],"abstract_inverted_index":{"Multimodal":[0],"sentiment":[1,9,127,198],"analysis":[2,10],"(MSA),":[3],"which":[4,167],"supposes":[5],"to":[6,23,122,190,224],"improve":[7],"text-based":[8],"with":[11,86,96,115,183],"associated":[12],"acoustic":[13,37,117,170,216],"and":[14,38,80,105,108,118,139,171,207,217,220,235],"visual":[15,39,119,172,218],"modalities,":[16],"is":[17,154],"an":[18,78,165],"emerging":[19],"research":[20,212],"area":[21],"due":[22],"its":[24,94],"potential":[25],"applications":[26,228],"in":[27,54,143],"Human-Computer":[28],"Interaction":[29],"(HCI).":[30],"However,":[31],"existing":[32],"researches":[33],"observe":[34],"that":[35,203],"the":[36,45,64,69,74,83,87,90,131,136,140,144,147,169,184,186,222],"modalities":[40,173],"contribute":[41],"much":[42],"less":[43],"than":[44],"textual":[46],"modality,":[47],"termed":[48],"as":[49,164],"text-predominant.":[50],"Under":[51],"such":[52],"circumstances,":[53],"this":[55],"work,":[56],"we":[57,72],"emphasize":[58],"making":[59],"non-verbal":[60,124,195],"cues":[61,125,219],"matter":[62],"for":[63,126,197,213,229,239],"MSA":[65],"task.":[66],"Firstly,":[67],"from":[68,130,135,174],"resource":[70],"perspective,":[71,133],"present":[73],"CH-SIMS":[75,91,145,205],"v2.0":[76,92,206],"dataset,":[77,89],"extension":[79],"enhancement":[81],"of":[82,193],"CH-SIMS.":[84],"Compared":[85],"original":[88],"doubles":[93],"size":[95],"another":[97],"2121":[98],"refined":[99],"video":[100,113],"segments":[101,114],"containing":[102],"both":[103,204],"unimodal":[104,137],"multimodal":[106,180],"annotations":[107,138],"collects":[109],"10161":[110],"unlabelled":[111],"raw":[112],"rich":[116],"emotion-bearing":[120,215],"context":[121,181],"highlight":[123],"prediction.":[128,199],"Secondly,":[129],"model":[132,187],"benefiting":[134],"unsupervised":[141],"data":[142],"v2.0,":[146],"Acoustic":[148],"Visual":[149],"Mixup":[150],"Consistent":[151],"(AV-MC)":[152],"framework":[153,209],"proposed.":[155],"The":[156,232],"designed":[157],"modality":[158],"mixup":[159],"module":[160],"can":[161,188],"be":[162,191],"regarded":[163],"augmentation,":[166],"mixes":[168],"different":[175,194],"videos.":[176],"Through":[177],"drawing":[178],"unobserved":[179],"along":[182],"text,":[185],"learn":[189],"aware":[192],"contexts":[196],"Our":[200],"evaluations":[201],"demonstrate":[202],"AV-MC":[208],"enable":[210],"further":[211],"discovering":[214],"pave":[221],"path":[223],"interpretable":[225],"end-to-end":[226],"HCI":[227],"real-world":[230],"scenarios.":[231],"full":[233],"dataset":[234],"code":[236],"are":[237],"available":[238],"use":[240],"at":[241],"https://github.com/thuiar/ch-sims-v2.":[242]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":10}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}