{"id":"https://openalex.org/W4413785908","doi":"https://doi.org/10.1109/tpami.2025.3603653","title":"Creating Multimodal Interactive Digital Twin Characters From Videos: A Dataset and Baseline","display_name":"Creating Multimodal Interactive Digital Twin Characters From Videos: A Dataset and Baseline","publication_year":2025,"publication_date":"2025-08-28","ids":{"openalex":"https://openalex.org/W4413785908","doi":"https://doi.org/10.1109/tpami.2025.3603653","pmid":"https://pubmed.ncbi.nlm.nih.gov/40875422"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3603653","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3603653","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/tpami.2025.3603653","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056182986","display_name":"Meidai Xuanyuan","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Meidai Xuanyuan","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-1581-8743","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102942427","display_name":"Yuwang Wang","orcid":"https://orcid.org/0000-0002-6880-959X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuwang Wang","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6880-959X","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001909797","display_name":"Honglei Guo","orcid":"https://orcid.org/0000-0002-1485-1987"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Honglei Guo","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-1485-1987","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hanshi Qu","orcid":"https://orcid.org/0009-0000-1968-8691"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanshi Qu","raw_affiliation_strings":["University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-1968-8691","affiliations":[{"raw_affiliation_string":"University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119011488","display_name":"Kun Zhang","orcid":"https://orcid.org/0000-0001-8915-1769"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Zhang","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051061923","display_name":"LI Zhong-ming","orcid":"https://orcid.org/0009-0005-6525-0793"},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhongming Li","raw_affiliation_strings":["University of Edinburgh, Edinburgh, U.K","The University of Edinburgh, Edinburgh, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh, Edinburgh, U.K","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"The University of Edinburgh, Edinburgh, U.K","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Danping Yan","orcid":"https://orcid.org/0009-0008-7748-1806"},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danping Yan","raw_affiliation_strings":["University of North Carolina at Chapel Hill, Chapel Hill, NC, USA","The University of North Carolina at Chapel Hill, Chapel Hill, USA"],"raw_orcid":"https://orcid.org/0009-0008-7748-1806","affiliations":[{"raw_affiliation_string":"University of North Carolina at Chapel Hill, Chapel Hill, NC, USA","institution_ids":["https://openalex.org/I114027177"]},{"raw_affiliation_string":"The University of North Carolina at Chapel Hill, Chapel Hill, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100707702","display_name":"Tao Yu","orcid":"https://orcid.org/0009-0003-5701-9642"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao Yu","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112613657","display_name":"Jianhua Tao","orcid":"https://orcid.org/0000-0002-9344-6428"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianhua Tao","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9344-6428","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080722708","display_name":"Qionghai Dai","orcid":"https://orcid.org/0000-0001-7043-3061"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qionghai Dai","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5056182986"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.23565678,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"48","issue":"1","first_page":"92","last_page":"108"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10731","display_name":"Educational Games and Gamification","score":0.9014999866485596,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10731","display_name":"Educational Games and Gamification","score":0.9014999866485596,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.7861925363540649},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7578972578048706},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6435725688934326},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4547925591468811},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3429945707321167}],"concepts":[{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.7861925363540649},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7578972578048706},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6435725688934326},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4547925591468811},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3429945707321167},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D007091","descriptor_name":"Image Processing, Computer-Assisted","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D013690","descriptor_name":"Television","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D013690","descriptor_name":"Television","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D013690","descriptor_name":"Television","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D014584","descriptor_name":"User-Computer Interface","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D014584","descriptor_name":"User-Computer Interface","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D014584","descriptor_name":"User-Computer Interface","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D014741","descriptor_name":"Video Recording","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D014741","descriptor_name":"Video Recording","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D014741","descriptor_name":"Video Recording","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016208","descriptor_name":"Databases, Factual","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2025.3603653","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3603653","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40875422","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40875422","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":{"id":"doi:10.1109/tpami.2025.3603653","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3603653","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":42,"referenced_works":["https://openalex.org/W2133012565","https://openalex.org/W2143614671","https://openalex.org/W2146334809","https://openalex.org/W2160473997","https://openalex.org/W2325939864","https://openalex.org/W2556418146","https://openalex.org/W2594690981","https://openalex.org/W2769666294","https://openalex.org/W2890394457","https://openalex.org/W2962770929","https://openalex.org/W2979894294","https://openalex.org/W2999528291","https://openalex.org/W3034323190","https://openalex.org/W3048664667","https://openalex.org/W3097792222","https://openalex.org/W3099284785","https://openalex.org/W3154411171","https://openalex.org/W3197199219","https://openalex.org/W3207849023","https://openalex.org/W4200295417","https://openalex.org/W4221155337","https://openalex.org/W4291127200","https://openalex.org/W4312933868","https://openalex.org/W4372340947","https://openalex.org/W4375869161","https://openalex.org/W4386066083","https://openalex.org/W4386066256","https://openalex.org/W4386071653","https://openalex.org/W4386072021","https://openalex.org/W4386075984","https://openalex.org/W4386076250","https://openalex.org/W4388488609","https://openalex.org/W4389518991","https://openalex.org/W4389519254","https://openalex.org/W4389519488","https://openalex.org/W4390872742","https://openalex.org/W4390872774","https://openalex.org/W4390873123","https://openalex.org/W4402671594","https://openalex.org/W4402716313","https://openalex.org/W4407450206","https://openalex.org/W4413147584"],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"In":[0],"this":[1,64],"paper,":[2],"we":[3,67,125],"introduce":[4],"a":[5,127,174],"novel":[6],"framework":[7,129],"for":[8,177,191],"creating":[9],"multimodal":[10,81],"interactive":[11,196],"digital":[12,23,132,178,199,210],"twin":[13,24,133],"characters,":[14,134],"from":[15],"dialogue":[16,82,137],"videos":[17],"of":[18,28,94,112,120,136,194,209],"TV":[19],"shows.":[20],"Specifically,":[21],"these":[22],"characters":[25],"are":[26],"capable":[27],"responding":[29],"to":[30,130,204],"user":[31],"inputs":[32],"with":[33,84,152],"harmonious":[34],"textual,":[35],"vocal,":[36],"and":[37,50,59,79,91,149,170,184,197,206],"visual":[38,150],"content.":[39],"They":[40],"not":[41],"only":[42],"replicate":[43],"the":[44,69,99,117,189,192,202],"external":[45],"characteristics,":[46],"such":[47],"as":[48],"appearance":[49],"tone,":[51],"but":[52],"also":[53],"capture":[54],"internal":[55],"attributes,":[56],"including":[57],"personality":[58],"habitual":[60],"behaviors.":[61],"To":[62],"support":[63],"ambitious":[65],"task,":[66],"collect":[68],"Multimodal":[70],"Character-Centric":[71],"Conversation":[72],"Dataset,":[73],"named":[74],"MCCCD,":[75],"which":[76],"includes":[77],"character-specific":[78,171],"high-quality":[80],"data":[83,114],"detailed":[85,118],"annotations,":[86],"featuring":[87],"6.8":[88],"k":[89],"utterances":[90],"4.6":[92],"hours":[93],"audio/video":[95],"per":[96],"character.":[97],"Notably,":[98],"MCCCD":[100],"dataset":[101,183],"is":[102],"approximately":[103],"ten":[104],"times":[105],"larger":[106],"than":[107],"existing":[108,165],"datasets":[109],"in":[110,167],"terms":[111],"per-character":[113],"volume,":[115],"facilitating":[116],"modeling":[119],"complex":[121],"character-centric":[122],"traits.":[123],"Further,":[124],"propose":[126],"baseline":[128,186],"create":[131],"consists":[135],"generation":[138,144],"through":[139],"large":[140],"language":[141],"models,":[142,148],"voice":[143],"via":[145],"speech":[146],"synthesis":[147],"representation":[151],"3D":[153],"talking":[154],"head":[155],"models.":[156],"Experimental":[157],"results":[158],"demonstrate":[159],"that":[160],"our":[161],"approach":[162],"significantly":[163],"outperforms":[164],"methods":[166],"generating":[168],"consistent":[169],"responses,":[172],"setting":[173],"new":[175],"benchmark":[176],"character":[179],"creation.":[180],"Our":[181],"collected":[182],"proposed":[185],"have":[187],"paved":[188],"way":[190],"creation":[193],"highly":[195],"natural":[198],"avatars,":[200],"opening":[201],"door":[203],"extensive":[205],"practical":[207],"applications":[208],"humans.":[211]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}