{"id":"https://openalex.org/W7166508477","doi":"https://doi.org/10.48550/arxiv.2606.28215","title":"HAT-4D: Lifting Monocular Video for 4D Multi-Object Interactions via Human-Agent Collaboration","display_name":"HAT-4D: Lifting Monocular Video for 4D Multi-Object Interactions via Human-Agent Collaboration","publication_year":2026,"publication_date":"2026-06-26","ids":{"openalex":"https://openalex.org/W7166508477","doi":"https://doi.org/10.48550/arxiv.2606.28215"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.28215","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.28215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.28215","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139631373","display_name":"Jiaxin Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jiaxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139581724","display_name":"Yuxiang Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Yuxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139602735","display_name":"Zhenkai Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhenkai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139595775","display_name":"Xinrui Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Xinrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139598231","display_name":"Haoyuan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haoyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139613959","display_name":"Yichen Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139602164","display_name":"Su Linxiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Linxiang, Su","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043761995","display_name":"C Shijia Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Chenyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139603670","display_name":"Mingyu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Mingyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049726719","display_name":"Yan Ding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043456325","display_name":"Boran Wen","orcid":"https://orcid.org/0000-0002-0987-4711"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wen, Boran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139561817","display_name":"Li Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139588530","display_name":"Ruiyang Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Ruiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139590511","display_name":"Yong-Lu Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yong-Lu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.3880000114440918,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.3880000114440918,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.22210000455379486,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.07660000026226044,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/monocular","display_name":"Monocular","score":0.8604000210762024},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6547999978065491},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5978999733924866},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5616999864578247},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4918000102043152},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.4025000035762787},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.38940000534057617},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.38440001010894775}],"concepts":[{"id":"https://openalex.org/C65909025","wikidata":"https://www.wikidata.org/wiki/Q1945033","display_name":"Monocular","level":2,"score":0.8604000210762024},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7894999980926514},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6547999978065491},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6172999739646912},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5978999733924866},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5616999864578247},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4941999912261963},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4918000102043152},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.4025000035762787},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.38940000534057617},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.38440001010894775},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.3806999921798706},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3659000098705292},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3474000096321106},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.3472000062465668},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.3431999981403351},{"id":"https://openalex.org/C109950114","wikidata":"https://www.wikidata.org/wiki/Q4464732","display_name":"3D reconstruction","level":2,"score":0.32519999146461487},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.2930999994277954},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.26460000872612},{"id":"https://openalex.org/C2776977519","wikidata":"https://www.wikidata.org/wiki/Q4636326","display_name":"3D interaction","level":3,"score":0.2639999985694885},{"id":"https://openalex.org/C48007421","wikidata":"https://www.wikidata.org/wiki/Q676252","display_name":"Motion capture","level":3,"score":0.25920000672340393},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.28215","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.28215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.28215","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.28215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.42731529474258423}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Extracting":[0],"dynamic":[1],"4D":[2,27,98,126],"object":[3],"interactions":[4,69],"from":[5,73],"massive,":[6],"in-the-wild":[7],"monocular":[8,26,125],"videos":[9],"offers":[10],"a":[11,74,81,111,131,165],"highly":[12],"efficient":[13],"data":[14,113,176,189],"collection":[15],"pathway":[16],"for":[17,124,186],"scaling":[18],"Embodied":[19],"AI":[20],"and":[21,41,67,91,97,140,190],"training":[22],"VLAs.":[23],"However,":[24],"existing":[25],"reconstruction":[28],"methods":[29],"primarily":[30],"focus":[31],"on":[32,106,137,151],"isolated":[33],"objects,":[34],"often":[35],"failing":[36],"under":[37],"the":[38,55,62,117,175],"severe":[39],"occlusions":[40,93],"complex":[42],"dynamics":[43],"inherent":[44],"in":[45],"multi-object":[46],"interactions.":[47],"To":[48],"bridge":[49],"this":[50],"gap,":[51],"we":[52],"propose":[53],"HAT-4D,":[54],"first":[56],"agentic":[57],"framework":[58],"designed":[59],"to":[60],"reconstruct":[61],"3D":[63,95],"geometry,":[64],"temporal":[65,141],"dynamics,":[66],"physical":[68,138],"of":[70,119,168],"multiple":[71],"objects":[72],"single":[75],"video.":[76],"By":[77],"integrating":[78],"VLMs":[79],"with":[80],"multi-level":[82],"human-in-the-loop":[83],"feedback":[84,170],"mechanism,":[85],"HAT-4D":[86,115,147,179],"efficiently":[87],"resolves":[88],"depth":[89],"ambiguities":[90],"interaction-induced":[92],"during":[94],"generation":[96],"propagation,":[99],"yielding":[100],"physically":[101],"plausible":[102],"assets":[103],"without":[104],"relying":[105],"expensive":[107],"multicamera":[108],"rigs.":[109],"As":[110],"scalable":[112],"engine,":[114],"facilitates":[116],"creation":[118],"MVOIK-4D,":[120],"an":[121],"open-world":[122],"benchmark":[123],"interaction":[127,172],"reconstruction,":[128],"accompanied":[129],"by":[130,178],"novel":[132],"multi-dimensional":[133],"evaluation":[134,153],"protocol":[135],"focused":[136],"plausibility":[139],"consistency.":[142],"Extensive":[143],"experiments":[144],"demonstrate":[145],"that":[146,163],"achieves":[148],"SOTA":[149],"performance":[150,183],"most":[152],"metrics,":[154],"while":[155],"maintaining":[156],"competitive":[157],"semantic":[158],"alignment.":[159],"Ablation":[160],"studies":[161],"show":[162],"introducing":[164],"small":[166],"amount":[167],"human":[169],"improves":[171,181],"reconstruction.":[173],"Moreover,":[174],"produced":[177],"effectively":[180],"baseline":[182],"when":[184],"used":[185],"fine-tuning.":[187],"Our":[188],"code":[191],"are":[192],"available":[193],"at":[194],"https://lijiaxin0111.github.io/HAT4D/":[195]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-30T00:00:00"}