{"id":"https://openalex.org/W4417125193","doi":"https://doi.org/10.1145/3757377.3763966","title":"Unifying Latent Action and Latent State Pre-training for Policy Learning from Videos","display_name":"Unifying Latent Action and Latent State Pre-training for Policy Learning from Videos","publication_year":2025,"publication_date":"2025-12-08","ids":{"openalex":"https://openalex.org/W4417125193","doi":"https://doi.org/10.1145/3757377.3763966"},"language":null,"primary_location":{"id":"doi:10.1145/3757377.3763966","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3757377.3763966","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SIGGRAPH Asia 2025 Conference Papers","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050719461","display_name":"Guangyan Chen","orcid":"https://orcid.org/0000-0002-2903-1957"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guangyan Chen","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2903-1957","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100416064","display_name":"Meiling Wang","orcid":"https://orcid.org/0000-0002-3618-7423"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meiling Wang","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3618-7423","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109736425","display_name":"Te Cui","orcid":"https://orcid.org/0009-0002-3265-3148"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Te Cui","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-3265-3148","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111172887","display_name":"Luojie Yang","orcid":"https://orcid.org/0009-0006-5429-2207"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Luojie Yang","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0006-5429-2207","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qi Shao","orcid":"https://orcid.org/0009-0009-5971-4773"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Shao","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-5971-4773","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101797837","display_name":"Lin Zhao","orcid":"https://orcid.org/0000-0001-8381-3397"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Zhao","raw_affiliation_strings":["JD, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8381-3397","affiliations":[{"raw_affiliation_string":"JD, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074422885","display_name":"Tianle Zhang","orcid":"https://orcid.org/0000-0002-0779-5905"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianle Zhang","raw_affiliation_strings":["JD, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-0779-5905","affiliations":[{"raw_affiliation_string":"JD, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050724832","display_name":"Yihang Li","orcid":"https://orcid.org/0000-0001-8548-2499"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yihang Li","raw_affiliation_strings":["JD, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8548-2499","affiliations":[{"raw_affiliation_string":"JD, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062124960","display_name":"Yi Yang","orcid":"https://orcid.org/0000-0003-3964-2433"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Yang","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3964-2433","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068949782","display_name":"Yufeng Yue","orcid":"https://orcid.org/0000-0001-6628-7946"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yufeng Yue","raw_affiliation_strings":["Beijing Institute of Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6628-7946","affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.20409999787807465,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.12770000100135803,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5995000004768372},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.592199981212616},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5249000191688538},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5023000240325928},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.382999986410141},{"id":"https://openalex.org/keywords/latent-variable","display_name":"Latent variable","score":0.375},{"id":"https://openalex.org/keywords/prior-probability","display_name":"Prior probability","score":0.3698999881744385}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.714900016784668},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.64410001039505},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5995000004768372},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.592199981212616},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5249000191688538},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5052000284194946},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5023000240325928},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.382999986410141},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.375},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.3698999881744385},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.3698999881744385},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.3325999975204468},{"id":"https://openalex.org/C104114177","wikidata":"https://www.wikidata.org/wiki/Q79782","display_name":"Motion (physics)","level":2,"score":0.3181000053882599},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.3140000104904175},{"id":"https://openalex.org/C2987834672","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Action recognition","level":3,"score":0.3098999857902527},{"id":"https://openalex.org/C65965080","wikidata":"https://www.wikidata.org/wiki/Q1806885","display_name":"Latent variable model","level":3,"score":0.3077999949455261},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2944999933242798},{"id":"https://openalex.org/C2780440489","wikidata":"https://www.wikidata.org/wiki/Q5227278","display_name":"Data-driven","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C112933361","wikidata":"https://www.wikidata.org/wiki/Q2845258","display_name":"Probabilistic latent semantic analysis","level":2,"score":0.25839999318122864}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3757377.3763966","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3757377.3763966","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SIGGRAPH Asia 2025 Conference Papers","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5876114600","display_name":null,"funder_award_id":"624B2025,62473050,92370203","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W2963802910","https://openalex.org/W3038298277","https://openalex.org/W3122520957","https://openalex.org/W3175995235","https://openalex.org/W3205786327","https://openalex.org/W3207832698","https://openalex.org/W4210560636","https://openalex.org/W4214570817","https://openalex.org/W4283788863","https://openalex.org/W4285218426","https://openalex.org/W4313156423","https://openalex.org/W4401415287","https://openalex.org/W4402702968","https://openalex.org/W4413144282","https://openalex.org/W4415302385","https://openalex.org/W4415797243","https://openalex.org/W4415797250","https://openalex.org/W4415798132"],"related_works":[],"abstract_inverted_index":{"Video":[0],"data":[1,12,28],"provides":[2],"an":[3,147],"accessible":[4],"and":[5,99,137,175,190,194],"rich":[6,62],"source":[7],"beyond":[8],"expensive":[9],"action-labeled":[10,148],"robot":[11,70,149,158],"for":[13,48,81],"advancing":[14],"robotic":[15,30],"learning":[16],"paradigms.":[17],"Motivated":[18],"by":[19,169],"this":[20,88],"potential,":[21],"researchers":[22],"investigate":[23],"methods":[24],"to":[25,55,71,111,132,151,156,192],"exploit":[26],"video":[27,116],"in":[29,171,183,196],"learning.":[31],"Recent":[32],"approaches":[33,42,52],"can":[34],"be":[35],"primarily":[36],"divided":[37],"into":[38,83],"two":[39],"categories:":[40],"Action-based":[41],"tokenize":[43],"latent":[44,97,100,113,120,135,139],"actions":[45,121,136],"from":[46,103,115,188],"videos":[47,131],"policy":[49,127,143],"pre-training.":[50],"State-based":[51],"pre-train":[53],"models":[54],"predict":[56,133],"subsequent":[57,138],"states.":[58,140],"The":[59],"former":[60],"establishes":[61],"motion":[63],"priors,":[64],"while":[65,118],"the":[66,69,126,142,153,167,172,177,197,201],"latter":[67],"empowers":[68],"anticipate":[72],"future":[73],"events.":[74],"These":[75],"complementary":[76],"capabilities":[77],"suggest":[78],"significant":[79],"potential":[80],"integration":[82],"a":[84,93,108,184],"unified":[85,109],"framework.":[86],"In":[87,200],"paper,":[89],"we":[90],"propose":[91],"UniMimic,":[92],"novel":[94],"approach":[95],"unifying":[96],"action":[98],"state":[101,123],"pre-training":[102,164],"videos.":[104],"We":[105],"first":[106],"train":[107],"tokenizer":[110],"learn":[112],"states":[114],"frames":[117],"deriving":[119],"between":[122],"tokens.":[124],"Subsequently,":[125],"is":[128,144],"pre-trained":[129],"on":[130,146],"these":[134],"Finally,":[141],"fine-tuned":[145],"dataset":[150],"transfer":[152],"learned":[154],"priors":[155],"precise":[157],"execution.":[159],"Experiments":[160],"exhibit":[161],"that":[162],"our":[163,204],"stage":[165],"enhances":[166],"performance":[168],"19%":[170],"Libero":[173],"benchmark":[174],"improves":[176],"average":[178],"number":[179],"of":[180,186],"tasks":[181],"completed":[182],"row":[185],"5":[187],"2.50":[189],"2.35":[191],"3.89":[193],"3.73":[195],"CALVIN":[198],"benchmark.":[199],"real-world":[202],"experiments,":[203],"method":[205],"still":[206],"delivers":[207],"improvements":[208],"exceeding":[209],"36%.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-12-08T00:00:00"}