{"id":"https://openalex.org/W4389339279","doi":"https://doi.org/10.1145/3618338","title":"Scene-Aware Activity Program Generation with Language Guidance","display_name":"Scene-Aware Activity Program Generation with Language Guidance","publication_year":2023,"publication_date":"2023-12-05","ids":{"openalex":"https://openalex.org/W4389339279","doi":"https://doi.org/10.1145/3618338"},"language":"en","primary_location":{"id":"doi:10.1145/3618338","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3618338","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078942499","display_name":"Zejia Su","orcid":"https://orcid.org/0000-0001-9121-6779"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zejia Su","raw_affiliation_strings":["Shenzhen University, China"],"raw_orcid":"https://orcid.org/0000-0001-9121-6779","affiliations":[{"raw_affiliation_string":"Shenzhen University, China","institution_ids":["https://openalex.org/I180726961"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030729327","display_name":"Qingnan Fan","orcid":"https://orcid.org/0000-0003-1249-2826"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qingnan Fan","raw_affiliation_strings":["Vivo, China"],"raw_orcid":"https://orcid.org/0000-0003-1249-2826","affiliations":[{"raw_affiliation_string":"Vivo, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087161777","display_name":"Xuelin Chen","orcid":"https://orcid.org/0009-0007-0158-9469"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuelin Chen","raw_affiliation_strings":["Tencent AI Lab, China"],"raw_orcid":"https://orcid.org/0009-0007-0158-9469","affiliations":[{"raw_affiliation_string":"Tencent AI Lab, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067529252","display_name":"Oliver van Kaick","orcid":"https://orcid.org/0000-0001-9869-6832"},"institutions":[{"id":"https://openalex.org/I67031392","display_name":"Carleton University","ror":"https://ror.org/02qtvee93","country_code":"CA","type":"education","lineage":["https://openalex.org/I67031392"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Oliver Van Kaick","raw_affiliation_strings":["Carleton University, Canada"],"raw_orcid":"https://orcid.org/0000-0001-9869-6832","affiliations":[{"raw_affiliation_string":"Carleton University, Canada","institution_ids":["https://openalex.org/I67031392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034304352","display_name":"Hui Huang","orcid":"https://orcid.org/0000-0003-3212-0544"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hui Huang","raw_affiliation_strings":["Shenzhen University, China"],"raw_orcid":"https://orcid.org/0000-0003-3212-0544","affiliations":[{"raw_affiliation_string":"Shenzhen University, China","institution_ids":["https://openalex.org/I180726961"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013892799","display_name":"Ruizhen Hu","orcid":"https://orcid.org/0000-0002-6798-0336"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruizhen Hu","raw_affiliation_strings":["Shenzhen University, China"],"raw_orcid":"https://orcid.org/0000-0002-6798-0336","affiliations":[{"raw_affiliation_string":"Shenzhen University, China","institution_ids":["https://openalex.org/I180726961"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.4759,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.70158034,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"42","issue":"6","first_page":"1","last_page":"16"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8666069507598877},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.7264777421951294},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.6481038331985474},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5344347953796387},{"id":"https://openalex.org/keywords/scene-graph","display_name":"Scene graph","score":0.5211660861968994},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.49448370933532715},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4790281355381012},{"id":"https://openalex.org/keywords/rationality","display_name":"Rationality","score":0.44833311438560486},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.43209755420684814},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.41659823060035706},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4123358130455017},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3794783353805542},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3572933077812195},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3422146439552307},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3332095146179199},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.30166131258010864}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8666069507598877},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.7264777421951294},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.6481038331985474},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5344347953796387},{"id":"https://openalex.org/C179372163","wikidata":"https://www.wikidata.org/wiki/Q1406181","display_name":"Scene graph","level":3,"score":0.5211660861968994},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.49448370933532715},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4790281355381012},{"id":"https://openalex.org/C201717286","wikidata":"https://www.wikidata.org/wiki/Q938185","display_name":"Rationality","level":2,"score":0.44833311438560486},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.43209755420684814},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.41659823060035706},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4123358130455017},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3794783353805542},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3572933077812195},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3422146439552307},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3332095146179199},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.30166131258010864},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3618338","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3618338","pdf_url":null,"source":{"id":"https://openalex.org/S185367456","display_name":"ACM Transactions on Graphics","issn_l":"0730-0301","issn":["0730-0301","1557-7368"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Graphics","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5600000023841858,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G4480005133","display_name":null,"funder_award_id":"62322207, U21B2023, U2001206","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W1613163714","https://openalex.org/W1985795534","https://openalex.org/W2079252443","https://openalex.org/W2189089430","https://openalex.org/W2229480318","https://openalex.org/W2251648989","https://openalex.org/W2561529111","https://openalex.org/W2594519801","https://openalex.org/W2610034660","https://openalex.org/W2799002257","https://openalex.org/W2948589675","https://openalex.org/W2964935470","https://openalex.org/W2986068433","https://openalex.org/W2998116985","https://openalex.org/W3012588100","https://openalex.org/W3109097593","https://openalex.org/W3168640669","https://openalex.org/W3191240065","https://openalex.org/W4214625308","https://openalex.org/W4285378361","https://openalex.org/W4310463627"],"related_works":["https://openalex.org/W2118717649","https://openalex.org/W410723623","https://openalex.org/W2413243053","https://openalex.org/W2015341305","https://openalex.org/W4225593417","https://openalex.org/W2035068594","https://openalex.org/W2059783833","https://openalex.org/W2573498121","https://openalex.org/W3160494304","https://openalex.org/W1549477351"],"abstract_inverted_index":{"We":[0],"address":[1],"the":[2,29,36,63,70,82,103,119,142,146,152,157,162,170,174,202],"problem":[3],"of":[4,73,81,118,176,195,204],"scene-aware":[5],"activity":[6,14,104],"program":[7],"generation,":[8],"which":[9],"requires":[10],"decomposing":[11],"a":[12,24,54,59,74,107,113,138],"given":[13],"task":[15],"into":[16],"instructions":[17,86,101],"that":[18],"can":[19],"be":[20],"sequentially":[21],"performed":[22],"within":[23],"target":[25,83],"scene":[26,84,128,158],"to":[27,38,67,89,123,140,200],"complete":[28],"activity.":[30],"While":[31],"existing":[32],"methods":[33],"have":[34],"shown":[35],"ability":[37],"generate":[39,90,141],"rational":[40],"or":[41],"executable":[42],"programs,":[43],"generating":[44],"programs":[45,91],"with":[46,78,92],"both":[47],"high":[48,93],"rationality":[49,72,94],"and":[50,95,115,127,155,190],"executability":[51],"still":[52],"remains":[53],"challenge.":[55],"Hence,":[56],"we":[57],"propose":[58],"novel":[60],"method":[61,98,178],"where":[62,85],"key":[64],"idea":[65],"is":[66],"explicitly":[68],"combine":[69],"language":[71,76,125],"powerful":[75],"model":[77],"dynamic":[79],"perception":[80,160],"are":[87,134,167,185,197],"executed,":[88],"executability.":[96],"Our":[97],"iteratively":[99],"generates":[100],"for":[102,159],"program.":[105,147],"Specifically,":[106],"two-branch":[108],"feature":[109],"encoder":[110],"operates":[111],"on":[112,169,192],"language-based":[114],"graph-based":[116],"representation":[117],"current":[120],"generation":[121],"progress":[122],"extract":[124],"features":[126,133],"graph":[129],"features,":[130],"respectively.":[131],"These":[132],"then":[135],"used":[136],"by":[137],"predictor":[139],"next":[143,163],"instruction":[144],"in":[145,161],"Subsequently,":[148],"another":[149],"module":[150],"performs":[151],"predicted":[153],"action":[154],"updates":[156],"iteration.":[164],"Extensive":[165],"evaluations":[166],"conducted":[168],"VirtualHome-Env":[171],"dataset,":[172],"showing":[173],"advantages":[175],"our":[177,205],"over":[179],"previous":[180],"work.":[181],"Key":[182],"algorithmic":[183],"designs":[184],"validated":[186],"through":[187],"ablation":[188],"studies,":[189],"results":[191],"other":[193],"types":[194],"inputs":[196],"also":[198],"presented":[199],"show":[201],"generalizability":[203],"method.":[206]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}