{"id":"https://openalex.org/W4389665702","doi":"https://doi.org/10.1109/iros55552.2023.10342381","title":"PACT: Perception-Action Causal Transformer for Autoregressive Robotics Pre-Training","display_name":"PACT: Perception-Action Causal Transformer for Autoregressive Robotics Pre-Training","publication_year":2023,"publication_date":"2023-10-01","ids":{"openalex":"https://openalex.org/W4389665702","doi":"https://doi.org/10.1109/iros55552.2023.10342381"},"language":"en","primary_location":{"id":"doi:10.1109/iros55552.2023.10342381","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros55552.2023.10342381","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062522127","display_name":"Rogerio Bonatti","orcid":"https://orcid.org/0000-0003-3015-9613"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Rogerio Bonatti","raw_affiliation_strings":["Microsoft"],"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050296324","display_name":"Sai Vemprala","orcid":"https://orcid.org/0000-0001-7554-5417"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sai Vemprala","raw_affiliation_strings":["Scaled Foundations"],"affiliations":[{"raw_affiliation_string":"Scaled Foundations","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100529011","display_name":"Shuang Ma","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107260","display_name":"Apple (United Kingdom)","ror":"https://ror.org/01vpeym60","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210107260"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Shuang Ma","raw_affiliation_strings":["Apple"],"affiliations":[{"raw_affiliation_string":"Apple","institution_ids":["https://openalex.org/I4210107260"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049915877","display_name":"Felipe Vieira Frujeri","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Felipe Frujeri","raw_affiliation_strings":["Microsoft"],"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043320256","display_name":"Shuhang Chen","orcid":"https://orcid.org/0000-0003-4634-3220"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shuhang Chen","raw_affiliation_strings":["Scaled Foundations"],"affiliations":[{"raw_affiliation_string":"Scaled Foundations","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101933711","display_name":"Ashish Kapoor","orcid":"https://orcid.org/0009-0004-3764-8449"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ashish Kapoor","raw_affiliation_strings":["Scaled Foundations"],"affiliations":[{"raw_affiliation_string":"Scaled Foundations","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5062522127"],"corresponding_institution_ids":["https://openalex.org/I4210164937"],"apc_list":null,"apc_paid":null,"fwci":2.452,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.91369208,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"3621","last_page":"3627"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7260839343070984},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.717528223991394},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.6448912620544434},{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.5980561375617981},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.49848341941833496},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4702051281929016},{"id":"https://openalex.org/keywords/mobile-robot","display_name":"Mobile robot","score":0.44720447063446045},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.13282686471939087}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7260839343070984},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.717528223991394},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6448912620544434},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.5980561375617981},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.49848341941833496},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4702051281929016},{"id":"https://openalex.org/C19966478","wikidata":"https://www.wikidata.org/wiki/Q4810574","display_name":"Mobile robot","level":3,"score":0.44720447063446045},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13282686471939087},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iros55552.2023.10342381","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros55552.2023.10342381","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W2194775991","https://openalex.org/W2896457183","https://openalex.org/W2962744691","https://openalex.org/W2963800628","https://openalex.org/W2966715458","https://openalex.org/W2969748094","https://openalex.org/W2969876226","https://openalex.org/W2970340522","https://openalex.org/W2981851019","https://openalex.org/W2983104849","https://openalex.org/W2991391304","https://openalex.org/W2997591391","https://openalex.org/W3009928773","https://openalex.org/W3034500398","https://openalex.org/W3114554915","https://openalex.org/W3154596443","https://openalex.org/W3160050461","https://openalex.org/W3169291081","https://openalex.org/W3190965961","https://openalex.org/W3195577433","https://openalex.org/W3205261317","https://openalex.org/W3209493138","https://openalex.org/W3215626407","https://openalex.org/W4221159977","https://openalex.org/W4221165505","https://openalex.org/W4221167110","https://openalex.org/W4234552385","https://openalex.org/W4283721947","https://openalex.org/W4287112297","https://openalex.org/W4287126489","https://openalex.org/W4287209565","https://openalex.org/W4292779060","https://openalex.org/W4312289535","https://openalex.org/W4312301659","https://openalex.org/W4385245566","https://openalex.org/W6739901393","https://openalex.org/W6752941136","https://openalex.org/W6755207826","https://openalex.org/W6755668676","https://openalex.org/W6763422710","https://openalex.org/W6765527290","https://openalex.org/W6766850857","https://openalex.org/W6766904570","https://openalex.org/W6767211374","https://openalex.org/W6778883912","https://openalex.org/W6790188114","https://openalex.org/W6790271839","https://openalex.org/W6791353385","https://openalex.org/W6793736971","https://openalex.org/W6793853427","https://openalex.org/W6796289742","https://openalex.org/W6797065961","https://openalex.org/W6799838802","https://openalex.org/W6800751262","https://openalex.org/W6801810553","https://openalex.org/W6802977921","https://openalex.org/W6803872405","https://openalex.org/W6804244202","https://openalex.org/W6810080435","https://openalex.org/W6810296985","https://openalex.org/W6810536332","https://openalex.org/W6839204832"],"related_works":["https://openalex.org/W1508899372","https://openalex.org/W2039460805","https://openalex.org/W4250956039","https://openalex.org/W4240485100","https://openalex.org/W2056130799","https://openalex.org/W2045758229","https://openalex.org/W405964254","https://openalex.org/W4254970379","https://openalex.org/W2122871747","https://openalex.org/W3114279067"],"abstract_inverted_index":{"Robotics":[0],"has":[1],"long":[2],"been":[3],"a":[4,35,39,47,54,64,78,98,121,141,146,154,185,199,209],"field":[5],"riddled":[6],"with":[7],"complex":[8],"systems":[9],"architectures":[10],"whose":[11],"modules":[12],"and":[13,23,86,95,134,153,194,221],"connections,":[14],"whether":[15],"traditional":[16],"or":[17],"learning-based,":[18],"require":[19],"significant":[20],"human":[21],"expertise":[22],"prior":[24],"knowledge.":[25],"Inspired":[26],"by":[27],"large":[28,201],"pre-trained":[29],"language":[30],"models,":[31],"this":[32,115],"work":[33],"introduces":[34],"paradigm":[36],"for":[37,50,97,190,203],"pretraining":[38],"general":[40],"purpose":[41],"representation":[42,117,212],"that":[43,68,114,144,157,165],"can":[44,118,216],"serve":[45],"as":[46,120,130,149],"starting":[48,123],"point":[49,124],"multiple":[51],"tasks":[52,128,192,214],"on":[53,105,170],"given":[55],"robot.":[56,100],"We":[57,136,163],"present":[58],"the":[59,106,173,224],"Perception-Action":[60],"Causal":[61],"Transformer":[62],"(PACT),":[63],"generative":[65],"transformer-based":[66],"architecture":[67],"aims":[69],"to":[70,125,183,197],"build":[71],"representations":[72],"directly":[73],"from":[74,188],"robot":[75,143],"data":[76],"in":[77,178],"self-supervised":[79],"fashion.":[80],"Through":[81],"autoregressive":[82],"prediction":[83],"of":[84,108,172,227],"states":[85],"actions":[87],"over":[88],"time,":[89],"our":[90],"model":[91,176,187,202,219],"implicitly":[92],"encodes":[93],"dynamics":[94],"behaviors":[96],"particular":[99],"Our":[101],"experimental":[102],"evaluation":[103],"focuses":[104],"domain":[107],"mobile":[109],"agents,":[110],"where":[111],"we":[112,215],"show":[113,164],"robot-specific":[116],"function":[119],"single":[122,186],"achieve":[126],"distinct":[127],"such":[129,228],"safe":[131],"navigation,":[132],"localization":[133],"mapping.":[135],"evaluate":[137],"two":[138],"form":[139],"factors:":[140],"wheeled":[142],"uses":[145,158],"LiDAR":[147],"sensor":[148],"perception":[150],"input":[151],"(MuSHR),":[152],"simulated":[155],"agent":[156],"first-person":[159],"RGB":[160],"images":[161],"(Habitat).":[162],"finetuning":[166],"small":[167],"task-specific":[168],"networks":[169],"top":[171],"larger":[174],"pretrained":[175],"results":[177],"significantly":[179],"better":[180],"performance":[181,196],"compared":[182],"training":[184,198],"scratch":[189],"all":[191],"simultaneously,":[193],"comparable":[195],"separate":[200],"each":[204],"task":[205],"independently.":[206],"By":[207],"sharing":[208],"common":[210],"good-quality":[211],"across":[213],"lower":[217],"overall":[218],"capacity":[220],"speed":[222],"up":[223],"real-time":[225],"deployment":[226],"systems.":[229]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}