{"id":"https://openalex.org/W4388328933","doi":"https://doi.org/10.48550/arxiv.2311.00926","title":"M2T2: Multi-Task Masked Transformer for Object-centric Pick and Place","display_name":"M2T2: Multi-Task Masked Transformer for Object-centric Pick and Place","alternate_titles":[],"publication_year":2023,"publication_date":"2023-11-02","ids":{"openalex":"https://openalex.org/W4388328933","doi":"https://doi.org/10.48550/arxiv.2311.00926"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2311.00926","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.00926","pdf_url":"https://arxiv.org/pdf/2311.00926","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4388328933"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2311.00926","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058911745","display_name":"Wentao Yuan","orcid":"https://orcid.org/0000-0002-3836-8877","observed_orcids":["https://orcid.org/0000-0002-3836-8877"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Yuan, Wentao","raw_affiliation_strings":["University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114179160","display_name":"Adithyavairavan Murali","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Murali, Adithyavairavan","raw_affiliation_strings":["University of Washington , NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington , NVIDIA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210127875"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021728625","display_name":"Arsalan Mousavian","orcid":"https://orcid.org/0000-0001-9356-9455","observed_orcids":["https://orcid.org/0000-0001-9356-9455"]},"institutions":[{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Mousavian, Arsalan","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":["https://openalex.org/I4210127875"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108257764","display_name":"Dieter Fox","orcid":"https://orcid.org/0009-0009-4694-9127","observed_orcids":["https://orcid.org/0009-0009-4694-9127"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210127875","display_name":"Nvidia (United States)","ror":"https://ror.org/03jdj4y14","country_code":"US","type":"company","lineage":["https://openalex.org/I4210127875"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Fox, Dieter","raw_affiliation_strings":["University of Washington , NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington , NVIDIA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210127875"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":46,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10719","display_name":"3D Shape Modeling and Analysis","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-task-transformer","display_name":"multi-task transformer","score":0.6159999966621399},{"id":"https://openalex.org/keywords/robotic-manipulation","display_name":"robotic manipulation","score":0.5189999938011169},{"id":"https://openalex.org/keywords/point-clouds","display_name":"point clouds","score":0.45500001311302185},{"id":"https://openalex.org/keywords/sim-to-real-transfer","display_name":"sim-to-real transfer","score":0.875},{"id":"https://openalex.org/keywords/zero-shot-generalization","display_name":"zero-shot generalization","score":0.7379999756813049},{"id":"https://openalex.org/keywords/rlbench","display_name":"RLBench","score":0.7639999985694885},{"id":"https://openalex.org/keywords/cluttered-scenes","display_name":"cluttered scenes","score":0.675000011920929}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8242782950401306},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6590353846549988},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6503232717514038},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6185268759727478},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5595516562461853},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.5378898978233337},{"id":"https://openalex.org/C131979681","wikidata":"https://www.wikidata.org/wiki/Q1899648","display_name":"Point cloud","level":2,"score":0.5009660720825195},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.49387437105178833},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.43946966528892517},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32517874240875244},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08345615863800049},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2311.00926","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.00926","pdf_url":"https://arxiv.org/pdf/2311.00926","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2311.00926","doi":"https://doi.org/10.48550/arxiv.2311.00926","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2311.00926","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2311.00926","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.00926","pdf_url":"https://arxiv.org/pdf/2311.00926","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G399295477","display_name":"Collaborative Research: NRI: FND: Graph Neural Networks for Multi-Object Manipulation","funder_award_id":"2024057","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4388328933.pdf","grobid_xml":"https://content.openalex.org/works/W4388328933.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2432481613","https://openalex.org/W2524545028","https://openalex.org/W2624503621","https://openalex.org/W2908510526","https://openalex.org/W2945413402","https://openalex.org/W2970710335","https://openalex.org/W2991687685","https://openalex.org/W3007769740","https://openalex.org/W3035198432","https://openalex.org/W3037767967","https://openalex.org/W3102191234","https://openalex.org/W3105769864","https://openalex.org/W3108425990","https://openalex.org/W3135367836","https://openalex.org/W3150774829","https://openalex.org/W3175388581","https://openalex.org/W3193576605","https://openalex.org/W3206916018","https://openalex.org/W3209621543","https://openalex.org/W4225544038","https://openalex.org/W4303648971","https://openalex.org/W4382319504","https://openalex.org/W4383066118","https://openalex.org/W4383097638","https://openalex.org/W4383108457","https://openalex.org/W4385430679"],"related_works":["https://openalex.org/W7135180683","https://openalex.org/W4402727730","https://openalex.org/W4413918030","https://openalex.org/W4409262207","https://openalex.org/W7129014066","https://openalex.org/W7220832861","https://openalex.org/W6949276541","https://openalex.org/W4297812239","https://openalex.org/W7138311089","https://openalex.org/W7135406014"],"abstract_inverted_index":{"With":[0],"the":[1,45,123,140,144,163],"advent":[2],"of":[3,47,60,70,86,122,180,187],"large":[4],"language":[5,32,181],"models":[6,24,55,150],"and":[7,108,157,197],"large-scale":[8,128],"robotic":[9],"datasets,":[10],"there":[11],"has":[12],"been":[13],"tremendous":[14],"progress":[15],"in":[16,57,95,154,159,184,193],"high-level":[17],"decision-making":[18],"for":[19,66,113,169],"object":[20,164],"manipulation.":[21],"These":[22],"generic":[23],"are":[25,199],"able":[26],"to":[27,40,44,166],"interpret":[28],"complex":[29],"tasks":[30,183],"using":[31],"commands,":[33],"but":[34,63],"they":[35],"often":[36],"have":[37],"difficulties":[38],"generalizing":[39],"out-of-distribution":[41],"objects":[42,94,192],"due":[43],"inability":[46],"low-level":[48,58,87],"action":[49,115],"primitives.":[50],"In":[51],"contrast,":[52],"existing":[53],"task-specific":[54,149],"excel":[56],"manipulation":[59],"unknown":[61],"objects,":[62],"only":[64],"work":[65,90],"a":[67,79,100,118,127,178],"single":[68,80],"type":[69],"action.":[71],"To":[72],"bridge":[73],"this":[74],"gap,":[75],"we":[76],"present":[77],"M2T2,":[78],"model":[81,102],"that":[82,89],"supplies":[83],"different":[84,114],"types":[85],"actions":[88],"robustly":[91],"on":[92,126,139,177,190,201],"arbitrary":[93],"cluttered":[96],"scenes.":[97],"M2T2":[98,134,172],"is":[99],"transformer":[101],"which":[103],"reasons":[104],"about":[105,152],"contact":[106],"points":[107],"predicts":[109],"valid":[110],"gripper":[111],"poses":[112],"modes":[116],"given":[117],"raw":[119],"point":[120],"cloud":[121],"scene.":[124],"Trained":[125],"synthetic":[129],"dataset":[130],"with":[131,147],"128K":[132],"scenes,":[133],"achieves":[135,174],"zero-shot":[136],"sim2real":[137],"transfer":[138],"real":[141,195],"robot,":[142],"outperforming":[143],"baseline":[145],"system":[146],"state-of-the-art":[148,175],"by":[151],"19%":[153],"overall":[155],"performance":[156],"37.5%":[158],"challenging":[160],"scenes":[161],"where":[162],"needs":[165],"be":[167],"re-oriented":[168],"collision-free":[170],"placement.":[171],"also":[173],"results":[176],"subset":[179],"conditioned":[182],"RLBench.":[185],"Videos":[186],"robot":[188],"experiments":[189],"unseen":[191],"both":[194],"world":[196],"simulation":[198],"available":[200],"our":[202],"project":[203],"website":[204],"https://m2-t2.github.io.":[205]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4388328933","counts_by_year":[{"year":2026,"cited_by_count":19},{"year":2025,"cited_by_count":22},{"year":2024,"cited_by_count":5}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2023-11-04T00:00:00"}
