{"id":"https://openalex.org/W3052028990","doi":"https://doi.org/10.1109/iros45743.2020.9341677","title":"Integrated Benchmarking and Design for Reproducible and Accessible Evaluation of Robotic Agents","display_name":"Integrated Benchmarking and Design for Reproducible and Accessible Evaluation of Robotic Agents","publication_year":2020,"publication_date":"2020-10-24","ids":{"openalex":"https://openalex.org/W3052028990","doi":"https://doi.org/10.1109/iros45743.2020.9341677","mag":"3052028990"},"language":"en","primary_location":{"id":"doi:10.1109/iros45743.2020.9341677","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros45743.2020.9341677","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2020 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019764202","display_name":"Jacopo Tani","orcid":"https://orcid.org/0000-0001-7419-4352"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Jacopo Tani","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000150746","display_name":"Andrea F. Daniele","orcid":"https://orcid.org/0000-0002-2977-0207"},"institutions":[{"id":"https://openalex.org/I160992636","display_name":"Toyota Technological Institute at Chicago","ror":"https://ror.org/02sn5gb64","country_code":"US","type":"education","lineage":["https://openalex.org/I160992636"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrea F. Daniele","raw_affiliation_strings":["Toyota Technological Institute at Chicago, Chicago, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Technological Institute at Chicago, Chicago, USA","institution_ids":["https://openalex.org/I160992636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038503162","display_name":"Gianmarco Bernasconi","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Gianmarco Bernasconi","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063292429","display_name":"Amaury Camus","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Amaury Camus","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103052945","display_name":"Aleksandar Petrov","orcid":"https://orcid.org/0000-0002-2128-4298"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Aleksandar Petrov","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090399194","display_name":"Anthony Courchesne","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Anthony Courchesne","raw_affiliation_strings":["Mila, Universit\u00e9 de Montr\u00e9al, Montr\u00e9al, Canada"],"affiliations":[{"raw_affiliation_string":"Mila, Universit\u00e9 de Montr\u00e9al, Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020127805","display_name":"Bhairav Mehta","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bhairav Mehta","raw_affiliation_strings":["Mila, Universit\u00e9 de Montr\u00e9al, Montr\u00e9al, Canada"],"affiliations":[{"raw_affiliation_string":"Mila, Universit\u00e9 de Montr\u00e9al, Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109584950","display_name":"Rohit Suri","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Rohit Suri","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045565540","display_name":"Tomasz Zaluska","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Tomasz Zaluska","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103153703","display_name":"Matthew R. Walter","orcid":"https://orcid.org/0000-0003-1425-6050"},"institutions":[{"id":"https://openalex.org/I160992636","display_name":"Toyota Technological Institute at Chicago","ror":"https://ror.org/02sn5gb64","country_code":"US","type":"education","lineage":["https://openalex.org/I160992636"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew R. Walter","raw_affiliation_strings":["Toyota Technological Institute at Chicago, Chicago, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Technological Institute at Chicago, Chicago, USA","institution_ids":["https://openalex.org/I160992636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029069358","display_name":"Emilio Frazzoli","orcid":null},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]},{"id":"https://openalex.org/I126452172","display_name":"Aptiv (United States)","ror":"https://ror.org/02967gp58","country_code":"US","type":"company","lineage":["https://openalex.org/I126452172","https://openalex.org/I4210107152"]}],"countries":["CH","US"],"is_corresponding":false,"raw_author_name":"Emilio Frazzoli","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland","nuTonomy, an Aptiv company, Boston, USA"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]},{"raw_affiliation_string":"nuTonomy, an Aptiv company, Boston, USA","institution_ids":["https://openalex.org/I126452172"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037065865","display_name":"Liam Paull","orcid":"https://orcid.org/0000-0003-2492-6660"},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Liam Paull","raw_affiliation_strings":["Mila, Universit\u00e9 de Montr\u00e9al, Montr\u00e9al, Canada"],"affiliations":[{"raw_affiliation_string":"Mila, Universit\u00e9 de Montr\u00e9al, Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057460079","display_name":"Andrea Censi","orcid":"https://orcid.org/0000-0001-5162-0398"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Andrea Censi","raw_affiliation_strings":["ETH Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":13,"corresponding_author_ids":["https://openalex.org/A5019764202"],"corresponding_institution_ids":["https://openalex.org/I35440088"],"apc_list":null,"apc_paid":null,"fwci":2.9091,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.93431112,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"6229","last_page":"6236"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12784","display_name":"Modular Robots and Swarm Intelligence","score":0.9829999804496765,"subfield":{"id":"https://openalex.org/subfields/2210","display_name":"Mechanical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.9697999954223633,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.9463787078857422},{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.7895045280456543},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7190654873847961},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5520569682121277},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5498093962669373},{"id":"https://openalex.org/keywords/autonomy","display_name":"Autonomy","score":0.49397164583206177},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.47048237919807434},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4696720540523529},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.443869948387146},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.43840456008911133},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.39797383546829224},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.38363274931907654},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33347588777542114},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.3215292990207672},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.1572672426700592},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11409041285514832}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9463787078857422},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.7895045280456543},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7190654873847961},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5520569682121277},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5498093962669373},{"id":"https://openalex.org/C65414064","wikidata":"https://www.wikidata.org/wiki/Q484105","display_name":"Autonomy","level":2,"score":0.49397164583206177},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.47048237919807434},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4696720540523529},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.443869948387146},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.43840456008911133},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.39797383546829224},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.38363274931907654},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33347588777542114},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.3215292990207672},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.1572672426700592},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11409041285514832},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iros45743.2020.9341677","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros45743.2020.9341677","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2020 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.6299999952316284}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1504992226","https://openalex.org/W1559961358","https://openalex.org/W1574680346","https://openalex.org/W1820821744","https://openalex.org/W2074070730","https://openalex.org/W2125188427","https://openalex.org/W2150066425","https://openalex.org/W2270443308","https://openalex.org/W2340897893","https://openalex.org/W2414283705","https://openalex.org/W2415595313","https://openalex.org/W2505763307","https://openalex.org/W2529170537","https://openalex.org/W2553603302","https://openalex.org/W2604704868","https://openalex.org/W2615547864","https://openalex.org/W2724857106","https://openalex.org/W2737347195","https://openalex.org/W2754517384","https://openalex.org/W2901136733","https://openalex.org/W2948013437","https://openalex.org/W2953482357","https://openalex.org/W2962867954","https://openalex.org/W2963120839","https://openalex.org/W2964138223","https://openalex.org/W2998660800","https://openalex.org/W3003535010","https://openalex.org/W3035172746","https://openalex.org/W3035574168","https://openalex.org/W4285719527","https://openalex.org/W4295719664","https://openalex.org/W6729999887","https://openalex.org/W6735676390","https://openalex.org/W6744123322","https://openalex.org/W6745935785","https://openalex.org/W6756486208","https://openalex.org/W6760782946","https://openalex.org/W6762919599","https://openalex.org/W6780559895"],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699","https://openalex.org/W2992534160","https://openalex.org/W1595058678","https://openalex.org/W3022812046"],"abstract_inverted_index":{"As":[0],"robotics":[1,61],"matures":[2],"and":[3,43,66,93,127,167,178],"increases":[4],"in":[5],"complexity,":[6],"it":[7],"is":[8,71,112,122,171],"more":[9],"necessary":[10],"than":[11],"ever":[12],"that":[13,63,69,98,121,169],"robot":[14,176],"autonomy":[15],"research":[16,62],"be":[17],"reproducible.":[18,128],"Compared":[19],"to":[20,27,89,138],"other":[21],"sciences,":[22],"there":[23,170],"are":[24],"specific":[25],"challenges":[26],"benchmarking":[28],"autonomy,":[29],"such":[30],"as":[31],"the":[32,35,38,41,44,76,79,85,102,106,113,155,159,165],"complexity":[33],"of":[34,40,78,105,109,134,161],"software":[36],"stacks,":[37],"variability":[39],"hardware":[42,151,177],"reliance":[45],"on":[46],"data-driven":[47],"techniques,":[48],"amongst":[49],"others.":[50],"In":[51],"this":[52,91,110],"paper,":[53],"we":[54,99],"describe":[55],"a":[56,95,116],"new":[57],"concept":[58],"for":[59],"reproducible":[60],"integrates":[64],"development":[65],"benchmarking,":[67],"so":[68],"reproducibility":[70],"obtained":[72],"\"by":[73],"design\"":[74],"from":[75],"beginning":[77],"research/development":[80],"processes.":[81],"We":[82,153],"first":[83],"provide":[84],"overall":[86],"conceptual":[87],"objectives":[88],"achieve":[90],"goal":[92],"then":[94],"concrete":[96],"instance":[97],"have":[100],"built:":[101],"DUCKIENet.":[103],"One":[104],"central":[107],"components":[108],"setup":[111,120],"Duckietown":[114],"Autolab,":[115],"remotely":[117],"accessible":[118],"standardized":[119],"itself":[123],"also":[124],"relatively":[125],"low-cost":[126],"When":[129],"evaluating":[130],"agents,":[131],"careful":[132],"definition":[133],"interfaces":[135],"allows":[136],"users":[137],"choose":[139],"among":[140],"local":[141],"versus":[142],"remote":[143,149,181],"evaluation":[144],"using":[145,164],"simulation,":[146],"logs,":[147],"or":[148],"automated":[150],"setups.":[152],"validate":[154],"system":[156],"by":[157],"analyzing":[158],"repeatability":[160],"experiments":[162],"conducted":[163],"infrastructure":[166],"show":[168],"low":[172],"variance":[173],"across":[174,179],"different":[175,180],"labs.":[182],"<sup":[183],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[184],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">\u2020</sup>":[185]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}