{"id":"https://openalex.org/W4390480935","doi":"https://doi.org/10.1109/mra.2023.3336473","title":"Digital Robot Judge: Building a Task-Centric Performance Database of Real-World Manipulation With Electronic Task Boards","display_name":"Digital Robot Judge: Building a Task-Centric Performance Database of Real-World Manipulation With Electronic Task Boards","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4390480935","doi":"https://doi.org/10.1109/mra.2023.3336473"},"language":"en","primary_location":{"id":"doi:10.1109/mra.2023.3336473","is_oa":true,"landing_page_url":"https://doi.org/10.1109/mra.2023.3336473","pdf_url":null,"source":{"id":"https://openalex.org/S132261465","display_name":"IEEE Robotics & Automation Magazine","issn_l":"1070-9932","issn":["1070-9932","1558-223X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics & Automation Magazine","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/mra.2023.3336473","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022127549","display_name":"Peter So","orcid":"https://orcid.org/0000-0002-2762-7407"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Peter So","raw_affiliation_strings":["Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037556368","display_name":"Andriy Sarabakha","orcid":"https://orcid.org/0000-0002-3629-0674"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Andriy Sarabakha","raw_affiliation_strings":["School of Electrical and Electronic Engineering, Nanyang Technological University, Singapore"],"affiliations":[{"raw_affiliation_string":"School of Electrical and Electronic Engineering, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100380478","display_name":"Fan Wu","orcid":"https://orcid.org/0000-0001-5051-3005"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Fan Wu","raw_affiliation_strings":["Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082830894","display_name":"Utku \u00c7ulha","orcid":"https://orcid.org/0000-0002-7788-6124"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Utku Culha","raw_affiliation_strings":["Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017337432","display_name":"Fares J. Abu\u2010Dakka","orcid":"https://orcid.org/0000-0001-9062-9416"},"institutions":[{"id":"https://openalex.org/I162361429","display_name":"Mondragon Unibertsitatea","ror":"https://ror.org/00wvqgd19","country_code":"ES","type":"education","lineage":["https://openalex.org/I162361429"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Fares J. Abu-Dakka","raw_affiliation_strings":["Department of Electronic and Informatics, Faculty of Engineering, Mondragon Unibertsitatea, Arrasate, Spain"],"affiliations":[{"raw_affiliation_string":"Department of Electronic and Informatics, Faculty of Engineering, Mondragon Unibertsitatea, Arrasate, Spain","institution_ids":["https://openalex.org/I162361429"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024171209","display_name":"Sami Haddadin","orcid":"https://orcid.org/0000-0001-7696-4955"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sami Haddadin","raw_affiliation_strings":["Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Munich Institute of Robotics and Machine Intelligence, Technical University of Munich, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5022127549"],"corresponding_institution_ids":["https://openalex.org/I62916508"],"apc_list":null,"apc_paid":null,"fwci":6.7214,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.96888817,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"31","issue":"4","first_page":"32","last_page":"44"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9869999885559082,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9869999885559082,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9768999814987183,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7336559295654297},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.6795287132263184},{"id":"https://openalex.org/keywords/robotics","display_name":"Robotics","score":0.635744035243988},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.612737238407135},{"id":"https://openalex.org/keywords/dashboard","display_name":"Dashboard","score":0.6023720502853394},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5957072973251343},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5466896891593933},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.474604070186615},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.4613288938999176},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.4367358386516571},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.43245309591293335},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4260454773902893},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.33543121814727783},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.16276252269744873},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.16096508502960205},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.08791062235832214},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.08644163608551025}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7336559295654297},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.6795287132263184},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.635744035243988},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.612737238407135},{"id":"https://openalex.org/C33499554","wikidata":"https://www.wikidata.org/wiki/Q1417134","display_name":"Dashboard","level":2,"score":0.6023720502853394},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5957072973251343},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5466896891593933},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.474604070186615},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.4613288938999176},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.4367358386516571},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.43245309591293335},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4260454773902893},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.33543121814727783},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.16276252269744873},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.16096508502960205},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.08791062235832214},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.08644163608551025},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/mra.2023.3336473","is_oa":true,"landing_page_url":"https://doi.org/10.1109/mra.2023.3336473","pdf_url":null,"source":{"id":"https://openalex.org/S132261465","display_name":"IEEE Robotics & Automation Magazine","issn_l":"1070-9932","issn":["1070-9932","1558-223X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics & Automation Magazine","raw_type":"journal-article"},{"id":"pmh:oai:ebiltegia.mondragon.edu:20.500.11984/6531","is_oa":true,"landing_page_url":"https://katalogoa.mondragon.edu/janium-bin/janium_login_opac.pl?find&ficha_no=174290","pdf_url":"http://ebiltegia.mondragon.edu/xmlui/bitstream/20.500.11984/6531/1/Digital_Robot_Judge_Building_a_Task-centric_Performance_Database_of_Real-World_Manipulation_With_Electronic_Task_Boards.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Robotics & Automation Magazine Vol. 31. N. 4. December 2024","raw_type":null},{"id":"pmh:oai:mediatum.ub.tum.de:node/1737922","is_oa":false,"landing_page_url":"https://mediatum.ub.tum.de/1737922","pdf_url":null,"source":{"id":"https://openalex.org/S4377196330","display_name":"mediaTUM  (Technical University of Munich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I62916508","host_organization_name":"Technical University of Munich","host_organization_lineage":["https://openalex.org/I62916508"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"},{"id":"pmh:oai:mediatum.ub.tum.de:node/1737929","is_oa":false,"landing_page_url":"https://mediatum.ub.tum.de/1737929","pdf_url":null,"source":{"id":"https://openalex.org/S4377196330","display_name":"mediaTUM  (Technical University of Munich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I62916508","host_organization_name":"Technical University of Munich","host_organization_lineage":["https://openalex.org/I62916508"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"},{"id":"pmh:oai:mediatum.ub.tum.de:node/1737946","is_oa":false,"landing_page_url":"https://mediatum.ub.tum.de/1737946","pdf_url":null,"source":{"id":"https://openalex.org/S4377196330","display_name":"mediaTUM  (Technical University of Munich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I62916508","host_organization_name":"Technical University of Munich","host_organization_lineage":["https://openalex.org/I62916508"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/mra.2023.3336473","is_oa":true,"landing_page_url":"https://doi.org/10.1109/mra.2023.3336473","pdf_url":null,"source":{"id":"https://openalex.org/S132261465","display_name":"IEEE Robotics & Automation Magazine","issn_l":"1070-9932","issn":["1070-9932","1558-223X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics & Automation Magazine","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.4300000071525574}],"awards":[{"id":"https://openalex.org/G1375572575","display_name":null,"funder_award_id":"390696704","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G1933938177","display_name":null,"funder_award_id":"EXC 2050/1","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G3962681082","display_name":null,"funder_award_id":"ID 390696704","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G4021765181","display_name":null,"funder_award_id":"2050/1","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G543305895","display_name":null,"funder_award_id":"EXC 2050/1-Project ID 390696704","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G7453299778","display_name":null,"funder_award_id":"90696704","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G762232396","display_name":null,"funder_award_id":"Project","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320323383","display_name":"Technische Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/02kkvpp62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W1559961358","https://openalex.org/W2034587361","https://openalex.org/W2055669065","https://openalex.org/W2126791158","https://openalex.org/W2202110990","https://openalex.org/W2529170537","https://openalex.org/W2556468286","https://openalex.org/W2748289595","https://openalex.org/W2967393806","https://openalex.org/W2967717386","https://openalex.org/W2996999711","https://openalex.org/W3000439273","https://openalex.org/W3007769740","https://openalex.org/W3099587965","https://openalex.org/W3202310555","https://openalex.org/W3213354518","https://openalex.org/W3216908904","https://openalex.org/W4200452682","https://openalex.org/W6782766965"],"related_works":["https://openalex.org/W2607236244","https://openalex.org/W2378211422","https://openalex.org/W2382083831","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W4386891374","https://openalex.org/W972276598","https://openalex.org/W4246352526","https://openalex.org/W2028665553"],"abstract_inverted_index":{"Robotics":[0],"aims":[1],"to":[2,30,47,75,78],"develop":[3],"manipulation":[4,49,121],"skills":[5,122],"approaching":[6],"human":[7],"performance.":[8],"However,":[9],"skill":[10],"complexity":[11],"is":[12,26],"often":[13],"over-":[14],"or":[15,28,59],"underestimated":[16],"based":[17],"on":[18,65,104],"individual":[19],"experience,":[20],"and":[21,81,87,92],"the":[22,55,66,129],"real-world":[23,120],"performance":[24,50,69,113],"gap":[25],"difficult":[27],"expensive":[29],"measure":[31,48],"through":[32,68],"in-person":[33,79],"competitions.":[34],"To":[35],"bridge":[36],"this":[37],"gap,":[38],"we":[39,52],"propose":[40],"a":[41,84,105,111],"compact,":[42],"Internet-connected,":[43],"electronic":[44],"task":[45],"board":[46,67],"remotely;":[51],"call":[53],"it":[54],"digital":[56],"robot":[57,124],"judge,":[58],"\u201cDR.J.\u201d":[60],"By":[61],"detecting":[62],"key":[63],"events":[64],"circuitry,":[70],"DR.J":[71],"provides":[72,110],"an":[73],"alternative":[74],"transporting":[76],"equipment":[77],"competitions":[80],"serves":[82],"as":[83],"portable":[85],"test":[86],"data-generation":[88],"system":[89],"that":[90,109,115],"captures":[91],"grades":[93],"performances,":[94],"making":[95],"comparisons":[96],"less":[97],"expensive.":[98],"Data":[99],"collected":[100],"are":[101],"automatically":[102],"published":[103],"web":[106],"dashboard":[107],"(WD)":[108],"living":[112],"benchmark":[114],"can":[116],"visualize":[117],"improvements":[118],"in":[119],"of":[123],"platforms":[125],"over":[126],"time":[127],"across":[128],"globe.":[130]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":4}],"updated_date":"2026-03-21T08:13:44.787528","created_date":"2024-01-02T00:00:00"}