{"id":"https://openalex.org/W4285345015","doi":"https://doi.org/10.55417/fr.2021010","title":"Perception Robustness Testing at Different Levels of Generality","display_name":"Perception Robustness Testing at Different Levels of Generality","publication_year":2021,"publication_date":"2021-12-01","ids":{"openalex":"https://openalex.org/W4285345015","doi":"https://doi.org/10.55417/fr.2021010"},"language":"en","primary_location":{"id":"doi:10.55417/fr.2021010","is_oa":true,"landing_page_url":"https://doi.org/10.55417/fr.2021010","pdf_url":"https://doi.org/10.55417/fr.2021010","source":{"id":"https://openalex.org/S4387279083","display_name":"Field Robotics","issn_l":"2771-3989","issn":["2771-3989"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Field Robotics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.55417/fr.2021010","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040620353","display_name":"Zachary Pezzementi","orcid":"https://orcid.org/0000-0002-1137-7378"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zachary Pezzementi","raw_affiliation_strings":["National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0000-0002-1137-7378","affiliations":[{"raw_affiliation_string":"National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035658063","display_name":"Trenton Tabor","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Trenton Tabor","raw_affiliation_strings":["National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112330765","display_name":"Jonathan Chang","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jonathan K. Chang","raw_affiliation_strings":["National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043772306","display_name":"Carter Tiernan","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carter Tiernan","raw_affiliation_strings":["National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015283427","display_name":"Bill Drozd","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bill Drozd","raw_affiliation_strings":["National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045655492","display_name":"Eric Sample","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eric Sample","raw_affiliation_strings":["National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111876485","display_name":"Chris Hazard","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Hazard","raw_affiliation_strings":["National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Robotics Engineering Center (NREC), Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013993834","display_name":"Michael Wagner","orcid":"https://orcid.org/0000-0002-9778-7684"},"institutions":[{"id":"https://openalex.org/I4210146707","display_name":"Edge Case Research (United States)","ror":"https://ror.org/03zrkam54","country_code":"US","type":"company","lineage":["https://openalex.org/I4210146707"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Wagner","raw_affiliation_strings":["Edge Case Research LLC, Pittsburgh, Pennsylvania, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edge Case Research LLC, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I4210146707"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036158209","display_name":"Philip Koopman","orcid":"https://orcid.org/0000-0003-1658-2386"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Philip Koopman","raw_affiliation_strings":["Electrical and Computer Engineering Department, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0000-0003-1658-2386","affiliations":[{"raw_affiliation_string":"Electrical and Computer Engineering Department, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1941,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.52565281,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":96},"biblio":{"volume":"1","issue":null,"first_page":"253","last_page":"286"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.8846137523651123},{"id":"https://openalex.org/keywords/generality","display_name":"Generality","score":0.7793636322021484},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7429214715957642},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5437551140785217},{"id":"https://openalex.org/keywords/robustness-testing","display_name":"Robustness testing","score":0.530535101890564},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.43771764636039734},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42297428846359253},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.41284799575805664},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3701304495334625},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.14599695801734924}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.8846137523651123},{"id":"https://openalex.org/C2780767217","wikidata":"https://www.wikidata.org/wiki/Q5532421","display_name":"Generality","level":2,"score":0.7793636322021484},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7429214715957642},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5437551140785217},{"id":"https://openalex.org/C137726913","wikidata":"https://www.wikidata.org/wiki/Q7353550","display_name":"Robustness testing","level":3,"score":0.530535101890564},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43771764636039734},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42297428846359253},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.41284799575805664},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3701304495334625},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14599695801734924},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C58166","wikidata":"https://www.wikidata.org/wiki/Q224821","display_name":"Fuzzy logic","level":2,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.55417/fr.2021010","is_oa":true,"landing_page_url":"https://doi.org/10.55417/fr.2021010","pdf_url":"https://doi.org/10.55417/fr.2021010","source":{"id":"https://openalex.org/S4387279083","display_name":"Field Robotics","issn_l":"2771-3989","issn":["2771-3989"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Field Robotics","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.55417/fr.2021010","is_oa":true,"landing_page_url":"https://doi.org/10.55417/fr.2021010","pdf_url":"https://doi.org/10.55417/fr.2021010","source":{"id":"https://openalex.org/S4387279083","display_name":"Field Robotics","issn_l":"2771-3989","issn":["2771-3989"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Field Robotics","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/11","display_name":"Sustainable cities and communities","score":0.6399999856948853}],"awards":[],"funders":[{"id":"https://openalex.org/F4320332808","display_name":"Smithsonian Tropical Research Institute","ror":"https://ror.org/035jbxr46"},{"id":"https://openalex.org/F4320337657","display_name":"Naval Air Systems Command","ror":"https://ror.org/054ajk512"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4285345015.pdf","grobid_xml":"https://content.openalex.org/works/W4285345015.grobid-xml"},"referenced_works_count":67,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1518931405","https://openalex.org/W1921093919","https://openalex.org/W1993850091","https://openalex.org/W2053083163","https://openalex.org/W2099318124","https://openalex.org/W2128254161","https://openalex.org/W2138948150","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2201145266","https://openalex.org/W2254462240","https://openalex.org/W2274287116","https://openalex.org/W2328067583","https://openalex.org/W2397830550","https://openalex.org/W2407521645","https://openalex.org/W2479714754","https://openalex.org/W2490270993","https://openalex.org/W2496266747","https://openalex.org/W2557728737","https://openalex.org/W2561397319","https://openalex.org/W2570343428","https://openalex.org/W2601564443","https://openalex.org/W2616028256","https://openalex.org/W2626967530","https://openalex.org/W2736899637","https://openalex.org/W2748021867","https://openalex.org/W2767459365","https://openalex.org/W2778115935","https://openalex.org/W2784465508","https://openalex.org/W2797527871","https://openalex.org/W2798302089","https://openalex.org/W2888339491","https://openalex.org/W2892362080","https://openalex.org/W2941175503","https://openalex.org/W2942630857","https://openalex.org/W2949736877","https://openalex.org/W2950787360","https://openalex.org/W2950800384","https://openalex.org/W2951940613","https://openalex.org/W2963351448","https://openalex.org/W2963542245","https://openalex.org/W2963730616","https://openalex.org/W2964350391","https://openalex.org/W2969454448","https://openalex.org/W2981495453","https://openalex.org/W2989924675","https://openalex.org/W2990955823","https://openalex.org/W2996490176","https://openalex.org/W2999862950","https://openalex.org/W3046704329","https://openalex.org/W3101442004","https://openalex.org/W3106250896","https://openalex.org/W3121281282","https://openalex.org/W3121342653","https://openalex.org/W3139294479","https://openalex.org/W4230519770","https://openalex.org/W4246256082","https://openalex.org/W4288281368","https://openalex.org/W4288359148","https://openalex.org/W4288359223","https://openalex.org/W4288404646","https://openalex.org/W4293584584","https://openalex.org/W4293586251","https://openalex.org/W4297775537","https://openalex.org/W4381304672","https://openalex.org/W6739651123"],"related_works":["https://openalex.org/W2385695368","https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W1980755708","https://openalex.org/W2369604612","https://openalex.org/W4224009465","https://openalex.org/W2160571967","https://openalex.org/W4286629047","https://openalex.org/W4205364923","https://openalex.org/W2789584252"],"abstract_inverted_index":{"Ensuring":[0],"the":[1,24,34,93,154],"success":[2],"and":[3,99,145],"safety":[4],"of":[5,18,23,36,44,48,95,117,149],"deployed":[6],"robotic":[7],"systems":[8],"often":[9,152],"requires":[10],"predicting":[11],"their":[12],"behavior":[13],"in":[14,40,137],"a":[15,37,41,89,105,127],"wide":[16],"range":[17,43],"conditions,":[19],"precluding":[20],"practical":[21],"testing":[22],"physical":[25,59],"system":[26,39],"itself.":[27],"We":[28,77,102,121],"introduce":[29,104],"an":[30],"approach":[31],"to":[32,58,71,73,86,115],"predict":[33],"robustness":[35,107,150],"perception":[38],"broad":[42],"conditions":[45,51],"through":[46],"simulation":[47],"challenging":[49],"real-world":[50],"achieved":[52],"by":[53],"modifying":[54],"real":[55],"images":[56],"according":[57],"phenomena,":[60],"producing":[61],"physically-realistic,":[62],"mutated":[63],"images.":[64],"Typical":[65],"perception-system":[66],"performance":[67,119],"metrics":[68,80],"are":[69,82],"difficult":[70],"relate":[72],"system-level":[74],"application-or":[75],"safety-requirements.":[76],"propose":[78],"performance-evaluating":[79],"that":[81,111],"more":[83],"closely":[84],"tied":[85],"these":[87,118],"targets,":[88],"relationship":[90],"dependent":[91],"on":[92,125,131],"amount":[94],"application-specific":[96],"context":[97],"available":[98],"assumptions":[100],"imposed.":[101],"also":[103],"novel":[106],"metric,":[108],"Robustness":[109],"ROC,":[110],"can":[112,151],"be":[113],"applied":[114],"any":[116],"metrics.":[120],"demonstrate":[122],"this":[123],"method":[124],"NREC-AgPD,":[126],"large":[128],"dataset":[129],"focused":[130],"person":[132,143],"detection":[133],"for":[134],"off-road":[135],"driving":[136],"orchard":[138],"environments,":[139],"assessing":[140],"several":[141],"recent":[142],"detectors,":[144],"showing":[146],"how":[147],"considerations":[148],"change":[153],"best-choice":[155],"algorithm.":[156]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}