{"id":"https://openalex.org/W4308080086","doi":"https://doi.org/10.1109/itsc55140.2022.9922202","title":"Certifiable Evaluation for Autonomous Vehicle Perception Systems using Deep Importance Sampling (Deep IS)","display_name":"Certifiable Evaluation for Autonomous Vehicle Perception Systems using Deep Importance Sampling (Deep IS)","publication_year":2022,"publication_date":"2022-10-08","ids":{"openalex":"https://openalex.org/W4308080086","doi":"https://doi.org/10.1109/itsc55140.2022.9922202"},"language":"en","primary_location":{"id":"doi:10.1109/itsc55140.2022.9922202","is_oa":false,"landing_page_url":"https://doi.org/10.1109/itsc55140.2022.9922202","pdf_url":null,"source":{"id":"https://openalex.org/S4363607737","display_name":"2022 IEEE 25th International Conference on Intelligent Transportation Systems (ITSC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE 25th International Conference on Intelligent Transportation Systems (ITSC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024932692","display_name":"Mansur Arief","orcid":"https://orcid.org/0000-0002-4636-3451"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Mansur Arief","raw_affiliation_strings":["Carnegie Mellon University,Department of Mechanical Engineering,Pittsburgh,PA,USA,15213"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Department of Mechanical Engineering,Pittsburgh,PA,USA,15213","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061106766","display_name":"Zhepeng Cen","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhepeng Cen","raw_affiliation_strings":["Carnegie Mellon University,Department of Mechanical Engineering,Pittsburgh,PA,USA,15213"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Department of Mechanical Engineering,Pittsburgh,PA,USA,15213","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036473194","display_name":"Zhenyuan Liu","orcid":"https://orcid.org/0000-0002-7032-9560"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhenyuan Liu","raw_affiliation_strings":["Columbia University,Department of Industrial Engineering and Operations Research,New York,NY,USA,10027"],"affiliations":[{"raw_affiliation_string":"Columbia University,Department of Industrial Engineering and Operations Research,New York,NY,USA,10027","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101445344","display_name":"Zhiyuan Huang","orcid":"https://orcid.org/0000-0003-1284-2128"},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyuan Huang","raw_affiliation_strings":["Tongji University,Department of Management Science and Engineering,Shanghai,China","Department of Management Science and Engineering, Tongji University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Tongji University,Department of Management Science and Engineering,Shanghai,China","institution_ids":["https://openalex.org/I116953780"]},{"raw_affiliation_string":"Department of Management Science and Engineering, Tongji University, Shanghai, China","institution_ids":["https://openalex.org/I116953780"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100738534","display_name":"\u0411\u043e \u041b\u0438","orcid":"https://orcid.org/0000-0002-8019-8891"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bo Li","raw_affiliation_strings":["University of Illinois Urbana Champaign,Department of Computer Science,Urbana Champaign,IL,USA","Department of Computer Science, University of Illinois Urbana Champaign, Urbana Champaign, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana Champaign,Department of Computer Science,Urbana Champaign,IL,USA","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"Department of Computer Science, University of Illinois Urbana Champaign, Urbana Champaign, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071064725","display_name":"Henry Lam","orcid":"https://orcid.org/0000-0002-3193-563X"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Henry Lam","raw_affiliation_strings":["Columbia University,Department of Industrial Engineering and Operations Research,New York,NY,USA,10027"],"affiliations":[{"raw_affiliation_string":"Columbia University,Department of Industrial Engineering and Operations Research,New York,NY,USA,10027","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037644321","display_name":"Ding Zhao","orcid":"https://orcid.org/0000-0002-9400-8446"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ding Zhao","raw_affiliation_strings":["Carnegie Mellon University,Department of Mechanical Engineering,Pittsburgh,PA,USA,15213"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Department of Mechanical Engineering,Pittsburgh,PA,USA,15213","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5024932692"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":2.9882,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.91648352,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1736","last_page":"1742"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11357","display_name":"Risk and Safety Analysis","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11357","display_name":"Risk and Safety Analysis","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10780","display_name":"Reliability and Maintenance Optimization","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/2213","display_name":"Safety, Risk, Reliability and Quality"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7777203917503357},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7472175359725952},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.6319006681442261},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.613632082939148},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.6003490090370178},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5668919086456299},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5564068555831909},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5207967162132263},{"id":"https://openalex.org/keywords/upper-and-lower-bounds","display_name":"Upper and lower bounds","score":0.4532325565814972},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.4475197196006775},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.432498961687088},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.23712879419326782},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.13152936100959778},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12172001600265503}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7777203917503357},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7472175359725952},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.6319006681442261},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.613632082939148},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.6003490090370178},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5668919086456299},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5564068555831909},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5207967162132263},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.4532325565814972},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4475197196006775},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.432498961687088},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.23712879419326782},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.13152936100959778},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12172001600265503},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/itsc55140.2022.9922202","is_oa":false,"landing_page_url":"https://doi.org/10.1109/itsc55140.2022.9922202","pdf_url":null,"source":{"id":"https://openalex.org/S4363607737","display_name":"2022 IEEE 25th International Conference on Intelligent Transportation Systems (ITSC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE 25th International Conference on Intelligent Transportation Systems (ITSC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W1525133041","https://openalex.org/W1672082250","https://openalex.org/W1972339550","https://openalex.org/W2080328084","https://openalex.org/W2117876524","https://openalex.org/W2162501664","https://openalex.org/W2312206206","https://openalex.org/W2328067583","https://openalex.org/W2438413413","https://openalex.org/W2462906003","https://openalex.org/W2488996366","https://openalex.org/W2511072509","https://openalex.org/W2584223592","https://openalex.org/W2584539651","https://openalex.org/W2891468099","https://openalex.org/W2897535010","https://openalex.org/W2948153973","https://openalex.org/W2963290958","https://openalex.org/W2963338863","https://openalex.org/W2963690607","https://openalex.org/W3008659207","https://openalex.org/W3024414633","https://openalex.org/W3037100357","https://openalex.org/W3127647470","https://openalex.org/W3161583653","https://openalex.org/W3209451589","https://openalex.org/W3210932196","https://openalex.org/W4230519770","https://openalex.org/W4242178966","https://openalex.org/W4251113929","https://openalex.org/W4301187031","https://openalex.org/W6718836005","https://openalex.org/W6751834733","https://openalex.org/W6763392207","https://openalex.org/W6779832048","https://openalex.org/W6803068164","https://openalex.org/W6803207957"],"related_works":["https://openalex.org/W4298388782","https://openalex.org/W2950066684","https://openalex.org/W4310989423","https://openalex.org/W4312831135","https://openalex.org/W3082895349","https://openalex.org/W4288853838","https://openalex.org/W4317565044","https://openalex.org/W3179488938","https://openalex.org/W2909645158","https://openalex.org/W3158264953"],"abstract_inverted_index":{"Evaluating":[0],"the":[1,23,65,84,130,134,138,152,166,179,192],"performance":[2],"of":[3,129,141,151],"autonomous":[4],"vehicles":[5],"(AV)":[6],"and":[7,183],"their":[8],"complex":[9],"AI-driven":[10],"functionalities":[11],"to":[12,43,62,112,160,178,191,205],"high":[13,45],"precision":[14],"under":[15,87],"naturalistic":[16,215],"conditions":[17],"remains":[18],"a":[19,40,75,108,142,156,201],"challenge,":[20],"especially":[21],"when":[22],"failure":[24,216],"or":[25],"dangerous":[26],"cases":[27,217],"are":[28,219],"rare.":[29],"Rarity":[30],"does":[31],"not":[32],"only":[33,148,184],"require":[34],"an":[35,80,114,121],"enormous":[36],"sample":[37],"size":[38],"for":[39,83],"naive":[41,157],"method":[42,159,204],"achieve":[44,161],"confidence":[46],"residual":[47],"risk":[48,56,86,118,180],"estimation,":[49],"but":[50],"it":[51],"can":[52,78],"also":[53],"cause":[54],"serious":[55],"underestimation":[57],"issues":[58],"that":[59,72,106,123,128,218],"is":[60,124,172],"hard":[61],"detect.":[63],"Meanwhile,":[64],"state-of-the-art":[66,131],"rare":[67,214],"safety-critical":[68],"event":[69],"evaluation":[70],"approach":[71],"comes":[73],"with":[74,120,127,207,213],"correctness":[76],"guarantee":[77],"compute":[79],"upper":[81,181],"bound":[82,182],"true":[85,193],"certain":[88],"conditions,":[89],"which":[90],"limits":[91],"its":[92],"practical":[93],"uses.":[94],"In":[95,133],"this":[96],"work,":[97],"we":[98],"propose":[99],"Deep":[100,146,170],"Importance":[101],"Sampling":[102],"(Deep":[103],"IS)":[104],"framework":[105],"utilizes":[107],"deep":[109],"neural":[110],"network":[111],"obtain":[113],"efficient":[115,196,203],"less":[116,175],"biased":[117],"estimate,":[119],"efficiency":[122],"on":[125],"par":[126],"method.":[132],"numerical":[135],"experiment":[136],"evaluating":[137],"misclassification":[139],"rate":[140],"traffic":[143],"sign":[144],"classifier,":[145],"IS":[147,171,198],"needs":[149],"1/40-th":[150],"samples":[153],"required":[154],"by":[155,169,186],"sampling":[158],"10%":[162,189],"relative":[163],"error.":[164],"Furthermore,":[165],"estimate":[167],"produced":[168],"10":[173],"times":[174],"conservative":[176],"compared":[177],"off":[185],"at":[187],"most":[188],"difference":[190],"target.":[194],"This":[195],"deep-learning-based":[197],"procedure":[199],"promises":[200],"highly":[202],"deal":[206],"often":[208],"high-dimensional":[209],"functional":[210],"safety":[211],"problems":[212],"prevalent":[220],"in":[221],"AV":[222],"domains.":[223]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-04T07:04:00.330322","created_date":"2025-10-10T00:00:00"}