{"id":"https://openalex.org/W4206784906","doi":"https://doi.org/10.1109/ivworkshops54471.2021.9669240","title":"Quantitative Evaluation of Autonomous Driving in CARLA","display_name":"Quantitative Evaluation of Autonomous Driving in CARLA","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W4206784906","doi":"https://doi.org/10.1109/ivworkshops54471.2021.9669240"},"language":"en","primary_location":{"id":"doi:10.1109/ivworkshops54471.2021.9669240","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ivworkshops54471.2021.9669240","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE Intelligent Vehicles Symposium Workshops (IV Workshops)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.osti.gov/biblio/1814365","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047655591","display_name":"Shang Gao","orcid":"https://orcid.org/0000-0003-1803-1457"},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shang Gao","raw_affiliation_strings":["Oak Ridge National Laboratory, Oak Ridge, TN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002826974","display_name":"Spencer Paulissen","orcid":"https://orcid.org/0000-0003-4238-6020"},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Spencer Paulissen","raw_affiliation_strings":["Oak Ridge National Laboratory, Oak Ridge, TN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068431797","display_name":"Mark Coletti","orcid":"https://orcid.org/0000-0003-1020-531X"},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Coletti","raw_affiliation_strings":["Oak Ridge National Laboratory, Oak Ridge, TN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021986280","display_name":"Robert M. Patton","orcid":"https://orcid.org/0000-0002-8101-0571"},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert Patton","raw_affiliation_strings":["Oak Ridge National Laboratory, Oak Ridge, TN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oak Ridge National Laboratory, Oak Ridge, TN","institution_ids":["https://openalex.org/I1289243028"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1289243028"],"apc_list":null,"apc_paid":null,"fwci":0.6158,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.66837216,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"257","last_page":"263"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11099","display_name":"Autonomous Vehicle Technology and Safety","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/2203","display_name":"Automotive Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11099","display_name":"Autonomous Vehicle Technology and Safety","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/2203","display_name":"Automotive Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10524","display_name":"Traffic control and management","score":0.9602000117301941,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spearmans-rank-correlation-coefficient","display_name":"Spearman's rank correlation coefficient","score":0.8069754838943481},{"id":"https://openalex.org/keywords/pearson-product-moment-correlation-coefficient","display_name":"Pearson product-moment correlation coefficient","score":0.7838008999824524},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.7076998949050903},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6083632111549377},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5292438864707947},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.4305403530597687},{"id":"https://openalex.org/keywords/positive-correlation","display_name":"Positive correlation","score":0.42665308713912964},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.410685658454895},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3904833197593689},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37007108330726624},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.25883978605270386}],"concepts":[{"id":"https://openalex.org/C159744936","wikidata":"https://www.wikidata.org/wiki/Q1126730","display_name":"Spearman's rank correlation coefficient","level":2,"score":0.8069754838943481},{"id":"https://openalex.org/C55078378","wikidata":"https://www.wikidata.org/wiki/Q1136628","display_name":"Pearson product-moment correlation coefficient","level":2,"score":0.7838008999824524},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.7076998949050903},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6083632111549377},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5292438864707947},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.4305403530597687},{"id":"https://openalex.org/C2986115478","wikidata":"https://www.wikidata.org/wiki/Q186290","display_name":"Positive correlation","level":2,"score":0.42665308713912964},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.410685658454895},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3904833197593689},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37007108330726624},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.25883978605270386},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/ivworkshops54471.2021.9669240","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ivworkshops54471.2021.9669240","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE Intelligent Vehicles Symposium Workshops (IV Workshops)","raw_type":"proceedings-article"},{"id":"pmh:oai:osti.gov:1814365","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1814365","pdf_url":null,"source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:osti.gov:1814365","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1814365","pdf_url":null,"source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"sustainable_development_goals":[{"display_name":"Gender equality","score":0.5099999904632568,"id":"https://metadata.un.org/sdg/5"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320332359","display_name":"Office of Science","ror":"https://ror.org/00mmn6b08"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W2109797278","https://openalex.org/W2342840547","https://openalex.org/W2396217537","https://openalex.org/W2483814582","https://openalex.org/W2559767995","https://openalex.org/W2748895148","https://openalex.org/W2768997245","https://openalex.org/W2790640584","https://openalex.org/W2799352588","https://openalex.org/W2804452702","https://openalex.org/W2806163172","https://openalex.org/W2837605352","https://openalex.org/W2890235476","https://openalex.org/W2891056855","https://openalex.org/W2905173465","https://openalex.org/W2948153973","https://openalex.org/W2952175120","https://openalex.org/W2962894046","https://openalex.org/W2963041685","https://openalex.org/W2963657083","https://openalex.org/W2967895468","https://openalex.org/W2970900401","https://openalex.org/W2990123902","https://openalex.org/W2992742966","https://openalex.org/W2996474936","https://openalex.org/W3003329097","https://openalex.org/W3012432597","https://openalex.org/W4295719664","https://openalex.org/W4299805595","https://openalex.org/W6704559304","https://openalex.org/W6712453370","https://openalex.org/W6722063826","https://openalex.org/W6745935785","https://openalex.org/W6750749703","https://openalex.org/W6751474528","https://openalex.org/W6752781648","https://openalex.org/W6755103542","https://openalex.org/W6763392207","https://openalex.org/W6764072708","https://openalex.org/W6765535111","https://openalex.org/W6771352390"],"related_works":["https://openalex.org/W1911095394","https://openalex.org/W2749680699","https://openalex.org/W2146096861","https://openalex.org/W2999441357","https://openalex.org/W2773390159","https://openalex.org/W1478087083","https://openalex.org/W3152154711","https://openalex.org/W2963320501","https://openalex.org/W1996619012","https://openalex.org/W4232364410"],"abstract_inverted_index":{"There":[0],"have":[1],"been":[2],"many":[3],"recent":[4],"advancements":[5],"in":[6,60,98,196],"imitation":[7],"and":[8,29,68,75,94,121,136,155,171,194],"reinforcement":[9],"learning":[10],"for":[11,45,192],"autonomous":[12],"driving,":[13,185],"but":[14],"existing":[15],"metrics":[16],"generally":[17],"lack":[18],"the":[19,31,56,61,64,72,99,130,165,197],"means":[20],"to":[21,58,129],"capture":[22,179],"a":[23],"wide":[24],"range":[25],"of":[26,33,52,63],"driving":[27,53,101],"behaviors":[28],"compare":[30,79],"severity":[32],"different":[34,50,92,96],"failure":[35],"cases.":[36],"To":[37],"address":[38],"this":[39],"shortcoming,":[40],"we":[41,186],"introduce":[42],"Quan-titative":[43],"Evaluation":[44],"Driving":[46],"(QED),":[47],"which":[48],"assesses":[49],"aspects":[51],"behavior":[54],"including":[55],"ability":[57],"stay":[59],"center":[62],"lane,":[65],"avoid":[66,76],"weaving":[67],"erratic":[69],"behavior,":[70],"follow":[71],"speed":[73],"limit,":[74],"collisions.":[77],"We":[78],"scores":[80,85],"generated":[81],"by":[82,87],"QED":[83,116,150,176],"against":[84],"assigned":[86],"human":[88,126,160],"evaluators":[89],"on":[90],"30":[91],"drivers":[93,109,146],"6":[95],"towns":[97],"CARLA":[100],"simulator.":[102],"In":[103,140],"\"easy\"":[104],"evaluation":[105,142],"scenarios":[106,143],"where":[107,144],"better":[108],"are":[110],"easily":[111],"distinguished":[112],"from":[113],"worse":[114],"drivers,":[115],"attains":[117,151],"0.96":[118,133],"Pearson":[119,134,153,169],"correlation":[120,124,135,154,158,170],"0.97":[122],"Spearman":[123,138,157,173],"with":[125,159],"evaluators,":[127,161],"similar":[128],"baseline":[131,166],"inter-human-evaluator":[132,167],"0.95":[137],"correlation.":[139,174],"\"hard\"":[141],"ranking":[145],"is":[147],"more":[148],"ambiguous,":[149],"0.84":[152],"0.74":[156],"slighter":[162],"higher":[163],"than":[164],"0.78":[168],"0.7":[172],"While":[175],"may":[177],"not":[178],"every":[180],"characteristic":[181],"that":[182],"defines":[183],"good":[184],"consider":[187],"it":[188],"an":[189],"important":[190],"foundation":[191],"reproducibility":[193],"standardization":[195],"community.":[198]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}