{"id":"https://openalex.org/W4410949633","doi":"https://doi.org/10.1109/tits.2025.3571213","title":"Methodology and Benchmark for Automated Driving Theory Test of Large Language Models","display_name":"Methodology and Benchmark for Automated Driving Theory Test of Large Language Models","publication_year":2025,"publication_date":"2025-06-02","ids":{"openalex":"https://openalex.org/W4410949633","doi":"https://doi.org/10.1109/tits.2025.3571213"},"language":"en","primary_location":{"id":"doi:10.1109/tits.2025.3571213","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2025.3571213","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070527876","display_name":"Dashuai Pei","orcid":"https://orcid.org/0000-0002-8633-9931"},"institutions":[{"id":"https://openalex.org/I196699116","display_name":"Wuhan University of Technology","ror":"https://ror.org/03fe7t173","country_code":"CN","type":"education","lineage":["https://openalex.org/I196699116"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dashuai Pei","raw_affiliation_strings":["School of Navigation, Wuhan University of Technology (WUT), Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-8633-9931","affiliations":[{"raw_affiliation_string":"School of Navigation, Wuhan University of Technology (WUT), Wuhan, China","institution_ids":["https://openalex.org/I196699116"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102737709","display_name":"Yiwen Wu","orcid":"https://orcid.org/0000-0003-0241-7988"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiwen Wu","raw_affiliation_strings":["School of Computer Science, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014610561","display_name":"Jianhua He","orcid":"https://orcid.org/0000-0002-5738-8507"},"institutions":[{"id":"https://openalex.org/I110002522","display_name":"University of Essex","ror":"https://ror.org/02nkf1q06","country_code":"GB","type":"education","lineage":["https://openalex.org/I110002522"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jianhua He","raw_affiliation_strings":["School of Computer Science and Electronic Engineering, University of Essex, Colchester, U.K"],"raw_orcid":"https://orcid.org/0000-0002-5738-8507","affiliations":[{"raw_affiliation_string":"School of Computer Science and Electronic Engineering, University of Essex, Colchester, U.K","institution_ids":["https://openalex.org/I110002522"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088177440","display_name":"Kezhong Liu","orcid":"https://orcid.org/0000-0002-1085-7556"},"institutions":[{"id":"https://openalex.org/I196699116","display_name":"Wuhan University of Technology","ror":"https://ror.org/03fe7t173","country_code":"CN","type":"education","lineage":["https://openalex.org/I196699116"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kezhong Liu","raw_affiliation_strings":["School of Navigation and the State Key Laboratory of Maritime Technology and Safety, Wuhan University of Technology (WUT), Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-1085-7556","affiliations":[{"raw_affiliation_string":"School of Navigation and the State Key Laboratory of Maritime Technology and Safety, Wuhan University of Technology (WUT), Wuhan, China","institution_ids":["https://openalex.org/I196699116"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065545898","display_name":"Mozi Chen","orcid":"https://orcid.org/0000-0002-6758-3314"},"institutions":[{"id":"https://openalex.org/I196699116","display_name":"Wuhan University of Technology","ror":"https://ror.org/03fe7t173","country_code":"CN","type":"education","lineage":["https://openalex.org/I196699116"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mozi Chen","raw_affiliation_strings":["School of Navigation and the State Key Laboratory of Maritime Technology and Safety, Wuhan University of Technology (WUT), Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-6758-3314","affiliations":[{"raw_affiliation_string":"School of Navigation and the State Key Laboratory of Maritime Technology and Safety, Wuhan University of Technology (WUT), Wuhan, China","institution_ids":["https://openalex.org/I196699116"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005567793","display_name":"Xuedou Xiao","orcid":"https://orcid.org/0000-0002-8724-7254"},"institutions":[{"id":"https://openalex.org/I196699116","display_name":"Wuhan University of Technology","ror":"https://ror.org/03fe7t173","country_code":"CN","type":"education","lineage":["https://openalex.org/I196699116"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuedou Xiao","raw_affiliation_strings":["School of Navigation, Wuhan University of Technology (WUT), Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-8724-7254","affiliations":[{"raw_affiliation_string":"School of Navigation, Wuhan University of Technology (WUT), Wuhan, China","institution_ids":["https://openalex.org/I196699116"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021707628","display_name":"Shengkai Zhang","orcid":"https://orcid.org/0000-0003-0154-4897"},"institutions":[{"id":"https://openalex.org/I196699116","display_name":"Wuhan University of Technology","ror":"https://ror.org/03fe7t173","country_code":"CN","type":"education","lineage":["https://openalex.org/I196699116"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shengkai Zhang","raw_affiliation_strings":["School of Information Engineering and the State Key Laboratory of Maritime Technology and Safety, Wuhan University of Technology (WUT), Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0003-0154-4897","affiliations":[{"raw_affiliation_string":"School of Information Engineering and the State Key Laboratory of Maritime Technology and Safety, Wuhan University of Technology (WUT), Wuhan, China","institution_ids":["https://openalex.org/I196699116"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062501978","display_name":"Juanjuan Zheng","orcid":"https://orcid.org/0009-0001-6818-0381"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Zheng","raw_affiliation_strings":["School of Electronic Engineering, State Key Laboratory of Information Photonics and Optical Communications, Beijing University of Posts and Telecommunications, Beijing, China","State Key Laboratory of Information Photonics and Optical Communications, School of Electronic Engineering, Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-6818-0381","affiliations":[{"raw_affiliation_string":"School of Electronic Engineering, State Key Laboratory of Information Photonics and Optical Communications, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]},{"raw_affiliation_string":"State Key Laboratory of Information Photonics and Optical Communications, School of Electronic Engineering, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.7066,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.97888857,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"26","issue":"10","first_page":"17013","last_page":"17026"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.6804999709129333,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.6804999709129333,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5768027305603027},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5483000874519348},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.519078254699707},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43403980135917664},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.41246938705444336},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3227817416191101},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.09372174739837646}],"concepts":[{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5768027305603027},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5483000874519348},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.519078254699707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43403980135917664},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.41246938705444336},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3227817416191101},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.09372174739837646},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tits.2025.3571213","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2025.3571213","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.4399999976158142}],"awards":[{"id":"https://openalex.org/G3022527671","display_name":null,"funder_award_id":"52401423","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4237402247","display_name":null,"funder_award_id":"2024HBB079","funder_id":"https://openalex.org/F4320317780","funder_display_name":"Postdoctoral\u00a0Science\u00a0Foundation\u00a0of\u00a0Hubei\u00a0Province"},{"id":"https://openalex.org/G8024316946","display_name":null,"funder_award_id":"62402353","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320317780","display_name":"Postdoctoral\u00a0Science\u00a0Foundation\u00a0of\u00a0Hubei\u00a0Province","ror":null},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526","https://openalex.org/W2121910908","https://openalex.org/W915438175","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs),":[3],"with":[4],"their":[5,108,260],"strong":[6],"generalization":[7],"and":[8,39,47,63,84,96,129,154,172,182,264,274],"inference":[9],"capabilities,":[10],"have":[11],"been":[12],"increasingly":[13],"leveraged":[14],"to":[15,100,255,276],"address":[16,51],"the":[17,33,56,75,101,137,151,203,217,221,225,229,249],"challenges":[18],"of":[19,35,42,163,199,216],"handling":[20],"corner":[21],"cases":[22],"in":[23,74,81,93,202,211,262,272],"autonomous":[24,82],"driving":[25,44,59,83,94,104,140,242,278],"(AD).":[26],"However,":[27,210],"a":[28,36,69,120],"critical":[29,111],"unresolved":[30],"issue":[31],"remains:":[32],"lack":[34],"comprehensive":[37,121],"understanding":[38,263],"formal":[40],"assessment":[41,170],"LLMs\u2019":[43,91],"theory":[45,60,95,105,141],"knowledge":[46],"practical":[48],"skills.":[49],"To":[50,114],"this":[52,208],"issue,":[53],"we":[54,118,144,167],"propose":[55],"first":[57],"dedicated":[58],"test":[61,165,251],"framework":[62,88],"benchmark":[64],"for":[65,78,110,240,284],"LLMs.":[66],"That":[67],"is":[68,247],"crucial":[70],"yet":[71],"unexplored":[72],"area":[73],"literature,":[76],"particularly":[77],"safety-critical":[79],"applications":[80],"driver":[85],"assistance.":[86],"Our":[87,244],"systematically":[89],"evaluates":[90],"competence":[92],"hazard":[97,131,212],"perception,":[98],"akin":[99],"official":[102,138],"UK":[103,139],"test,":[106,205],"ensuring":[107],"qualification":[109],"driving-related":[112],"tasks.":[113],"facilitate":[115],"rigorous":[116],"benchmarking,":[117],"construct":[119],"dataset":[122],"comprising":[123],"over":[124],"700":[125],"multiple-choice":[126],"questions":[127,252],"(MCQs)":[128],"54":[130],"perception":[132,213],"video":[133],"tests":[134],"sourced":[135],"from":[136,150],"examination.":[142],"Additionally,":[143],"incorporate":[145],"two":[146,161],"standardized":[147],"MCQ":[148],"sets":[149],"UK\u2019s":[152],"Driver":[153],"Vehicle":[155],"Standards":[156],"Agency":[157],"(DVSA).":[158],"For":[159],"these":[160,234],"types":[162],"theoretical":[164],"items,":[166],"design":[168],"tailored":[169],"methodologies":[171],"evaluation":[173],"metrics,":[174],"including":[175],"accuracy,":[176],"recall,":[177],"precision,":[178],"F1-score,":[179],"real-time":[180],"performance,":[181],"computational":[183],"efficiency.":[184],"The":[185],"experimental":[186],"results":[187],"reveal":[188],"that":[189,248],"among":[190],"all":[191],"LLMs":[192,253],"tested,":[193],"only":[194],"GPT-4o":[195],"achieved":[196],"an":[197],"accuracy":[198],"88.":[200],"21%":[201],"MCQs":[204],"successfully":[206],"passing":[207,222],"component.":[209],"testing,":[214],"none":[215],"evaluated":[218],"models":[219,235],"met":[220],"criteria":[223],"under":[224],"given":[226],"settings,":[227],"highlighting":[228],"substantial":[230],"improvements":[231],"required":[232],"before":[233],"can":[236],"be":[237],"practically":[238],"deployed":[239],"real-world":[241],"applications.":[243],"key":[245],"insight":[246],"specific":[250],"fail":[254],"answer":[256],"correctly":[257],"directly":[258],"reflect":[259],"deficiencies":[261],"flexibly":[265],"applying":[266],"traffic":[267],"regulations,":[268],"as":[269,271],"well":[270],"analyzing":[273],"responding":[275],"complex":[277],"scenarios.":[279],"This":[280],"provides":[281],"clear":[282],"directions":[283],"future":[285],"improvements.":[286]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2025-10-10T00:00:00"}