{"id":"https://openalex.org/W4410949934","doi":"https://doi.org/10.1109/mnet.2025.3576035","title":"TeleQnA: A Benchmark Dataset to Assess Large Language Models Telecommunications Knowledge","display_name":"TeleQnA: A Benchmark Dataset to Assess Large Language Models Telecommunications Knowledge","publication_year":2025,"publication_date":"2025-06-02","ids":{"openalex":"https://openalex.org/W4410949934","doi":"https://doi.org/10.1109/mnet.2025.3576035"},"language":"en","primary_location":{"id":"doi:10.1109/mnet.2025.3576035","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mnet.2025.3576035","pdf_url":null,"source":{"id":"https://openalex.org/S186584794","display_name":"IEEE Network","issn_l":"0890-8044","issn":["0890-8044","1558-156X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Network","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072222232","display_name":"Ali Maatouk","orcid":"https://orcid.org/0000-0002-3436-7068"},"institutions":[{"id":"https://openalex.org/I4210123571","display_name":"Huawei Technologies (France)","ror":"https://ror.org/02rbzf697","country_code":"FR","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210123571"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Ali Maatouk","raw_affiliation_strings":["Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France"],"raw_orcid":"https://orcid.org/0000-0002-3436-7068","affiliations":[{"raw_affiliation_string":"Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France","institution_ids":["https://openalex.org/I4210123571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064737739","display_name":"Fadhel Ayed","orcid":"https://orcid.org/0009-0008-4234-7682"},"institutions":[{"id":"https://openalex.org/I4210123571","display_name":"Huawei Technologies (France)","ror":"https://ror.org/02rbzf697","country_code":"FR","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210123571"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Fadhel Ayed","raw_affiliation_strings":["Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France"],"raw_orcid":"https://orcid.org/0009-0008-4234-7682","affiliations":[{"raw_affiliation_string":"Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France","institution_ids":["https://openalex.org/I4210123571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048382270","display_name":"Nicola Piovesan","orcid":"https://orcid.org/0000-0001-5397-5248"},"institutions":[{"id":"https://openalex.org/I4210123571","display_name":"Huawei Technologies (France)","ror":"https://ror.org/02rbzf697","country_code":"FR","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210123571"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Nicola Piovesan","raw_affiliation_strings":["Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France"],"raw_orcid":"https://orcid.org/0000-0001-5397-5248","affiliations":[{"raw_affiliation_string":"Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France","institution_ids":["https://openalex.org/I4210123571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045740906","display_name":"Antonio De Domenico","orcid":"https://orcid.org/0000-0003-1229-4045"},"institutions":[{"id":"https://openalex.org/I4210123571","display_name":"Huawei Technologies (France)","ror":"https://ror.org/02rbzf697","country_code":"FR","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210123571"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Antonio De Domenico","raw_affiliation_strings":["Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France"],"raw_orcid":"https://orcid.org/0000-0003-1229-4045","affiliations":[{"raw_affiliation_string":"Paris Research Center, Huawei Technologies, Boulogne-Billancourt, France","institution_ids":["https://openalex.org/I4210123571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056145687","display_name":"M\u00e9rouane Debbah","orcid":"https://orcid.org/0000-0001-8941-8080"},"institutions":[{"id":"https://openalex.org/I176601375","display_name":"Khalifa University of Science and Technology","ror":"https://ror.org/05hffr360","country_code":"AE","type":"education","lineage":["https://openalex.org/I176601375"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Merouane Debbah","raw_affiliation_strings":["Department of Computer and Information Engineering, KU 6G Research Center, Khalifa University of Science and Technology, Abu Dhabi, United Arab Emirates","Khalifa University of Science and Technology, Abu Dhabi, UAE"],"raw_orcid":"https://orcid.org/0000-0001-8941-8080","affiliations":[{"raw_affiliation_string":"Department of Computer and Information Engineering, KU 6G Research Center, Khalifa University of Science and Technology, Abu Dhabi, United Arab Emirates","institution_ids":["https://openalex.org/I176601375"]},{"raw_affiliation_string":"Khalifa University of Science and Technology, Abu Dhabi, UAE","institution_ids":["https://openalex.org/I176601375"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101766226","display_name":"Zhi\u2010Quan Luo","orcid":"https://orcid.org/0000-0003-3995-914X"},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhi-Quan Luo","raw_affiliation_strings":["School of Science and Engineering, The Chinese University of Hong Kong, Shenzhen, China","The Chinese University of Hong Kong, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-3995-914X","affiliations":[{"raw_affiliation_string":"School of Science and Engineering, The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":14.3467,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.98715192,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"40","issue":"2","first_page":"253","last_page":"260"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.84579998254776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.84579998254776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7677160501480103},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7506175637245178},{"id":"https://openalex.org/keywords/telecommunications-service","display_name":"Telecommunications service","score":0.5868750214576721},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.5675951242446899},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.3325514793395996},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33246108889579773}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7677160501480103},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7506175637245178},{"id":"https://openalex.org/C33271190","wikidata":"https://www.wikidata.org/wiki/Q3467747","display_name":"Telecommunications service","level":2,"score":0.5868750214576721},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.5675951242446899},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3325514793395996},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33246108889579773},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mnet.2025.3576035","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mnet.2025.3576035","pdf_url":null,"source":{"id":"https://openalex.org/S186584794","display_name":"IEEE Network","issn_l":"0890-8044","issn":["0890-8044","1558-156X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Network","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W2125436846","https://openalex.org/W2946609015","https://openalex.org/W2970062726","https://openalex.org/W3096831136","https://openalex.org/W4384071683","https://openalex.org/W4385572697","https://openalex.org/W4392152396","https://openalex.org/W4408325149","https://openalex.org/W4415725257"],"related_works":["https://openalex.org/W2747796271","https://openalex.org/W2153684536","https://openalex.org/W2310885894","https://openalex.org/W2084934336","https://openalex.org/W3123532826","https://openalex.org/W2092374979","https://openalex.org/W1021961827","https://openalex.org/W2181612434","https://openalex.org/W1643527116","https://openalex.org/W2117921669"],"abstract_inverted_index":{"We":[0],"introduce":[1],"TeleQnA,":[2],"the":[3,10,38,62,65,69,78,124,133,149,158,177],"first":[4],"benchmark":[5],"dataset":[6,25,47,134,185],"designed":[7],"to":[8,60,76,167,170],"evaluate":[9],"knowledge":[11,114],"of":[12,64,80,148,160,174,179],"Large":[13],"Language":[14],"Models":[15],"(LLMs)":[16],"in":[17,102,163],"telecommunications.":[18],"Comprising":[19],"10,000":[20],"questions":[21,98],"and":[22,32,48,86],"answers,":[23],"this":[24,46,182],"draws":[26],"from":[27],"diverse":[28],"sources,":[29],"including":[30,83],"standards":[31],"research":[33],"manuscripts.":[34],"This":[35],"paper":[36],"outlines":[37],"automated":[39],"question":[40],"generation":[41],"framework":[42],"responsible":[43],"for":[44,126],"creating":[45],"provides":[49],"details":[50],"on":[51,123,191],"how":[52,111],"we":[53],"integrated":[54],"human":[55],"input":[56],"at":[57],"various":[58],"stages":[59],"ensure":[61],"quality":[63],"questions.":[66],"Afterwards,":[67],"using":[68],"provided":[70],"dataset,":[71],"an":[72],"evaluation":[73],"is":[74,135,143],"conducted":[75],"assess":[77],"capabilities":[79],"several":[81],"LLMs,":[82],"GPT-3.5,":[84],"GPT-4,":[85],"Mixtral.":[87],"The":[88,151,184],"results":[89,109],"highlight":[90],"that":[91,147,154],"these":[92],"models":[93],"struggle":[94],"with":[95,137],"complex":[96],"standards-related":[97],"but":[99],"exhibit":[100],"proficiency":[101],"addressing":[103],"general":[104],"telecom-related":[105],"inquiries.":[106],"Additionally,":[107],"our":[108],"showcase":[110],"incorporating":[112],"telecom":[113,129,139,164],"context":[115],"significantly":[116],"enhances":[117],"their":[118,168],"performance,":[119],"thus":[120],"shedding":[121],"light":[122],"need":[125],"a":[127],"specialized":[128],"foundation":[130],"model.":[131],"Finally,":[132],"shared":[136],"active":[138,161],"professionals,":[140],"whose":[141],"performance":[142,159],"subsequently":[144],"benchmarked":[145],"against":[146],"LLMs.":[150],"findings":[152],"illustrate":[153],"LLMs":[155,180],"can":[156],"rival":[157],"professionals":[162],"knowledge,":[165],"thanks":[166],"capacity":[169],"process":[171],"vast":[172],"amounts":[173],"information,":[175],"underscoring":[176],"potential":[178],"within":[181],"domain.":[183],"has":[186],"been":[187],"made":[188],"publicly":[189],"accessible":[190],"GitHub<sup":[192],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[193],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">1</sup>.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":9}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}