{"id":"https://openalex.org/W4391759634","doi":"https://doi.org/10.1109/tse.2024.3363611","title":"A Systematic Review of IoT Systems Testing: Objectives, Approaches, Tools, and Challenges","display_name":"A Systematic Review of IoT Systems Testing: Objectives, Approaches, Tools, and Challenges","publication_year":2024,"publication_date":"2024-02-12","ids":{"openalex":"https://openalex.org/W4391759634","doi":"https://doi.org/10.1109/tse.2024.3363611"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3363611","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3363611","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032494599","display_name":"Jean Baptiste Minani","orcid":"https://orcid.org/0000-0002-9164-6645"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jean Baptiste Minani","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Concordia University, Montreal, QC, Canada"],"raw_orcid":"https://orcid.org/0000-0002-9164-6645","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Concordia University, Montreal, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042078753","display_name":"Fatima Sabir","orcid":"https://orcid.org/0000-0001-8707-6603"},"institutions":[{"id":"https://openalex.org/I172780181","display_name":"University of the Punjab","ror":"https://ror.org/011maz450","country_code":"PK","type":"education","lineage":["https://openalex.org/I172780181"]},{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA","PK"],"is_corresponding":false,"raw_author_name":"Fatima Sabir","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Concordia University, Canada","Department of Computer Sciences, University of the Punjab, Lahore, Pakistan"],"raw_orcid":"https://orcid.org/0000-0001-8707-6603","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Concordia University, Canada","institution_ids":["https://openalex.org/I60158472"]},{"raw_affiliation_string":"Department of Computer Sciences, University of the Punjab, Lahore, Pakistan","institution_ids":["https://openalex.org/I172780181"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007508017","display_name":"Naouel Moha","orcid":"https://orcid.org/0000-0001-9252-9937"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Naouel Moha","raw_affiliation_strings":["Department of Computer Science, École de Technologie Supérieure (ÉTS) \u2013 Université du Québec, Montreal, QC, Canada"],"raw_orcid":"https://orcid.org/0000-0001-9252-9937","affiliations":[{"raw_affiliation_string":"Department of Computer Science, École de Technologie Supérieure (ÉTS) \u2013 Université du Québec, Montreal, QC, Canada","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048594006","display_name":"Yann\u2010Ga\u00ebl Gu\u00e9h\u00e9neuc","orcid":"https://orcid.org/0000-0002-4361-2563"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Yann-Ga\u00ebl Gu\u00e9h\u00e9neuc","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Concordia University, Montreal, QC, Canada"],"raw_orcid":"https://orcid.org/0000-0002-4361-2563","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Concordia University, Montreal, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.2008,"has_fulltext":false,"cited_by_count":32,"citation_normalized_percentile":{"value":0.98378186,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"50","issue":"4","first_page":"785","last_page":"815"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7874116897583008},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.6187340617179871},{"id":"https://openalex.org/keywords/internet-of-things","display_name":"Internet of Things","score":0.5311678051948547},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5184785723686218},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4938684105873108},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.45948800444602966},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.3438665568828583},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3265337347984314},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.204708993434906},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.1357850730419159}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7874116897583008},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.6187340617179871},{"id":"https://openalex.org/C81860439","wikidata":"https://www.wikidata.org/wiki/Q251212","display_name":"Internet of Things","level":2,"score":0.5311678051948547},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5184785723686218},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4938684105873108},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.45948800444602966},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3438665568828583},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3265337347984314},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.204708993434906},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.1357850730419159},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tse.2024.3363611","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3363611","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:espace2.etsmtl.ca:28501","is_oa":false,"landing_page_url":"https://espace2.etsmtl.ca/id/eprint/28501/","pdf_url":null,"source":{"id":"https://openalex.org/S4306402392","display_name":"Espace \u00c9TS (ETS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1341030882","host_organization_name":"Educational Testing Service","host_organization_lineage":["https://openalex.org/I1341030882"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article publi\u00e9 dans une revue, r\u00e9vis\u00e9 par les pairs"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8500000238418579,"id":"https://metadata.un.org/sdg/7","display_name":"Affordable and clean energy"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":143,"referenced_works":["https://openalex.org/W1508143344","https://openalex.org/W1531836108","https://openalex.org/W1590942805","https://openalex.org/W2037789405","https://openalex.org/W2043510950","https://openalex.org/W2048041296","https://openalex.org/W2102586032","https://openalex.org/W2111335329","https://openalex.org/W2118167481","https://openalex.org/W2140243388","https://openalex.org/W2163621605","https://openalex.org/W2202090419","https://openalex.org/W2247871685","https://openalex.org/W2295601601","https://openalex.org/W2492337228","https://openalex.org/W2529828338","https://openalex.org/W2573848005","https://openalex.org/W2581042172","https://openalex.org/W2587997002","https://openalex.org/W2588970946","https://openalex.org/W2612533336","https://openalex.org/W2615918556","https://openalex.org/W2618974155","https://openalex.org/W2718978864","https://openalex.org/W2744812396","https://openalex.org/W2744867331","https://openalex.org/W2749021693","https://openalex.org/W2755418175","https://openalex.org/W2771407264","https://openalex.org/W2775312613","https://openalex.org/W2777551158","https://openalex.org/W2779475380","https://openalex.org/W2782743160","https://openalex.org/W2783690368","https://openalex.org/W2788922520","https://openalex.org/W2791199353","https://openalex.org/W2792171464","https://openalex.org/W2801733426","https://openalex.org/W2805204587","https://openalex.org/W2805511363","https://openalex.org/W2806778556","https://openalex.org/W2809697408","https://openalex.org/W2883476426","https://openalex.org/W2885063984","https://openalex.org/W2885304496","https://openalex.org/W2888657876","https://openalex.org/W2891795716","https://openalex.org/W2896499546","https://openalex.org/W2896785522","https://openalex.org/W2898101126","https://openalex.org/W2899429604","https://openalex.org/W2899592006","https://openalex.org/W2900337970","https://openalex.org/W2902216132","https://openalex.org/W2902473403","https://openalex.org/W2903812886","https://openalex.org/W2908873467","https://openalex.org/W2909043639","https://openalex.org/W2910778426","https://openalex.org/W2915007698","https://openalex.org/W2941765909","https://openalex.org/W2946339517","https://openalex.org/W2946433174","https://openalex.org/W2949704390","https://openalex.org/W2955029264","https://openalex.org/W2955481530","https://openalex.org/W2960144990","https://openalex.org/W2971926888","https://openalex.org/W2971930899","https://openalex.org/W2973697807","https://openalex.org/W2974769630","https://openalex.org/W2979491945","https://openalex.org/W2990231074","https://openalex.org/W2991175020","https://openalex.org/W2991315158","https://openalex.org/W2991345810","https://openalex.org/W2991435551","https://openalex.org/W2992100963","https://openalex.org/W3003749772","https://openalex.org/W3006477812","https://openalex.org/W3008301453","https://openalex.org/W3010530047","https://openalex.org/W3010936976","https://openalex.org/W3011754649","https://openalex.org/W3013398655","https://openalex.org/W3021110750","https://openalex.org/W3025710795","https://openalex.org/W3028244571","https://openalex.org/W3028511678","https://openalex.org/W3029349360","https://openalex.org/W3036314835","https://openalex.org/W3037478491","https://openalex.org/W3043039374","https://openalex.org/W3087788272","https://openalex.org/W3093879603","https://openalex.org/W3095441340","https://openalex.org/W3105687553","https://openalex.org/W3105888471","https://openalex.org/W3106580438","https://openalex.org/W3111264080","https://openalex.org/W3123893780","https://openalex.org/W3128203310","https://openalex.org/W3135742583","https://openalex.org/W3146142859","https://openalex.org/W3156207907","https://openalex.org/W3160783671","https://openalex.org/W3161919951","https://openalex.org/W3165093798","https://openalex.org/W3166552890","https://openalex.org/W3189886031","https://openalex.org/W3191432102","https://openalex.org/W3194947008","https://openalex.org/W3200618075","https://openalex.org/W3201451281","https://openalex.org/W3203564925","https://openalex.org/W3210865440","https://openalex.org/W4200210069","https://openalex.org/W4206112135","https://openalex.org/W4206880630","https://openalex.org/W4210678207","https://openalex.org/W4214808596","https://openalex.org/W4224440113","https://openalex.org/W4230313442","https://openalex.org/W4233449030","https://openalex.org/W4253439110","https://openalex.org/W4285248702","https://openalex.org/W4289656789","https://openalex.org/W4292825917","https://openalex.org/W4296056485","https://openalex.org/W4297102271","https://openalex.org/W4300972769","https://openalex.org/W4304172242","https://openalex.org/W4312543091","https://openalex.org/W4317642351","https://openalex.org/W4378882743","https://openalex.org/W4382658138","https://openalex.org/W4386399631","https://openalex.org/W4387453400","https://openalex.org/W6695307027","https://openalex.org/W6756832365","https://openalex.org/W6777891483","https://openalex.org/W6903871005","https://openalex.org/W7046478302"],"related_works":["https://openalex.org/W4245926026","https://openalex.org/W4311097251","https://openalex.org/W2586548817","https://openalex.org/W2625093826","https://openalex.org/W4200598720","https://openalex.org/W2921026492","https://openalex.org/W4247463117","https://openalex.org/W4361251261","https://openalex.org/W3031181660","https://openalex.org/W4238100021"],"abstract_inverted_index":{"Internet":[0],"of":[1,35,44,73,213],"Things":[2],"(IoT)":[3],"systems":[4,18,37,136,143,170,198],"are":[5,173],"becoming":[6],"prevalent":[7],"in":[8,21,218,227],"various":[9],"domains,":[10],"from":[11,131],"healthcare":[12],"to":[13,53,99,117,133,175],"smart":[14],"homes.":[15],"Testing":[16],"IoT":[17,36,46,79,135,142,169,197,248,260],"is":[19],"critical":[20],"ensuring":[22],"their":[23,200],"reliability.":[24],"Previous":[25],"papers":[26],"studied":[27],"separately":[28],"the":[29,41,45,71,88,129,203,223,239,252],"objectives,":[30,81,241],"approaches,":[31,82,242,267],"tools,":[32,83,243],"and":[33,69,84,95,103,114,123,127,162,178,192,244,266],"challenges":[34,245],"testing.":[38,137,185],"However,":[39],"despite":[40],"rapid":[42],"evolution":[43],"domain,":[47],"no":[48],"review":[49,65],"has":[50],"been":[51],"undertaken":[52],"investigate":[54],"all":[55],"four":[56],"aspects":[57,177],"collectively.":[58],"This":[59],"paper":[60],"presents":[61],"a":[62,231],"systematic":[63],"literature":[64],"that":[66,141,167],"aggregates,":[67],"synthesizes,":[68],"discusses":[70],"results":[72,139],"83":[74],"primary":[75],"studies":[76,119],"(PSs)":[77],"concerning":[78],"testing":[80,144,171,190,196,226,240],"challenges.":[85],"We":[86,125],"followed":[87],"Preferred":[89],"Reporting":[90],"Items":[91],"for":[92,182,195,207,225,259,274],"Systematic":[93],"Review":[94],"Meta-Analysis":[96],"(PRISMA)":[97],"protocol":[98],"report":[100],"our":[101,234],"findings":[102],"answer":[104],"research":[105,272],"questions":[106],"(RQs).":[107],"To":[108],"select":[109],"PSs,":[110],"we":[111,254],"applied":[112],"inclusion":[113],"exclusion":[115],"criteria":[116],"relevant":[118],"published":[120],"between":[121],"2012":[122],"2022.":[124],"extracted":[126],"analyzed":[128],"data":[130,217],"PSs":[132],"understand":[134],"The":[138,210],"reveal":[140],"embraces":[145],"traditional":[146],"software":[147],"quality":[148],"attributes":[149],"but":[150],"also":[151,165,187,255,269],"introduces":[152],"new":[153,271],"ones":[154],"like":[155],"connectivity,":[156],"energy":[157],"efficiency,":[158],"device":[159],"lifespan,":[160],"distributivity,":[161],"dynamicity.":[163],"They":[164,186],"show":[166,188],"existing":[168,264],"approaches":[172],"limited":[174],"specific":[176],"should":[179],"be":[180],"expanded":[181],"more":[183],"comprehensive":[184],"19":[189],"tools":[191,265],"15":[193],"testbeds":[194],"with":[199,222,247],"limitations,":[201],"necessitating":[202],"development":[204],"or":[205],"enhancement":[206],"wider":[208],"coverage.":[209],"large":[211],"number":[212],"heterogeneous":[214],"devices":[215],"generating":[216],"different":[219],"formats,":[220],"along":[221],"need":[224],"real-world":[228],"scenarios,":[229],"poses":[230],"challenge.":[232],"Thus,":[233],"study":[235],"offers":[236],"insights":[237],"into":[238],"associated":[246],"systems.":[249],"Based":[250],"on":[251],"results,":[253],"provide":[256],"practical":[257],"guidance":[258],"practitioners":[261],"by":[262],"cataloging":[263],"while":[268],"identifying":[270],"opportunities":[273],"interested":[275],"researchers.":[276]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":5}],"updated_date":"2026-07-02T09:51:11.867554","created_date":"2025-10-10T00:00:00"}