{"id":"https://openalex.org/W7140314544","doi":"https://doi.org/10.48550/arxiv.2603.23142","title":"Can Language Models Pass Software Testing Certification Exams? a case study","display_name":"Can Language Models Pass Software Testing Certification Exams? a case study","publication_year":2026,"publication_date":"2026-03-24","ids":{"openalex":"https://openalex.org/W7140314544","doi":"https://doi.org/10.48550/arxiv.2603.23142"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.23142","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23142","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.23142","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009828484","display_name":"Fitash Ul Haq","orcid":"https://orcid.org/0000-0003-2253-9085"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haq, Fitash Ul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5124090227","display_name":"Jordi Cabot","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cabot, Jordi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8098999857902527,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8098999857902527,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.04430000111460686,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.023900000378489494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/certification","display_name":"Certification","score":0.6783000230789185},{"id":"https://openalex.org/keywords/competence","display_name":"Competence (human resources)","score":0.45489999651908875},{"id":"https://openalex.org/keywords/system-integration-testing","display_name":"System integration testing","score":0.4462999999523163},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.39309999346733093},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.3912999927997589},{"id":"https://openalex.org/keywords/manual-testing","display_name":"Manual testing","score":0.38199999928474426},{"id":"https://openalex.org/keywords/conceptual-model","display_name":"Conceptual model","score":0.37209999561309814},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.3700000047683716},{"id":"https://openalex.org/keywords/software-reliability-testing","display_name":"Software reliability testing","score":0.3691999912261963}],"concepts":[{"id":"https://openalex.org/C46304622","wikidata":"https://www.wikidata.org/wiki/Q374814","display_name":"Certification","level":2,"score":0.6783000230789185},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.618399977684021},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.578000009059906},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.45489999651908875},{"id":"https://openalex.org/C111524372","wikidata":"https://www.wikidata.org/wiki/Q7663718","display_name":"System integration testing","level":5,"score":0.4462999999523163},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.39309999346733093},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.3912999927997589},{"id":"https://openalex.org/C182122060","wikidata":"https://www.wikidata.org/wiki/Q6752328","display_name":"Manual testing","level":5,"score":0.38199999928474426},{"id":"https://openalex.org/C13606891","wikidata":"https://www.wikidata.org/wiki/Q2623243","display_name":"Conceptual model","level":2,"score":0.37209999561309814},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.3700000047683716},{"id":"https://openalex.org/C52928878","wikidata":"https://www.wikidata.org/wiki/Q7554226","display_name":"Software reliability testing","level":5,"score":0.3691999912261963},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.34439998865127563},{"id":"https://openalex.org/C178059732","wikidata":"https://www.wikidata.org/wiki/Q1982529","display_name":"Software performance testing","level":5,"score":0.33660000562667847},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.32600000500679016},{"id":"https://openalex.org/C74579156","wikidata":"https://www.wikidata.org/wiki/Q7554342","display_name":"Software peer review","level":5,"score":0.3197000026702881},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.3158999979496002},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.299699991941452},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29649999737739563},{"id":"https://openalex.org/C188329197","wikidata":"https://www.wikidata.org/wiki/Q6554613","display_name":"Software quality analyst","level":5,"score":0.2953000068664551},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.2833000123500824},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.25949999690055847},{"id":"https://openalex.org/C166007726","wikidata":"https://www.wikidata.org/wiki/Q4536542","display_name":"Language assessment","level":2,"score":0.2542000114917755},{"id":"https://openalex.org/C86469151","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Non-regression testing","level":5,"score":0.2540999948978424},{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.2538999915122986},{"id":"https://openalex.org/C14224292","wikidata":"https://www.wikidata.org/wiki/Q13600188","display_name":"Conceptual framework","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.23142","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23142","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.23142","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23142","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"play":[4],"a":[5],"pivotal":[6],"role":[7],"in":[8,20,179],"both":[9,97],"academic":[10],"research":[11],"and":[12,30,71,100,120,200],"broader":[13],"societal":[14],"applications.":[15],"LLMs":[16,39,55],"are":[17,131,157],"increasingly":[18],"used":[19,132],"software":[21,44,49,61,65,77,208],"testing":[22,45,50,62,66,78,209],"activities":[23],"such":[24],"as":[25],"test":[26],"case":[27],"generation,":[28],"selection,":[29],"repair.":[31],"However,":[32],"several":[33],"important":[34],"questions":[35,67,79,162],"remain:":[36],"(1)":[37],"do":[38,54,73],"possess":[40,56],"enough":[41],"information":[42],"about":[43],"principles":[46],"to":[47,63,133,150],"perform":[48],"tasks":[51],"effectively?":[52],"(2)":[53],"sufficient":[57,152],"conceptual":[58,153],"understanding":[59],"of":[60,76,83,112,137,181,207],"answer":[64,86],"under":[68],"metamorphic":[69,166],"transformations?":[70],"(3)":[72],"certain":[74],"properties":[75],"influence":[80],"the":[81,101,123,135,155,172,182,195,205],"performance":[82],"LLMs?":[84],"To":[85],"these":[87],"questions,":[88],"this":[89],"study":[90],"evaluates":[91],"60":[92],"multimodal":[93],"language":[94],"models":[95,156,169,188],"from":[96,122],"commercial":[98,187],"vendors":[99],"open-source":[102,191],"community.":[103],"The":[104],"evaluation":[105],"is":[106,144],"performed":[107],"using":[108,164],"30":[109,183],"sample":[110],"exams":[111],"different":[113],"types":[114],"(core":[115],"foundation,":[116],"core":[117],"advanced,":[118],"specialist,":[119],"expert)":[121],"International":[124],"Software":[125],"Testing":[126],"Qualifications":[127],"Board":[128],"(ISTQB),":[129],"which":[130],"assess":[134],"competence":[136],"human":[138],"testers.":[139],"In":[140],"total,":[141],"each":[142],"model":[143],"evaluated":[145],"on":[146,160],"1,171":[147],"questions.":[148],"Furthermore,":[149],"ensure":[151],"understanding,":[154],"also":[158],"tested":[159],"exam":[161],"transformed":[163],"context-preserving":[165],"techniques.":[167],"Two":[168],"passed":[170],"all":[171,180],"certifications":[173],"by":[174],"scoring":[175],"at":[176],"least":[177],"65%":[178],"certification":[184,210],"exams,":[185],"with":[186],"generally":[189],"outperforming":[190],"ones.":[192],"We":[193],"analyze":[194],"reasons":[196],"behind":[197],"incorrect":[198],"answers":[199],"provide":[201],"recommendations":[202],"for":[203],"improving":[204],"design":[206],"exams.":[211]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-26T00:00:00"}