{"id":"https://openalex.org/W7163693454","doi":"https://doi.org/10.1016/j.infsof.2026.108215","title":"An empirical investigation of Large Language Models for automated software compliance testing: Evidence from web accessibility","display_name":"An empirical investigation of Large Language Models for automated software compliance testing: Evidence from web accessibility","publication_year":2026,"publication_date":"2026-06-05","ids":{"openalex":"https://openalex.org/W7163693454","doi":"https://doi.org/10.1016/j.infsof.2026.108215"},"language":"en","primary_location":{"id":"doi:10.1016/j.infsof.2026.108215","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108215","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.infsof.2026.108215","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137767140","display_name":"Juan-Miguel L\u00f3pez-Gil","orcid":"https://orcid.org/0000-0001-7730-0472"},"institutions":[{"id":"https://openalex.org/I169108374","display_name":"University of the Basque Country","ror":"https://ror.org/000xsnr85","country_code":"ES","type":"education","lineage":["https://openalex.org/I169108374"]},{"id":"https://openalex.org/I47686490","display_name":"Donostia International Physics Center","ror":"https://ror.org/02e24yw40","country_code":"ES","type":"other","lineage":["https://openalex.org/I47686490"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Juan-Miguel L\u00f3pez-Gil","raw_affiliation_strings":["Department of Computer Languages and Systems, University of the Basque Country (UPV/EHU), Manuel Lardizabal 1, Donostia-San Sebastian, 20018, Spain"],"raw_orcid":"https://orcid.org/0000-0001-7730-0472","affiliations":[{"raw_affiliation_string":"Department of Computer Languages and Systems, University of the Basque Country (UPV/EHU), Manuel Lardizabal 1, Donostia-San Sebastian, 20018, Spain","institution_ids":["https://openalex.org/I169108374","https://openalex.org/I47686490"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5135861387","display_name":"Juanan Pereira","orcid":"https://orcid.org/0000-0002-7935-3612"},"institutions":[{"id":"https://openalex.org/I169108374","display_name":"University of the Basque Country","ror":"https://ror.org/000xsnr85","country_code":"ES","type":"education","lineage":["https://openalex.org/I169108374"]},{"id":"https://openalex.org/I47686490","display_name":"Donostia International Physics Center","ror":"https://ror.org/02e24yw40","country_code":"ES","type":"other","lineage":["https://openalex.org/I47686490"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Juanan Pereira","raw_affiliation_strings":["Department of Computer Languages and Systems, University of the Basque Country (UPV/EHU), Manuel Lardizabal 1, Donostia-San Sebastian, 20018, Spain"],"raw_orcid":"https://orcid.org/0000-0002-7935-3612","affiliations":[{"raw_affiliation_string":"Department of Computer Languages and Systems, University of the Basque Country (UPV/EHU), Manuel Lardizabal 1, Donostia-San Sebastian, 20018, Spain","institution_ids":["https://openalex.org/I169108374","https://openalex.org/I47686490"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5137767140"],"corresponding_institution_ids":["https://openalex.org/I169108374","https://openalex.org/I47686490"],"apc_list":{"value":3350,"currency":"USD","value_usd":3350},"apc_paid":{"value":3350,"currency":"USD","value_usd":3350},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.93391762,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":"198","issue":null,"first_page":"108215","last_page":"108215"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12481","display_name":"Digital Accessibility for Disabilities","score":0.4799000024795532,"subfield":{"id":"https://openalex.org/subfields/3307","display_name":"Human Factors and Ergonomics"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12481","display_name":"Digital Accessibility for Disabilities","score":0.4799000024795532,"subfield":{"id":"https://openalex.org/subfields/3307","display_name":"Human Factors and Ergonomics"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.20069999992847443,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.056699998676776886,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.475600004196167},{"id":"https://openalex.org/keywords/compliance","display_name":"Compliance (psychology)","score":0.40959998965263367},{"id":"https://openalex.org/keywords/web-accessibility","display_name":"Web accessibility","score":0.40779998898506165},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.3610999882221222},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.33809998631477356},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.31450000405311584}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6517000198364258},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.475600004196167},{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.40959998965263367},{"id":"https://openalex.org/C194669863","wikidata":"https://www.wikidata.org/wiki/Q808932","display_name":"Web accessibility","level":4,"score":0.40779998898506165},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.4011000096797943},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.3610999882221222},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.34929999709129333},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.33809998631477356},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.31450000405311584},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.296099990606308},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.2896000146865845},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.28459998965263367},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.2502000033855438}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.infsof.2026.108215","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108215","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.infsof.2026.108215","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108215","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321919","display_name":"Euskal Herriko Unibertsitatea","ror":"https://ror.org/000xsnr85"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W2062015732","https://openalex.org/W2108059865","https://openalex.org/W2143845053","https://openalex.org/W2168745915","https://openalex.org/W2999363798","https://openalex.org/W3003264646","https://openalex.org/W3176115515","https://openalex.org/W4292092805","https://openalex.org/W4297910414","https://openalex.org/W4306411323","https://openalex.org/W4389068312","https://openalex.org/W4389980560","https://openalex.org/W4392881840","https://openalex.org/W4394015350","https://openalex.org/W4404302073","https://openalex.org/W4410202262","https://openalex.org/W4411718336","https://openalex.org/W4412930207"],"related_works":[],"abstract_inverted_index":{"Context:":[0],"Automated":[1],"compliance":[2,18],"testing":[3,37],"faces":[4],"challenges":[5],"when":[6],"requirements":[7],"demand":[8],"semantic":[9],"understanding.":[10],"Traditional":[11,113],"tools":[12,114,144],"evaluate":[13],"at":[14],"most":[15],"50%":[16],"of":[17],"criteria,":[19,44],"necessitating":[20],"expensive":[21],"manual":[22],"verification.":[23],"Objectives:":[24],"This":[25],"study":[26],"evaluates":[27],"Large":[28],"Language":[29],"Model":[30],"(LLM)":[31],"capabilities":[32],"for":[33,220],"web":[34],"accessibility":[35],"conformance":[36],"across":[38],"39":[39],"WCAG":[40],"2":[41],"AA":[42],"success":[43],"examining":[45],"performance,":[46],"prompt":[47,189],"engineering,":[48],"and":[49,124,156,183,193],"proposing":[50],"an":[51,163],"interpretive":[52,197],"framework.":[53],"Methods:":[54],"We":[55],"evaluated":[56],"seven":[57],"cost-effective":[58],"LLMs":[59,78,128,147,182,216],"on":[60,200],"isolated":[61],"HTML":[62],"snippets":[63],"from":[64,89],"384":[65],"W3C":[66],"ACT":[67],"cases.":[68],"Three":[69],"prompting":[70],"strategies":[71],"were":[72],"tested":[73],"with":[74,103,149],"five":[75],"replications,":[76],"comparing":[77],"against":[79],"traditional":[80],"tools.":[81],"Results":[82],"characterise":[83],"snippet-level":[84],"evaluation,":[85],"which":[86],"may":[87],"differ":[88],"full-page":[90],"testing.":[91],"Results:":[92],"The":[93],"best":[94,218],"configuration":[95],"(deepseek-reasoner":[96],"under":[97],"provider-default":[98],"settings)":[99],"achieved":[100,115],"71.52%":[101],"accuracy":[102,118],"example-based":[104],"prompts\u2014an":[105],"8.2%":[106],"cross-model":[107],"average":[108],"improvement":[109],"over":[110],"basic":[111],"prompts.":[112],"22.9\u201333.9%":[116],"real":[117],"(treating":[119],"coverage":[120],"gaps":[121],"as":[122],"errors)":[123],"22.9\u201344.4%":[125],"decision":[126],"accuracy.":[127],"classified":[129],"every":[130],"test":[131,151],"case":[132],"(100%":[133],"coverage),":[134],"reflecting":[135],"their":[136],"always-answering":[137,181],"design":[138],"rather":[139],"than":[140],"superior":[141],"capability,":[142],"whereas":[143],"selectively":[145],"abstain.":[146],"struggled":[148],"determining":[150],"applicability":[152],"(45%\u201367%":[153],"error":[154],"rates)":[155],"Understandable":[157],"criteria.":[158],"Conclusion:":[159],"Contributions":[160],"include:":[161],"(1)":[162],"unprecedented":[164],"39-criteria":[165],"empirical":[166],"evaluation;":[167],"(2)":[168],"a":[169,195],"dual-metric":[170],"framework":[171,198],"(Real":[172],"vs.":[173],"Decision":[174],"Accuracy)":[175],"clarifying":[176],"the":[177],"structural":[178],"difference":[179],"between":[180],"abstaining":[184],"tools;":[185],"(3)":[186],"evidence":[187],"that":[188],"engineering":[190],"improves":[191],"performance;":[192],"(4)":[194],"post-hoc":[196],"(based":[199],"expert":[201],"judgment,":[202],"not":[203],"validated":[204],"measurement)":[205],"relating":[206],"criterion":[207],"characteristics":[208],"to":[209,212],"LLM":[210],"amenability":[211],"generate":[213],"future":[214],"hypotheses.":[215],"remain":[217],"suited":[219],"human-in-the-loop":[221],"workflows.":[222]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-06-06T00:00:00"}