{"id":"https://openalex.org/W4407170451","doi":"https://doi.org/10.1016/j.array.2026.100834","title":"Towards safer chatbots: Automated policy compliance evaluation of custom GPTs","display_name":"Towards safer chatbots: Automated policy compliance evaluation of custom GPTs","publication_year":2026,"publication_date":"2026-04-21","ids":{"openalex":"https://openalex.org/W4407170451","doi":"https://doi.org/10.1016/j.array.2026.100834"},"language":"en","primary_location":{"id":"doi:10.1016/j.array.2026.100834","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.array.2026.100834","pdf_url":null,"source":{"id":"https://openalex.org/S4210194039","display_name":"Array","issn_l":"2590-0056","issn":["2590-0056"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Array","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1016/j.array.2026.100834","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100609859","display_name":"David Rodr\u00edguez","orcid":"https://orcid.org/0000-0002-0911-4608"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"David Rodriguez","raw_affiliation_strings":["Information Processing and Telecommunications Center, Universidad Polit\u00e9cnica de Madrid, ETSI Telecomunicaci\u00f3n, 28040, Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0002-0911-4608","affiliations":[{"raw_affiliation_string":"Information Processing and Telecommunications Center, Universidad Polit\u00e9cnica de Madrid, ETSI Telecomunicaci\u00f3n, 28040, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024514553","display_name":"William Seymour","orcid":"https://orcid.org/0000-0002-0256-6740"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"William Seymour","raw_affiliation_strings":["King\u2019s College London, London, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-0256-6740","affiliations":[{"raw_affiliation_string":"King\u2019s College London, London, United Kingdom","institution_ids":["https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090208333","display_name":"Jos\u00e9 M. Del \u00c1lamo","orcid":"https://orcid.org/0000-0002-6513-0303"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Jose M. Del Alamo","raw_affiliation_strings":["Information Processing and Telecommunications Center, Universidad Polit\u00e9cnica de Madrid, ETSI Telecomunicaci\u00f3n, 28040, Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0002-6513-0303","affiliations":[{"raw_affiliation_string":"Information Processing and Telecommunications Center, Universidad Polit\u00e9cnica de Madrid, ETSI Telecomunicaci\u00f3n, 28040, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052525506","display_name":"Jos\u00e9 M. Such","orcid":"https://orcid.org/0000-0002-6041-178X"},"institutions":[{"id":"https://openalex.org/I4210117240","display_name":"Instituto de Gesti\u00f3n de la Innovaci\u00f3n y del Conocimiento","ror":"https://ror.org/02rgtxm82","country_code":"ES","type":"facility","lineage":["https://openalex.org/I134820265","https://openalex.org/I4210117240","https://openalex.org/I60053951"]},{"id":"https://openalex.org/I60053951","display_name":"Universitat Polit\u00e8cnica de Val\u00e8ncia","ror":"https://ror.org/01460j859","country_code":"ES","type":"education","lineage":["https://openalex.org/I60053951"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Jose Such","raw_affiliation_strings":["INGENIO (CSIC-Universitat Polit\u00e8cnica de Val\u00e8ncia), Valencia, Spain"],"raw_orcid":"https://orcid.org/0000-0002-6041-178X","affiliations":[{"raw_affiliation_string":"INGENIO (CSIC-Universitat Polit\u00e8cnica de Val\u00e8ncia), Valencia, Spain","institution_ids":["https://openalex.org/I60053951","https://openalex.org/I4210117240"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5090208333","https://openalex.org/A5100609859"],"corresponding_institution_ids":["https://openalex.org/I88060688"],"apc_list":{"value":1350,"currency":"USD","value_usd":1350},"apc_paid":{"value":1350,"currency":"USD","value_usd":1350},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.00476687,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"30","issue":null,"first_page":"100834","last_page":"100834"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/safer","display_name":"SAFER","score":0.9536703824996948},{"id":"https://openalex.org/keywords/compliance","display_name":"Compliance (psychology)","score":0.7487050294876099},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.4127788543701172},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.3214373290538788},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.32071948051452637},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.2956122159957886},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1789470911026001},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.1692633032798767},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.061802834272384644}],"concepts":[{"id":"https://openalex.org/C2776654903","wikidata":"https://www.wikidata.org/wiki/Q2601463","display_name":"SAFER","level":2,"score":0.9536703824996948},{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.7487050294876099},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.4127788543701172},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3214373290538788},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.32071948051452637},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2956122159957886},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1789470911026001},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.1692633032798767},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.061802834272384644}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1016/j.array.2026.100834","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.array.2026.100834","pdf_url":null,"source":{"id":"https://openalex.org/S4210194039","display_name":"Array","issn_l":"2590-0056","issn":["2590-0056"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Array","raw_type":"journal-article"},{"id":"pmh:oai:kclpure.kcl.ac.uk:openaire/43b89ed8-ebc2-42bc-88ca-dff283c45f48","is_oa":false,"landing_page_url":"https://kclpure.kcl.ac.uk/portal/en/publications/43b89ed8-ebc2-42bc-88ca-dff283c45f48","pdf_url":null,"source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Rodriguez, D, Seymour, W, Del Alamo, J & Such, J 2026, 'Towards safer chatbots : Automated policy compliance evaluation of custom GPTs', Array, vol. 30, 100834. https://doi.org/10.1016/j.array.2026.100834","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:arXiv.org:2502.01436","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.01436","pdf_url":"https://arxiv.org/pdf/2502.01436","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.01436","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.01436","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.1016/j.array.2026.100834","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.array.2026.100834","pdf_url":null,"source":{"id":"https://openalex.org/S4210194039","display_name":"Array","issn_l":"2590-0056","issn":["2590-0056"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Array","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320315062","display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","ror":null},{"id":"https://openalex.org/F4320321864","display_name":"Generalitat Valenciana","ror":"https://ror.org/0097mvx21"},{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320337994","display_name":"HORIZON EUROPE Global Challenges and European Industrial Competitiveness","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4230063333","https://openalex.org/W4238355463","https://openalex.org/W1498800420","https://openalex.org/W1515663861","https://openalex.org/W2085855985","https://openalex.org/W2042788600","https://openalex.org/W2007572891","https://openalex.org/W3013707207","https://openalex.org/W2108420737","https://openalex.org/W2775420868"],"abstract_inverted_index":{"User-configured":[0],"chatbots":[1,39,49,216],"built":[2],"on":[3,99],"top":[4],"of":[5,37,70,129,146,160,221,235,255,260,269],"large":[6],"language":[7],"models":[8,181,279],"are":[9,275],"increasingly":[10],"available":[11],"through":[12],"centralized":[13],"marketplaces":[14],"such":[15],"as":[16],"OpenAI\u2019s":[17,106],"GPT":[18,85,153],"Store.":[19,154],"While":[20],"these":[21,198],"platforms":[22],"enforce":[23],"usage":[24,76,107],"policies":[25],"intended":[26],"to":[27,51,196],"prevent":[28],"harmful":[29],"or":[30],"inappropriate":[31],"behavior,":[32,192],"the":[33,68,139,152,161,179,219],"scale":[34],"and":[35,90,111,183,217],"opacity":[36],"customized":[38],"make":[40],"systematic":[41],"policy":[42,77,133,174,224,232,265,273],"enforcement":[43],"challenging.":[44],"As":[45],"a":[46,62,121,142],"result,":[47],"policy-violating":[48,168],"continue":[50],"remain":[52],"publicly":[53],"accessible":[54],"despite":[55],"existing":[56],"review":[57,212],"processes.":[58],"This":[59],"paper":[60],"presents":[61],"fully":[63],"automated":[64,91],"method":[65,82,140,229],"for":[66,131,214,230],"evaluating":[67],"compliance":[69,92,117,225,233],"Custom":[71,148,236,262],"GPTs":[72,149,163,263],"with":[73,170,178,193,242,252],"its":[74],"marketplace":[75],"using":[78,94],"black-box":[79],"interaction.":[80],"The":[81,155],"combines":[83],"large-scale":[84,143,231],"discovery,":[86],"policy-driven":[87],"red-teaming":[88,240],"prompts,":[89],"assessment":[93,118],"an":[95,126],"LLM-as-a-judge.":[96],"We":[97,114,136],"focus":[98],"three":[100],"policy-relevant":[101],"domains":[102],"explicitly":[103],"addressed":[104],"in":[105,141,210,267],"policies:":[108],"Romantic,":[109],"Cybersecurity,":[110],"Academic":[112],"GPTs.":[113,237],"validate":[115],"our":[116],"component":[119],"against":[120,249],"human-annotated":[122],"ground-truth":[123],"dataset,":[124],"achieving":[125],"F1":[127,253],"score":[128,254],"0.975":[130],"binary":[132],"violation":[134],"detection.":[135],"then":[137],"apply":[138],"empirical":[144],"study":[145,259],"782":[147,261],"retrieved":[150],"from":[151,190,277],"results":[156],"show":[157],"that":[158,186],"58.7%":[159,268],"evaluated":[162],"exhibit":[164],"at":[165],"least":[166],"one":[167],"response,":[169],"substantial":[171],"variation":[172],"across":[173],"domains.":[175],"A":[176],"comparison":[177],"base":[180,278],"(GPT-4":[182],"GPT-4o)":[184],"indicates":[185],"most":[187],"violations":[188,266,274],"originate":[189],"model-level":[191],"customization":[194],"tending":[195],"amplify":[197],"tendencies":[199],"rather":[200,280],"than":[201,281],"create":[202],"new":[203],"failure":[204],"modes.":[205],"Our":[206],"findings":[207],"reveal":[208],"limitations":[209],"current":[211],"mechanisms":[213],"user-configured":[215],"demonstrate":[218],"feasibility":[220],"scalable,":[222],"behavior-based":[223],"evaluation.":[226],"\u2022":[227,238,245,257,271],"Automated":[228],"evaluation":[234],"Policy-driven":[239],"combined":[241],"LLM-as-a-judge":[243],"assessment.":[244],"Compliance":[246],"detection":[247],"validated":[248],"human":[250],"labels":[251],"0.975.":[256],"Large-scale":[258],"identifies":[264],"cases.":[270],"Most":[272],"inherited":[276],"customization.":[282]},"counts_by_year":[],"updated_date":"2026-06-19T17:40:00.097472","created_date":"2025-10-10T00:00:00"}