{"id":"https://openalex.org/W4416839008","doi":"https://doi.org/10.48550/arxiv.2507.14719","title":"Policy-Grounded Safety Evaluation of 20 Large Language Models","display_name":"Policy-Grounded Safety Evaluation of 20 Large Language Models","publication_year":2025,"publication_date":"2025-07-19","ids":{"openalex":"https://openalex.org/W4416839008","doi":"https://doi.org/10.48550/arxiv.2507.14719"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2507.14719","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2507.14719","pdf_url":"https://arxiv.org/pdf/2507.14719","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2507.14719","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120672831","display_name":"Juan Manuel Contreras","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Contreras, Juan Manuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5120672831"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5199000239372253,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5199000239372253,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0997999981045723,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.062300000339746475,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/misinformation","display_name":"Misinformation","score":0.5371999740600586},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.4108000099658966},{"id":"https://openalex.org/keywords/occupational-safety-and-health","display_name":"Occupational safety and health","score":0.40639999508857727},{"id":"https://openalex.org/keywords/human-factors-and-ergonomics","display_name":"Human factors and ergonomics","score":0.3968999981880188},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.375},{"id":"https://openalex.org/keywords/poison-control","display_name":"Poison control","score":0.37299999594688416}],"concepts":[{"id":"https://openalex.org/C2776990098","wikidata":"https://www.wikidata.org/wiki/Q13579947","display_name":"Misinformation","level":2,"score":0.5371999740600586},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4643000066280365},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.4205999970436096},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.4108000099658966},{"id":"https://openalex.org/C187155963","wikidata":"https://www.wikidata.org/wiki/Q629029","display_name":"Occupational safety and health","level":2,"score":0.40639999508857727},{"id":"https://openalex.org/C166735990","wikidata":"https://www.wikidata.org/wiki/Q1750812","display_name":"Human factors and ergonomics","level":3,"score":0.3968999981880188},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.376800000667572},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.375},{"id":"https://openalex.org/C3017944768","wikidata":"https://www.wikidata.org/wiki/Q1450463","display_name":"Poison control","level":2,"score":0.37299999594688416},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.35569998621940613},{"id":"https://openalex.org/C526869908","wikidata":"https://www.wikidata.org/wiki/Q3298118","display_name":"Suicide prevention","level":3,"score":0.3431999981403351},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3319999873638153},{"id":"https://openalex.org/C190385971","wikidata":"https://www.wikidata.org/wiki/Q373494","display_name":"Injury prevention","level":3,"score":0.31630000472068787},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2953000068664551},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.2879999876022339}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2507.14719","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2507.14719","pdf_url":"https://arxiv.org/pdf/2507.14719","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2507.14719","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2507.14719","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2507.14719","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2507.14719","pdf_url":"https://arxiv.org/pdf/2507.14719","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416839008.pdf","grobid_xml":"https://content.openalex.org/works/W4416839008.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"large":[1],"language":[2],"models":[3,93,133],"(LLMs)":[4],"become":[5],"increasingly":[6],"integrated":[7],"into":[8,40],"real-world":[9,75],"applications,":[10],"scalable":[11],"and":[12,28,43,64,134,144,150,166],"rigorous":[13],"safety":[14,32,38,76,85,98,127,149],"evaluation":[15],"is":[16],"essential.":[17],"This":[18],"paper":[19],"introduces":[20],"Aymara":[21,34,61,159],"AI,":[22],"a":[23],"programmatic":[24],"platform":[25],"for":[26,154],"generating":[27],"administering":[29],"customized,":[30],"policy-grounded":[31],"evaluations.":[33],"AI":[35,160,164],"transforms":[36],"natural-language":[37],"policies":[39],"adversarial":[41],"prompts":[42],"scores":[44,86,128],"model":[45],"responses":[46],"using":[47],"an":[48],"AI-based":[49],"rater":[50],"validated":[51],"against":[52],"human":[53],"judgments.":[54],"We":[55],"demonstrate":[56],"its":[57],"capabilities":[58],"through":[59],"the":[60,142,152],"LLM":[62,148],"Risk":[63],"Responsibility":[65],"Matrix,":[66],"which":[67],"evaluates":[68],"20":[69],"commercially":[70],"available":[71],"LLMs":[72],"across":[73,131],"10":[74],"domains.":[77],"Results":[78],"reveal":[79],"wide":[80],"performance":[81],"disparities,":[82],"with":[83],"mean":[84],"ranging":[87],"from":[88],"86.2%":[89],"to":[90,161],"52.4%.":[91],"While":[92],"performed":[94],"well":[95],"in":[96,109],"well-established":[97],"domains":[99,135],"such":[100],"as":[101],"Misinformation":[102],"(mean":[103,119],"=":[104,120],"95.7%),":[105],"they":[106],"consistently":[107],"failed":[108],"more":[110],"complex":[111],"or":[112],"underspecified":[113],"domains,":[114],"notably":[115],"Privacy":[116],"&":[117],"Impersonation":[118],"24.3%).":[121],"Analyses":[122],"of":[123,147],"Variance":[124],"confirmed":[125],"that":[126],"differed":[129],"significantly":[130],"both":[132],"(p":[136],"<":[137],".05).":[138],"These":[139],"findings":[140],"underscore":[141],"inconsistent":[143],"context-dependent":[145],"nature":[146],"highlight":[151],"need":[153],"scalable,":[155],"customizable":[156],"tools":[157],"like":[158],"support":[162],"responsible":[163],"development":[165],"oversight.":[167]},"counts_by_year":[],"updated_date":"2026-08-18T07:49:30.821534","created_date":"2025-12-01T00:00:00"}