{"id":"https://openalex.org/W7160627183","doi":"https://doi.org/10.48550/arxiv.2605.06652","title":"When No Benchmark Exists: Validating Comparative LLM Safety Scoring Without Ground-Truth Labels","display_name":"When No Benchmark Exists: Validating Comparative LLM Safety Scoring Without Ground-Truth Labels","publication_year":2026,"publication_date":"2026-05-07","ids":{"openalex":"https://openalex.org/W7160627183","doi":"https://doi.org/10.48550/arxiv.2605.06652"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.06652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.06652","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089611382","display_name":"Sushant Gautam","orcid":"https://orcid.org/0000-0001-9232-2661"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gautam, Sushant","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093868877","display_name":"Finn Schwall","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schwall, Finn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135704978","display_name":"Annika Willoch Olstad","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Olstad, Annika Willoch","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135657668","display_name":"Fernando Vallecillos Ruiz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ruiz, Fernando Vallecillos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082684945","display_name":"Birk Torpmann-Hagen","orcid":"https://orcid.org/0000-0002-9685-9906"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Torpmann-Hagen, Birk","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031403078","display_name":"Sunniva Maria Stordal Bj\u00f8rklund","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bj\u00f8rklund, Sunniva Maria Stordal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056189345","display_name":"Leon Moonen","orcid":"https://orcid.org/0000-0002-1761-6771"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moonen, Leon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135683677","display_name":"Klas Pettersen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pettersen, Klas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135714161","display_name":"Michael A. Riegler","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Riegler, Michael A.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13295","display_name":"Safety Systems Engineering in Autonomy","score":0.13490000367164612,"subfield":{"id":"https://openalex.org/subfields/2213","display_name":"Safety, Risk, Reliability and Quality"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13295","display_name":"Safety Systems Engineering in Autonomy","score":0.13490000367164612,"subfield":{"id":"https://openalex.org/subfields/2213","display_name":"Safety, Risk, Reliability and Quality"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10809","display_name":"Occupational Health and Safety Research","score":0.10019999742507935,"subfield":{"id":"https://openalex.org/subfields/3614","display_name":"Radiological and Ultrasound Technology"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.04050000011920929,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5809999704360962},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.5561000108718872},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.4950000047683716},{"id":"https://openalex.org/keywords/enforcement","display_name":"Enforcement","score":0.4343999922275543},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.39160001277923584},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.36309999227523804},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.3472999930381775}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6736000180244446},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5809999704360962},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.5561000108718872},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.4950000047683716},{"id":"https://openalex.org/C2779777834","wikidata":"https://www.wikidata.org/wiki/Q4202277","display_name":"Enforcement","level":2,"score":0.4343999922275543},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.39160001277923584},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.36309999227523804},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.3472999930381775},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33869999647140503},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3382999897003174},{"id":"https://openalex.org/C63428231","wikidata":"https://www.wikidata.org/wiki/Q9043","display_name":"Norwegian","level":2,"score":0.33230000734329224},{"id":"https://openalex.org/C2776654903","wikidata":"https://www.wikidata.org/wiki/Q2601463","display_name":"SAFER","level":2,"score":0.322299987077713},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3154999911785126},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3111000061035156},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3061000108718872},{"id":"https://openalex.org/C201650216","wikidata":"https://www.wikidata.org/wiki/Q829492","display_name":"Procurement","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C2778355321","wikidata":"https://www.wikidata.org/wiki/Q17079427","display_name":"Identity (music)","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.25220000743865967},{"id":"https://openalex.org/C49261128","wikidata":"https://www.wikidata.org/wiki/Q1132455","display_name":"Hazard","level":2,"score":0.2515999972820282}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.06652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.06652","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06652","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.5080989599227905,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Many":[0],"deployments":[1],"must":[2,207],"compare":[3],"candidate":[4],"language":[5],"models":[6],"for":[7,14],"safety":[8,29,111],"before":[9],"a":[10,37,51,78,101,109,215],"labeled":[11],"benchmark":[12],"exists":[13],"the":[15,33,97,128,143,160,179,184,202],"relevant":[16],"language,":[17],"sector,":[18],"or":[19],"regulatory":[20],"regime.":[21],"We":[22,95],"formalize":[23],"this":[24],"setting":[25],"as":[26,43],"benchmarkless":[27],"comparative":[28],"scoring":[30,103],"and":[31,60,88,91,105,114,123,135,165,175,191,201,204],"specify":[32],"contract":[34],"under":[35,50],"which":[36],"scenario-based":[38],"audit":[39],"can":[40],"be":[41,208],"interpreted":[42],"deployment":[44,166],"evidence.":[45],"Scores":[46],"are":[47,66],"valid":[48],"only":[49],"fixed":[52],"scenario":[53,189],"pack,":[54],"rubric,":[55],"auditor,":[56],"judge,":[57],"sampling":[58],"configuration,":[59],"rerun":[61],"budget.":[62],"Because":[63],"no":[64],"labels":[65],"available,":[67],"we":[68],"replace":[69],"ground-truth":[70],"agreement":[71],"with":[72,118],"an":[73],"instrumental-validity":[74],"chain:":[75],"responsiveness":[76],"to":[77,146],"controlled":[79],"safe-versus-abliterated":[80],"contrast,":[81],"dominance":[82],"of":[83,159],"target-driven":[84],"variance":[85,130],"over":[86],"auditor":[87,203],"judge":[89,205],"artifacts,":[90],"stability":[92],"across":[93],"reruns.":[94,141],"instantiate":[96],"chain":[98,145],"in":[99,162,182],"SimpleAudit,":[100],"local-first":[102],"instrument,":[104],"validate":[106],"it":[107,150],"on":[108,188],"Norwegian":[110,169],"pack.":[112],"Safe":[113],"abliterated":[115],"targets":[116],"separate":[117],"AUROC":[119],"values":[120],"between":[121],"0.89":[122],"1.00,":[124],"target":[125],"identity":[126],"is":[127],"dominant":[129],"component":[131],"($\u03b7^2":[132],"\\approx":[133],"0.52$),":[134],"severity":[136],"profiles":[137],"stabilize":[138],"by":[139],"ten":[140],"Applying":[142],"same":[144],"Petri":[147],"shows":[148],"that":[149],"admits":[151],"both":[152],"tools.":[153],"The":[154],"substantial":[155],"differences":[156],"arise":[157],"upstream":[158],"chain,":[161],"claim-contract":[163],"enforcement":[164],"fit.":[167],"A":[168],"public-sector":[170],"procurement":[171],"case":[172],"comparing":[173],"Borealis":[174],"Gemma":[176],"3":[177],"demonstrates":[178],"resulting":[180],"evidence":[181],"practice:":[183],"safer":[185],"model":[186],"depends":[187],"category":[190],"risk":[192],"measure.":[193],"Consequently,":[194],"scores,":[195],"matched":[196],"deltas,":[197],"critical":[198],"rates,":[199],"uncertainty,":[200],"used":[206],"reported":[209],"together":[210],"rather":[211],"than":[212],"collapsed":[213],"into":[214],"single":[216],"ranking.":[217]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-09T00:00:00"}