{"id":"https://openalex.org/W4415230919","doi":"https://doi.org/10.1609/aies.v8i1.36583","title":"\"Do Your Guardrails Even Guard?'' Method for Evaluating Effectiveness of Moderation Guardrails in Aligning LLM Outputs with Expert User Expectations","display_name":"\"Do Your Guardrails Even Guard?'' Method for Evaluating Effectiveness of Moderation Guardrails in Aligning LLM Outputs with Expert User Expectations","publication_year":2025,"publication_date":"2025-10-15","ids":{"openalex":"https://openalex.org/W4415230919","doi":"https://doi.org/10.1609/aies.v8i1.36583"},"language":"en","primary_location":{"id":"doi:10.1609/aies.v8i1.36583","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aies.v8i1.36583","pdf_url":"https://ojs.aaai.org/index.php/AIES/article/download/36583/38721","source":{"id":"https://openalex.org/S5407048695","display_name":"Proceedings of the AAAI/ACM Conference on AI Ethics and Society","issn_l":"3065-8365","issn":["3065-8365"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://ojs.aaai.org/index.php/AIES/article/download/36583/38721","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043408232","display_name":"Anindya Das Antar","orcid":"https://orcid.org/0000-0001-9912-8757"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anindya Das Antar","raw_affiliation_strings":["University of Michigan, Ann Arbor"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027689637","display_name":"Xun Huan","orcid":"https://orcid.org/0000-0001-6544-2764"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xun Huan","raw_affiliation_strings":["University of Michigan, Ann Arbor"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060671078","display_name":"Nikola Banovi\u0107","orcid":"https://orcid.org/0000-0002-2790-3264"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nikola Banovic","raw_affiliation_strings":["University of Michigan, Ann Arbor"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I27837315"],"apc_list":null,"apc_paid":null,"fwci":2.4854,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.90045888,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"8","issue":"1","first_page":"705","last_page":"718"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14056","display_name":"Safety Warnings and Signage","score":0.36489999294281006,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14056","display_name":"Safety Warnings and Signage","score":0.36489999294281006,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11006","display_name":"BIM and Construction Integration","score":0.33090001344680786,"subfield":{"id":"https://openalex.org/subfields/2215","display_name":"Building and Construction"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/guard","display_name":"Guard (computer science)","score":0.5967000126838684},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.49129998683929443},{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.46050000190734863},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.36500000953674316},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.31529998779296875},{"id":"https://openalex.org/keywords/subject-matter-expert","display_name":"Subject-matter expert","score":0.29820001125335693}],"concepts":[{"id":"https://openalex.org/C141141315","wikidata":"https://www.wikidata.org/wiki/Q2379942","display_name":"Guard (computer science)","level":2,"score":0.5967000126838684},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.49129998683929443},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.46050000190734863},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3977000117301941},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3675000071525574},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3287000060081482},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.31529998779296875},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.29820001125335693},{"id":"https://openalex.org/C91262260","wikidata":"https://www.wikidata.org/wiki/Q528074","display_name":"End user","level":2,"score":0.296999990940094},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2964000105857849},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.28459998965263367},{"id":"https://openalex.org/C2776090404","wikidata":"https://www.wikidata.org/wiki/Q1420643","display_name":"Recidivism","level":2,"score":0.26600000262260437}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aies.v8i1.36583","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aies.v8i1.36583","pdf_url":"https://ojs.aaai.org/index.php/AIES/article/download/36583/38721","source":{"id":"https://openalex.org/S5407048695","display_name":"Proceedings of the AAAI/ACM Conference on AI Ethics and Society","issn_l":"3065-8365","issn":["3065-8365"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/36583","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AIES/article/view/36583","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"3065-8365","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aies.v8i1.36583","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aies.v8i1.36583","pdf_url":"https://ojs.aaai.org/index.php/AIES/article/download/36583/38721","source":{"id":"https://openalex.org/S5407048695","display_name":"Proceedings of the AAAI/ACM Conference on AI Ethics and Society","issn_l":"3065-8365","issn":["3065-8365"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309652","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415230919.pdf","grobid_xml":"https://content.openalex.org/works/W4415230919.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Ensuring":[0],"that":[1,85,110,121],"large":[2],"language":[3],"models":[4],"(LLMs)":[5],"align":[6,46,87],"with":[7,55,90,98],"human":[8],"values":[9],"and":[10,45,56,70,82,105],"goals":[11],"is":[12],"crucial":[13],"for":[14],"their":[15],"adoption":[16],"in":[17,118],"high-stakes":[18],"decision-making.":[19],"To":[20],"guard":[21],"against":[22],"incorrect,":[23],"misleading,":[24],"or":[25,28],"otherwise":[26],"unexpected":[27],"undesirable":[29],"LLM":[30,53,88],"outputs,":[31],"guardrail":[32,69,124],"engineers":[33,125],"implement":[34],"guardrails":[35,84,117,130],"based":[36],"on":[37,73],"expert":[38,94],"knowledge":[39],"from":[40],"subject-matter":[41],"authorities":[42],"to":[43,80,131],"steer":[44],"pre-trained":[47],"LLMs.":[48],"Existing":[49],"evaluation":[50,97],"methods":[51],"assess":[52],"performance,":[54],"without":[57],"guardrails,":[58],"but":[59],"provide":[60],"limited":[61],"insight":[62],"into":[63],"the":[64],"contribution":[65],"of":[66,102,128],"each":[67],"individual":[68],"its":[71],"interactions":[72],"alignment.":[74,133],"Here,":[75],"we":[76,108],"present":[77],"a":[78,119],"method":[79,112],"evaluate":[81],"select":[83],"best":[86],"outputs":[89],"empirical":[91],"evidence":[92],"representing":[93],"knowledge.":[95],"Through":[96],"real-world":[99],"illustrative":[100],"examples":[101],"resume":[103],"quality":[104],"recidivism":[106],"prediction,":[107],"show":[109],"our":[111],"effectively":[113],"identifies":[114],"useful":[115],"moderation":[116],"way":[120],"could":[122],"help":[123],"interpret":[126],"contributions":[127],"different":[129],"\"user-LLM\"":[132]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-16T00:00:00"}