{"id":"https://openalex.org/W7225394160","doi":"https://doi.org/10.5281/zenodo.23997204","title":"Governing Production LLM Systems: A Use-Case Study of Observability, Guardrails and Evaluation Gates in Agentic AI Delivery","display_name":"Governing Production LLM Systems: A Use-Case Study of Observability, Guardrails and Evaluation Gates in Agentic AI Delivery","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225394160","doi":"https://doi.org/10.5281/zenodo.23997204"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23997204","doi":"https://doi.org/10.5281/zenodo.23997204","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23997204","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23997204","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5161006785","display_name":"Aarthi Arulraj","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Arulraj, Aarthi","raw_affiliation_strings":["Technical Program Manager, India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical Program Manager, India","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5161006785"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.8320000171661377,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.8320000171661377,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.08910000324249268,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.02590000070631504,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/telemetry","display_name":"telemetry","score":0.9620000123977661},{"id":"https://openalex.org/keywords/hallucination-detection","display_name":"hallucination detection","score":0.9890000224113464},{"id":"https://openalex.org/keywords/agentic-ai","display_name":"agentic AI","score":0.703000009059906},{"id":"https://openalex.org/keywords/multi-agent-systems","display_name":"multi-agent systems","score":0.7599999904632568},{"id":"https://openalex.org/keywords/software-delivery-pipelines","display_name":"software delivery pipelines","score":0.6330000162124634},{"id":"https://openalex.org/keywords/ai-risk-management","display_name":"AI risk management","score":0.35600000619888306}],"concepts":[{"id":"https://openalex.org/C36299963","wikidata":"https://www.wikidata.org/wiki/Q1369844","display_name":"Observability","level":2,"score":0.8797000050544739},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.728600025177002},{"id":"https://openalex.org/C2780821815","wikidata":"https://www.wikidata.org/wiki/Q5340806","display_name":"Portfolio","level":2,"score":0.5586000084877014},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.5149999856948853},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.44200000166893005},{"id":"https://openalex.org/C39389867","wikidata":"https://www.wikidata.org/wiki/Q380767","display_name":"Corporate governance","level":2,"score":0.4336000084877014},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.41620001196861267},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.40070000290870667},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.3894999921321869},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3880999982357025},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.37310001254081726},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35600000619888306},{"id":"https://openalex.org/C2776842983","wikidata":"https://www.wikidata.org/wiki/Q581319","display_name":"Production system (computer science)","level":3,"score":0.34119999408721924},{"id":"https://openalex.org/C2992770021","wikidata":"https://www.wikidata.org/wiki/Q7247850","display_name":"Production model","level":3,"score":0.335999995470047},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.329800009727478},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.3125999867916107},{"id":"https://openalex.org/C37945671","wikidata":"https://www.wikidata.org/wiki/Q7336207","display_name":"Risk-based testing","level":5,"score":0.2989000082015991},{"id":"https://openalex.org/C62354387","wikidata":"https://www.wikidata.org/wiki/Q875399","display_name":"Boundary (topology)","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.28529998660087585},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.27790001034736633},{"id":"https://openalex.org/C108170787","wikidata":"https://www.wikidata.org/wiki/Q3951828","display_name":"Agency (philosophy)","level":2,"score":0.2768000066280365},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.2761000096797943},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.2759000062942505},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.27559998631477356}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23997204","doi":"https://doi.org/10.5281/zenodo.23997204","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23997204","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23997205","doi":"https://doi.org/10.5281/zenodo.23997205","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23997205","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23997204","doi":"https://doi.org/10.5281/zenodo.23997204","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23997204","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415162365","https://openalex.org/W7222628263","https://openalex.org/W4417025163","https://openalex.org/W7169516010","https://openalex.org/W7158586692","https://openalex.org/W7172231021","https://openalex.org/W7171713090","https://openalex.org/W7213469301","https://openalex.org/W7138863281","https://openalex.org/W7207738045"],"abstract_inverted_index":{"This":[0],"paper":[1,41,151],"presents":[2],"a":[3,45,88,95,123],"use-case":[4],"study":[5],"identified":[6],"from":[7,112],"the":[8,63,116,132,145,150],"author's":[9,146],"delivery":[10,38,64],"practice:":[11],"how":[12],"should":[13],"an":[14,81],"organisation":[15],"govern":[16],"production":[17,77,103],"large":[18],"language":[19],"model":[20],"(LLM)":[21],"systems":[22],"whose":[23],"failure":[24],"modes":[25],"-":[26,32],"hallucination,":[27],"unsafe":[28],"outputs,":[29],"behavioural":[30],"drift":[31],"are":[33,144],"invisible":[34],"to":[35,114],"conventional":[36],"software":[37],"governance?":[39],"The":[40,71],"proposes":[42],"and":[43,51,58,94,131],"evaluates":[44],"governance":[46],"framework":[47,72],"combining":[48],"runtime":[49],"observability":[50,83],"telemetry,":[52],"continuous":[53],"hallucination":[54],"detection,":[55],"boundary":[56],"guardrails,":[57],"evaluation":[59],"gates":[60],"embedded":[61],"in":[62],"pipeline,":[65],"operated":[66],"through":[67],"sprint-reviewed":[68],"risk":[69,105],"registers.":[70],"was":[73],"applied":[74],"across":[75,138],"three":[76],"AI":[78,91,104,117],"platform":[79,84],"programs:":[80],"LLM":[82],"for":[85],"agentic":[86],"AI,":[87],"multimodal":[89],"automated":[90,129],"testing":[92,118],"platform,":[93],"low-code":[96],"multi-agent":[97],"builder.":[98],"Across":[99],"these":[100],"programs,":[101],"measured":[102],"fell":[106],"by":[107],"90%,":[108],"program":[109,148],"completion":[110],"rose":[111],"78%":[113],"96%,":[115],"lifecycle":[119],"accelerated":[120],"threefold":[121],"on":[122],"base":[124],"of":[125],"more":[126],"than":[127],"300":[128],"tests,":[130],"portfolio":[133],"recorded":[134],"zero":[135],"missed":[136],"milestones":[137],"two":[139],"consecutive":[140],"quarters.":[141],"All":[142],"figures":[143],"internal":[147],"metrics;":[149],"states":[152],"its":[153],"limitations":[154],"explicitly.":[155]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-11-07T00:00:00"}