{"id":"https://openalex.org/W4409721409","doi":"https://doi.org/10.1145/3706599.3719914","title":"Design Principles and Guidelines for LLM Observability: Insights from Developers","display_name":"Design Principles and Guidelines for LLM Observability: Insights from Developers","publication_year":2025,"publication_date":"2025-04-23","ids":{"openalex":"https://openalex.org/W4409721409","doi":"https://doi.org/10.1145/3706599.3719914"},"language":"en","primary_location":{"id":"doi:10.1145/3706599.3719914","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3706599.3719914","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xin Chen","orcid":"https://orcid.org/0009-0007-6825-6212"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Chen","raw_affiliation_strings":["Alibaba Cloud Computing, Alibaba Group, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0007-6825-6212","affiliations":[{"raw_affiliation_string":"Alibaba Cloud Computing, Alibaba Group, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yan Li","orcid":"https://orcid.org/0009-0003-9443-9777"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan Li","raw_affiliation_strings":["Alibaba Cloud Computing, Alibaba Group, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0003-9443-9777","affiliations":[{"raw_affiliation_string":"Alibaba Cloud Computing, Alibaba Group, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109849797","display_name":"X. Sean Wang","orcid":"https://orcid.org/0009-0001-1050-8364"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoming Wang","raw_affiliation_strings":["Alibaba Cloud Computing, Alibaba Group, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0001-1050-8364","affiliations":[{"raw_affiliation_string":"Alibaba Cloud Computing, Alibaba Group, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I45928872"],"apc_list":null,"apc_paid":null,"fwci":19.3739,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.98802933,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9557999968528748,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9557999968528748,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9404000043869019,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/observability","display_name":"Observability","score":0.9314007759094238},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6157629489898682},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.4643018841743469},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.38502436876296997},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3778805136680603},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.23253285884857178},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11679080128669739}],"concepts":[{"id":"https://openalex.org/C36299963","wikidata":"https://www.wikidata.org/wiki/Q1369844","display_name":"Observability","level":2,"score":0.9314007759094238},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6157629489898682},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.4643018841743469},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.38502436876296997},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3778805136680603},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.23253285884857178},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11679080128669739},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3706599.3719914","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3706599.3719914","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":42,"referenced_works":["https://openalex.org/W2147839365","https://openalex.org/W2148357053","https://openalex.org/W2162598207","https://openalex.org/W2394669110","https://openalex.org/W2514759358","https://openalex.org/W2732203370","https://openalex.org/W2795398890","https://openalex.org/W2795559220","https://openalex.org/W2804878416","https://openalex.org/W2916904544","https://openalex.org/W2981731882","https://openalex.org/W3004940685","https://openalex.org/W3032875465","https://openalex.org/W3035371891","https://openalex.org/W3197347140","https://openalex.org/W4251284059","https://openalex.org/W4313650676","https://openalex.org/W4317603347","https://openalex.org/W4321013654","https://openalex.org/W4321786089","https://openalex.org/W4378647241","https://openalex.org/W4385732922","https://openalex.org/W4386142022","https://openalex.org/W4388624604","https://openalex.org/W4388886073","https://openalex.org/W4392849749","https://openalex.org/W4394064026","https://openalex.org/W4401329115","https://openalex.org/W4401641057","https://openalex.org/W4402356211","https://openalex.org/W4403319956","https://openalex.org/W4403334466","https://openalex.org/W4403536740","https://openalex.org/W4403848149","https://openalex.org/W4404257675","https://openalex.org/W4404386233","https://openalex.org/W4405092785","https://openalex.org/W4405387258","https://openalex.org/W4405756264","https://openalex.org/W4405897463","https://openalex.org/W4406302454","https://openalex.org/W4406756422"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2046459260","https://openalex.org/W2967463586","https://openalex.org/W2074679142","https://openalex.org/W2765830098","https://openalex.org/W1971989957","https://openalex.org/W2517338020","https://openalex.org/W3157641275"],"abstract_inverted_index":{"The":[0,33,84,109,130],"growing":[1],"number":[2],"of":[3,50,55,64,144,151],"large":[4,23],"model":[5,25,128],"development":[6],"tools":[7,139],"has":[8],"increased":[9],"the":[10,29,38,47,142],"demand":[11],"for":[12,22,90,135],"transparency":[13],"and":[14,94,107,117,126,140,149],"accountability.":[15],"We":[16],"present":[17],"four":[18],"developer-centric":[19],"design":[20,43,88],"principles":[21,44,116],"language":[24],"(LLM)":[26],"observability":[27,138],"across":[28,73],"entire":[30],"LLMOps":[31],"lifecycle.":[32],"research":[34],"focuses":[35],"on":[36,98],"improving":[37],"developer":[39,99,118,146],"experience":[40,100],"by":[41,78],"integrating":[42],"that":[45],"address":[46],"unique":[48],"needs":[49],"developers":[51,72],"at":[52],"various":[53],"stages":[54],"LLM":[56],"development.":[57],"A":[58],"mixed-methods":[59],"methodology":[60],"was":[61],"implemented":[62],"consisting":[63],"three":[65],"focus":[66],"groups":[67],"(n=10":[68],"per":[69],"group)":[70],"with":[71,120],"different":[74],"proficiency":[75],"levels,":[76],"complemented":[77],"a":[79,121],"small-scale":[80],"designer":[81],"rating":[82],"survey.":[83],"study":[85,131],"highlights":[86],"key":[87],"principles\u2014Design":[89],"Awareness,":[91],"Monitoring,":[92],"Intervention,":[93],"Operability\u2014and":[95],"their":[96],"impact":[97],"factors":[101],"like":[102],"mental":[103,127],"models,":[104],"efficiency,":[105],"visualization,":[106],"understandability.":[108],"findings":[110],"reveal":[111],"significant":[112],"correlations":[113],"between":[114,124],"these":[115],"needs,":[119],"particular":[122],"trade-off":[123],"efficiency":[125],"clarity.":[129],"provides":[132],"actionable":[133],"insights":[134],"designing":[136],"effective":[137],"emphasizes":[141],"importance":[143],"balancing":[145],"control,":[147],"transparency,":[148],"ease":[150],"use.":[152]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}