{"id":"https://openalex.org/W4390992306","doi":"https://doi.org/10.1109/bibm58861.2023.10385617","title":"Probabilistic Performance Bounds for Evaluating Depression Models Given Noisy Self-Report Labels","display_name":"Probabilistic Performance Bounds for Evaluating Depression Models Given Noisy Self-Report Labels","publication_year":2023,"publication_date":"2023-12-05","ids":{"openalex":"https://openalex.org/W4390992306","doi":"https://doi.org/10.1109/bibm58861.2023.10385617"},"language":"en","primary_location":{"id":"doi:10.1109/bibm58861.2023.10385617","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/bibm58861.2023.10385617","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027452433","display_name":"Robert R\u00f3\u017ca\u0144ski","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert R\u00f3\u017ca\u0144ski","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107174154","display_name":"Elizabeth Shriberg","orcid":"https://orcid.org/0009-0004-3779-4956"},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Elizabeth Shriberg","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037087707","display_name":"Yang L\u00fc","orcid":"https://orcid.org/0000-0001-9887-7078"},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang Lu","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043581685","display_name":"Amir Harati","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amir Harati","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004023433","display_name":"Tomasz Rutowski","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tomasz Rutowski","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068244618","display_name":"Piotr Chlebek","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Piotr Chlebek","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093740446","display_name":"Tulio Goulart","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tulio Goulart","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067556320","display_name":"Ricardo Oliveira","orcid":"https://orcid.org/0000-0001-5167-1523"},"institutions":[{"id":"https://openalex.org/I4210100173","display_name":"San Francisco Department of Public Health","ror":"https://ror.org/017ztfb41","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210100173"]},{"id":"https://openalex.org/I4210106430","display_name":"Ellipsis","ror":"https://ror.org/01pmtzj45","country_code":"US","type":"other","lineage":["https://openalex.org/I4210106430"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ricardo Oliveira","raw_affiliation_strings":["Ellipsis Health,San Francisco,CA,USA","Ellipsis Health, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ellipsis Health,San Francisco,CA,USA","institution_ids":["https://openalex.org/I4210100173"]},{"raw_affiliation_string":"Ellipsis Health, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210106430"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"5","issue":null,"first_page":"4827","last_page":"4833"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9348000288009644,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9348000288009644,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.807892918586731},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.614310622215271},{"id":"https://openalex.org/keywords/depression","display_name":"Depression (economics)","score":0.5028218626976013},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4076893925666809},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33489930629730225},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.07645571231842041}],"concepts":[{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.807892918586731},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.614310622215271},{"id":"https://openalex.org/C2776867660","wikidata":"https://www.wikidata.org/wiki/Q1814941","display_name":"Depression (economics)","level":2,"score":0.5028218626976013},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4076893925666809},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33489930629730225},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.07645571231842041},{"id":"https://openalex.org/C139719470","wikidata":"https://www.wikidata.org/wiki/Q39680","display_name":"Macroeconomics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bibm58861.2023.10385617","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/bibm58861.2023.10385617","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7200000286102295,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W1454884387","https://openalex.org/W1965090383","https://openalex.org/W1983963519","https://openalex.org/W1999271084","https://openalex.org/W2003502731","https://openalex.org/W2050360441","https://openalex.org/W2098021754","https://openalex.org/W2122111042","https://openalex.org/W2162207080","https://openalex.org/W2218423201","https://openalex.org/W2766211031","https://openalex.org/W2939765894","https://openalex.org/W3004047823","https://openalex.org/W3024540227","https://openalex.org/W3100361261","https://openalex.org/W3111789454","https://openalex.org/W3144340278","https://openalex.org/W3161135402","https://openalex.org/W3196761683","https://openalex.org/W3197275371","https://openalex.org/W3197466182","https://openalex.org/W3197811449","https://openalex.org/W4220931217","https://openalex.org/W4221152443","https://openalex.org/W4221161909","https://openalex.org/W4223610593","https://openalex.org/W4225294421","https://openalex.org/W4283219344","https://openalex.org/W4283704665","https://openalex.org/W4285818382","https://openalex.org/W4287018912","https://openalex.org/W4296069164","https://openalex.org/W4297841453","https://openalex.org/W4297841728","https://openalex.org/W4297841808","https://openalex.org/W4297841836","https://openalex.org/W4377292557","https://openalex.org/W4387393037","https://openalex.org/W6784099588","https://openalex.org/W6810523572","https://openalex.org/W6840158260"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Advances":[0],"in":[1,45,164,196],"AI":[2],"for":[3,31,94,229],"health":[4],"applications":[5],"rely":[6],"on":[7,213],"evaluating":[8],"performance":[9,84,210],"against":[10],"labeled":[11],"test":[12,63,180,193],"data.":[13],"In":[14],"the":[15,26,51,170,218],"area":[16],"of":[17,53,222],"mental":[18],"health,":[19],"self-report":[20],"labels":[21],"from":[22,143,161,183],"surveys":[23],"such":[24],"as":[25],"Patient":[27],"Health":[28],"Questionnaire":[29],"(PHQ)":[30],"depression,":[32],"are":[33,235],"useful":[34],"but":[35],"noisy.":[36],"This":[37],"\"fuzzy":[38],"label\"":[39],"problem":[40],"is":[41],"not":[42],"currently":[43],"reflected":[44],"reporting":[46],"model":[47],"performance,":[48],"adding":[49],"to":[50,77,115,126],"challenge":[52],"comparing":[54],"results":[55,133,160],"across":[56],"diverse":[57],"corpora,":[58],"data":[59,181],"sizes,":[60],"metrics,":[61],"and":[62,82,97,108,129,152,201,220,226,232],"label":[64,113,156,215],"distributions.To":[65],"address":[66],"this":[67],"issue,":[68],"we":[69,122,158],"develop":[70],"an":[71],"approach":[72,90],"inspired":[73],"by":[74,174],"Bayes":[75],"Error":[76],"estimate":[78],"a":[79,103,144,162,175],"model\u2019s":[80],"upper":[81],"lower":[83],"bounds.":[85],"Unlike":[86],"past":[87],"work,":[88],"our":[89],"can":[91],"be":[92],"used":[93],"both":[95,150],"regression":[96],"classification.":[98],"The":[99,192],"method":[100],"starts":[101],"with":[102],"perfect":[104],"match":[105],"between":[106],"target":[107,130],"prediction":[109,128],"vectors,":[110],"then":[111],"applies":[112],"noise":[114,216],"degrade":[116],"performance.":[117],"To":[118],"obtain":[119],"confidence":[120],"intervals,":[121],"use":[123],"test-set":[124],"bootstrapping":[125],"produce":[127],"vectors.We":[131],"present":[132],"using":[134],"voice-based":[135],"deep":[136],"learning":[137],"models":[138,223],"that":[139],"predict":[140],"depression":[141],"risk":[142],"conversational":[145],"speech":[146,197,199],"sample.":[147],"Models":[148],"capture":[149],"language":[151],"acoustic":[153],"information.":[154],"For":[155],"noise,":[157],"introduce":[159],"corpus":[163],"which":[165],"5625":[166],"unique":[167],"subjects":[168],"completed":[169],"PHQ-8":[171,214],"twice,":[172],"separated":[173],"short":[176],"distraction":[177],"task.":[178],"Speech":[179],"come":[182],"three":[184],"real-world":[185],"corpora":[186,225],"encompassing":[187],"over":[188,224],"3500":[189],"total":[190],"datapoints.":[191],"sets":[194],"differ":[195],"elicitation,":[198],"length,":[200],"speaker":[202],"demographics":[203],"among":[204],"other":[205],"factors.Results":[206],"illustrate":[207],"how":[208],"probabilistic":[209],"bounds":[211],"based":[212],"affect":[217],"interpretation":[219],"comparison":[221],"metrics.":[227],"Implications":[228],"science,":[230],"technology,":[231],"future":[233],"directions":[234],"discussed.":[236]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}