{"id":"https://openalex.org/W3179835674","doi":"https://doi.org/10.1145/3463934","title":"How Teams Communicate about the Quality of ML Models: A Case Study at an International Technology Company","display_name":"How Teams Communicate about the Quality of ML Models: A Case Study at an International Technology Company","publication_year":2021,"publication_date":"2021-07-08","ids":{"openalex":"https://openalex.org/W3179835674","doi":"https://doi.org/10.1145/3463934","mag":"3179835674"},"language":"en","primary_location":{"id":"doi:10.1145/3463934","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3463934","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3463934","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3463934","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084450288","display_name":"Jumana Almahmoud","orcid":"https://orcid.org/0000-0002-5482-2232"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jumana Almahmoud","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge , MA, USA"],"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge , MA, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026655012","display_name":"Robert DeLine","orcid":"https://orcid.org/0000-0001-8885-8367"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert DeLine","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010119044","display_name":"Steven M. Drucker","orcid":"https://orcid.org/0000-0002-5022-9343"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Steven M. Drucker","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5084450288"],"corresponding_institution_ids":["https://openalex.org/I63966007"],"apc_list":null,"apc_paid":null,"fwci":1.2761,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.8294094,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"5","issue":"GROUP","first_page":"1","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9846000075340271,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6428242921829224},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6416592001914978},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5218821167945862},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4724702537059784},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4685412645339966},{"id":"https://openalex.org/keywords/team-software-process","display_name":"Team software process","score":0.42867526412010193},{"id":"https://openalex.org/keywords/face","display_name":"Face (sociological concept)","score":0.4166587293148041},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.38480040431022644},{"id":"https://openalex.org/keywords/engineering-management","display_name":"Engineering management","score":0.36490219831466675},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.33871105313301086},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.3103444576263428},{"id":"https://openalex.org/keywords/software-development-process","display_name":"Software development process","score":0.23680326342582703},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.14150577783584595}],"concepts":[{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6428242921829224},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6416592001914978},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5218821167945862},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4724702537059784},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4685412645339966},{"id":"https://openalex.org/C51845450","wikidata":"https://www.wikidata.org/wiki/Q1421651","display_name":"Team software process","level":5,"score":0.42867526412010193},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.4166587293148041},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.38480040431022644},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.36490219831466675},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.33871105313301086},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3103444576263428},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.23680326342582703},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.14150577783584595},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3463934","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3463934","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3463934","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3463934","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3463934","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3463934","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","score":0.4399999976158142,"display_name":"Partnerships for the goals"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3179835674.pdf","grobid_xml":"https://content.openalex.org/works/W3179835674.grobid-xml"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W2014380238","https://openalex.org/W2079990928","https://openalex.org/W2097273107","https://openalex.org/W2108816886","https://openalex.org/W2137406659","https://openalex.org/W2145860152","https://openalex.org/W2182687482","https://openalex.org/W2201853392","https://openalex.org/W2290526371","https://openalex.org/W2398344594","https://openalex.org/W2405607598","https://openalex.org/W2512274390","https://openalex.org/W2610589002","https://openalex.org/W2611748211","https://openalex.org/W2757656223","https://openalex.org/W2782864149","https://openalex.org/W2789415069","https://openalex.org/W2807910285","https://openalex.org/W2808450727","https://openalex.org/W2886324022","https://openalex.org/W2891628559","https://openalex.org/W2897042519","https://openalex.org/W2900861633","https://openalex.org/W2905034244","https://openalex.org/W2910548926","https://openalex.org/W2913615284","https://openalex.org/W2922234936","https://openalex.org/W2923421605","https://openalex.org/W2927610422","https://openalex.org/W2938508218","https://openalex.org/W2940731426","https://openalex.org/W2943238809","https://openalex.org/W2943629455","https://openalex.org/W2962851501","https://openalex.org/W2969788738","https://openalex.org/W2971867419","https://openalex.org/W2972278312","https://openalex.org/W2974071289","https://openalex.org/W2983901715","https://openalex.org/W3001363766","https://openalex.org/W3001807593","https://openalex.org/W3014011525","https://openalex.org/W3019489177","https://openalex.org/W3021831361","https://openalex.org/W3099361686","https://openalex.org/W3100279624","https://openalex.org/W3101243562","https://openalex.org/W3105992983","https://openalex.org/W3106372104","https://openalex.org/W3122778363","https://openalex.org/W3123186970","https://openalex.org/W3125798375","https://openalex.org/W4231729184","https://openalex.org/W4288083705","https://openalex.org/W4288359825","https://openalex.org/W4289219086"],"related_works":["https://openalex.org/W4211055703","https://openalex.org/W2108773545","https://openalex.org/W2041798199","https://openalex.org/W2188678893","https://openalex.org/W2084819355","https://openalex.org/W2134109003","https://openalex.org/W2023088167","https://openalex.org/W2257437395","https://openalex.org/W2087711777","https://openalex.org/W2131708986"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"(ML)":[2],"has":[3,50],"become":[4],"a":[5,98,135,139,147,164],"crucial":[6],"component":[7],"in":[8,58,67,116,146,185],"software":[9,22,141],"products,":[10],"either":[11],"as":[12,89],"part":[13],"of":[14,54,72,85,120,149,171,179,200],"the":[15,62,70,76,118,175,198,206],"user":[16,39],"experience":[17,40],"or":[18],"used":[19],"internally":[20],"by":[21],"teams.":[23],"Prior":[24],"studies":[25],"have":[26],"explored":[27],"how":[28,55],"ML":[29,121,181],"is":[30,100],"affecting":[31,96],"development":[32],"team":[33,56,63,207],"roles":[34,60],"beyond":[35,82,188],"data":[36],"scientists,":[37],"including":[38,104,163,197],"designers,":[41],"program":[42],"managers,":[43],"developers":[44],"and":[45,91,108,168,177,183],"operations":[46],"engineers.":[47],"However,":[48],"there":[49],"been":[51],"little":[52],"investigation":[53],"members":[57],"different":[59],"on":[61],"communicate":[64],"about":[65,69,174],"ML,":[66],"particular":[68],"quality":[71,79,119],"models.":[73],"We":[74,159,192],"use":[75,199],"general":[77],"term":[78],"to":[80,93,155,204],"look":[81],"technical":[83],"issues":[84],"model":[86,99],"evaluation,":[87],"such":[88],"accuracy":[90],"overfitting,":[92],"any":[94],"issue":[95],"whether":[97],"suitable":[101],"for":[102],"use,":[103],"ethical,":[105],"engineering,":[106],"operations,":[107],"legal":[109],"considerations.":[110],"What":[111,123],"challenges":[112],"do":[113],"teams":[114],"face":[115],"discussing":[117],"models?":[122],"work":[124],"practices":[125],"mitigate":[126],"those":[127],"challenges?":[128],"To":[129],"address":[130],"these":[131],"questions,":[132],"we":[133],"conducted":[134],"mixed-methods":[136],"study":[137],"at":[138],"large":[140],"company,":[142],"first":[143],"interviewing15":[144],"employees":[145,154],"variety":[148],"roles,":[150],"then":[151],"surveying":[152],"168":[153],"broaden":[156],"our":[157],"understanding.":[158],"found":[160,193],"several":[161,194],"challenges,":[162],"mismatch":[165],"between":[166],"user-focused":[167],"model-focused":[169],"notions":[170],"performance,":[172],"misunderstandings":[173],"capabilities":[176],"limitations":[178],"evolving":[180],"technology,":[182],"difficulties":[184],"understanding":[186],"concerns":[187],"one's":[189],"own":[190],"role.":[191],"mitigation":[195],"strategies,":[196],"demos":[201],"during":[202],"discussions":[203],"keep":[205],"customer-focused.":[208]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}