{"id":"https://openalex.org/W4383428053","doi":"https://doi.org/10.1109/tem.2023.3287759","title":"Practices for Managing Machine Learning Products: A Multivocal Literature Review","display_name":"Practices for Managing Machine Learning Products: A Multivocal Literature Review","publication_year":2023,"publication_date":"2023-07-06","ids":{"openalex":"https://openalex.org/W4383428053","doi":"https://doi.org/10.1109/tem.2023.3287759"},"language":"en","primary_location":{"id":"doi:10.1109/tem.2023.3287759","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tem.2023.3287759","pdf_url":null,"source":{"id":"https://openalex.org/S154533451","display_name":"IEEE Transactions on Engineering Management","issn_l":"0018-9391","issn":["0018-9391","1558-0040"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Engineering Management","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.1109/TEM.2023.3287759","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041595102","display_name":"Isaque Alves","orcid":"https://orcid.org/0000-0002-7254-0378"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Isaque Alves","raw_affiliation_strings":["Department of Computer Science, University of São Paulo, São Paulo, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-7254-0378","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of São Paulo, São Paulo, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063313857","display_name":"Leonardo Leite","orcid":"https://orcid.org/0000-0001-5762-5413"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leonardo A. F. Leite","raw_affiliation_strings":["Department of Computer Science, University of São Paulo, São Paulo, Brazil"],"raw_orcid":"https://orcid.org/0000-0001-5762-5413","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of São Paulo, São Paulo, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032725260","display_name":"Paulo Meirelles","orcid":"https://orcid.org/0000-0002-8923-2814"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paulo Meirelles","raw_affiliation_strings":["Department of Computer Science, University of São Paulo, São Paulo, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-8923-2814","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of São Paulo, São Paulo, Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069259459","display_name":"F\u00e1bio Kon","orcid":"https://orcid.org/0000-0003-3888-7340"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fabio Kon","raw_affiliation_strings":["Department of Computer Science, University of São Paulo, São Paulo, Brazil"],"raw_orcid":"https://orcid.org/0000-0003-3888-7340","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of São Paulo, São Paulo, Brazil","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087617102","display_name":"Carla Rocha","orcid":"https://orcid.org/0000-0003-3102-5166"},"institutions":[{"id":"https://openalex.org/I150729083","display_name":"Universidade de Bras\u00edlia","ror":"https://ror.org/02xfp8v59","country_code":"BR","type":"education","lineage":["https://openalex.org/I150729083"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Carla Silva Rocha Aguiar","raw_affiliation_strings":["Faculty of Gama, University of Brasilia, Brasilia, Brazil"],"raw_orcid":"https://orcid.org/0000-0003-3102-5166","affiliations":[{"raw_affiliation_string":"Faculty of Gama, University of Brasilia, Brasilia, Brazil","institution_ids":["https://openalex.org/I150729083"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.6993,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.93850522,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"71","issue":null,"first_page":"7425","last_page":"7455"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9729999899864197,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6325439810752869},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5700036287307739},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.5603548288345337},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.5237136483192444},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.5223085284233093},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.4870932102203369},{"id":"https://openalex.org/keywords/grey-literature","display_name":"Grey literature","score":0.41008245944976807},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36081913113594055},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3391450047492981},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.3293966054916382},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.22180387377738953},{"id":"https://openalex.org/keywords/management","display_name":"Management","score":0.1628095805644989},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.10481289029121399}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6325439810752869},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5700036287307739},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.5603548288345337},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.5237136483192444},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.5223085284233093},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.4870932102203369},{"id":"https://openalex.org/C61904560","wikidata":"https://www.wikidata.org/wiki/Q1148359","display_name":"Grey literature","level":3,"score":0.41008245944976807},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36081913113594055},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3391450047492981},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3293966054916382},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.22180387377738953},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.1628095805644989},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.10481289029121399},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tem.2023.3287759","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tem.2023.3287759","pdf_url":null,"source":{"id":"https://openalex.org/S154533451","display_name":"IEEE Transactions on Engineering Management","issn_l":"0018-9391","issn":["0018-9391","1558-0040"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Engineering Management","raw_type":"journal-article"},{"id":"pmh:003150207","is_oa":true,"landing_page_url":"https://doi.org/10.1109/TEM.2023.3287759","pdf_url":null,"source":{"id":"https://openalex.org/S4306402641","display_name":"LA Referencia (Red Federada de Repositorios Institucionales de Publicaciones Cient\u00edficas)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4383465926","host_organization_name":"LA Referencia","host_organization_lineage":["https://openalex.org/I4383465926"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"instacron:USP","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:003150207","is_oa":true,"landing_page_url":"https://doi.org/10.1109/TEM.2023.3287759","pdf_url":null,"source":{"id":"https://openalex.org/S4306402641","display_name":"LA Referencia (Red Federada de Repositorios Institucionales de Publicaciones Cient\u00edficas)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4383465926","host_organization_name":"LA Referencia","host_organization_lineage":["https://openalex.org/I4383465926"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"instacron:USP","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W1590137713","https://openalex.org/W1977543826","https://openalex.org/W2032860265","https://openalex.org/W2094198095","https://openalex.org/W2113895566","https://openalex.org/W2182687482","https://openalex.org/W2336383631","https://openalex.org/W2546786407","https://openalex.org/W2553353452","https://openalex.org/W2612139288","https://openalex.org/W2613597870","https://openalex.org/W2735864825","https://openalex.org/W2794670651","https://openalex.org/W2796163371","https://openalex.org/W2810669927","https://openalex.org/W2888360199","https://openalex.org/W2888649985","https://openalex.org/W2896627448","https://openalex.org/W2896671475","https://openalex.org/W2922234936","https://openalex.org/W2926314329","https://openalex.org/W2944264045","https://openalex.org/W2953027140","https://openalex.org/W2955440819","https://openalex.org/W2970433196","https://openalex.org/W2979417040","https://openalex.org/W2980482960","https://openalex.org/W2986642681","https://openalex.org/W2989644209","https://openalex.org/W2990954032","https://openalex.org/W2995558939","https://openalex.org/W2996510345","https://openalex.org/W3004493192","https://openalex.org/W3011399955","https://openalex.org/W3023326764","https://openalex.org/W3030051638","https://openalex.org/W3033381930","https://openalex.org/W3039522238","https://openalex.org/W3045719304","https://openalex.org/W3087529357","https://openalex.org/W3089691379","https://openalex.org/W3090753246","https://openalex.org/W3091407142","https://openalex.org/W3092923367","https://openalex.org/W3098910265","https://openalex.org/W3104217200","https://openalex.org/W3119305695","https://openalex.org/W3119696503","https://openalex.org/W3120666308","https://openalex.org/W3120727620","https://openalex.org/W3129250023","https://openalex.org/W3130255873","https://openalex.org/W3131717598","https://openalex.org/W3134402730","https://openalex.org/W3138805148","https://openalex.org/W3162893999","https://openalex.org/W3165033489","https://openalex.org/W3200195527","https://openalex.org/W4210494082","https://openalex.org/W4220891756","https://openalex.org/W4220913617","https://openalex.org/W4234473732","https://openalex.org/W4281909185","https://openalex.org/W4285141508","https://openalex.org/W4285675053","https://openalex.org/W4286331441","https://openalex.org/W4288359825","https://openalex.org/W4289821803","https://openalex.org/W4293769249","https://openalex.org/W4382987554","https://openalex.org/W6687241523","https://openalex.org/W6729659067","https://openalex.org/W6753598056","https://openalex.org/W6754752432","https://openalex.org/W6760887788","https://openalex.org/W6783446307","https://openalex.org/W6783457582","https://openalex.org/W6784422159","https://openalex.org/W6791570380","https://openalex.org/W6801630174"],"related_works":["https://openalex.org/W2368605798","https://openalex.org/W2518037665","https://openalex.org/W2348524959","https://openalex.org/W2477036161","https://openalex.org/W2368049389","https://openalex.org/W2384861574","https://openalex.org/W4294565801","https://openalex.org/W2170801710","https://openalex.org/W2952704802","https://openalex.org/W2741781807"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"(ML)":[2],"has":[3],"grown":[4],"in":[5,7,51,154],"popularity":[6],"the":[8,47,52,60,79,83,96,104],"software":[9],"industry":[10,53],"due":[11],"to":[12,15,45,82,131,151],"its":[13],"ability":[14],"solve":[16],"complex":[17],"problems.":[18],"Developing":[19],"ML":[20,56,65,84,137,157,169],"systems":[21],"involves":[22],"more":[23],"uncertainty":[24],"and":[25,34,39,58,73,91,107,115,135,159,161,167],"risk":[26],"because":[27],"it":[28],"requires":[29],"identifying":[30],"a":[31,69,76,142,162],"business":[32],"opportunity":[33],"managing":[35,168],"source":[36],"code,":[37],"data,":[38,121],"trained":[40],"models.":[41],"Our":[42],"research":[43],"aims":[44],"identify":[46,152],"existing":[48],"practices":[49,80,125,160],"used":[50],"for":[54,164],"building":[55],"applications":[57],"comprehending":[59],"organizational":[61],"complexity":[62],"of":[63,78,95,144],"adopting":[64],"systems.":[66,138,170],"We":[67],"conducted":[68],"multivocal":[70],"literature":[71,106],"review":[72],"then":[74],"created":[75],"taxonomy":[77,143],"applied":[81],"system":[85],"life":[86],"cycle":[87],"discussed":[88],"among":[89],"practitioners":[90],"researchers.":[92],"The":[93,139],"core":[94,128],"study":[97],"emerged":[98],"from":[99,103],"41":[100],"selected":[101,109],"posts":[102],"grey":[105],"37":[108],"scientific":[110],"papers.":[111],"Applying":[112],"Initial":[113],"Coding":[114,117],"Focused":[116],"techniques":[118],"into":[119,126],"these":[120],"we":[122],"mapped":[123],"91":[124],"six":[127],"categories":[129],"related":[130],"designing,":[132],"developing,":[133],"testing,":[134],"deploying":[136],"results,":[140],"including":[141],"practices,":[145],"provide":[146],"organizations":[147],"with":[148],"valuable":[149],"insights":[150],"gaps":[153],"their":[155],"current":[156],"processes":[158],"roadmap":[163],"improving,":[165],"optimizing,":[166]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}