{"id":"https://openalex.org/W4283217613","doi":"https://doi.org/10.1145/3531146.3533118","title":"Towards a multi-stakeholder value-based assessment framework for algorithmic systems","display_name":"Towards a multi-stakeholder value-based assessment framework for algorithmic systems","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4283217613","doi":"https://doi.org/10.1145/3531146.3533118"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2205.04525","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.04525","pdf_url":"https://arxiv.org/pdf/2205.04525","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"article","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2205.04525","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091303182","display_name":"Mireia Yurrita","orcid":"https://orcid.org/0000-0002-9685-4873"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Yurrita Semperena, M. (author)","raw_affiliation_strings":["Human Information Communication Design, TU Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human Information Communication Design, TU Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022409031","display_name":"Dave Murray-Rust","orcid":"https://orcid.org/0000-0001-6098-7861"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Murray-Rust, D.S. (author)","raw_affiliation_strings":["Human Information Communication Design, TU Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human Information Communication Design, TU Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088069213","display_name":"Agathe Balayn","orcid":"https://orcid.org/0000-0003-2725-5305"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Balayn, A.M.A. (author)","raw_affiliation_strings":["Web Information Systems, TU Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Web Information Systems, TU Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028101938","display_name":"Alessandro Bozzon","orcid":"https://orcid.org/0000-0002-3300-2913"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Bozzon, A. (author)","raw_affiliation_strings":["Knowledge and Intelligence Design, TU Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Knowledge and Intelligence Design, TU Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5091303182"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":7.0898,"has_fulltext":true,"cited_by_count":34,"citation_normalized_percentile":{"value":0.97201382,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9761000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/operationalization","display_name":"Operationalization","score":0.8508203625679016},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.7474679946899414},{"id":"https://openalex.org/keywords/deliberation","display_name":"Deliberation","score":0.6672277450561523},{"id":"https://openalex.org/keywords/negotiation","display_name":"Negotiation","score":0.6612910628318787},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6530830264091492},{"id":"https://openalex.org/keywords/stakeholder","display_name":"Stakeholder","score":0.6032408475875854},{"id":"https://openalex.org/keywords/closeness","display_name":"Closeness","score":0.5670819878578186},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.5334304571151733},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5230703949928284},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.49372586607933044},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.4795553386211395},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.3959217369556427},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.35671162605285645},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.13724184036254883},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.12694230675697327},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.12139177322387695},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.11827075481414795},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.10149389505386353},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09809812903404236}],"concepts":[{"id":"https://openalex.org/C9354725","wikidata":"https://www.wikidata.org/wiki/Q286017","display_name":"Operationalization","level":2,"score":0.8508203625679016},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.7474679946899414},{"id":"https://openalex.org/C2776946740","wikidata":"https://www.wikidata.org/wiki/Q358652","display_name":"Deliberation","level":3,"score":0.6672277450561523},{"id":"https://openalex.org/C199776023","wikidata":"https://www.wikidata.org/wiki/Q202875","display_name":"Negotiation","level":2,"score":0.6612910628318787},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6530830264091492},{"id":"https://openalex.org/C201305675","wikidata":"https://www.wikidata.org/wiki/Q852998","display_name":"Stakeholder","level":2,"score":0.6032408475875854},{"id":"https://openalex.org/C2779545769","wikidata":"https://www.wikidata.org/wiki/Q5135364","display_name":"Closeness","level":2,"score":0.5670819878578186},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.5334304571151733},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5230703949928284},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.49372586607933044},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.4795553386211395},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3959217369556427},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35671162605285645},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.13724184036254883},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.12694230675697327},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.12139177322387695},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.11827075481414795},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.10149389505386353},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09809812903404236},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2205.04525","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.04525","pdf_url":"https://arxiv.org/pdf/2205.04525","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:tudelft.nl:uuid:844e3e54-e50e-404e-912d-a1cd5a09b687","is_oa":false,"landing_page_url":"http://resolver.tudelft.nl/uuid:844e3e54-e50e-404e-912d-a1cd5a09b687","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2205.04525","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.04525","pdf_url":"https://arxiv.org/pdf/2205.04525","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.4099999964237213}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":193,"referenced_works":["https://openalex.org/W174580382","https://openalex.org/W1498112155","https://openalex.org/W1583837637","https://openalex.org/W1853708643","https://openalex.org/W1873763122","https://openalex.org/W1945616565","https://openalex.org/W1968683068","https://openalex.org/W1977024621","https://openalex.org/W1982966188","https://openalex.org/W1986829999","https://openalex.org/W2026028666","https://openalex.org/W2100960835","https://openalex.org/W2103154003","https://openalex.org/W2106460240","https://openalex.org/W2114296159","https://openalex.org/W2137406659","https://openalex.org/W2183546783","https://openalex.org/W2240086230","https://openalex.org/W2461642564","https://openalex.org/W2489837444","https://openalex.org/W2522104760","https://openalex.org/W2530395818","https://openalex.org/W2535690855","https://openalex.org/W2559010931","https://openalex.org/W2560296050","https://openalex.org/W2575439561","https://openalex.org/W2588194244","https://openalex.org/W2599025709","https://openalex.org/W2621160224","https://openalex.org/W2623360248","https://openalex.org/W2742760489","https://openalex.org/W2753845591","https://openalex.org/W2756188387","https://openalex.org/W2756490226","https://openalex.org/W2768894107","https://openalex.org/W2781332141","https://openalex.org/W2785729136","https://openalex.org/W2787955716","https://openalex.org/W2788481061","https://openalex.org/W2791170418","https://openalex.org/W2796077430","https://openalex.org/W2809878087","https://openalex.org/W2884943453","https://openalex.org/W2889357760","https://openalex.org/W2889989236","https://openalex.org/W2899579542","https://openalex.org/W2902634493","https://openalex.org/W2909551589","https://openalex.org/W2911227954","https://openalex.org/W2912269676","https://openalex.org/W2913756371","https://openalex.org/W2914202940","https://openalex.org/W2916797267","https://openalex.org/W2932004331","https://openalex.org/W2941486823","https://openalex.org/W2942073295","https://openalex.org/W2942730552","https://openalex.org/W2944988415","https://openalex.org/W2945295328","https://openalex.org/W2949611699","https://openalex.org/W2949678053","https://openalex.org/W2951923023","https://openalex.org/W2956281901","https://openalex.org/W2957285709","https://openalex.org/W2958109080","https://openalex.org/W2963116854","https://openalex.org/W2963290659","https://openalex.org/W2963341057","https://openalex.org/W2963524705","https://openalex.org/W2964031043","https://openalex.org/W2964082701","https://openalex.org/W2964892778","https://openalex.org/W2967778900","https://openalex.org/W2969580338","https://openalex.org/W2972735048","https://openalex.org/W2973631113","https://openalex.org/W2974817986","https://openalex.org/W2981731882","https://openalex.org/W2982915530","https://openalex.org/W2983140679","https://openalex.org/W2985543011","https://openalex.org/W2996238829","https://openalex.org/W2996541402","https://openalex.org/W2998192266","https://openalex.org/W2998546532","https://openalex.org/W2999331643","https://openalex.org/W3001221685","https://openalex.org/W3001807593","https://openalex.org/W3003162384","https://openalex.org/W3003809373","https://openalex.org/W3005057496","https://openalex.org/W3010300896","https://openalex.org/W3013778941","https://openalex.org/W3014011525","https://openalex.org/W3023001449","https://openalex.org/W3029022080","https://openalex.org/W3029022390","https://openalex.org/W3030081171","https://openalex.org/W3031866031","https://openalex.org/W3032916997","https://openalex.org/W3035078899","https://openalex.org/W3035310383","https://openalex.org/W3036061622","https://openalex.org/W3037042691","https://openalex.org/W3040416015","https://openalex.org/W3042309366","https://openalex.org/W3042707343","https://openalex.org/W3043315258","https://openalex.org/W3088587572","https://openalex.org/W3092626593","https://openalex.org/W3094140017","https://openalex.org/W3094233077","https://openalex.org/W3100121331","https://openalex.org/W3100279624","https://openalex.org/W3101427066","https://openalex.org/W3104149808","https://openalex.org/W3104847483","https://openalex.org/W3108359080","https://openalex.org/W3114537318","https://openalex.org/W3118297265","https://openalex.org/W3124415856","https://openalex.org/W3127141043","https://openalex.org/W3133702157","https://openalex.org/W3133759846","https://openalex.org/W3133893548","https://openalex.org/W3133980122","https://openalex.org/W3134798742","https://openalex.org/W3135328939","https://openalex.org/W3135371071","https://openalex.org/W3146248196","https://openalex.org/W3146668661","https://openalex.org/W3153619806","https://openalex.org/W3155775763","https://openalex.org/W3158471737","https://openalex.org/W3160037564","https://openalex.org/W3160152376","https://openalex.org/W3160343688","https://openalex.org/W3160735296","https://openalex.org/W3160872964","https://openalex.org/W3161120666","https://openalex.org/W3173027639","https://openalex.org/W3174220540","https://openalex.org/W3175257734","https://openalex.org/W3176964086","https://openalex.org/W3181414820","https://openalex.org/W3185440164","https://openalex.org/W3190455919","https://openalex.org/W3194678924","https://openalex.org/W3203474437","https://openalex.org/W3204947859","https://openalex.org/W3205290952","https://openalex.org/W3207830467","https://openalex.org/W3209901185","https://openalex.org/W3210239237","https://openalex.org/W3212368439","https://openalex.org/W3212464620","https://openalex.org/W3212896246","https://openalex.org/W3213556453","https://openalex.org/W3217525002","https://openalex.org/W4206429981","https://openalex.org/W4210595892","https://openalex.org/W4229999961","https://openalex.org/W4234516313","https://openalex.org/W4236082062","https://openalex.org/W4247200422","https://openalex.org/W4287025910","https://openalex.org/W4287332073","https://openalex.org/W4287332853","https://openalex.org/W4287760010","https://openalex.org/W4287817742","https://openalex.org/W4288083797","https://openalex.org/W4288083801","https://openalex.org/W4288083802","https://openalex.org/W4288103499","https://openalex.org/W4288414189","https://openalex.org/W4289293239","https://openalex.org/W4291497561","https://openalex.org/W4292081290","https://openalex.org/W4293507378","https://openalex.org/W4293580221","https://openalex.org/W4294152847","https://openalex.org/W4296978576","https://openalex.org/W4297780677","https://openalex.org/W4297820713","https://openalex.org/W4299313519","https://openalex.org/W4299337115","https://openalex.org/W4300596510","https://openalex.org/W4308410483","https://openalex.org/W4310492983","https://openalex.org/W4312663109","https://openalex.org/W4313559136","https://openalex.org/W4320464992","https://openalex.org/W4386564359"],"related_works":["https://openalex.org/W2944277854","https://openalex.org/W2258359646","https://openalex.org/W2493324121","https://openalex.org/W4234402960","https://openalex.org/W4210958110","https://openalex.org/W2483613126","https://openalex.org/W4210994448","https://openalex.org/W4386824558","https://openalex.org/W4394927648","https://openalex.org/W2097838441"],"abstract_inverted_index":{"<p>In":[0],"an":[1,199],"effort":[2],"to":[3,24,70,104,134,154,161,188,193,215,226],"regulate":[4],"Machine":[5],"Learning-driven":[6],"(ML)":[7],"systems,":[8],"current":[9,194],"auditing":[10,42,72,137,196],"processes":[11],"mostly":[12],"focus":[13],"on":[14,142,213],"detecting":[15],"harmful":[16],"algorithmic":[17,80],"biases.":[18],"While":[19],"these":[20,106,123],"strategies":[21],"have":[22],"proven":[23],"be":[25,54],"impactful,":[26],"some":[27,121,135],"values":[28,46,89,109,208],"outlined":[29],"in":[30,35,41],"documents":[31],"dealing":[32],"with":[33,49,90,176,198],"ethics":[34],"ML-driven":[36],"systems":[37],"are":[38,110,126],"still":[39],"underrepresented":[40],"processes.":[43],"Such":[44],"unaddressed":[45],"mainly":[47],"deal":[48],"contextual":[50],"factors":[51],"that":[52,66,74,94,139,150,157,171,202],"cannot":[53],"easily":[55],"quantified.":[56],"In":[57,102],"this":[58],"paper,":[59],"we":[60,148,173,210],"develop":[61],"a":[62,85,227],"value-based":[63],"assessment":[64,200],"framework":[65,83,201],"is":[67,152],"not":[68],"limited":[69],"bias":[71],"and":[73,98,117,129,145,164,167,180,205,209,223],"covers":[75],"prominent":[76],"ethical":[77],"principles":[78],"for":[79,184],"systems.":[81],"Our":[82],"presents":[84],"circular":[86],"arrangement":[87],"of":[88,122,230],"two":[91],"bipolar":[92],"dimensions":[93],"make":[95],"common":[96],"motivations":[97],"potential":[99],"tensions":[100,206],"explicit.":[101],"order":[103],"operationalize":[105,216],"high-level":[107],"principles,":[108],"then":[111],"broken":[112],"down":[113],"into":[114],"specific":[115],"criteria":[116,125,168],"their":[118],"manifestations.":[119],"However,":[120],"value-specific":[124],"mutually":[127],"exclusive":[128],"require":[130],"negotiation.":[131],"As":[132],"opposed":[133],"other":[136],"frameworks":[138],"merely":[140],"rely":[141],"ML":[143,195],"researchers'":[144],"practitioners'":[146],"input,":[147],"argue":[149],"it":[151],"necessary":[153],"include":[155],"stakeholders":[156,175],"present":[158],"diverse":[159],"standpoints":[160],"systematically":[162],"negotiate":[163],"consolidate":[165],"value":[166,186],"tensions.":[169],"To":[170],"end,":[172],"map":[174],"different":[177],"insight":[178],"needs,":[179],"assign":[181],"tailored":[182],"means":[183],"communicating":[185],"manifestations":[187],"them.":[189],"We,":[190],"therefore,":[191],"contribute":[192],"practices":[197],"visualizes":[203],"closeness":[204],"between":[207],"give":[211],"guidelines":[212],"how":[214],"them,":[217],"while":[218],"opening":[219],"up":[220],"the":[221],"evaluation":[222],"deliberation":[224],"process":[225],"wide":[228],"range":[229],"stakeholders.</p>":[231]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-28T14:05:53.105641","created_date":"2022-06-22T00:00:00"}