{"id":"https://openalex.org/W2998862821","doi":"https://doi.org/10.1145/3313831.3376783","title":"Keeping Community in the Loop: Understanding Wikipedia Stakeholder Values for Machine Learning-Based Systems","display_name":"Keeping Community in the Loop: Understanding Wikipedia Stakeholder Values for Machine Learning-Based Systems","publication_year":2020,"publication_date":"2020-04-21","ids":{"openalex":"https://openalex.org/W2998862821","doi":"https://doi.org/10.1145/3313831.3376783","mag":"2998862821"},"language":"en","primary_location":{"id":"doi:10.1145/3313831.3376783","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3313831.3376783","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376783","source":null,"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376783","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058678661","display_name":"C. Estelle Smith","orcid":"https://orcid.org/0000-0002-4981-7105"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"C. Estelle Smith","raw_affiliation_strings":["University of Minnesota, Minneapolis, MN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Minnesota, Minneapolis, MN, USA","institution_ids":["https://openalex.org/I130238516"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100604886","display_name":"Bowen Yu","orcid":"https://orcid.org/0000-0002-6804-1859"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bowen Yu","raw_affiliation_strings":["University of Minnesota, Minneapolis, MN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Minnesota, Minneapolis, MN, USA","institution_ids":["https://openalex.org/I130238516"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049457740","display_name":"Anjali Srivastava","orcid":"https://orcid.org/0000-0002-8052-639X"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anjali Srivastava","raw_affiliation_strings":["University of Minnesota, Minneapolis, MN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Minnesota, Minneapolis, MN, USA","institution_ids":["https://openalex.org/I130238516"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051442967","display_name":"Aaron Halfaker","orcid":"https://orcid.org/0000-0001-8907-6367"},"institutions":[{"id":"https://openalex.org/I4210126245","display_name":"Wikimedia Foundation","ror":"https://ror.org/032q98j12","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210126245"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaron Halfaker","raw_affiliation_strings":["Wikimedia Foundation, San Francisco, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Wikimedia Foundation, San Francisco, CA, USA","institution_ids":["https://openalex.org/I4210126245"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058856783","display_name":"Loren Terveen","orcid":"https://orcid.org/0000-0002-8843-4035"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Loren Terveen","raw_affiliation_strings":["University of Minnesota, Minneapolis, MN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Minnesota, Minneapolis, MN, USA","institution_ids":["https://openalex.org/I130238516"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051842323","display_name":"Haiyi Zhu","orcid":"https://orcid.org/0000-0001-7271-9100"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haiyi Zhu","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":74,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"14"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9553999900817871,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9498000144958496,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.7287954092025757},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7046613097190857},{"id":"https://openalex.org/keywords/stakeholder","display_name":"Stakeholder","score":0.6935212016105652},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6751714944839478},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6002780795097351},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.5727939605712891},{"id":"https://openalex.org/keywords/peer-production","display_name":"Peer production","score":0.4935358762741089},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.47069644927978516},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.38931411504745483},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3867643475532532},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3463061451911926},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32821184396743774},{"id":"https://openalex.org/keywords/public-relations","display_name":"Public relations","score":0.14208349585533142},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.12614870071411133},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.09484383463859558}],"concepts":[{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.7287954092025757},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7046613097190857},{"id":"https://openalex.org/C201305675","wikidata":"https://www.wikidata.org/wiki/Q852998","display_name":"Stakeholder","level":2,"score":0.6935212016105652},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6751714944839478},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6002780795097351},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.5727939605712891},{"id":"https://openalex.org/C2779507501","wikidata":"https://www.wikidata.org/wiki/Q797769","display_name":"Peer production","level":2,"score":0.4935358762741089},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.47069644927978516},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.38931411504745483},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3867643475532532},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3463061451911926},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32821184396743774},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.14208349585533142},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.12614870071411133},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.09484383463859558},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3313831.3376783","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3313831.3376783","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376783","source":null,"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2001.04879","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.04879","pdf_url":"https://arxiv.org/pdf/2001.04879","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3313831.3376783","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3313831.3376783","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376783","source":null,"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.6700000166893005,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G4209333995","display_name":"EAGER:AI-DCL:Capture, Explain and Negotiate the Inherent Trade-offs in Machine Learning Algorithms","funder_award_id":"2000782","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6242458504","display_name":"FAI: Advancing Fairness in AI with Human-Algorithm Collaborations","funder_award_id":"1939606","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7304765949","display_name":"CHS:Small: Incorporating and Balancing Stakeholder Values in Algorithm Design","funder_award_id":"2001851","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G835961829","display_name":null,"funder_award_id":"IIS-2001851,IIS-2000782,IIS-1939606","funder_id":"https://openalex.org/F4320309856","funder_display_name":"National Youth Science Foundation"},{"id":"https://openalex.org/G8707501984","display_name":null,"funder_award_id":"IIS-1939606","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309636","display_name":"University of Minnesota","ror":"https://ror.org/03grvy078"},{"id":"https://openalex.org/F4320309856","display_name":"National Youth Science Foundation","ror":"https://ror.org/054yz2f06"},{"id":"https://openalex.org/F4320310207","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2998862821.pdf","grobid_xml":"https://content.openalex.org/works/W2998862821.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W1778277763","https://openalex.org/W1919323716","https://openalex.org/W1972067873","https://openalex.org/W1981475687","https://openalex.org/W2007232175","https://openalex.org/W2019031481","https://openalex.org/W2036291768","https://openalex.org/W2054551600","https://openalex.org/W2055904836","https://openalex.org/W2097519322","https://openalex.org/W2100960835","https://openalex.org/W2115973081","https://openalex.org/W2117493442","https://openalex.org/W2118110456","https://openalex.org/W2133369086","https://openalex.org/W2145602677","https://openalex.org/W2147603330","https://openalex.org/W2332488709","https://openalex.org/W2361256855","https://openalex.org/W2395678419","https://openalex.org/W2405896853","https://openalex.org/W2418507704","https://openalex.org/W2420245003","https://openalex.org/W2490228895","https://openalex.org/W2530395818","https://openalex.org/W2584805976","https://openalex.org/W2594475271","https://openalex.org/W2597603852","https://openalex.org/W2599514293","https://openalex.org/W2610178560","https://openalex.org/W2618493064","https://openalex.org/W2749945618","https://openalex.org/W2758861434","https://openalex.org/W2786242872","https://openalex.org/W2790025105","https://openalex.org/W2795580035","https://openalex.org/W2795582800","https://openalex.org/W2897978524","https://openalex.org/W2898911770","https://openalex.org/W2900885930","https://openalex.org/W2910732263","https://openalex.org/W2942073295","https://openalex.org/W2963095307","https://openalex.org/W2963104135","https://openalex.org/W2963178340","https://openalex.org/W2963473400","https://openalex.org/W2963749936","https://openalex.org/W2963917042","https://openalex.org/W2988207158","https://openalex.org/W3100046612","https://openalex.org/W3106521870","https://openalex.org/W3122302244","https://openalex.org/W4289383937","https://openalex.org/W4292807893"],"related_works":["https://openalex.org/W2886802431","https://openalex.org/W4388216822","https://openalex.org/W1761762290","https://openalex.org/W4391191813","https://openalex.org/W4387575966","https://openalex.org/W2384262901","https://openalex.org/W2975214487","https://openalex.org/W651304006","https://openalex.org/W2345720417","https://openalex.org/W4240005889"],"abstract_inverted_index":{"On":[0],"Wikipedia,":[1],"sophisticated":[2],"algorithmic":[3],"tools":[4],"are":[5],"used":[6,69],"to":[7,21,50,142],"assess":[8],"the":[9,23,33,60,92,102,128],"quality":[10,66],"of":[11,35,94,123],"edits":[12],"and":[13,54,74,119,125,136],"take":[14,44],"corrective":[15],"actions.":[16],"However,":[17],"algorithms":[18,126,144],"can":[19],"fail":[20],"solve":[22],"problems":[24],"they":[25,30],"were":[26],"designed":[27],"for":[28,139],"if":[29],"conflict":[31],"with":[32,115],"values":[34,78,135],"communities":[36],"who":[37],"use":[38],"them.":[39],"In":[40],"this":[41],"study,":[42],"we":[43],"a":[45,52],"Value-Sensitive":[46],"Algorithm":[47],"Design":[48],"approach":[49],"understanding":[51],"community-created":[53],"-maintained":[55],"machine":[56],"learning-based":[57],"algorithm":[58],"called":[59],"Objective":[61],"Revision":[62],"Evaluation":[63],"System":[64],"(ORES)---a":[65],"prediction":[67],"system":[68],"in":[70],"numerous":[71],"Wikipedia":[72],"applications":[73],"contexts.":[75],"Five":[76],"major":[77],"converged":[79],"across":[80],"stakeholder":[81],"groups":[82],"that":[83],"ORES":[84],"(and":[85],"its":[86],"dependent":[87],"applications)":[88],"should:":[89],"(1)":[90],"reduce":[91],"effort":[93],"community":[95],"maintenance,":[96],"(2)":[97],"maintain":[98],"human":[99],"judgement":[100],"as":[101],"final":[103],"authority,":[104],"(3)":[105],"support":[106],"differing":[107,109],"peoples'":[108],"workflows,":[110],"(4)":[111],"encourage":[112],"positive":[113],"engagement":[114],"diverse":[116],"editor":[117],"groups,":[118],"(5)":[120],"establish":[121],"trustworthiness":[122],"people":[124],"within":[127],"community.":[129],"We":[130],"reveal":[131],"tensions":[132],"between":[133],"these":[134],"discuss":[137],"implications":[138],"future":[140],"research":[141],"improve":[143],"like":[145],"ORES.":[146]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":17},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2020-01-23T00:00:00"}