{"id":"https://openalex.org/W4392120502","doi":"https://doi.org/10.1145/3613904.3642278","title":"Wikibench: Community-Driven Data Curation for AI Evaluation on Wikipedia","display_name":"Wikibench: Community-Driven Data Curation for AI Evaluation on Wikipedia","publication_year":2024,"publication_date":"2024-05-11","ids":{"openalex":"https://openalex.org/W4392120502","doi":"https://doi.org/10.1145/3613904.3642278"},"language":"en","primary_location":{"id":"doi:10.1145/3613904.3642278","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642278","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642278","source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642278","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056050477","display_name":"Tzu-Sheng Kuo","orcid":"https://orcid.org/0000-0002-1504-7640"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Tzu-Sheng Kuo","raw_affiliation_strings":["Human-Computer Interaction Institute, Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051442967","display_name":"Aaron Halfaker","orcid":"https://orcid.org/0000-0001-8907-6367"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaron Lee Halfaker","raw_affiliation_strings":["Office of Applied Research, Microsoft, United States"],"affiliations":[{"raw_affiliation_string":"Office of Applied Research, Microsoft, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090170925","display_name":"Zirui Cheng","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zirui Cheng","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101414964","display_name":"Jiwoo Kim","orcid":"https://orcid.org/0009-0008-0898-8371"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiwoo Kim","raw_affiliation_strings":["Columbia University, United States"],"affiliations":[{"raw_affiliation_string":"Columbia University, United States","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007691163","display_name":"Meng-Hsin Wu","orcid":"https://orcid.org/0009-0007-4919-0801"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Meng-Hsin Wu","raw_affiliation_strings":["School of Computer Science, Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"School of Computer Science, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004225142","display_name":"Tongshuang Wu","orcid":"https://orcid.org/0000-0003-1630-0588"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tongshuang Wu","raw_affiliation_strings":["Human-Computer Interaction Institute, Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022664382","display_name":"Kenneth Holstein","orcid":"https://orcid.org/0000-0001-6730-922X"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenneth Holstein","raw_affiliation_strings":["Human-Computer Interaction Institute, Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051842323","display_name":"Haiyi Zhu","orcid":"https://orcid.org/0000-0001-7271-9100"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haiyi Zhu","raw_affiliation_strings":["Human-Computer Interaction Institute, Carnegie Mellon University, United States"],"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5056050477"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":14.8301,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.99095949,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9614999890327454,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-curation","display_name":"Data curation","score":0.8181391954421997},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.784722089767456},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.60673987865448},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5547788143157959},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5371264815330505},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5189340114593506},{"id":"https://openalex.org/keywords/digital-curation","display_name":"Digital curation","score":0.4881325662136078},{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.4823424518108368},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.48033657670021057},{"id":"https://openalex.org/keywords/inclusion","display_name":"Inclusion (mineral)","score":0.46271201968193054},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.43187665939331055},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3936240077018738},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.22323963046073914},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.1436232626438141}],"concepts":[{"id":"https://openalex.org/C91632574","wikidata":"https://www.wikidata.org/wiki/Q15088675","display_name":"Data curation","level":2,"score":0.8181391954421997},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.784722089767456},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.60673987865448},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5547788143157959},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5371264815330505},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5189340114593506},{"id":"https://openalex.org/C2775953033","wikidata":"https://www.wikidata.org/wiki/Q5276060","display_name":"Digital curation","level":2,"score":0.4881325662136078},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.4823424518108368},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.48033657670021057},{"id":"https://openalex.org/C109359841","wikidata":"https://www.wikidata.org/wiki/Q728944","display_name":"Inclusion (mineral)","level":2,"score":0.46271201968193054},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.43187665939331055},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3936240077018738},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.22323963046073914},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.1436232626438141},{"id":"https://openalex.org/C107993555","wikidata":"https://www.wikidata.org/wiki/Q1662673","display_name":"Gender studies","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3613904.3642278","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642278","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642278","source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2402.14147","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14147","pdf_url":"https://arxiv.org/pdf/2402.14147","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3613904.3642278","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642278","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642278","source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7699999809265137,"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G1338404456","display_name":null,"funder_award_id":"19520","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5786289080","display_name":null,"funder_award_id":"1952085","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7304765949","display_name":"CHS:Small: Incorporating and Balancing Stakeholder Values in Algorithm Design","funder_award_id":"2001851","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392120502.pdf"},"referenced_works_count":71,"referenced_works":["https://openalex.org/W1972067873","https://openalex.org/W1975739711","https://openalex.org/W2019031481","https://openalex.org/W2019502441","https://openalex.org/W2036291768","https://openalex.org/W2054551600","https://openalex.org/W2059362837","https://openalex.org/W2063185719","https://openalex.org/W2093753056","https://openalex.org/W2099769844","https://openalex.org/W2132764451","https://openalex.org/W2145602677","https://openalex.org/W2147603330","https://openalex.org/W2153635508","https://openalex.org/W2164765478","https://openalex.org/W2566547698","https://openalex.org/W2621141770","https://openalex.org/W2796134555","https://openalex.org/W2810857251","https://openalex.org/W2898944906","https://openalex.org/W2899027170","https://openalex.org/W2920807444","https://openalex.org/W2940680165","https://openalex.org/W2940962650","https://openalex.org/W2941766203","https://openalex.org/W2949678053","https://openalex.org/W2964034671","https://openalex.org/W2966491090","https://openalex.org/W2983996708","https://openalex.org/W2998862821","https://openalex.org/W3003928769","https://openalex.org/W3081464307","https://openalex.org/W3094328607","https://openalex.org/W3105435131","https://openalex.org/W3105662186","https://openalex.org/W3108126053","https://openalex.org/W3160887439","https://openalex.org/W3163078977","https://openalex.org/W3163469193","https://openalex.org/W3163680969","https://openalex.org/W3197587259","https://openalex.org/W3204393347","https://openalex.org/W3205290952","https://openalex.org/W3206218807","https://openalex.org/W3206420877","https://openalex.org/W3212368439","https://openalex.org/W4224992683","https://openalex.org/W4225006216","https://openalex.org/W4225085211","https://openalex.org/W4231384729","https://openalex.org/W4234249932","https://openalex.org/W4288083800","https://openalex.org/W4288083802","https://openalex.org/W4289383937","https://openalex.org/W4300003134","https://openalex.org/W4309618884","https://openalex.org/W4309618902","https://openalex.org/W4310419543","https://openalex.org/W4313139696","https://openalex.org/W4321473289","https://openalex.org/W4321610533","https://openalex.org/W4321853859","https://openalex.org/W4327990559","https://openalex.org/W4366547557","https://openalex.org/W4366588458","https://openalex.org/W4380319026","https://openalex.org/W4385571444","https://openalex.org/W4386270115","https://openalex.org/W4387607025","https://openalex.org/W6769410160","https://openalex.org/W6804645036"],"related_works":["https://openalex.org/W2400390830","https://openalex.org/W2598248896","https://openalex.org/W2907605431","https://openalex.org/W842882028","https://openalex.org/W2138555252","https://openalex.org/W2807479512","https://openalex.org/W2277259138","https://openalex.org/W3135458263","https://openalex.org/W4231012597","https://openalex.org/W3200944274"],"abstract_inverted_index":{"AI":[0,13,31,49,80],"tools":[1,67],"are":[2,14],"increasingly":[3],"deployed":[4],"in":[5,88],"community":[6,61,106],"contexts.":[7],"However,":[8],"datasets":[9,47,99],"used":[10,114],"to":[11,38,77,116],"evaluate":[12],"typically":[15],"created":[16],"by":[17],"developers":[18],"and":[19,43,86,109,131],"annotators":[20],"outside":[21],"a":[22,72],"given":[23],"community,":[24],"which":[25],"can":[26,103],"yield":[27],"misleading":[28],"conclusions":[29],"about":[30],"performance.":[32],"How":[33],"might":[34],"we":[35,139],"empower":[36],"communities":[37,76],"drive":[39],"the":[40,118],"intentional":[41],"design":[42],"curation":[44,121],"of":[45],"evaluation":[46,81],"for":[48,143],"that":[50,74,98,145],"impacts":[51],"them?":[52],"We":[53,69],"investigate":[54],"this":[55],"question":[56],"on":[57,95,136],"Wikipedia,":[58],"an":[59],"online":[60],"with":[62],"multiple":[63],"AI-based":[64],"content":[65],"moderation":[66],"deployed.":[68],"introduce":[70],"Wikibench,":[71],"system":[73],"enables":[75],"collaboratively":[78],"curate":[79],"datasets,":[82],"while":[83],"navigating":[84],"ambiguities":[85],"differences":[87],"perspective":[89],"through":[90],"discussion.":[91],"A":[92],"field":[93],"study":[94,112],"Wikipedia":[96],"shows":[97],"curated":[100],"using":[101],"Wikibench":[102,115],"effectively":[104],"capture":[105],"consensus,":[107],"disagreement,":[108],"uncertainty.":[110],"Furthermore,":[111],"participants":[113],"shape":[117],"overall":[119],"data":[120,128,133,148],"process,":[122],"including":[123],"refining":[124],"label":[125],"definitions,":[126],"determining":[127],"inclusion":[129],"criteria,":[130],"authoring":[132],"statements.":[134],"Based":[135],"our":[137],"findings,":[138],"propose":[140],"future":[141],"directions":[142],"systems":[144],"support":[146],"community-driven":[147],"curation.":[149]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":15}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}