{"id":"https://openalex.org/W2898956098","doi":"https://doi.org/10.14778/3275536.3275541","title":"Cost-effective data annotation using game-based crowdsourcing","display_name":"Cost-effective data annotation using game-based crowdsourcing","publication_year":2018,"publication_date":"2018-09-01","ids":{"openalex":"https://openalex.org/W2898956098","doi":"https://doi.org/10.14778/3275536.3275541","mag":"2898956098"},"language":"en","primary_location":{"id":"doi:10.14778/3275536.3275541","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3275536.3275541","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103028400","display_name":"Jingru Yang","orcid":"https://orcid.org/0000-0002-8780-3919"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jingru Yang","raw_affiliation_strings":["Renmin University of China, China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100739546","display_name":"Ju Fan","orcid":"https://orcid.org/0000-0003-4729-9903"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ju Fan","raw_affiliation_strings":["Renmin University of China, China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074858555","display_name":"Zhewei Wei","orcid":"https://orcid.org/0000-0003-3620-5086"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhewei Wei","raw_affiliation_strings":["Renmin University of China, China and Beihang University, China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China, China and Beihang University, China","institution_ids":["https://openalex.org/I78988378","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100451576","display_name":"Guoliang Li","orcid":"https://orcid.org/0000-0002-1398-0621"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guoliang Li","raw_affiliation_strings":["Tsinghua University, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua University, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100627124","display_name":"Tongyu Liu","orcid":"https://orcid.org/0009-0002-7600-5476"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tongyu Liu","raw_affiliation_strings":["Renmin University of China, China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008721449","display_name":"Xiaoyong Du","orcid":"https://orcid.org/0000-0002-5757-9135"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyong Du","raw_affiliation_strings":["Renmin University of China, China"],"affiliations":[{"raw_affiliation_string":"Renmin University of China, China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5103028400"],"corresponding_institution_ids":["https://openalex.org/I78988378"],"apc_list":null,"apc_paid":null,"fwci":6.9453,"has_fulltext":false,"cited_by_count":43,"citation_normalized_percentile":{"value":0.96679666,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"12","issue":"1","first_page":"57","last_page":"70"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8342831134796143},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7886053323745728},{"id":"https://openalex.org/keywords/tuple","display_name":"Tuple","score":0.7753757238388062},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.624032199382782},{"id":"https://openalex.org/keywords/generator","display_name":"Generator (circuit theory)","score":0.5686286687850952},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5451033711433411},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5285964012145996},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48257461190223694},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4594922661781311},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.10489624738693237},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08275246620178223}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8342831134796143},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7886053323745728},{"id":"https://openalex.org/C118930307","wikidata":"https://www.wikidata.org/wiki/Q600590","display_name":"Tuple","level":2,"score":0.7753757238388062},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.624032199382782},{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.5686286687850952},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5451033711433411},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5285964012145996},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48257461190223694},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4594922661781311},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.10489624738693237},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08275246620178223},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.14778/3275536.3275541","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3275536.3275541","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W658020064","https://openalex.org/W1593045043","https://openalex.org/W1991137221","https://openalex.org/W2011907632","https://openalex.org/W2012668444","https://openalex.org/W2025721839","https://openalex.org/W2035274534","https://openalex.org/W2053653724","https://openalex.org/W2056748234","https://openalex.org/W2068645020","https://openalex.org/W2073068515","https://openalex.org/W2076357412","https://openalex.org/W2081118310","https://openalex.org/W2081261296","https://openalex.org/W2099471712","https://openalex.org/W2106675345","https://openalex.org/W2107598941","https://openalex.org/W2107966677","https://openalex.org/W2108991785","https://openalex.org/W2125101937","https://openalex.org/W2125943921","https://openalex.org/W2132679783","https://openalex.org/W2149713870","https://openalex.org/W2163051926","https://openalex.org/W2168144930","https://openalex.org/W2252115292","https://openalex.org/W2295333720","https://openalex.org/W2329105431","https://openalex.org/W2404161646","https://openalex.org/W2428834396","https://openalex.org/W2430704425","https://openalex.org/W2469104253","https://openalex.org/W2535778619","https://openalex.org/W2585226541","https://openalex.org/W2612526608","https://openalex.org/W2612619809","https://openalex.org/W2612964472","https://openalex.org/W2619206542","https://openalex.org/W2741649465","https://openalex.org/W2750620035","https://openalex.org/W2769041395","https://openalex.org/W2785650693","https://openalex.org/W2811367565","https://openalex.org/W2919115771","https://openalex.org/W2949312134","https://openalex.org/W2950133940","https://openalex.org/W2950577311","https://openalex.org/W2950833813","https://openalex.org/W2952935871","https://openalex.org/W2959716049","https://openalex.org/W2962843773","https://openalex.org/W3101023724","https://openalex.org/W3121928352","https://openalex.org/W3146259567","https://openalex.org/W6752967837"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"Large-scale":[0],"data":[1,13,131],"annotation":[2,17,35],"is":[3,108,133,198,221,240],"indispensable":[4],"for":[5,23,250],"many":[6],"applications,":[7],"such":[8],"as":[9],"machine":[10],"learning":[11],"and":[12,37,69,87,159,205,234,259,271,274],"integration.":[14],"However,":[15],"existing":[16],"solutions":[18],"either":[19],"incur":[20],"expensive":[21],"cost":[22,56],"large":[24,157],"datasets":[25],"or":[26],"produce":[27],"noisy":[28],"results.":[29],"This":[30,185],"paper":[31,186],"introduces":[32],"a":[33,72,106,112,130,137,148,212,256],"cost-effective":[34],"approach,":[36],"focuses":[38],"on":[39,268],"the":[40,54,62,118,126,144,183,188,201,209,216,247,252,275],"labeling":[41,55],"rule":[42,102,107,115,140,151,162,169,213,222,232],"generation":[43],"problem":[44],"that":[45,175,278],"aims":[46],"to":[47,51,80,110,135,165,179,199,229,241,245],"generate":[48,66],"high-quality":[49,82,154],"rules":[50,83,155,181],"largely":[52],"reduce":[53],"while":[57,117,161],"preserving":[58],"quality.":[59],"To":[60],"address":[61],"problem,":[63],"we":[64],"first":[65,197],"candidate":[67],"rules,":[68],"then":[70],"devise":[71],"game-based":[73,248],"crowdsourcing":[74,243],"approach":[75],"C":[76,89,191],"ROWD":[77,90,192],"G":[78,91,193],"AME":[79,92,194],"select":[81,242],"by":[84,171,214],"considering":[85,215],"coverage":[86,158,204],"precision.":[88,206],"employs":[93],"two":[94,145,217],"groups":[95,146],"of":[96,114,129,139,211],"crowd":[97],"workers:":[98],"one":[99],"group":[100,120],"answers":[101,121],"validation":[103,233],"tasks":[104,124,244],"(whether":[105,125],"valid)":[109],"play":[111,136,147],"role":[113,138],"generator,":[116],"other":[119],"tuple":[122,132,235],"checking":[123,172,236],"annotated":[127],"label":[128],"correct)":[134],"refuter.":[141],"We":[142,207,225,254,265],"let":[143],"two-player":[149],"game:":[150],"generator":[152,170],"identifies":[153],"with":[156],"precision,":[160],"refuter":[163],"tries":[164],"refute":[166],"its":[167],"opponent":[168],"some":[173],"tuples":[174],"provide":[176],"enough":[177],"evidence":[178],"reject":[180],"covering":[182],"tuples.":[184],"studies":[187],"challenges":[189],"in":[190],".":[195],"The":[196,219,238],"balance":[200],"trade-off":[202],"between":[203],"define":[208],"loss":[210],"factors.":[218],"second":[220],"precision":[223],"estimation.":[224],"utilize":[226],"Bayesian":[227],"estimation":[228],"combine":[230],"both":[231],"tasks.":[237],"third":[239],"fulfill":[246],"framework":[249],"minimizing":[251],"loss.":[253],"introduce":[255],"minimax":[257],"strategy":[258],"develop":[260],"efficient":[261],"task":[262],"selection":[263],"algorithms.":[264],"conduct":[266],"experiments":[267],"entity":[269],"matching":[270],"relation":[272],"extraction,":[273],"results":[276],"show":[277],"our":[279],"method":[280],"outperforms":[281],"state-of-the-art":[282],"solutions.":[283]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":1}],"updated_date":"2026-04-05T06:14:27.290980","created_date":"2025-10-10T00:00:00"}