{"id":"https://openalex.org/W3029299514","doi":"https://doi.org/10.1145/3318464.3383127","title":"Crowdsourcing Practice for Efficient Data Labeling: Aggregation, Incremental Relabeling, and Pricing","display_name":"Crowdsourcing Practice for Efficient Data Labeling: Aggregation, Incremental Relabeling, and Pricing","publication_year":2020,"publication_date":"2020-05-29","ids":{"openalex":"https://openalex.org/W3029299514","doi":"https://doi.org/10.1145/3318464.3383127","mag":"3029299514"},"language":"en","primary_location":{"id":"doi:10.1145/3318464.3383127","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3318464.3383127","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 ACM SIGMOD International Conference on Management of Data","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109286797","display_name":"Alexey Drutsa","orcid":null},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Alexey Drutsa","raw_affiliation_strings":["Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110323743","display_name":"Valentina Fedorova","orcid":null},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Valentina Fedorova","raw_affiliation_strings":["Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030955986","display_name":"Dmitry Ustalov","orcid":"https://orcid.org/0000-0002-9979-2188"},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Dmitry Ustalov","raw_affiliation_strings":["Yandex, Saint Petersburg, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Saint Petersburg, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003717172","display_name":"Olga Megorskaya","orcid":null},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Olga Megorskaya","raw_affiliation_strings":["Yandex, Saint Petersburg, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Saint Petersburg, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031735055","display_name":"Evfrosiniya Zerminova","orcid":null},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Evfrosiniya Zerminova","raw_affiliation_strings":["Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029223327","display_name":"Daria Baidakova","orcid":null},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Daria Baidakova","raw_affiliation_strings":["Yandex, Moscow, Russian Fed"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":["https://openalex.org/I58957048"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I58957048"],"apc_list":null,"apc_paid":null,"fwci":3.3154,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.9275442,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"2623","last_page":"2627"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.977400004863739,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9878802299499512},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.8431905508041382},{"id":"https://openalex.org/keywords/session","display_name":"Session (web analytics)","score":0.7882668972015381},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7688052654266357},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.6495568752288818},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5735191106796265},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.5503494143486023},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5299828052520752},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.48109954595565796},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4721548557281494},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.43523895740509033},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.34037578105926514},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.088433176279068}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9878802299499512},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.8431905508041382},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.7882668972015381},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7688052654266357},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6495568752288818},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5735191106796265},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.5503494143486023},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5299828052520752},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.48109954595565796},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4721548557281494},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.43523895740509033},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.34037578105926514},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.088433176279068},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3318464.3383127","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3318464.3383127","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 ACM SIGMOD International Conference on Management of Data","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6299999952316284,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1815682670","https://openalex.org/W1970381522","https://openalex.org/W1999308248","https://openalex.org/W2078277979","https://openalex.org/W2127008633","https://openalex.org/W2164545125","https://openalex.org/W2260880659","https://openalex.org/W2896881735"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W187181332","https://openalex.org/W1490650877"],"abstract_inverted_index":{"In":[0],"this":[1],"tutorial,":[2],"we":[3,113],"present":[4,40,115],"a":[5,53,157],"portion":[6],"of":[7,44,61,83],"unique":[8],"industry":[9],"experience":[10],"in":[11,120,151],"efficient":[12,45,121],"data":[13,32,187],"labeling":[14,33,73],"via":[15,34],"crowdsourcing":[16,36,86],"shared":[17],"by":[18,52,110],"both":[19],"leading":[20],"researchers":[21,178],"and":[22,38,75,125,134,148,162,177,188],"engineers":[23],"from":[24],"Yandex.":[25],"We":[26,128,172],"will":[27,39,49,58,90,114,129,155],"make":[28,168],"an":[29],"introduction":[30],"to":[31,140,167,179,182],"public":[35],"marketplaces":[37],"the":[41,62,72,84,97,101,106,111,116],"key":[42],"components":[43],"label":[46,64,78],"collection.":[47],"This":[48],"be":[50,91],"followed":[51],"practice":[54],"session,":[55],"where":[56],"participants":[57,154],"choose":[59],"one":[60,82],"real":[63,94],"collection":[65,79],"tasks,":[66,142],"experiment":[67],"with":[68],"selecting":[69],"settings":[70],"for":[71],"process,":[74],"launch":[76],"their":[77,132,160],"project":[80,107],"on":[81,93,165],"largest":[85],"marketplaces.":[87],"The":[88],"projects":[89,161],"run":[92],"crowds":[95],"within":[96],"tutorial":[98],"session.":[99],"While":[100],"crowd":[102],"performers":[103],"are":[104],"annotating":[105],"set":[108],"up":[109],"attendees,":[112],"major":[117],"theoretical":[118],"results":[119],"aggregation,":[122],"incremental":[123],"relabeling,":[124],"dynamic":[126],"pricing.":[127],"also":[130],"discuss":[131],"strengths":[133],"weaknesses":[135],"as":[136,138],"well":[137],"applicability":[139],"real-world":[141],"summarizing":[143],"our":[144],"five":[145],"year-long":[146],"research":[147],"industrial":[149],"expertise":[150],"crowdsourcing.":[152],"Finally,":[153],"receive":[156],"feedback":[158],"about":[159],"practical":[163],"advice":[164],"how":[166,181],"them":[169],"more":[170],"efficient.":[171],"invite":[173],"beginners,":[174],"advanced":[175],"specialists,":[176],"learn":[180],"collect":[183],"high":[184],"quality":[185],"labeled":[186],"do":[189],"it":[190],"efficiently.":[191]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}