{"id":"https://openalex.org/W4213112230","doi":"https://doi.org/10.1145/3488560.3510011","title":"Challenges in Data Production for AI with Human-in-the-Loop","display_name":"Challenges in Data Production for AI with Human-in-the-Loop","publication_year":2022,"publication_date":"2022-02-11","ids":{"openalex":"https://openalex.org/W4213112230","doi":"https://doi.org/10.1145/3488560.3510011"},"language":"en","primary_location":{"id":"doi:10.1145/3488560.3510011","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3488560.3510011","pdf_url":null,"source":{"id":"https://openalex.org/S4363608885","display_name":"Proceedings of the Fifteenth ACM International Conference on Web Search and Data Mining","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Fifteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030955986","display_name":"Dmitry Ustalov","orcid":"https://orcid.org/0000-0002-9979-2188"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Dmitry Ustalov","raw_affiliation_strings":["Toloka, Saint Petersburg, Russian Fed"],"affiliations":[{"raw_affiliation_string":"Toloka, Saint Petersburg, Russian Fed","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5030955986"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7208,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.63377024,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1651","last_page":"1652"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8918940424919128},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7001284956932068},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.5799118280410767},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.541361927986145},{"id":"https://openalex.org/keywords/human-intelligence","display_name":"Human intelligence","score":0.5367717742919922},{"id":"https://openalex.org/keywords/production","display_name":"Production (economics)","score":0.5318018794059753},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5192630290985107},{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.5017123222351074},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.49788594245910645},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.4845696687698364},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4693760871887207},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4259941577911377},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.41041356325149536},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1688195765018463},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.13907694816589355}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8918940424919128},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7001284956932068},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.5799118280410767},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.541361927986145},{"id":"https://openalex.org/C105409693","wikidata":"https://www.wikidata.org/wiki/Q5937824","display_name":"Human intelligence","level":2,"score":0.5367717742919922},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.5318018794059753},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5192630290985107},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.5017123222351074},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49788594245910645},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.4845696687698364},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4693760871887207},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4259941577911377},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.41041356325149536},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1688195765018463},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.13907694816589355},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C139719470","wikidata":"https://www.wikidata.org/wiki/Q39680","display_name":"Macroeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3488560.3510011","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3488560.3510011","pdf_url":null,"source":{"id":"https://openalex.org/S4363608885","display_name":"Proceedings of the Fifteenth ACM International Conference on Web Search and Data Mining","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Fifteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5199999809265137,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":3,"referenced_works":["https://openalex.org/W2981852735","https://openalex.org/W4288089799","https://openalex.org/W4292341621"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W1689950805","https://openalex.org/W3131060711","https://openalex.org/W4382705708","https://openalex.org/W4390818809","https://openalex.org/W4390143763","https://openalex.org/W4200460104"],"abstract_inverted_index":{"Today,":[0],"successful":[1],"Artificial":[2],"Intelligence":[3],"applications":[4],"rely":[5],"on":[6],"three":[7],"pillars:":[8],"machine":[9,65],"learning":[10,66],"algorithms,":[11],"hardware":[12,26],"for":[13,18],"running":[14],"them,":[15],"and":[16,20,25,33],"data":[17,35],"training":[19],"evaluating":[21],"models.":[22],"Although":[23],"algorithms":[24],"have":[27],"already":[28],"become":[29],"commodities,":[30],"obtaining":[31],"up-to-date":[32],"high-quality":[34],"at":[36],"scale":[37],"is":[38],"still":[39],"challenging-but":[40],"possible":[41],"by":[42],"building":[43],"hybrid":[44],"human-computer":[45],"pipelines":[46,62],"called":[47],"human-in-the-loop.":[48],"This":[49],"talk":[50],"will":[51,70],"show":[52],"how":[53],"to":[54],"make":[55],"a":[56],"significant":[57],"business":[58],"impact":[59],"using":[60],"human-in-the-loop":[61],"that":[63],"combine":[64],"with":[67],"crowdsourcing.":[68],"We":[69],"share":[71],"the":[72,77],"experience":[73],"of":[74,76],"one":[75],"world's":[78],"largest":[79],"search":[80],"engines,":[81],"Yandex.":[82]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-03-28T08:17:26.163206","created_date":"2025-10-10T00:00:00"}