{"id":"https://openalex.org/W4313030344","doi":"https://doi.org/10.1109/icpr56361.2022.9956439","title":"Mitigating Observation Biases in Crowdsourced Label Aggregation","display_name":"Mitigating Observation Biases in Crowdsourced Label Aggregation","publication_year":2022,"publication_date":"2022-08-21","ids":{"openalex":"https://openalex.org/W4313030344","doi":"https://doi.org/10.1109/icpr56361.2022.9956439"},"language":"en","primary_location":{"id":"doi:10.1109/icpr56361.2022.9956439","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpr56361.2022.9956439","pdf_url":null,"source":{"id":"https://openalex.org/S4363607731","display_name":"2022 26th International Conference on Pattern Recognition (ICPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 26th International Conference on Pattern Recognition (ICPR)","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2302.13100","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088543566","display_name":"Ryosuke Ueda","orcid":"https://orcid.org/0009-0001-3892-9857"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Ryosuke Ueda","raw_affiliation_strings":["Kyoto University,Kyoto,Japan","Kyoto University, Kyoto, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyoto University,Kyoto,Japan","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"Kyoto University, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088800219","display_name":"Koh Takeuchi","orcid":"https://orcid.org/0000-0002-3245-888X"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Koh Takeuchi","raw_affiliation_strings":["Kyoto University,Kyoto,Japan","Kyoto University, Kyoto, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyoto University,Kyoto,Japan","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"Kyoto University, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031707680","display_name":"Hisashi Kashima","orcid":"https://orcid.org/0000-0002-2770-0184"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hisashi Kashima","raw_affiliation_strings":["Kyoto University,Kyoto,Japan","Kyoto University, Kyoto, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyoto University,Kyoto,Japan","institution_ids":["https://openalex.org/I22299242"]},{"raw_affiliation_string":"Kyoto University, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I22299242"],"apc_list":null,"apc_paid":null,"fwci":0.7163,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.6867284,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1171","last_page":"1177"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9821000099182129,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9842449426651001},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7482936978340149},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7020485401153564},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6745759844779968},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5277988314628601},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5137113332748413},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.44902855157852173},{"id":"https://openalex.org/keywords/observational-study","display_name":"Observational study","score":0.43909403681755066},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4345758855342865},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4235653877258301},{"id":"https://openalex.org/keywords/data-aggregator","display_name":"Data aggregator","score":0.4179297387599945},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4141022264957428},{"id":"https://openalex.org/keywords/data-quality","display_name":"Data quality","score":0.4123859405517578},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.15543872117996216},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09575995802879333},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08265367150306702}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9842449426651001},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7482936978340149},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7020485401153564},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6745759844779968},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5277988314628601},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5137113332748413},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.44902855157852173},{"id":"https://openalex.org/C23131810","wikidata":"https://www.wikidata.org/wiki/Q818574","display_name":"Observational study","level":2,"score":0.43909403681755066},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4345758855342865},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4235653877258301},{"id":"https://openalex.org/C82578977","wikidata":"https://www.wikidata.org/wiki/Q16773055","display_name":"Data aggregator","level":3,"score":0.4179297387599945},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4141022264957428},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.4123859405517578},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.15543872117996216},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09575995802879333},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08265367150306702},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C24590314","wikidata":"https://www.wikidata.org/wiki/Q336038","display_name":"Wireless sensor network","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icpr56361.2022.9956439","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icpr56361.2022.9956439","pdf_url":null,"source":{"id":"https://openalex.org/S4363607731","display_name":"2022 26th International Conference on Pattern Recognition (ICPR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 26th International Conference on Pattern Recognition (ICPR)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2302.13100","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.13100","pdf_url":"https://arxiv.org/pdf/2302.13100","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2302.13100","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.13100","pdf_url":"https://arxiv.org/pdf/2302.13100","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.41999998688697815}],"awards":[{"id":"https://openalex.org/G8063491431","display_name":null,"funder_award_id":"JPMJCR21D1","funder_id":"https://openalex.org/F4320338075","funder_display_name":"Core Research for Evolutional Science and Technology"}],"funders":[{"id":"https://openalex.org/F4320338075","display_name":"Core Research for Evolutional Science and Technology","ror":"https://ror.org/00097mb19"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4313030344.pdf"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W941378780","https://openalex.org/W1459599406","https://openalex.org/W1501025848","https://openalex.org/W1510804179","https://openalex.org/W1543648998","https://openalex.org/W1970381522","https://openalex.org/W2058892254","https://openalex.org/W2076357412","https://openalex.org/W2119830539","https://openalex.org/W2141649520","https://openalex.org/W2142518823","https://openalex.org/W2150291618","https://openalex.org/W2151401338","https://openalex.org/W2294447324","https://openalex.org/W2329105431","https://openalex.org/W2405546739","https://openalex.org/W2585226541","https://openalex.org/W2605991684","https://openalex.org/W2783668259","https://openalex.org/W2789956525","https://openalex.org/W2807854944","https://openalex.org/W2903257353","https://openalex.org/W2914980526","https://openalex.org/W2941486823","https://openalex.org/W2945436856","https://openalex.org/W2949246411","https://openalex.org/W2950833813","https://openalex.org/W2964200481","https://openalex.org/W3016244823","https://openalex.org/W3033235490","https://openalex.org/W3038236330","https://openalex.org/W3150893739","https://openalex.org/W4233216783","https://openalex.org/W4293876646","https://openalex.org/W6600399172","https://openalex.org/W6628785978","https://openalex.org/W6677811664","https://openalex.org/W6680957539","https://openalex.org/W6695055480","https://openalex.org/W6697452742","https://openalex.org/W6736502011","https://openalex.org/W6763041513","https://openalex.org/W6763903876"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"Crowdsourcing":[0],"has":[1],"been":[2],"widely":[3],"used":[4,127],"to":[5,59,165],"efficiently":[6],"obtain":[7],"labeled":[8],"datasets":[9,138],"for":[10,114],"supervised":[11],"learning":[12],"from":[13,32],"large":[14],"numbers":[15],"of":[16,24,83,89,105,159],"human":[17],"resources":[18],"at":[19],"low":[20],"cost.":[21],"However,":[22],"one":[23],"the":[25,37,43,50,61,74,81,87,95,103,106,149,153,157],"technical":[26],"challenges":[27],"in":[28,77,80,128,156],"obtaining":[29],"high-quality":[30],"results":[31,97],"crowdsourcing":[33,115],"is":[34,47],"dealing":[35],"with":[36,102,120],"variability":[38],"and":[39,52,86,136,143,163,168],"bias":[40,76,124],"caused":[41],"by":[42,63],"fact":[44],"that":[45,117,148],"it":[46],"humans":[48],"execute":[49],"work,":[51],"various":[53],"studies":[54],"have":[55],"addressed":[56],"this":[57,69],"issue":[58],"improve":[60],"quality":[62,104],"integrating":[64],"redundantly":[65],"collected":[66],"responses.":[67,107],"In":[68],"study,":[70],"we":[71,146],"focus":[72],"on":[73],"observation":[75,161],"crowdsourcing.":[78],"Variations":[79],"frequency":[82],"worker":[84],"responses":[85,116],"complexity":[88],"tasks":[90],"occur,":[91],"which":[92],"may":[93],"affect":[94],"aggregation":[96,112,154],"when":[98],"they":[99],"are":[100,118],"correlated":[101],"We":[108],"also":[109],"propose":[110],"statistical":[111],"methods":[113],"combined":[119],"an":[121],"observational":[122],"data":[123],"removal":[125],"method":[126,151],"causal":[129],"inference.":[130],"Through":[131],"experiments":[132],"using":[133],"both":[134,166],"synthetic":[135],"real":[137],"with/without":[139],"artificially":[140],"injected":[141],"spam":[142,167],"colluding":[144,169],"workers,":[145],"verify":[147],"proposed":[150],"improves":[152],"accuracy":[155],"presence":[158],"strong":[160],"biases":[162],"robustness":[164],"workers.":[170]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2025-10-10T00:00:00"}