{"id":"https://openalex.org/W2003497265","doi":"https://doi.org/10.1145/1743384.1743478","title":"How reliable are annotations via crowdsourcing","display_name":"How reliable are annotations via crowdsourcing","publication_year":2010,"publication_date":"2010-03-29","ids":{"openalex":"https://openalex.org/W2003497265","doi":"https://doi.org/10.1145/1743384.1743478","mag":"2003497265"},"language":"en","primary_location":{"id":"doi:10.1145/1743384.1743478","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1743384.1743478","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the international conference on Multimedia information retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005712172","display_name":"Stefanie Nowak","orcid":null},"institutions":[{"id":"https://openalex.org/I4210138578","display_name":"Fraunhofer Institute for Digital Media Technology","ror":"https://ror.org/04gp0yb49","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210138578","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Stefanie Nowak","raw_affiliation_strings":["Fraunhofer IDMT, Ilmenau, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IDMT, Ilmenau, Germany","institution_ids":["https://openalex.org/I4210138578"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112279721","display_name":"Stefan R\u00fcger","orcid":null},"institutions":[{"id":"https://openalex.org/I204136569","display_name":"The Open University","ror":"https://ror.org/05mzfcs16","country_code":"GB","type":"education","lineage":["https://openalex.org/I204136569"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Stefan R\u00fcger","raw_affiliation_strings":["Open University, Milton Keynes, England UK","Open Univ., Milton Keynes, England, UK#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Open University, Milton Keynes, England UK","institution_ids":["https://openalex.org/I204136569"]},{"raw_affiliation_string":"Open Univ., Milton Keynes, England, UK#TAB#","institution_ids":["https://openalex.org/I204136569"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":42.0989,"has_fulltext":false,"cited_by_count":311,"citation_normalized_percentile":{"value":0.99728316,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"557","last_page":"566"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9642000198364258,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9520999789237976,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9209124445915222},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8406327962875366},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.824349582195282},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7246893644332886},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6446545124053955},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6132385730743408},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5904065370559692},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5859910249710083},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5562978982925415},{"id":"https://openalex.org/keywords/filter","display_name":"Filter (signal processing)","score":0.4872383177280426},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4058299660682678},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.33752119541168213},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.14469707012176514},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.0664113461971283}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9209124445915222},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8406327962875366},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.824349582195282},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7246893644332886},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6446545124053955},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6132385730743408},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5904065370559692},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5859910249710083},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5562978982925415},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.4872383177280426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4058299660682678},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33752119541168213},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.14469707012176514},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0664113461971283},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/1743384.1743478","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1743384.1743478","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the international conference on Multimedia information retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:oro.open.ac.uk:25874","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401187","display_name":"Open Research Online (The Open University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I204136569","host_organization_name":"The Open University","host_organization_lineage":["https://openalex.org/I204136569"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Conference or Workshop Item"},{"id":"pmh:oai:fraunhofer.de:N-151406","is_oa":false,"landing_page_url":"http://publica.fraunhofer.de/documents/N-151406.html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400801","display_name":"Publikationsdatenbank der Fraunhofer-Gesellschaft (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Fraunhofer IDMT","raw_type":"Conference Paper"},{"id":"pmh:oai:publica.fraunhofer.de:publica/368111","is_oa":false,"landing_page_url":"https://publica.fraunhofer.de/handle/publica/368111","pdf_url":null,"source":{"id":"https://openalex.org/S4306400318","display_name":"Fraunhofer-Publica (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4099999964237213,"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320320875","display_name":"Deutscher Akademischer Austauschdienst","ror":"https://ror.org/039djdh30"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W112197792","https://openalex.org/W189514963","https://openalex.org/W199384682","https://openalex.org/W1480777612","https://openalex.org/W1970381522","https://openalex.org/W1985514943","https://openalex.org/W1990892398","https://openalex.org/W2053154970","https://openalex.org/W2104858137","https://openalex.org/W2108598243","https://openalex.org/W2120308175","https://openalex.org/W2120665002","https://openalex.org/W2125943921","https://openalex.org/W2137935418","https://openalex.org/W2138928235","https://openalex.org/W2149489787","https://openalex.org/W2149490731","https://openalex.org/W2153292911","https://openalex.org/W2155803963","https://openalex.org/W2158394179","https://openalex.org/W2158880898","https://openalex.org/W2164777277","https://openalex.org/W2170931744","https://openalex.org/W3008021358","https://openalex.org/W3183522126","https://openalex.org/W4285719527","https://openalex.org/W6983592330","https://openalex.org/W7036679014"],"related_works":["https://openalex.org/W2163901716","https://openalex.org/W2103153231","https://openalex.org/W172170795","https://openalex.org/W7602594","https://openalex.org/W2187223183","https://openalex.org/W2971142271","https://openalex.org/W50774052","https://openalex.org/W4378377766","https://openalex.org/W1997653951","https://openalex.org/W4298819607"],"abstract_inverted_index":{"The":[0,96,196],"creation":[1],"of":[2,43,48,67,74,128,163,182,191,201,207],"golden":[3],"standard":[4],"datasets":[5],"is":[6,17,63,99,122,166,185,200],"a":[7,21,58,138],"costly":[8],"business.":[9],"Optimally":[10],"more":[11],"than":[12],"one":[13,178],"judgment":[14],"per":[15],"document":[16],"obtained":[18,56],"to":[19,65,124,176,187,193,204],"ensure":[20],"high":[22],"quality":[23,203],"on":[24,153,159],"annotations.":[25],"In":[26],"this":[27],"context,":[28],"we":[29],"explore":[30],"how":[31,40],"much":[32],"annotations":[33,44,53,66,206],"from":[34,37],"experts":[35,148],"differ":[36],"each":[38],"other,":[39],"different":[41,131,134],"sets":[42],"influence":[45,158],"the":[46,75,79,115,126,145,154,160,164,172,205],"ranking":[47,127],"systems":[49,129,165],"and":[50,91,104,111,118,133,149],"if":[51],"these":[52],"can":[54],"be":[55],"with":[57,69,93],"crowdsourcing":[59],"approach.":[60],"This":[61],"study":[62],"applied":[64,175],"images":[68,76],"multiple":[70],"concepts.":[71],"A":[72],"subset":[73],"employed":[77],"in":[78,137],"latest":[80],"ImageCLEF":[81],"Photo":[82],"Annotation":[83],"competition":[84],"was":[85],"manually":[86],"annotated":[87],"by":[88],"expert":[89],"annotators":[90],"non-experts":[92,150,192],"Mechanical":[94],"Turk.":[95],"inter-annotator":[97],"agreement":[98,146],"computed":[100],"at":[101],"an":[102],"image-based":[103],"concept-based":[105],"level":[106],"using":[107],"majority":[108,173],"vote,":[109],"accuracy":[110],"kappa":[112],"statistics.":[113],"Further,":[114],"Kendall":[116],"\u03c4":[117],"Kolmogorov-Smirnov":[119],"correlation":[120],"test":[121],"used":[123],"compare":[125],"regarding":[130],"ground-truths":[132],"evaluation":[135],"measures":[136],"benchmark":[139],"scenario.":[140],"Results":[141],"show":[142],"that":[143],"while":[144],"between":[147],"varies":[151],"depending":[152],"measure":[155],"used,":[156],"its":[157],"ranked":[161],"lists":[162],"rather":[167],"small.":[168],"To":[169],"sum":[170],"up,":[171],"vote":[174],"generate":[177],"annotation":[179,198],"set":[180,199],"out":[181],"several":[183],"opinions,":[184],"able":[186],"filter":[188],"noisy":[189],"judgments":[190],"some":[194],"extent.":[195],"resulting":[197],"comparable":[202],"experts.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":17},{"year":2020,"cited_by_count":17},{"year":2019,"cited_by_count":23},{"year":2018,"cited_by_count":24},{"year":2017,"cited_by_count":16},{"year":2016,"cited_by_count":29},{"year":2015,"cited_by_count":16},{"year":2014,"cited_by_count":40},{"year":2013,"cited_by_count":29},{"year":2012,"cited_by_count":21}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2016-06-24T00:00:00"}