{"id":"https://openalex.org/W1989360764","doi":"https://doi.org/10.1145/2660114.2660123","title":"Getting by with a Little Help from the Crowd","display_name":"Getting by with a Little Help from the Crowd","publication_year":2014,"publication_date":"2014-11-03","ids":{"openalex":"https://openalex.org/W1989360764","doi":"https://doi.org/10.1145/2660114.2660123","mag":"1989360764"},"language":"en","primary_location":{"id":"doi:10.1145/2660114.2660123","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2660114.2660123","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2014 International ACM Workshop on Crowdsourcing for Multimedia","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086566231","display_name":"Babak Loni","orcid":"https://orcid.org/0000-0002-2070-9696"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Babak Loni","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands","Delft University of Technology, Delft, Netherlands;"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]},{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands;","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067505586","display_name":"Jonathon Hare","orcid":"https://orcid.org/0000-0003-2921-4283"},"institutions":[{"id":"https://openalex.org/I43439940","display_name":"University of Southampton","ror":"https://ror.org/01ryk1543","country_code":"GB","type":"education","lineage":["https://openalex.org/I43439940"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jonathon Hare","raw_affiliation_strings":["University of Southampton, Southampton, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Southampton, Southampton, United Kingdom","institution_ids":["https://openalex.org/I43439940"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057095068","display_name":"Mihai G\u00e9orgescu","orcid":"https://orcid.org/0000-0003-1601-0580"},"institutions":[{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Mihai Georgescu","raw_affiliation_strings":["L3S Research Center, Hanover, Germany"],"affiliations":[{"raw_affiliation_string":"L3S Research Center, Hanover, Germany","institution_ids":["https://openalex.org/I4210136150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041662804","display_name":"Michael Riegler","orcid":"https://orcid.org/0000-0002-6937-986X"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Michael Riegler","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands","Delft University of Technology, Delft, Netherlands;"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]},{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands;","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021268896","display_name":"Xiaofei Zhu","orcid":"https://orcid.org/0000-0001-8239-7176"},"institutions":[{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Xiaofei Zhu","raw_affiliation_strings":["L3S Research Center, Hanover, Germany"],"affiliations":[{"raw_affiliation_string":"L3S Research Center, Hanover, Germany","institution_ids":["https://openalex.org/I4210136150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068392446","display_name":"Mohamed Morchid","orcid":"https://orcid.org/0000-0002-4427-2468"},"institutions":[{"id":"https://openalex.org/I198415970","display_name":"Universit\u00e9 d'Avignon et des Pays de Vaucluse","ror":"https://ror.org/00mfpxb84","country_code":"FR","type":"education","lineage":["https://openalex.org/I198415970"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Mohamed Morchid","raw_affiliation_strings":["LIA, University of Avignon, Avignon, France","LIA - University of Avignon, Avignon, France#TAB#"],"affiliations":[{"raw_affiliation_string":"LIA, University of Avignon, Avignon, France","institution_ids":["https://openalex.org/I198415970"]},{"raw_affiliation_string":"LIA - University of Avignon, Avignon, France#TAB#","institution_ids":["https://openalex.org/I198415970"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102920772","display_name":"Richard Dufour","orcid":"https://orcid.org/0000-0003-1203-9108"},"institutions":[{"id":"https://openalex.org/I198415970","display_name":"Universit\u00e9 d'Avignon et des Pays de Vaucluse","ror":"https://ror.org/00mfpxb84","country_code":"FR","type":"education","lineage":["https://openalex.org/I198415970"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Richard Dufour","raw_affiliation_strings":["LIA, University of Avignon, Avignon, France","LIA - University of Avignon, Avignon, France#TAB#"],"affiliations":[{"raw_affiliation_string":"LIA, University of Avignon, Avignon, France","institution_ids":["https://openalex.org/I198415970"]},{"raw_affiliation_string":"LIA - University of Avignon, Avignon, France#TAB#","institution_ids":["https://openalex.org/I198415970"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056272341","display_name":"Martha Larson","orcid":"https://orcid.org/0000-0003-4229-5866"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Martha Larson","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands","Delft University of Technology, Delft, Netherlands;"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]},{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands;","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5086566231"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":1.8699,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.88438234,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"69","last_page":"74"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9666593074798584},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7780889272689819},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6117671132087708},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6019526124000549},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.5756944417953491},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5737971067428589},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.46995070576667786},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4650273323059082},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4142218232154846},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3570500612258911},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3530399799346924},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.26283150911331177},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.12870106101036072}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9666593074798584},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7780889272689819},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6117671132087708},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6019526124000549},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.5756944417953491},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5737971067428589},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.46995070576667786},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4650273323059082},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4142218232154846},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3570500612258911},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3530399799346924},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.26283150911331177},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.12870106101036072},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2660114.2660123","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2660114.2660123","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2014 International ACM Workshop on Crowdsourcing for Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.soton.ac.uk:370278","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401019","display_name":"ePrints Soton (University of Southampton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I43439940","host_organization_name":"University of Southampton","host_organization_lineage":["https://openalex.org/I43439940"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Conference or Workshop Item"},{"id":"pmh:oai:HAL:hal-01318338v1","is_oa":false,"landing_page_url":"https://hal.science/hal-01318338","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"CrowdMM '14 International ACM Workshop on Crowdsourcing for Multimedia , Nov 2014, Orlando, United States. ⟨10.1145/2660114.2660123⟩","raw_type":"Conference papers"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3829048491","display_name":null,"funder_award_id":"FP7/2007-2013","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G3981126665","display_name":null,"funder_award_id":"Seventh Framework Programme","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5329144463","display_name":null,"funder_award_id":"287704","funder_id":"https://openalex.org/F4320334960","funder_display_name":"Seventh Framework Programme"},{"id":"https://openalex.org/G5334610876","display_name":null,"funder_award_id":"287704","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5593277320","display_name":null,"funder_award_id":"2007-2013","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5702163051","display_name":null,"funder_award_id":"FP7/2007","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320334960","display_name":"Seventh Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W140073476","https://openalex.org/W1543648998","https://openalex.org/W2000244138","https://openalex.org/W2003497265","https://openalex.org/W2006471615","https://openalex.org/W2016245010","https://openalex.org/W2032355985","https://openalex.org/W2048318462","https://openalex.org/W2103467726","https://openalex.org/W2108598243","https://openalex.org/W2109628656","https://openalex.org/W2125943921","https://openalex.org/W2136177758","https://openalex.org/W2136951832","https://openalex.org/W2142518823","https://openalex.org/W2149273804","https://openalex.org/W2170942078","https://openalex.org/W2172231696","https://openalex.org/W2289229463","https://openalex.org/W2402861908","https://openalex.org/W2404484773","https://openalex.org/W2408133538","https://openalex.org/W2522338500","https://openalex.org/W2596585349","https://openalex.org/W2597289420","https://openalex.org/W6682171051"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"Validating":[0],"user":[1,179],"tags":[2,180],"helps":[3],"to":[4,28,44,113,131,150,168],"refine":[5],"them,":[6],"making":[7,137],"them":[8],"more":[9],"useful":[10],"for":[11,181],"finding":[12],"images.":[13],"In":[14,174],"the":[15,36,58,61,74,84,88,92,192],"case":[16],"of":[17,54,63,76,91,105,139,153,166,177],"interpretation-sensitive":[18,154,178],"tags,":[19],"however,":[20],"automatic":[21],"(i.e.,":[22],"pixel-based)":[23],"approaches":[24,43],"cannot":[25],"be":[26],"expected":[27],"deliver":[29],"optimal":[30],"results.":[31],"Instead,":[32,161],"human":[33,55],"input":[34,56],"is":[35,111,184],"key.":[37],"This":[38],"paper":[39],"studies":[40],"how":[41],"crowdsourcing-based":[42],"image":[45,127],"tag":[46],"validation":[47,134,176],"can":[48,163],"achieve":[49],"parsimony":[50],"in":[51,60,73,145],"their":[52],"use":[53,138],"from":[57,66,171,191],"crowd,":[59],"form":[62],"votes":[64,107],"collected":[65],"workers":[67],"on":[68],"a":[69,102,115,188],"crowdsourcing":[70],"platform.":[71],"Experiments":[72],"domain":[75],"social":[77,182],"fashion":[78],"images":[79,183],"are":[80,108],"carried":[81],"out":[82],"using":[83],"dataset":[85],"published":[86],"by":[87],"Crowdsourcing":[89],"Task":[90],"Mediaeval":[93],"2013":[94],"Multimedia":[95],"Benchmark.":[96],"Experimental":[97],"results":[98],"reveal":[99],"that":[100],"when":[101],"larger":[103],"number":[104],"crowd-contributed":[106,141],"available,":[109],"it":[110],"difficult":[112],"beat":[114],"majority":[116],"vote.":[117],"However,":[118],"additional":[119],"information":[120],"sources,":[121],"i.e.,":[122],"crowdworker":[123],"history":[124],"and":[125],"visual":[126],"features,":[128],"allow":[129],"us":[130],"maintain":[132],"similar":[133],"performance":[135],"while":[136],"less":[140],"input.":[142],"Further,":[143],"investing":[144],"\"expensive\"":[146],"experts":[147,162],"who":[148],"collaborate":[149],"create":[151],"definitions":[152],"concepts":[155,167],"does":[156],"not":[157],"necessarily":[158],"pay":[159],"off.":[160],"cause":[164],"interpretations":[165],"drift":[169],"away":[170],"conventional":[172],"wisdom.":[173],"short,":[175],"possible,":[185],"with":[186],"\"just":[187],"little":[189],"help":[190],"crowd\".":[193]},"counts_by_year":[{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2016-06-24T00:00:00"}