{"id":"https://openalex.org/W2899257368","doi":"https://doi.org/10.1145/3274384","title":"When Crowds Give You Lemons","display_name":"When Crowds Give You Lemons","publication_year":2018,"publication_date":"2018-11-01","ids":{"openalex":"https://openalex.org/W2899257368","doi":"https://doi.org/10.1145/3274384","mag":"2899257368"},"language":"en","primary_location":{"id":"doi:10.1145/3274384","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3274384","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042333468","display_name":"Ioanna Lykourentzou","orcid":"https://orcid.org/0000-0002-4243-4128"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Ioanna Lykourentzou","raw_affiliation_strings":["Utrecht University, Utrecht, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Utrecht University, Utrecht, Netherlands","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040705509","display_name":"Faez Ahmed","orcid":"https://orcid.org/0000-0002-5227-2628"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Faez Ahmed","raw_affiliation_strings":["University of Maryland, College Park, MD, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland, College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050375725","display_name":"Costas Papastathis","orcid":null},"institutions":[{"id":"https://openalex.org/I158716096","display_name":"University of Peloponnese","ror":"https://ror.org/04d4d3c02","country_code":"GR","type":"education","lineage":["https://openalex.org/I158716096"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Costas Papastathis","raw_affiliation_strings":["University of Peloponnese, Tripoli-Arcadia, Greece"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Peloponnese, Tripoli-Arcadia, Greece","institution_ids":["https://openalex.org/I158716096"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084517014","display_name":"Irwyn Sadien","orcid":null},"institutions":[{"id":"https://openalex.org/I69356397","display_name":"Xi\u2019an Jiaotong-Liverpool University","ror":"https://ror.org/03zmrmn05","country_code":"CN","type":"education","lineage":["https://openalex.org/I69356397"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Irwyn Sadien","raw_affiliation_strings":["Xi'an Jiaotong - Liverpool University, Suzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong - Liverpool University, Suzhou, China","institution_ids":["https://openalex.org/I69356397"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062199049","display_name":"Konstantinos Papangelis","orcid":"https://orcid.org/0000-0002-7619-3190"},"institutions":[{"id":"https://openalex.org/I69356397","display_name":"Xi\u2019an Jiaotong-Liverpool University","ror":"https://ror.org/03zmrmn05","country_code":"CN","type":"education","lineage":["https://openalex.org/I69356397"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Konstantinos Papangelis","raw_affiliation_strings":["Xi'an Jiatong-Liverpool University, Suzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiatong-Liverpool University, Suzhou, China","institution_ids":["https://openalex.org/I69356397"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9453,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.80853485,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"2","issue":"CSCW","first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9861000180244446,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.7770119905471802},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7334892749786377},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.5815091729164124},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.502659797668457},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.4973738491535187},{"id":"https://openalex.org/keywords/adversary","display_name":"Adversary","score":0.48082435131073},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.4673472046852112},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.46391358971595764},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.4466024339199066},{"id":"https://openalex.org/keywords/ideation","display_name":"Ideation","score":0.4228765368461609},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3927789330482483},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34010863304138184},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.2896377146244049},{"id":"https://openalex.org/keywords/politics","display_name":"Politics","score":0.13475611805915833},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.12262406945228577},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.10258060693740845},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09641149640083313},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08310618996620178},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.08295917510986328}],"concepts":[{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.7770119905471802},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7334892749786377},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.5815091729164124},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.502659797668457},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.4973738491535187},{"id":"https://openalex.org/C41065033","wikidata":"https://www.wikidata.org/wiki/Q2825412","display_name":"Adversary","level":2,"score":0.48082435131073},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4673472046852112},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.46391358971595764},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.4466024339199066},{"id":"https://openalex.org/C170477896","wikidata":"https://www.wikidata.org/wiki/Q17039022","display_name":"Ideation","level":2,"score":0.4228765368461609},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3927789330482483},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34010863304138184},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2896377146244049},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.13475611805915833},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.12262406945228577},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.10258060693740845},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09641149640083313},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08310618996620178},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.08295917510986328},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3274384","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3274384","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:uu:oai:dspace.library.uu.nl:1874/375362","is_oa":false,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/375362","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction - CSCW, 2. ACM","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.4000000059604645}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W47951267","https://openalex.org/W92756571","https://openalex.org/W1511351087","https://openalex.org/W1516454196","https://openalex.org/W1546477643","https://openalex.org/W1600836884","https://openalex.org/W1605255363","https://openalex.org/W1607035479","https://openalex.org/W1680189815","https://openalex.org/W1976097260","https://openalex.org/W1976426554","https://openalex.org/W1985059878","https://openalex.org/W1988625484","https://openalex.org/W2001368603","https://openalex.org/W2007423920","https://openalex.org/W2013964050","https://openalex.org/W2042628596","https://openalex.org/W2043315106","https://openalex.org/W2045453095","https://openalex.org/W2054558047","https://openalex.org/W2071347005","https://openalex.org/W2073299727","https://openalex.org/W2090509708","https://openalex.org/W2127791224","https://openalex.org/W2130736456","https://openalex.org/W2132053410","https://openalex.org/W2139252194","https://openalex.org/W2144807535","https://openalex.org/W2144933361","https://openalex.org/W2156578941","https://openalex.org/W2168697680","https://openalex.org/W2172776241","https://openalex.org/W2290149535","https://openalex.org/W2559773866","https://openalex.org/W2560413501","https://openalex.org/W2584365390","https://openalex.org/W2587836231","https://openalex.org/W2606600550","https://openalex.org/W2645731340","https://openalex.org/W2739351760","https://openalex.org/W2795800176","https://openalex.org/W2798684679","https://openalex.org/W2963429606","https://openalex.org/W2963804993","https://openalex.org/W3104033643","https://openalex.org/W3113461067","https://openalex.org/W3118655244","https://openalex.org/W3121257585","https://openalex.org/W3121293133","https://openalex.org/W3122116788","https://openalex.org/W3124602076","https://openalex.org/W3126128963","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W4240200267","https://openalex.org/W1511510665","https://openalex.org/W1524661185","https://openalex.org/W2078823605","https://openalex.org/W2500095415","https://openalex.org/W4233026749","https://openalex.org/W2097922264","https://openalex.org/W2282342021","https://openalex.org/W4248106174","https://openalex.org/W627242580"],"abstract_inverted_index":{"Following":[0],"successful":[1],"crowd":[2],"ideation":[3],"contests,":[4],"organizations":[5],"in":[6,122,140],"search":[7],"of":[8,17,87,146],"the":[9,40,43,49,77,81,137,147],"\"next":[10],"big":[11],"thing\"":[12],"are":[13,29],"left":[14],"with":[15],"hundreds":[16],"ideas.":[18,149],"Expert-based":[19],"idea":[20,53,64,99,109,119,138],"filtering":[21,65,120],"is":[22,130],"lengthy":[23],"and":[24,45,91,111,135],"costly;":[25],"therefore,":[26],"crowd-based":[27,63],"strategies":[28,34,121],"often":[30],"employed.":[31],"Unfortunately,":[32],"these":[33,69],"typically":[35],"(1)":[36,72],"do":[37],"not":[38],"separate":[39],"mediocre":[41],"from":[42],"excellent,":[44],"(2)":[46,92],"direct":[47],"all":[48],"attention":[50],"to":[51,75,96,102],"certain":[52],"concepts,":[54],"while":[55,142],"others":[56],"starve.":[57],"We":[58,113],"introduce":[59],"DBLemons":[60,115,129],"-":[61],"a":[62,85,97,103,123],"strategy":[66],"that":[67,128],"addresses":[68],"issues":[70],"by":[71,93],"asking":[73],"voters":[74,95],"identify":[76],"worst":[78],"rather":[79],"than":[80],"best":[82],"ideas":[83],"using":[84],"\"bag":[86],"lemons''":[88],"voting":[89],"approach,":[90],"exposing":[94],"wider":[98],"spectrum,":[100],"thanks":[101],"dynamic":[104],"diversity-based":[105],"ranking":[106],"system":[107],"balancing":[108],"quality":[110],"coverage.":[112],"compare":[114],"against":[116],"two":[117],"state-of-the-art":[118],"real-world":[124],"setting.":[125],"Results":[126],"show":[127],"more":[131],"accurate,":[132],"less":[133],"time-consuming,":[134],"reduces":[136],"space":[139],"half":[141],"still":[143],"retaining":[144],"94%":[145],"top":[148]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2018-11-09T00:00:00"}