{"id":"https://openalex.org/W2922227747","doi":"https://doi.org/10.1109/tkde.2019.2904064","title":"A Review of Judgment Analysis Algorithms for Crowdsourced Opinions","display_name":"A Review of Judgment Analysis Algorithms for Crowdsourced Opinions","publication_year":2019,"publication_date":"2019-03-08","ids":{"openalex":"https://openalex.org/W2922227747","doi":"https://doi.org/10.1109/tkde.2019.2904064","mag":"2922227747"},"language":"en","primary_location":{"id":"doi:10.1109/tkde.2019.2904064","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2019.2904064","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011316577","display_name":"Sujoy Chatterjee","orcid":"https://orcid.org/0000-0002-6146-4779"},"institutions":[{"id":"https://openalex.org/I201841394","display_name":"Universit\u00e9 C\u00f4te d'Azur","ror":"https://ror.org/019tgvf94","country_code":"FR","type":"education","lineage":["https://openalex.org/I201841394"]},{"id":"https://openalex.org/I4210126779","display_name":"Observatoire de la C\u00f4te d\u2019Azur","ror":"https://ror.org/039fj2469","country_code":"FR","type":"facility","lineage":["https://openalex.org/I201841394","https://openalex.org/I4210126779"]},{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1294671590"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Sujoy Chatterjee","raw_affiliation_strings":["Universit\u00e9 C\u00f4te d'Azur, CNRS, I3S, France"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 C\u00f4te d'Azur, CNRS, I3S, France","institution_ids":["https://openalex.org/I4210126779","https://openalex.org/I201841394","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016363044","display_name":"Anirban Mukhopadhyay","orcid":"https://orcid.org/0000-0001-8128-3521"},"institutions":[{"id":"https://openalex.org/I77501641","display_name":"University of Kalyani","ror":"https://ror.org/03v783k16","country_code":"IN","type":"education","lineage":["https://openalex.org/I77501641"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Anirban Mukhopadhyay","raw_affiliation_strings":["Department of Computer Science & Engineering, University of Kalyani, Nadia, India"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering, University of Kalyani, Nadia, India","institution_ids":["https://openalex.org/I77501641"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040278918","display_name":"Malay Bhattacharyya","orcid":"https://orcid.org/0000-0001-7781-1264"},"institutions":[{"id":"https://openalex.org/I6498739","display_name":"Indian Statistical Institute","ror":"https://ror.org/00q2w1j53","country_code":"IN","type":"education","lineage":["https://openalex.org/I6498739"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Malay Bhattacharyya","raw_affiliation_strings":["Machine Intelligence Unit, Indian Statistical Institute, Kolkata, India"],"affiliations":[{"raw_affiliation_string":"Machine Intelligence Unit, Indian Statistical Institute, Kolkata, India","institution_ids":["https://openalex.org/I6498739"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5011316577"],"corresponding_institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I201841394","https://openalex.org/I4210126779"],"apc_list":null,"apc_paid":null,"fwci":2.7083,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.90612591,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"32","issue":"7","first_page":"1234","last_page":"1248"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9868999719619751,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8226168751716614},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6549755334854126},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.6323251724243164},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.5940483808517456},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5743728280067444},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5568957328796387},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4865378439426422},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.44784772396087646},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4352087080478668},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4328191876411438},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.17000091075897217}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8226168751716614},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6549755334854126},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6323251724243164},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.5940483808517456},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5743728280067444},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5568957328796387},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4865378439426422},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.44784772396087646},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4352087080478668},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4328191876411438},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.17000091075897217},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tkde.2019.2904064","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2019.2904064","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6600000262260437,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":138,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W108763474","https://openalex.org/W114565888","https://openalex.org/W615644768","https://openalex.org/W934246857","https://openalex.org/W1543648998","https://openalex.org/W1724528691","https://openalex.org/W1970381522","https://openalex.org/W1995973744","https://openalex.org/W2005966437","https://openalex.org/W2012668444","https://openalex.org/W2017288758","https://openalex.org/W2027600744","https://openalex.org/W2034771068","https://openalex.org/W2036431079","https://openalex.org/W2040758693","https://openalex.org/W2071497152","https://openalex.org/W2073068515","https://openalex.org/W2080942732","https://openalex.org/W2086413055","https://openalex.org/W2098865355","https://openalex.org/W2105268242","https://openalex.org/W2106476394","https://openalex.org/W2108026649","https://openalex.org/W2108501770","https://openalex.org/W2113878109","https://openalex.org/W2114268578","https://openalex.org/W2114269021","https://openalex.org/W2119830539","https://openalex.org/W2119853015","https://openalex.org/W2129345386","https://openalex.org/W2129444086","https://openalex.org/W2131864341","https://openalex.org/W2134910757","https://openalex.org/W2136150876","https://openalex.org/W2140890285","https://openalex.org/W2141282920","https://openalex.org/W2142518823","https://openalex.org/W2143890915","https://openalex.org/W2144660879","https://openalex.org/W2144807981","https://openalex.org/W2146266967","https://openalex.org/W2148440006","https://openalex.org/W2149273804","https://openalex.org/W2149489787","https://openalex.org/W2150612552","https://openalex.org/W2152009989","https://openalex.org/W2152124886","https://openalex.org/W2160473997","https://openalex.org/W2164545125","https://openalex.org/W2180771322","https://openalex.org/W2187444452","https://openalex.org/W2210284195","https://openalex.org/W2251818274","https://openalex.org/W2290431464","https://openalex.org/W2293201672","https://openalex.org/W2294591530","https://openalex.org/W2295086410","https://openalex.org/W2296044363","https://openalex.org/W2329105431","https://openalex.org/W2342391083","https://openalex.org/W2441563267","https://openalex.org/W2443200919","https://openalex.org/W2471221218","https://openalex.org/W2510401720","https://openalex.org/W2511082080","https://openalex.org/W2514000338","https://openalex.org/W2526647111","https://openalex.org/W2543178424","https://openalex.org/W2562389319","https://openalex.org/W2577737453","https://openalex.org/W2578777894","https://openalex.org/W2585226541","https://openalex.org/W2586075686","https://openalex.org/W2586614409","https://openalex.org/W2599032673","https://openalex.org/W2602918972","https://openalex.org/W2611973430","https://openalex.org/W2612925715","https://openalex.org/W2615551042","https://openalex.org/W2739555426","https://openalex.org/W2739753637","https://openalex.org/W2740579382","https://openalex.org/W2741237776","https://openalex.org/W2750691297","https://openalex.org/W2752461112","https://openalex.org/W2761817502","https://openalex.org/W2770277425","https://openalex.org/W2776588445","https://openalex.org/W2807923226","https://openalex.org/W2902499109","https://openalex.org/W2952595241","https://openalex.org/W2962686686","https://openalex.org/W2963414797","https://openalex.org/W2963573141","https://openalex.org/W2963685039","https://openalex.org/W3121257585","https://openalex.org/W4293503264","https://openalex.org/W4297825406","https://openalex.org/W6619251144","https://openalex.org/W6624509166","https://openalex.org/W6632436798","https://openalex.org/W6637626751","https://openalex.org/W6640607308","https://openalex.org/W6675944832","https://openalex.org/W6676096210","https://openalex.org/W6676527293","https://openalex.org/W6677811664","https://openalex.org/W6679158198","https://openalex.org/W6679767772","https://openalex.org/W6679950065","https://openalex.org/W6680834967","https://openalex.org/W6680880821","https://openalex.org/W6680957539","https://openalex.org/W6681400661","https://openalex.org/W6681495149","https://openalex.org/W6682171051","https://openalex.org/W6682518680","https://openalex.org/W6682576451","https://openalex.org/W6682675497","https://openalex.org/W6685463421","https://openalex.org/W6686975137","https://openalex.org/W6691296735","https://openalex.org/W6697072043","https://openalex.org/W6725509692","https://openalex.org/W6725813279","https://openalex.org/W6728819328","https://openalex.org/W6735212286","https://openalex.org/W6737088072","https://openalex.org/W6742117261","https://openalex.org/W6742521153","https://openalex.org/W6743611570","https://openalex.org/W6743760679","https://openalex.org/W6745215937","https://openalex.org/W6745712239","https://openalex.org/W6752478188","https://openalex.org/W6755917680","https://openalex.org/W6757170051"],"related_works":["https://openalex.org/W2617901822","https://openalex.org/W4297475189","https://openalex.org/W4287827046","https://openalex.org/W3011286422","https://openalex.org/W3032646019","https://openalex.org/W2738674657","https://openalex.org/W2329751363","https://openalex.org/W2922919311","https://openalex.org/W3138475137","https://openalex.org/W3153403682"],"abstract_inverted_index":{"The":[0],"crowd-powered":[1],"systems":[2],"have":[3,120],"been":[4,97,121],"shown":[5],"to":[6,14,66,71,104,163],"be":[7,29],"highly":[8],"successful":[9],"in":[10,41,80,99,123,171],"the":[11,46,73,100,133,141,147,183,198],"current":[12],"decade":[13],"manage":[15],"collective":[16],"contribution":[17],"of":[18,38,49,84,94,113,143,146,157,160,182,189],"online":[19,47],"workers":[20,51],"for":[21,31,131],"solving":[22],"different":[23,129,195],"complex":[24,165],"tasks.":[25],"It":[26],"can":[27],"also":[28],"used":[30],"soliciting":[32],"opinions":[33,162,199],"from":[34,89,136],"a":[35,42,59,82,179],"large":[36],"set":[37],"people":[39],"working":[40],"distributed":[43],"manner.":[44],"Unfortunately,":[45],"community":[48],"crowd":[50],"might":[52],"involve":[53],"non-experts":[54],"as":[55],"opinion":[56],"providers.":[57],"As":[58],"result,":[60],"such":[61],"approaches":[62],"may":[63],"give":[64],"rise":[65],"noise":[67],"making":[68],"it":[69],"hard":[70],"predict":[72],"appropriate":[74],"(gold)":[75],"judgment.":[76],"Judgment":[77],"analysis":[78,185],"is":[79],"general":[81],"way":[83],"learning":[85],"about":[86],"human":[87],"decision":[88],"multiple":[90],"opinions.":[91],"A":[92],"spectrum":[93],"algorithms":[95],"has":[96],"proposed":[98],"last":[101],"few":[102],"decades":[103],"address":[105],"this":[106,172,175],"problem.":[107],"They":[108],"are":[109,200],"broadly":[110],"made":[111],"up":[112],"supervised":[114],"or":[115],"unsupervised":[116],"types.":[117],"However,":[118],"they":[119],"readdressed":[122],"recent":[124],"years":[125],"having":[126],"focus":[127],"on":[128],"strategies":[130],"obtaining":[132],"gold":[134],"judgment":[135,184],"crowdsourced":[137,161],"opinions,":[138,144],"viz.,":[139],"estimating":[140],"accuracy":[142],"difficulties":[145],"problem,":[148],"spammer":[149],"identification,":[150],"handling":[151],"noise,":[152],"etc.":[153],"Besides":[154],"this,":[155],"investigation":[156],"various":[158],"types":[159],"solve":[164],"real-life":[166],"problems":[167],"provide":[168,178],"new":[169],"insights":[170],"domain.":[173],"In":[174],"survey,":[176],"we":[177],"comprehensive":[180],"overview":[181],"problem":[186],"and":[187],"some":[188],"its":[190],"novel":[191],"variants,":[192],"addressed":[193],"with":[194],"approaches,":[196],"where":[197],"crowdsourced.":[201]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}