{"id":"https://openalex.org/W3029413384","doi":null,"title":"Objective Assessment of Subjective Tasks in Crowdsourcing Applications.","display_name":"Objective Assessment of Subjective Tasks in Crowdsourcing Applications.","alternate_titles":[],"publication_year":2020,"publication_date":"2020-05-01","ids":{"openalex":"https://openalex.org/W3029413384","mag":"3029413384"},"language":"en","primary_location":{"id":"pmh:oai:https://nottingham-repository.worktribe.com/:4554093","doi":null,"is_oa":true,"landing_page_url":"https://nottingham-repository.worktribe.com/output/4554093","pdf_url":"https://nottingham-repository.worktribe.com/file/4554093/1/2020.cllrd-1.3%281%29","source":{"id":"https://openalex.org/S4306402483","display_name":"Repository@Nottingham (University of Nottingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I142263535","host_organization_name":"University of Nottingham","host_organization_lineage":["https://openalex.org/I142263535"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Presentation / Conference Contribution"},"parent_works":[],"lineage":["https://openalex.org/W3029413384"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://nottingham-repository.worktribe.com/file/4554093/1/2020.cllrd-1.3%281%29","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023468555","display_name":"Giannis Haralabopoulos","orcid":"https://orcid.org/0000-0002-2142-4975","observed_orcids":["https://orcid.org/0000-0002-2142-4975"]},"institutions":[{"id":"https://openalex.org/I142263535","display_name":"University of Nottingham","ror":"https://ror.org/01ee9ar58","country_code":"GB","type":"education","lineage":["https://openalex.org/I142263535"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Giannis Haralabopoulos","raw_affiliation_strings":["University of Nottingham, Nottingham, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Nottingham, Nottingham, United Kingdom","institution_ids":["https://openalex.org/I142263535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074257559","display_name":"Myron Tsikandilakis","orcid":"https://orcid.org/0000-0001-8829-7563","observed_orcids":["https://orcid.org/0000-0001-8829-7563"]},"institutions":[{"id":"https://openalex.org/I142263535","display_name":"University of Nottingham","ror":"https://ror.org/01ee9ar58","country_code":"GB","type":"education","lineage":["https://openalex.org/I142263535"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Myron Tsikandilakis","raw_affiliation_strings":["University of Nottingham, Nottingham, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Nottingham, Nottingham, United Kingdom","institution_ids":["https://openalex.org/I142263535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024981509","display_name":"Mercedes Torres Torres","orcid":"https://orcid.org/0000-0003-0452-3784","observed_orcids":["https://orcid.org/0000-0003-0452-3784"]},"institutions":[{"id":"https://openalex.org/I142263535","display_name":"University of Nottingham","ror":"https://ror.org/01ee9ar58","country_code":"GB","type":"education","lineage":["https://openalex.org/I142263535"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Mercedes Torres Torres","raw_affiliation_strings":["University of Nottingham, Nottingham, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Nottingham, Nottingham, United Kingdom","institution_ids":["https://openalex.org/I142263535"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027117910","display_name":"Derek McAuley","orcid":"https://orcid.org/0000-0001-9159-9933","observed_orcids":["https://orcid.org/0000-0001-9159-9933"]},"institutions":[{"id":"https://openalex.org/I142263535","display_name":"University of Nottingham","ror":"https://ror.org/01ee9ar58","country_code":"GB","type":"education","lineage":["https://openalex.org/I142263535"]}],"countries":["GB"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Derek McAuley","raw_affiliation_strings":["University of Nottingham, Nottingham, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Nottingham, Nottingham, United Kingdom","institution_ids":["https://openalex.org/I142263535"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I142263535"],"apc_list":null,"apc_paid":null,"fwci":2.0626,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":{"value":0.85940684,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":92,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"15","last_page":"25"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.007799999788403511,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.8109999895095825},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"sentiment analysis","score":0.5440000295639038},{"id":"https://openalex.org/keywords/vocabulary-acquisition","display_name":"vocabulary acquisition","score":0.8790000081062317},{"id":"https://openalex.org/keywords/emotion-lexicon","display_name":"emotion lexicon","score":0.6620000004768372},{"id":"https://openalex.org/keywords/gold-standard","display_name":"gold standard","score":0.621999979019165},{"id":"https://openalex.org/keywords/majority-voting","display_name":"majority voting","score":0.7639999985694885},{"id":"https://openalex.org/keywords/thresholding","display_name":"thresholding","score":0.9589999914169312}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8672260642051697},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7852115631103516},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.641453742980957},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5939009785652161},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5787028074264526},{"id":"https://openalex.org/C2778121359","wikidata":"https://www.wikidata.org/wiki/Q8096","display_name":"Lexicon","level":2,"score":0.5392154455184937},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5313057899475098},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5075756311416626},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49992871284484863},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.465109646320343},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.4549294710159302},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.44089260697364807},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3641151189804077},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.355986088514328},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3238537907600403},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.16416767239570618},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:https://nottingham-repository.worktribe.com/:4554093","doi":null,"is_oa":true,"landing_page_url":"https://nottingham-repository.worktribe.com/output/4554093","pdf_url":"https://nottingham-repository.worktribe.com/file/4554093/1/2020.cllrd-1.3%281%29","source":{"id":"https://openalex.org/S4306402483","display_name":"Repository@Nottingham (University of Nottingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I142263535","host_organization_name":"University of Nottingham","host_organization_lineage":["https://openalex.org/I142263535"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Presentation / Conference Contribution"},{"id":"pmh:oai:nottingham-repository.worktribe.com:4554093","doi":null,"is_oa":true,"landing_page_url":"https://nottingham-repository.worktribe.com/file/4554093/1/2020.cllrd-1.3%281%29","pdf_url":null,"source":{"id":"https://openalex.org/S4306402483","display_name":"Repository@Nottingham (University of Nottingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I142263535","host_organization_name":"University of Nottingham","host_organization_lineage":["https://openalex.org/I142263535"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Presentation / Conference Contribution"},{"id":"pmh:oai:centaur.reading.ac.uk:105383","doi":null,"is_oa":false,"landing_page_url":"https://centaur.reading.ac.uk/105383/","pdf_url":null,"source":{"id":"https://openalex.org/S4306402273","display_name":"CentAUR (University of Reading)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I71052956","host_organization_name":"University of Reading","host_organization_lineage":["https://openalex.org/I71052956"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Conference or Workshop Item"},{"id":"mag:3029413384","doi":null,"is_oa":false,"landing_page_url":"https://www.aclweb.org/anthology/2020.cllrd-1.3.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306424877","display_name":"Language Resources and Evaluation","issn_l":"1574-020X","issn":["1574-020X","1574-0218"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","dongbi-c","erih-plus","jufo-3","norway-2","poland-70","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Language Resources and Evaluation","raw_type":null}],"best_oa_location":{"id":"pmh:oai:https://nottingham-repository.worktribe.com/:4554093","doi":null,"is_oa":true,"landing_page_url":"https://nottingham-repository.worktribe.com/output/4554093","pdf_url":"https://nottingham-repository.worktribe.com/file/4554093/1/2020.cllrd-1.3%281%29","source":{"id":"https://openalex.org/S4306402483","display_name":"Repository@Nottingham (University of Nottingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I142263535","host_organization_name":"University of Nottingham","host_organization_lineage":["https://openalex.org/I142263535"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Presentation / Conference Contribution"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5558505907","display_name":"From Human Data to Personal Experience","funder_award_id":"EP/M02315X/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3029413384.pdf","grobid_xml":"https://content.openalex.org/works/W3029413384.grobid-xml"},"referenced_works_count":74,"referenced_works":["https://openalex.org/W14764284","https://openalex.org/W68510733","https://openalex.org/W81140812","https://openalex.org/W291753371","https://openalex.org/W1544484417","https://openalex.org/W1589554437","https://openalex.org/W1947985544","https://openalex.org/W1962580118","https://openalex.org/W1966821122","https://openalex.org/W1968215287","https://openalex.org/W1988240102","https://openalex.org/W2003238582","https://openalex.org/W2014639527","https://openalex.org/W2015525779","https://openalex.org/W2021097538","https://openalex.org/W2039117504","https://openalex.org/W2040397381","https://openalex.org/W2045793922","https://openalex.org/W2052600358","https://openalex.org/W2066064791","https://openalex.org/W2068019880","https://openalex.org/W2070248097","https://openalex.org/W2083433785","https://openalex.org/W2099813784","https://openalex.org/W2101217916","https://openalex.org/W2108950759","https://openalex.org/W2125387256","https://openalex.org/W2136527530","https://openalex.org/W2142262074","https://openalex.org/W2151401338","https://openalex.org/W2159190230","https://openalex.org/W2250966211","https://openalex.org/W2323843104","https://openalex.org/W2324495736","https://openalex.org/W2380448536","https://openalex.org/W2394629054","https://openalex.org/W2406430758","https://openalex.org/W2409945449","https://openalex.org/W2410553363","https://openalex.org/W2460077910","https://openalex.org/W2461785184","https://openalex.org/W2493922156","https://openalex.org/W2576746547","https://openalex.org/W2585292421","https://openalex.org/W2600553981","https://openalex.org/W2606593809","https://openalex.org/W2738482091","https://openalex.org/W2746748886","https://openalex.org/W2757016771","https://openalex.org/W2763288992","https://openalex.org/W2765126493","https://openalex.org/W2766644980","https://openalex.org/W2777811318","https://openalex.org/W2779206865","https://openalex.org/W2780698117","https://openalex.org/W2783668259","https://openalex.org/W2793625747","https://openalex.org/W2793929280","https://openalex.org/W2798989012","https://openalex.org/W2804000041","https://openalex.org/W2806912439","https://openalex.org/W2807493978","https://openalex.org/W2810665353","https://openalex.org/W2886624995","https://openalex.org/W2890955624","https://openalex.org/W2900256687","https://openalex.org/W2901156209","https://openalex.org/W2902496224","https://openalex.org/W2902690450","https://openalex.org/W2912611323","https://openalex.org/W2919464470","https://openalex.org/W2945811520","https://openalex.org/W2982986041","https://openalex.org/W4248517331"],"related_works":["https://openalex.org/W2474333055","https://openalex.org/W4387344951","https://openalex.org/W3030589078","https://openalex.org/W4401042630","https://openalex.org/W2951469843","https://openalex.org/W2256112857","https://openalex.org/W6939313912","https://openalex.org/W2612422652","https://openalex.org/W4297093004","https://openalex.org/W2993596934"],"abstract_inverted_index":{"Labelling,":[0],"or":[1],"annotation,":[2,94],"is":[3,31,47,74,125],"the":[4,29,53,197,202],"process":[5],"by":[6],"which":[7],"we":[8],"assign":[9],"labels":[10],"to":[11,16,32,68,82,92,116,139,214],"an":[12,194],"item":[13],"with":[14,57,227],"regards":[15],"a":[17,99,108,123,169,178,185],"task.":[18,188],"In":[19],"some":[20],"Artificial":[21],"Intelligence":[22],"problems,":[23],"such":[24,39,153],"as":[25,40,137,154],"Computer":[26],"Vision":[27],"tasks,":[28,152],"goal":[30],"obtain":[33],"objective":[34,208],"labels.":[35],"However,":[36,111],"in":[37,107,163,184,217],"problems":[38],"text":[41],"and":[42,76,134,157,167],"sentiment":[43,54,158],"analysis,":[44],"subjective":[45,151,218],"labelling":[46],"often":[48,114],"required.":[49],"More":[50],"so":[51,121],"when":[52,122],"analysis":[55,91],"deals":[56],"actual":[58],"emotions":[59],"instead":[60],"of":[61,102,180,199,204],"polarity":[62],"(positive/negative).":[63],"Scientists":[64],"employ":[65],"human":[66,164],"experts":[67],"create":[69],"these":[70,146],"labels,":[71],"but":[72],"it":[73],"costly":[75],"time":[77],"consuming.":[78],"Crowdsourcing":[79],"enables":[80],"researchers":[81,96],"utilise":[83],"non-expert":[84,112,141],"knowledge":[85],"for":[86,150],"scientific":[87],"tasks.":[88],"From":[89],"image":[90],"semantic":[93],"interested":[95],"can":[97,211],"gather":[98],"large":[100],"sample":[101],"answers":[103],"via":[104],"crowdsourcing":[105],"platforms":[106],"timely":[109],"manner.":[110],"contributions":[113],"need":[115],"be":[117,212],"thoroughly":[118],"assessed,":[119],"particularly":[120],"task":[124],"subjective.":[126],"Researchers":[127],"have":[128],"traditionally":[129],"used":[130,213],"'Gold":[131],"Standard',":[132],"'Thresholding'":[133],"'Majority":[135],"Voting'":[136],"methods":[138,147],"filter":[140],"contributions.":[142,205],"We":[143,160,189],"argue":[144],"that":[145,172,220],"are":[148],"unsuitable":[149],"lexicon":[155,186],"acquisition":[156,187],"analysis.":[159],"discuss":[161],"subjectivity":[162],"centered":[165],"tasks":[166,219],"present":[168],"filtering":[170],"method":[171,192,210],"defines":[173],"quality":[174,225],"contributors,":[175],"based":[176],"on":[177],"set":[179],"objectively":[181],"infused":[182],"terms":[183],"evaluate":[190],"our":[191],"against":[193],"established":[195],"lexicon,":[196],"diversity":[198],"emotions-i.e.":[200],"subjectivity-and":[201],"exclusion":[203],"Our":[206],"proposed":[207],"evaluation":[209],"assess":[215],"contributors":[216],"will":[221],"provide":[222],"domain":[223],"agnostic,":[224],"results,":[226],"at":[228],"least":[229],"7%":[230],"improvement":[231],"over":[232],"traditional":[233],"methods.":[234]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3029413384","counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":4}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2020-06-05T00:00:00"}
