{"id":"https://openalex.org/W4206971023","doi":"https://doi.org/10.1109/ssci50451.2021.9660043","title":"Estimating crowd-worker's reliability with interval-valued labels to improve the quality of crowdsourced work","display_name":"Estimating crowd-worker's reliability with interval-valued labels to improve the quality of crowdsourced work","publication_year":2021,"publication_date":"2021-12-05","ids":{"openalex":"https://openalex.org/W4206971023","doi":"https://doi.org/10.1109/ssci50451.2021.9660043"},"language":"en","primary_location":{"id":"doi:10.1109/ssci50451.2021.9660043","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ssci50451.2021.9660043","pdf_url":null,"source":{"id":"https://openalex.org/S4363604921","display_name":"2021 IEEE Symposium Series on Computational Intelligence (SSCI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE Symposium Series on Computational Intelligence (SSCI)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007121441","display_name":"Makenzie Spurling","orcid":null},"institutions":[{"id":"https://openalex.org/I4210152127","display_name":"Conway School of Landscape Design","ror":"https://ror.org/04q7y8a54","country_code":"US","type":"education","lineage":["https://openalex.org/I4210152127"]},{"id":"https://openalex.org/I32038505","display_name":"University of Central Arkansas","ror":"https://ror.org/029bp0k25","country_code":"US","type":"education","lineage":["https://openalex.org/I32038505"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Makenzie Spurling","raw_affiliation_strings":["University of Central Arkansas, Conway, AR, USA"],"affiliations":[{"raw_affiliation_string":"University of Central Arkansas, Conway, AR, USA","institution_ids":["https://openalex.org/I32038505","https://openalex.org/I4210152127"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011759044","display_name":"Chenyi Hu","orcid":"https://orcid.org/0000-0003-1982-7537"},"institutions":[{"id":"https://openalex.org/I4210152127","display_name":"Conway School of Landscape Design","ror":"https://ror.org/04q7y8a54","country_code":"US","type":"education","lineage":["https://openalex.org/I4210152127"]},{"id":"https://openalex.org/I32038505","display_name":"University of Central Arkansas","ror":"https://ror.org/029bp0k25","country_code":"US","type":"education","lineage":["https://openalex.org/I32038505"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenyi Hu","raw_affiliation_strings":["University of Central Arkansas, Conway, AR, USA"],"affiliations":[{"raw_affiliation_string":"University of Central Arkansas, Conway, AR, USA","institution_ids":["https://openalex.org/I32038505","https://openalex.org/I4210152127"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075369864","display_name":"Huixin Zhan","orcid":"https://orcid.org/0000-0001-8926-1941"},"institutions":[{"id":"https://openalex.org/I12315562","display_name":"Texas Tech University","ror":"https://ror.org/0405mnx93","country_code":"US","type":"education","lineage":["https://openalex.org/I12315562"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huixin Zhan","raw_affiliation_strings":["Texas Tech University, Lubbock, TX, USA"],"affiliations":[{"raw_affiliation_string":"Texas Tech University, Lubbock, TX, USA","institution_ids":["https://openalex.org/I12315562"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051706630","display_name":"Victor S. Sheng","orcid":"https://orcid.org/0000-0003-4960-174X"},"institutions":[{"id":"https://openalex.org/I12315562","display_name":"Texas Tech University","ror":"https://ror.org/0405mnx93","country_code":"US","type":"education","lineage":["https://openalex.org/I12315562"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Victor S. Sheng","raw_affiliation_strings":["Texas Tech University, Lubbock, TX, USA"],"affiliations":[{"raw_affiliation_string":"Texas Tech University, Lubbock, TX, USA","institution_ids":["https://openalex.org/I12315562"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5007121441"],"corresponding_institution_ids":["https://openalex.org/I32038505","https://openalex.org/I4210152127"],"apc_list":null,"apc_paid":null,"fwci":0.6459,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.65212264,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"01","last_page":"08"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11220","display_name":"Water Systems and Optimization","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/2205","display_name":"Civil and Structural Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9498181343078613},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7581640481948853},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7562470436096191},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7351152896881104},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.6533485651016235},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.5976424813270569},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5697715282440186},{"id":"https://openalex.org/keywords/predictability","display_name":"Predictability","score":0.5395724773406982},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5347676873207092},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.5278964638710022},{"id":"https://openalex.org/keywords/interval","display_name":"Interval (graph theory)","score":0.4987905025482178},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.4923022985458374},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46643877029418945},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12941259145736694},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1270298957824707},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.09138056635856628},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.08813390135765076},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.07840976119041443}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9498181343078613},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7581640481948853},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7562470436096191},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7351152896881104},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.6533485651016235},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.5976424813270569},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5697715282440186},{"id":"https://openalex.org/C197640229","wikidata":"https://www.wikidata.org/wiki/Q2534066","display_name":"Predictability","level":2,"score":0.5395724773406982},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5347676873207092},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.5278964638710022},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.4987905025482178},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4923022985458374},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46643877029418945},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12941259145736694},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1270298957824707},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.09138056635856628},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.08813390135765076},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.07840976119041443},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ssci50451.2021.9660043","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ssci50451.2021.9660043","pdf_url":null,"source":{"id":"https://openalex.org/S4363604921","display_name":"2021 IEEE Symposium Series on Computational Intelligence (SSCI)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE Symposium Series on Computational Intelligence (SSCI)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5099999904632568,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W56003785","https://openalex.org/W1535018738","https://openalex.org/W1557129622","https://openalex.org/W1584152251","https://openalex.org/W1642621435","https://openalex.org/W1882350379","https://openalex.org/W1979883556","https://openalex.org/W1991668886","https://openalex.org/W1995875735","https://openalex.org/W2002071019","https://openalex.org/W2028956919","https://openalex.org/W2042103421","https://openalex.org/W2057317659","https://openalex.org/W2060330798","https://openalex.org/W2066506151","https://openalex.org/W2082360522","https://openalex.org/W2086970615","https://openalex.org/W2124099811","https://openalex.org/W2125943921","https://openalex.org/W2142518823","https://openalex.org/W2150934113","https://openalex.org/W2285341298","https://openalex.org/W2294377781","https://openalex.org/W2337344967","https://openalex.org/W2475282416","https://openalex.org/W2533800767","https://openalex.org/W2582409643","https://openalex.org/W2964397271","https://openalex.org/W3010595911","https://openalex.org/W3025446445","https://openalex.org/W3033468048","https://openalex.org/W3033483740","https://openalex.org/W6600399172","https://openalex.org/W6639113821","https://openalex.org/W6680957539","https://openalex.org/W6695650742"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W2039876276","https://openalex.org/W2605569989","https://openalex.org/W3121841074","https://openalex.org/W2901018557","https://openalex.org/W2055572829","https://openalex.org/W1894159578","https://openalex.org/W2807400035","https://openalex.org/W4297796115","https://openalex.org/W3125086856"],"abstract_inverted_index":{"With":[0,106],"inputs":[1],"from":[2,103],"human":[3,24],"crowds,":[4],"usually":[5,60],"through":[6],"the":[7,52,77],"Internet,":[8],"crowdsourcing":[9,43,81],"has":[10,48],"become":[11],"a":[12],"promising":[13],"methodology":[14],"in":[15,42,94,142],"AI":[16],"and":[17,65,68,101,120,135,148],"machine":[18],"learning":[19,113,140],"for":[20],"applications":[21],"that":[22,46,89,132],"require":[23],"knowledge.":[25],"Researchers":[26],"have":[27,69],"recently":[28],"proposed":[29],"interval-valued":[30],"labels":[31],"(IVLs),":[32],"instead":[33],"of":[34,72,80,96,144],"commonly":[35],"used":[36],"binary-valued":[37],"ones,":[38],"to":[39],"manage":[40],"uncertainty":[41],"[19].":[44],"However,":[45],"work":[47,85],"not":[49],"yet":[50],"taken":[51],"crowd":[53],"worker's":[54,92,107],"reliability":[55,93],"into":[56],"consideration.":[57],"Crowd":[58],"workers":[59],"come":[61],"with":[62,82],"various":[63],"social":[64],"economic":[66],"backgrounds,":[67],"different":[70],"levels":[71],"reliability.":[73],"To":[74],"further":[75],"improve":[76,139],"overall":[78],"quality":[79],"IVLs,":[83],"this":[84,109],"presents":[86],"practical":[87],"methods":[88],"quantitatively":[90],"estimate":[91],"terms":[95,143],"his/her":[97,104],"correctness,":[98],"confidence,":[99],"stability,":[100],"predictability":[102],"IVLs.":[105],"reliability,":[108],"paper":[110],"proposes":[111],"two":[112],"schemes:":[114],"weighted":[115,121],"interval":[116],"majority":[117],"voting":[118],"(WIMV)":[119],"preferred":[122],"matching":[123],"probability":[124],"(WPMP).":[125],"Computational":[126],"experiments":[127],"on":[128],"sample":[129],"datasets":[130],"demonstrate":[131],"both":[133],"WIMV":[134],"WPMP":[136],"can":[137],"significantly":[138],"results":[141],"higher":[145],"precision.":[146],"accuracy.":[147],"F1":[149],"-score":[150],"than":[151],"other":[152],"methods.":[153]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}