{"id":"https://openalex.org/W3184788036","doi":"https://doi.org/10.3390/s21155007","title":"A Time-Series-Based New Behavior Trace Model for Crowd Workers That Ensures Quality Annotation","display_name":"A Time-Series-Based New Behavior Trace Model for Crowd Workers That Ensures Quality Annotation","publication_year":2021,"publication_date":"2021-07-23","ids":{"openalex":"https://openalex.org/W3184788036","doi":"https://doi.org/10.3390/s21155007","mag":"3184788036","pmid":"https://pubmed.ncbi.nlm.nih.gov/34372243"},"language":"en","primary_location":{"id":"doi:10.3390/s21155007","is_oa":true,"landing_page_url":"https://doi.org/10.3390/s21155007","pdf_url":"https://www.mdpi.com/1424-8220/21/15/5007/pdf?version=1635397049","source":{"id":"https://openalex.org/S101949793","display_name":"Sensors","issn_l":"1424-8220","issn":["1424-8220"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/1424-8220/21/15/5007/pdf?version=1635397049","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038576721","display_name":"Fattoh Al-Qershi","orcid":"https://orcid.org/0000-0002-9609-4472"},"institutions":[{"id":"https://openalex.org/I28022161","display_name":"King Saud University","ror":"https://ror.org/02f81g417","country_code":"SA","type":"education","lineage":["https://openalex.org/I28022161"]}],"countries":["SA"],"is_corresponding":true,"raw_author_name":"Fattoh Al-Qershi","raw_affiliation_strings":["Department of Information Systems, College of Computer and Information Sciences, King Saud University, Riyadh 11543, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-9609-4472","affiliations":[{"raw_affiliation_string":"Department of Information Systems, College of Computer and Information Sciences, King Saud University, Riyadh 11543, Saudi Arabia","institution_ids":["https://openalex.org/I28022161"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061632677","display_name":"Muhammad Al\u2010Qurishi","orcid":"https://orcid.org/0000-0002-7594-7325"},"institutions":[{"id":"https://openalex.org/I4210096041","display_name":"Riyadh Elm University","ror":"https://ror.org/00rz3mr26","country_code":"SA","type":"education","lineage":["https://openalex.org/I4210096041"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Muhammad Al-Qurishi","raw_affiliation_strings":["Research and Innovation Division, Research Department, Elm Company, Riyadh 12382, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-7594-7325","affiliations":[{"raw_affiliation_string":"Research and Innovation Division, Research Department, Elm Company, Riyadh 12382, Saudi Arabia","institution_ids":["https://openalex.org/I4210096041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015439725","display_name":"Mehmet Sabih Aksoy","orcid":"https://orcid.org/0000-0003-0118-9602"},"institutions":[{"id":"https://openalex.org/I28022161","display_name":"King Saud University","ror":"https://ror.org/02f81g417","country_code":"SA","type":"education","lineage":["https://openalex.org/I28022161"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Mehmet Aksoy","raw_affiliation_strings":["Department of Information Systems, College of Computer and Information Sciences, King Saud University, Riyadh 11543, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Systems, College of Computer and Information Sciences, King Saud University, Riyadh 11543, Saudi Arabia","institution_ids":["https://openalex.org/I28022161"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043253462","display_name":"Mohammed Faisal","orcid":"https://orcid.org/0000-0001-7720-0076"},"institutions":[{"id":"https://openalex.org/I28022161","display_name":"King Saud University","ror":"https://ror.org/02f81g417","country_code":"SA","type":"education","lineage":["https://openalex.org/I28022161"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Mohammed Faisal","raw_affiliation_strings":["College of Applied Computer Sciences, King Saud University, Riyadh 145111, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0001-7720-0076","affiliations":[{"raw_affiliation_string":"College of Applied Computer Sciences, King Saud University, Riyadh 145111, Saudi Arabia","institution_ids":["https://openalex.org/I28022161"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081962715","display_name":"Mohammed Algabri","orcid":"https://orcid.org/0000-0001-7962-8121"},"institutions":[{"id":"https://openalex.org/I28022161","display_name":"King Saud University","ror":"https://ror.org/02f81g417","country_code":"SA","type":"education","lineage":["https://openalex.org/I28022161"]}],"countries":["SA"],"is_corresponding":false,"raw_author_name":"Mohammed Algabri","raw_affiliation_strings":["Department of Computer Science, College of Computer and Information Sciences, King Saud University, Riyadh 11543, Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, College of Computer and Information Sciences, King Saud University, Riyadh 11543, Saudi Arabia","institution_ids":["https://openalex.org/I28022161"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5038576721"],"corresponding_institution_ids":["https://openalex.org/I28022161"],"apc_list":{"value":2400,"currency":"CHF","value_usd":2598},"apc_paid":{"value":2400,"currency":"CHF","value_usd":2598},"fwci":0.7312,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.76892285,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"21","issue":"15","first_page":"5007","last_page":"5007"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8455898761749268},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.7243227362632751},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.722355842590332},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6774781942367554},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5841122269630432},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5779343843460083},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5691437125205994},{"id":"https://openalex.org/keywords/tracing","display_name":"Tracing","score":0.44042670726776123},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4183924198150635},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.10080015659332275}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8455898761749268},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.7243227362632751},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.722355842590332},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6774781942367554},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5841122269630432},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5779343843460083},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5691437125205994},{"id":"https://openalex.org/C138673069","wikidata":"https://www.wikidata.org/wiki/Q322229","display_name":"Tracing","level":2,"score":0.44042670726776123},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4183924198150635},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.10080015659332275},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D063045","descriptor_name":"Crowdsourcing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D063045","descriptor_name":"Crowdsourcing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D063045","descriptor_name":"Crowdsourcing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":5,"locations":[{"id":"doi:10.3390/s21155007","is_oa":true,"landing_page_url":"https://doi.org/10.3390/s21155007","pdf_url":"https://www.mdpi.com/1424-8220/21/15/5007/pdf?version=1635397049","source":{"id":"https://openalex.org/S101949793","display_name":"Sensors","issn_l":"1424-8220","issn":["1424-8220"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors","raw_type":"journal-article"},{"id":"pmid:34372243","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/34372243","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors (Basel, Switzerland)","raw_type":null},{"id":"pmh:oai:doaj.org/article:2eb198a34a7741b592ea736d0f96c469","is_oa":true,"landing_page_url":"https://doaj.org/article/2eb198a34a7741b592ea736d0f96c469","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sensors, Vol 21, Iss 15, p 5007 (2021)","raw_type":"article"},{"id":"pmh:oai:mdpi.com:/1424-8220/21/15/5007/","is_oa":true,"landing_page_url":"https://dx.doi.org/10.3390/s21155007","pdf_url":null,"source":{"id":"https://openalex.org/S4306400947","display_name":"MDPI (MDPI AG)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210097602","host_organization_name":"Multidisciplinary Digital Publishing Institute (Switzerland)","host_organization_lineage":["https://openalex.org/I4210097602"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sensors; Volume 21; Issue 15; Pages: 5007","raw_type":"Text"},{"id":"pmh:oai:pubmedcentral.nih.gov:8347513","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/8347513","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sensors (Basel)","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.3390/s21155007","is_oa":true,"landing_page_url":"https://doi.org/10.3390/s21155007","pdf_url":"https://www.mdpi.com/1424-8220/21/15/5007/pdf?version=1635397049","source":{"id":"https://openalex.org/S101949793","display_name":"Sensors","issn_l":"1424-8220","issn":["1424-8220"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sensors","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.5299999713897705,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320321145","display_name":"King Saud University","ror":"https://ror.org/02f81g417"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3184788036.pdf","grobid_xml":"https://content.openalex.org/works/W3184788036.grobid-xml"},"referenced_works_count":82,"referenced_works":["https://openalex.org/W152645600","https://openalex.org/W170732776","https://openalex.org/W1692308832","https://openalex.org/W1964000773","https://openalex.org/W2004373800","https://openalex.org/W2007615281","https://openalex.org/W2008348094","https://openalex.org/W2012079387","https://openalex.org/W2014683958","https://openalex.org/W2020571458","https://openalex.org/W2024166834","https://openalex.org/W2030139727","https://openalex.org/W2036431079","https://openalex.org/W2056132907","https://openalex.org/W2074275752","https://openalex.org/W2094607109","https://openalex.org/W2099593264","https://openalex.org/W2101057528","https://openalex.org/W2101234009","https://openalex.org/W2113878109","https://openalex.org/W2115755118","https://openalex.org/W2124958607","https://openalex.org/W2124994029","https://openalex.org/W2125943921","https://openalex.org/W2141282920","https://openalex.org/W2144994235","https://openalex.org/W2150597317","https://openalex.org/W2167913131","https://openalex.org/W2170989440","https://openalex.org/W2182392283","https://openalex.org/W2187291759","https://openalex.org/W2271188064","https://openalex.org/W2293201672","https://openalex.org/W2294447324","https://openalex.org/W2296739333","https://openalex.org/W2315338392","https://openalex.org/W2339885376","https://openalex.org/W2397362117","https://openalex.org/W2494980014","https://openalex.org/W2508556482","https://openalex.org/W2522741018","https://openalex.org/W2525412388","https://openalex.org/W2535526893","https://openalex.org/W2579968393","https://openalex.org/W2582409643","https://openalex.org/W2601659907","https://openalex.org/W2625465398","https://openalex.org/W2767632622","https://openalex.org/W2780156044","https://openalex.org/W2787634879","https://openalex.org/W2792814840","https://openalex.org/W2794325666","https://openalex.org/W2802314367","https://openalex.org/W2811107472","https://openalex.org/W2897765576","https://openalex.org/W2925381077","https://openalex.org/W2947459187","https://openalex.org/W2948941454","https://openalex.org/W2953336832","https://openalex.org/W2963569749","https://openalex.org/W2964054038","https://openalex.org/W2965857035","https://openalex.org/W2967268202","https://openalex.org/W2973037965","https://openalex.org/W2982492177","https://openalex.org/W3025324209","https://openalex.org/W3025446445","https://openalex.org/W3033999739","https://openalex.org/W3039352388","https://openalex.org/W3047305092","https://openalex.org/W3088165622","https://openalex.org/W3102460643","https://openalex.org/W3121928352","https://openalex.org/W3159813332","https://openalex.org/W3205934028","https://openalex.org/W4230475127","https://openalex.org/W4285719527","https://openalex.org/W6675354045","https://openalex.org/W6675576870","https://openalex.org/W6767255924","https://openalex.org/W6769374068","https://openalex.org/W6779988842"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"Crowdsourcing":[0],"is":[1,40],"a":[2,83,101],"new":[3],"mode":[4],"of":[5,38,117,121],"value":[6],"creation":[7],"in":[8,115],"which":[9],"organizations":[10],"leverage":[11],"numerous":[12],"Internet":[13],"users":[14],"to":[15],"accomplish":[16],"tasks.":[17],"However,":[18],"because":[19],"these":[20],"workers":[21,39],"have":[22],"different":[23],"backgrounds":[24],"and":[25,49,73,99],"intentions,":[26],"crowdsourcing":[27],"suffers":[28],"from":[29,70],"quality":[30],"concerns.":[31],"In":[32,119],"the":[33,36,96,110],"literature,":[34],"tracing":[35],"behavior":[37,62],"preferred":[41],"over":[42],"other":[43],"methodologies":[44],"such":[45],"as":[46],"consensus":[47],"methods":[48],"gold":[50],"standard":[51],"approaches.":[52],"This":[53],"paper":[54],"proposes":[55],"two":[56],"novel":[57],"models":[58,67,108],"based":[59],"on":[60],"workers'":[61],"for":[63],"task":[64],"classification.":[65],"These":[66],"newly":[68],"benefit":[69],"time-series":[71,80],"features":[72,81],"characteristics.":[74],"The":[75,87,106],"first":[76],"model":[77,89,125,133],"uses":[78],"multiple":[79],"with":[82],"machine":[84],"learning":[85],"classifier.":[86,105],"second":[88],"converts":[90],"time":[91],"series":[92],"into":[93],"images":[94],"using":[95],"recurrent":[97],"characteristic":[98],"applies":[100],"convolutional":[102,130],"neural":[103,131],"network":[104,132],"proposed":[107],"surpass":[109],"current":[111],"state":[112],"of-the-art":[113],"baselines":[114],"terms":[116,120],"performance.":[118],"accuracy,":[122],"our":[123,129],"feature-based":[124],"achieved":[126,134],"83.8%,":[127],"whereas":[128],"76.6%.":[135]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}