{"id":"https://openalex.org/W4401613338","doi":"https://doi.org/10.3390/a17080357","title":"A System Design Perspective for Business Growth in a Crowdsourced Data Labeling Practice","display_name":"A System Design Perspective for Business Growth in a Crowdsourced Data Labeling Practice","publication_year":2024,"publication_date":"2024-08-15","ids":{"openalex":"https://openalex.org/W4401613338","doi":"https://doi.org/10.3390/a17080357"},"language":"en","primary_location":{"id":"doi:10.3390/a17080357","is_oa":true,"landing_page_url":"https://doi.org/10.3390/a17080357","pdf_url":"https://www.mdpi.com/1999-4893/17/8/357/pdf?version=1723728331","source":{"id":"https://openalex.org/S190629608","display_name":"Algorithms","issn_l":"1999-4893","issn":["1999-4893"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Algorithms","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/1999-4893/17/8/357/pdf?version=1723728331","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074949664","display_name":"Vahid Hajipour","orcid":"https://orcid.org/0000-0002-3974-6403"},"institutions":[{"id":"https://openalex.org/I136830121","display_name":"Islamic Azad University South Tehran Branch","ror":"https://ror.org/02xc21a77","country_code":"IR","type":"education","lineage":["https://openalex.org/I110525433","https://openalex.org/I136830121"]}],"countries":["IR"],"is_corresponding":false,"raw_author_name":"Vahid Hajipour","raw_affiliation_strings":["Department of Industrial Engineering, West Tehran Branch, Islamic Azad University, Tehran 1468763785, Iran","Research Center, FANAP Co., Tehran 1657245030, Iran"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Industrial Engineering, West Tehran Branch, Islamic Azad University, Tehran 1468763785, Iran","institution_ids":["https://openalex.org/I136830121"]},{"raw_affiliation_string":"Research Center, FANAP Co., Tehran 1657245030, Iran","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031487885","display_name":"Sajjad Jalali","orcid":"https://orcid.org/0000-0002-8023-0337"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sajjad Jalali","raw_affiliation_strings":["Research Center, FANAP Co., Tehran 1657245030, Iran"],"raw_orcid":"https://orcid.org/0000-0002-8023-0337","affiliations":[{"raw_affiliation_string":"Research Center, FANAP Co., Tehran 1657245030, Iran","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100778046","display_name":"Francisco J. Santos\u2010Arteaga","orcid":"https://orcid.org/0000-0003-2385-4781"},"institutions":[{"id":"https://openalex.org/I121748325","display_name":"Universidad Complutense de Madrid","ror":"https://ror.org/02p0gd045","country_code":"ES","type":"education","lineage":["https://openalex.org/I121748325"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Francisco Javier Santos-Arteaga","raw_affiliation_strings":["Department of Financial and Actuarial Economics & Statistics, Universidad Complutense de Madrid, 28223 Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0003-2385-4781","affiliations":[{"raw_affiliation_string":"Department of Financial and Actuarial Economics & Statistics, Universidad Complutense de Madrid, 28223 Madrid, Spain","institution_ids":["https://openalex.org/I121748325"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042026213","display_name":"Samira Vazifeh-Noshafagh","orcid":"https://orcid.org/0000-0002-8594-8049"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Samira Vazifeh Noshafagh","raw_affiliation_strings":["Department of Industrial Engineering, Doctoral Programme in Materials, Mechatronics and Systems Engineering, University of Trento, 38123 Trento, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Industrial Engineering, Doctoral Programme in Materials, Mechatronics and Systems Engineering, University of Trento, 38123 Trento, Italy","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091636789","display_name":"Debora Di Caprio","orcid":"https://orcid.org/0000-0002-6900-3977"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Debora Di Caprio","raw_affiliation_strings":["Department of Economics and Management, University of Trento, 38122 Trento, Italy"],"raw_orcid":"https://orcid.org/0000-0002-6900-3977","affiliations":[{"raw_affiliation_string":"Department of Economics and Management, University of Trento, 38122 Trento, Italy","institution_ids":["https://openalex.org/I193223587"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5100778046"],"corresponding_institution_ids":["https://openalex.org/I121748325"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":0.3713,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.71005515,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"17","issue":"8","first_page":"357","last_page":"357"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9822999835014343,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9593999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7959268093109131},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7874715328216553},{"id":"https://openalex.org/keywords/payment","display_name":"Payment","score":0.6223115921020508},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.6004300117492676},{"id":"https://openalex.org/keywords/profitability-index","display_name":"Profitability index","score":0.596903383731842},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5961413383483887},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5528141260147095},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5470205545425415},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5096118450164795},{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.48076292872428894},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.4341888129711151},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3533768057823181},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.35303646326065063},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33821117877960205},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3060455918312073},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1892470121383667},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.15538272261619568}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7959268093109131},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7874715328216553},{"id":"https://openalex.org/C145097563","wikidata":"https://www.wikidata.org/wiki/Q1148747","display_name":"Payment","level":2,"score":0.6223115921020508},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.6004300117492676},{"id":"https://openalex.org/C129361004","wikidata":"https://www.wikidata.org/wiki/Q2470236","display_name":"Profitability index","level":2,"score":0.596903383731842},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5961413383483887},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5528141260147095},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5470205545425415},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5096118450164795},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.48076292872428894},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.4341888129711151},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3533768057823181},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35303646326065063},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33821117877960205},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3060455918312073},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1892470121383667},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.15538272261619568},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3390/a17080357","is_oa":true,"landing_page_url":"https://doi.org/10.3390/a17080357","pdf_url":"https://www.mdpi.com/1999-4893/17/8/357/pdf?version=1723728331","source":{"id":"https://openalex.org/S190629608","display_name":"Algorithms","issn_l":"1999-4893","issn":["1999-4893"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Algorithms","raw_type":"journal-article"},{"id":"pmh:oai:iris.unitn.it:11572/425690","is_oa":true,"landing_page_url":"https://hdl.handle.net/11572/425690","pdf_url":"https://iris.unitn.it/bitstream/11572/425690/1/algorithms-17-00357-v2.pdf","source":{"id":"https://openalex.org/S4306401913","display_name":"Institutional Research Information System (Universit\u00e0 degli Studi di Trento)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193223587","host_organization_name":"University of Trento","host_organization_lineage":["https://openalex.org/I193223587"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:c471bebd3b3346988e9353dfe8bcbadf","is_oa":true,"landing_page_url":"https://doaj.org/article/c471bebd3b3346988e9353dfe8bcbadf","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Algorithms, Vol 17, Iss 8, p 357 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/a17080357","is_oa":true,"landing_page_url":"https://doi.org/10.3390/a17080357","pdf_url":"https://www.mdpi.com/1999-4893/17/8/357/pdf?version=1723728331","source":{"id":"https://openalex.org/S190629608","display_name":"Algorithms","issn_l":"1999-4893","issn":["1999-4893"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Algorithms","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401613338.pdf"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W802374166","https://openalex.org/W2121222025","https://openalex.org/W2150597317","https://openalex.org/W2168765606","https://openalex.org/W2475282416","https://openalex.org/W2562389319","https://openalex.org/W2574988924","https://openalex.org/W2580332477","https://openalex.org/W2615593621","https://openalex.org/W2728944333","https://openalex.org/W2782152208","https://openalex.org/W2912010834","https://openalex.org/W2939316233","https://openalex.org/W2962788669","https://openalex.org/W2981287880","https://openalex.org/W2983806390","https://openalex.org/W2990651917","https://openalex.org/W3023492949","https://openalex.org/W3044343366","https://openalex.org/W3087221916","https://openalex.org/W3128303581","https://openalex.org/W3191130514","https://openalex.org/W3212464620","https://openalex.org/W4200235661","https://openalex.org/W4226185393","https://openalex.org/W4292092773","https://openalex.org/W4297152001","https://openalex.org/W4376609317","https://openalex.org/W4385367710","https://openalex.org/W4385597668","https://openalex.org/W4387613669","https://openalex.org/W4392877722","https://openalex.org/W6638921482","https://openalex.org/W6748289949","https://openalex.org/W6798914965","https://openalex.org/W6841909175","https://openalex.org/W6850758467","https://openalex.org/W6862712754"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W4399160670"],"abstract_inverted_index":{"Data":[0],"labeling":[1,33,58,117,141],"systems":[2],"are":[3,64,126],"designed":[4],"to":[5,66,105],"facilitate":[6],"the":[7,16,57,72,75,113,116,123,134],"training":[8],"and":[9,45,70,94,109,136],"validation":[10],"of":[11,18,49,59,74,115,138],"machine":[12],"learning":[13],"algorithms":[14],"under":[15],"umbrella":[17],"crowdsourcing":[19],"practices.":[20],"The":[21,52],"current":[22],"paper":[23],"presents":[24],"a":[25,30,85,101],"novel":[26],"approach":[27],"for":[28,43],"designing":[29],"customized":[31],"data":[32],"system,":[34],"emphasizing":[35],"two":[36],"key":[37],"aspects:":[38],"an":[39,46],"innovative":[40],"payment":[41,80],"mechanism":[42,81],"users":[44],"efficient":[47],"configuration":[48],"output":[50],"results.":[51],"main":[53],"problem":[54],"addressed":[55],"is":[56,82],"datasets":[60],"where":[61],"golden":[62],"items":[63],"utilized":[65],"verify":[67],"user":[68,92,130],"performance":[69],"assure":[71],"quality":[73],"annotated":[76],"outputs.":[77],"Our":[78,119],"proposed":[79,124],"enhanced":[83],"through":[84],"modified":[86],"skip-based":[87],"golden-oriented":[88],"function":[89],"that":[90,122],"balances":[91],"penalties":[93],"prevents":[95],"spam":[96],"activities.":[97],"Additionally,":[98],"we":[99],"introduce":[100],"comprehensive":[102],"reporting":[103],"framework":[104],"measure":[106],"aggregated":[107],"results":[108],"accuracy":[110],"levels,":[111],"ensuring":[112],"reliability":[114],"output.":[118],"findings":[120],"indicate":[121],"solutions":[125],"pivotal":[127],"in":[128],"incentivizing":[129],"participation,":[131],"thereby":[132],"reinforcing":[133],"applicability":[135],"profitability":[137],"newly":[139],"launched":[140],"systems.":[142]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}