{"id":"https://openalex.org/W3007156732","doi":"https://doi.org/10.1109/bigdata47090.2019.9006114","title":"Towards Quality Assessment of Crowdworker Output Based on Behavioral Data","display_name":"Towards Quality Assessment of Crowdworker Output Based on Behavioral Data","publication_year":2019,"publication_date":"2019-12-01","ids":{"openalex":"https://openalex.org/W3007156732","doi":"https://doi.org/10.1109/bigdata47090.2019.9006114","mag":"3007156732"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata47090.2019.9006114","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9006114","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056999317","display_name":"Shigeaki Yuasa","orcid":null},"institutions":[{"id":"https://openalex.org/I15807432","display_name":"Osaka Prefecture University","ror":"https://ror.org/02cf1je33","country_code":"JP","type":"education","lineage":["https://openalex.org/I15807432"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Shigeaki Yuasa","raw_affiliation_strings":["Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan"],"affiliations":[{"raw_affiliation_string":"Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan","institution_ids":["https://openalex.org/I15807432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108834962","display_name":"Takumi Nakai","orcid":null},"institutions":[{"id":"https://openalex.org/I15807432","display_name":"Osaka Prefecture University","ror":"https://ror.org/02cf1je33","country_code":"JP","type":"education","lineage":["https://openalex.org/I15807432"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Takumi Nakai","raw_affiliation_strings":["Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan"],"affiliations":[{"raw_affiliation_string":"Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan","institution_ids":["https://openalex.org/I15807432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018167502","display_name":"Takanori Maruichi","orcid":null},"institutions":[{"id":"https://openalex.org/I15807432","display_name":"Osaka Prefecture University","ror":"https://ror.org/02cf1je33","country_code":"JP","type":"education","lineage":["https://openalex.org/I15807432"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Takanori Maruichi","raw_affiliation_strings":["Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan"],"affiliations":[{"raw_affiliation_string":"Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan","institution_ids":["https://openalex.org/I15807432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020049325","display_name":"Manuel Landsmann","orcid":null},"institutions":[{"id":"https://openalex.org/I15807432","display_name":"Osaka Prefecture University","ror":"https://ror.org/02cf1je33","country_code":"JP","type":"education","lineage":["https://openalex.org/I15807432"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Manuel Landsmann","raw_affiliation_strings":["Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan"],"affiliations":[{"raw_affiliation_string":"Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan","institution_ids":["https://openalex.org/I15807432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000232184","display_name":"Koichi Kise","orcid":"https://orcid.org/0000-0001-5779-6968"},"institutions":[{"id":"https://openalex.org/I15807432","display_name":"Osaka Prefecture University","ror":"https://ror.org/02cf1je33","country_code":"JP","type":"education","lineage":["https://openalex.org/I15807432"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Koichi Kise","raw_affiliation_strings":["Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan"],"affiliations":[{"raw_affiliation_string":"Dept. of Computer Science and Intelligent Systems, Osaka Prefecture University, Japan","institution_ids":["https://openalex.org/I15807432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049129001","display_name":"Masaki Matsubara","orcid":"https://orcid.org/0000-0003-1950-683X"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Masaki Matsubara","raw_affiliation_strings":["Faculty of Library, Information and Media Science, University of Tsukuba, Japan"],"affiliations":[{"raw_affiliation_string":"Faculty of Library, Information and Media Science, University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071533497","display_name":"Atsuyuki Morishima","orcid":"https://orcid.org/0000-0003-4606-9065"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Atsuyuki Morishima","raw_affiliation_strings":["Faculty of Library, Information and Media Science, University of Tsukuba, Japan"],"affiliations":[{"raw_affiliation_string":"Faculty of Library, Information and Media Science, University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5056999317"],"corresponding_institution_ids":["https://openalex.org/I15807432"],"apc_list":null,"apc_paid":null,"fwci":0.6019,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.77158892,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"4659","last_page":"4661"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11500","display_name":"Evacuation and Crowd Dynamics","score":0.9904000163078308,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9865000247955322,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5979084968566895},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5938997864723206},{"id":"https://openalex.org/keywords/data-quality","display_name":"Data quality","score":0.49974703788757324},{"id":"https://openalex.org/keywords/quality-assessment","display_name":"Quality assessment","score":0.42309486865997314},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.23239892721176147},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.14872193336486816},{"id":"https://openalex.org/keywords/operations-management","display_name":"Operations management","score":0.10094919800758362},{"id":"https://openalex.org/keywords/evaluation-methods","display_name":"Evaluation methods","score":0.09493228793144226}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5979084968566895},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5938997864723206},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.49974703788757324},{"id":"https://openalex.org/C3020001037","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assessment","level":3,"score":0.42309486865997314},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.23239892721176147},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14872193336486816},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.10094919800758362},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.09493228793144226},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata47090.2019.9006114","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9006114","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W2040762770","https://openalex.org/W2124994029","https://openalex.org/W2136120756","https://openalex.org/W2167913131","https://openalex.org/W2289052659","https://openalex.org/W2460082291","https://openalex.org/W2753614057","https://openalex.org/W2899112423","https://openalex.org/W2908879514","https://openalex.org/W3122383744","https://openalex.org/W4292341621"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W1963988314","https://openalex.org/W1924500548","https://openalex.org/W4237287867"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,62,110],"show":[4,81],"preliminary":[5,76],"results":[6],"on":[7,15],"the":[8,16,19,22,25,32,35,40,46,54,72,83,98,114,116,124],"quality":[9,41],"assessment":[10],"of":[11,18,48,75,85,92],"crowdworker":[12],"output":[13],"based":[14],"movements":[17],"mouse":[20,33],"and":[21,34,56,67,121],"eyes":[23,36],"while":[24],"task":[26,99],"is":[27,42,102,119],"performed.":[28],"We":[29],"assume":[30],"that":[31,82,113],"stop":[37],"longer":[38],"if":[39],"lower":[43],"due":[44],"to":[45,70,97],"lack":[47],"knowledge,":[49],"or":[50],"confidence,":[51],"etc.":[52],"Because":[53],"mouse-":[55],"eye-stopping":[57],"duration":[58,118],"follows":[59],"lognormal":[60],"distribution,":[61],"estimate":[63],"its":[64],"parameters":[65,84],"(mean":[66],"standard":[68],"deviation)":[69],"evaluate":[71],"quality.":[73],"Results":[74],"experiments":[77],"with":[78],"10":[79],"participants":[80],"correct":[86],"outputs":[87],"are":[88],"different":[89],"from":[90],"those":[91],"incorrect":[93],"ones.":[94],"As":[95],"compared":[96],"duration,":[100],"which":[101],"often":[103],"used":[104],"as":[105],"a":[106],"feature":[107],"for":[108,123],"assessment,":[109],"have":[111],"found":[112],"mouse-and":[115],"eyestopping":[117],"advantageous":[120],"complementary":[122],"assessment.":[125]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}