{"id":"https://openalex.org/W2997657386","doi":"https://doi.org/10.1080/10447318.2019.1709339","title":"Reliability of Perceived Usability Assessment via Crowdsourcing Platform: Retrospective Analysis and Novel Feedback Quality Inspection Method","display_name":"Reliability of Perceived Usability Assessment via Crowdsourcing Platform: Retrospective Analysis and Novel Feedback Quality Inspection Method","publication_year":2020,"publication_date":"2020-01-03","ids":{"openalex":"https://openalex.org/W2997657386","doi":"https://doi.org/10.1080/10447318.2019.1709339","mag":"2997657386"},"language":"en","primary_location":{"id":"doi:10.1080/10447318.2019.1709339","is_oa":false,"landing_page_url":"https://doi.org/10.1080/10447318.2019.1709339","pdf_url":null,"source":{"id":"https://openalex.org/S165559636","display_name":"International Journal of Human-Computer Interaction","issn_l":"1044-7318","issn":["1044-7318","1532-7590"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Human\u2013Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Wang Yuhui","orcid":null},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang Yuhui","raw_affiliation_strings":["Industrial Design Department, School of Mechanical Science and Engineering, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Industrial Design Department, School of Mechanical Science and Engineering, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100777330","display_name":"Lei Tian","orcid":"https://orcid.org/0000-0001-5449-3232"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Lei Tian","raw_affiliation_strings":["Industrial Design Department, School of Mechanical Science and Engineering, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Industrial Design Department, School of Mechanical Science and Engineering, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066570733","display_name":"Xinxiong Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu Xinxiong","raw_affiliation_strings":["Industrial Design Department, School of Mechanical Science and Engineering, Huazhong University of Science and Technology, Wuhan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Industrial Design Department, School of Mechanical Science and Engineering, Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5100777330"],"corresponding_institution_ids":["https://openalex.org/I47720641"],"apc_list":null,"apc_paid":null,"fwci":2.22,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.89282813,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":98},"biblio":{"volume":"36","issue":"11","first_page":"1070","last_page":"1084"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9635000228881836,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.9251474142074585},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8654947280883789},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7188617587089539},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6843719482421875},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6277462244033813},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.5142171382904053},{"id":"https://openalex.org/keywords/usability-inspection","display_name":"Usability inspection","score":0.4414176344871521},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.41967886686325073},{"id":"https://openalex.org/keywords/heuristic-evaluation","display_name":"Heuristic evaluation","score":0.2872021198272705},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.25847679376602173},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.18137845396995544},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.10954999923706055}],"concepts":[{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.9251474142074585},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8654947280883789},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7188617587089539},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6843719482421875},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6277462244033813},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.5142171382904053},{"id":"https://openalex.org/C23456302","wikidata":"https://www.wikidata.org/wiki/Q7901668","display_name":"Usability inspection","level":4,"score":0.4414176344871521},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.41967886686325073},{"id":"https://openalex.org/C3255780","wikidata":"https://www.wikidata.org/wiki/Q1616517","display_name":"Heuristic evaluation","level":3,"score":0.2872021198272705},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.25847679376602173},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.18137845396995544},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.10954999923706055},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1080/10447318.2019.1709339","is_oa":false,"landing_page_url":"https://doi.org/10.1080/10447318.2019.1709339","pdf_url":null,"source":{"id":"https://openalex.org/S165559636","display_name":"International Journal of Human-Computer Interaction","issn_l":"1044-7318","issn":["1044-7318","1532-7590"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Human\u2013Computer Interaction","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4399999976158142,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":66,"referenced_works":["https://openalex.org/W172711948","https://openalex.org/W173286940","https://openalex.org/W1496033039","https://openalex.org/W1509630249","https://openalex.org/W1628516742","https://openalex.org/W1635961553","https://openalex.org/W1872549940","https://openalex.org/W1897289387","https://openalex.org/W1975450585","https://openalex.org/W1991339231","https://openalex.org/W1998265429","https://openalex.org/W2002245046","https://openalex.org/W2005019823","https://openalex.org/W2014817805","https://openalex.org/W2017708996","https://openalex.org/W2021643730","https://openalex.org/W2030326602","https://openalex.org/W2032191478","https://openalex.org/W2043555414","https://openalex.org/W2050470102","https://openalex.org/W2051163483","https://openalex.org/W2055694118","https://openalex.org/W2058625228","https://openalex.org/W2059372305","https://openalex.org/W2066741390","https://openalex.org/W2067257553","https://openalex.org/W2069074793","https://openalex.org/W2071986193","https://openalex.org/W2080043525","https://openalex.org/W2080479333","https://openalex.org/W2089562763","https://openalex.org/W2098423543","https://openalex.org/W2099473180","https://openalex.org/W2104749423","https://openalex.org/W2105297890","https://openalex.org/W2117633598","https://openalex.org/W2124326568","https://openalex.org/W2126572790","https://openalex.org/W2136558449","https://openalex.org/W2138365794","https://openalex.org/W2141009959","https://openalex.org/W2142984484","https://openalex.org/W2145871833","https://openalex.org/W2151401338","https://openalex.org/W2168672239","https://openalex.org/W2171791132","https://openalex.org/W2211081391","https://openalex.org/W2255396083","https://openalex.org/W2341767795","https://openalex.org/W2474664182","https://openalex.org/W2478449778","https://openalex.org/W2513654055","https://openalex.org/W2522186013","https://openalex.org/W2591129357","https://openalex.org/W2594278025","https://openalex.org/W2776433122","https://openalex.org/W2784220379","https://openalex.org/W2790344272","https://openalex.org/W2795274753","https://openalex.org/W4200579496","https://openalex.org/W4230552874","https://openalex.org/W4230746704","https://openalex.org/W4245155777","https://openalex.org/W4247626774","https://openalex.org/W4298196011","https://openalex.org/W6609294999"],"related_works":["https://openalex.org/W1497983610","https://openalex.org/W65754041","https://openalex.org/W3206214086","https://openalex.org/W3210710594","https://openalex.org/W4244725308","https://openalex.org/W2550923876","https://openalex.org/W2057326359","https://openalex.org/W2485414097","https://openalex.org/W1978672487","https://openalex.org/W2077503414"],"abstract_inverted_index":{"This":[0],"paper":[1,140],"discusses":[2],"the":[3,49,64,95,125,143],"differences":[4],"in":[5,97,138,161],"perceived":[6,146],"usability":[7,147,163],"evaluations":[8],"between":[9],"crowdsourcing":[10,75],"platform":[11,76],"and":[12,55,57],"laboratory":[13,83],"testing.":[14,164],"A":[15,22],"retrospective":[16,162],"evaluation":[17],"measurement":[18],"method":[19,26,52,136,153],"was":[20,34,99],"adopted.":[21],"SUS":[23,133],"quality":[24,88,134],"inspection":[25,32,59,106,126,135],"with":[27],"two":[28],"new":[29],"sets":[30],"of":[31,48,69,124,145],"items":[33],"established":[35],"to":[36,62,118],"more":[37],"precisely":[38],"observe":[39,63],"certain":[40],"low-quality":[41],"feedback.":[42],"We":[43],"also":[44],"tested":[45],"a":[46,158],"combination":[47],"traditional":[50],"check":[51],"(Items":[53],"2":[54],"3)":[56],"all":[58],"item":[60,107,127],"pairs":[61,128],"cleaning":[65,112,152],"effect.":[66],"The":[67,121,132,150],"results":[68],"this":[70,139],"study":[71],"suggests":[72],"that":[73],"(i)":[74],"scores":[77],"(using":[78],"SUS)":[79],"are":[80],"higher":[81],"than":[82],"test":[84],"scores,":[85],"though":[86],"their":[87],"may":[89,116,154],"not":[90,155],"be":[91],"as":[92],"good,":[93],"while":[94],"difference":[96],"CGS":[98],"only":[100],"one":[101,105],"grade.":[102],"(ii)":[103],"Using":[104],"pair":[108],"is":[109,129],"an":[110],"acceptable":[111],"method,":[113],"but":[114],"it":[115],"lead":[117],"over-cleaning.":[119],"(iii)":[120],"internal":[122],"relevance":[123],"very":[130],"high.":[131],"proposed":[137],"can":[141],"improve":[142],"accuracy":[144],"evaluations.":[148],"(iv)":[149],"completion-time-based":[151],"require":[156],"setting":[157],"maximum":[159],"time":[160]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}