{"id":"https://openalex.org/W2395551797","doi":null,"title":"Mechanical Turk-based experiment vs laboratory-based experiment: A case study on the comparison of semantic transparency rating data","display_name":"Mechanical Turk-based experiment vs laboratory-based experiment: A case study on the comparison of semantic transparency rating data","publication_year":2015,"publication_date":"2015-01-01","ids":{"openalex":"https://openalex.org/W2395551797","mag":"2395551797"},"language":"en","primary_location":{"id":"pmh:oai:irdb.nii.ac.jp:00835:0003786839","doi":null,"is_oa":true,"landing_page_url":"https://waseda.repo.nii.ac.jp/records/42242","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th Pacific Asia Conference on Language, Information and Computation","raw_type":"conference paper"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://waseda.repo.nii.ac.jp/records/42242","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102971767","display_name":"Shichang Wang","orcid":"https://orcid.org/0000-0002-7633-8118","observed_orcids":["https://orcid.org/0000-0002-7633-8118"]},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shichang Wang","raw_affiliation_strings":["The Hong Kong Polytechnic University"],"raw_orcid":"https://orcid.org/0000-0002-7633-8118","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024924150","display_name":"Chu\u2010Ren Huang","orcid":"https://orcid.org/0000-0002-8526-5520","observed_orcids":["https://orcid.org/0000-0002-8526-5520"]},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chu\u2010Ren Huang","raw_affiliation_strings":["The Hong Kong Polytechnic University"],"raw_orcid":"https://orcid.org/0000-0002-8526-5520","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100389413","display_name":"Yao Yao","orcid":"https://orcid.org/0000-0001-8825-2680","observed_orcids":["https://orcid.org/0000-0001-8825-2680"]},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yao Yao","raw_affiliation_strings":["The Hong Kong Polytechnic University"],"raw_orcid":"https://orcid.org/0000-0001-8825-2680","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017602070","display_name":"Angel Chan","orcid":"https://orcid.org/0000-0002-9547-0210","observed_orcids":["https://orcid.org/0000-0002-9547-0210"]},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Angel Chan","raw_affiliation_strings":["The Hong Kong Polytechnic University"],"raw_orcid":"https://orcid.org/0000-0002-9547-0210","affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University","institution_ids":["https://openalex.org/I14243506"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I14243506"],"apc_list":null,"apc_paid":null,"fwci":1.0649,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.73881074,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"53","last_page":"62"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.6807000041007996,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.6807000041007996,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12694","display_name":"Categorization, perception, and language","score":0.11729999631643295,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.03779999911785126,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/semantic-transparency","display_name":"semantic transparency","score":0.9850000143051147},{"id":"https://openalex.org/keywords/mechanical-turk","display_name":"Mechanical Turk","score":0.7580000162124634},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.9359999895095825},{"id":"https://openalex.org/keywords/z-score-transformation","display_name":"z-score transformation","score":0.8429999947547913},{"id":"https://openalex.org/keywords/human-judgment","display_name":"human judgment","score":0.460999995470047}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8047199249267578},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.7928479313850403},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6194015741348267},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.534256100654602},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.48614785075187683},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.43626856803894043},{"id":"https://openalex.org/C204241405","wikidata":"https://www.wikidata.org/wiki/Q461499","display_name":"Transformation (genetics)","level":3,"score":0.42213237285614014},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3965160548686981},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3697025775909424},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32813403010368347},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2937247157096863},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22896885871887207},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"pmh:oai:irdb.nii.ac.jp:00835:0003786839","doi":null,"is_oa":true,"landing_page_url":"https://waseda.repo.nii.ac.jp/records/42242","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th Pacific Asia Conference on Language, Information and Computation","raw_type":"conference paper"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.709.2484","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.709.2484","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://bcmi.sjtu.edu.cn/%7Epaclic29/proceedings/PACLIC29-1007.193.pdf","raw_type":"text"},{"id":"pmh:oai:ira.lib.polyu.edu.hk:10397/68013","doi":null,"is_oa":false,"landing_page_url":"http://hdl.handle.net/10397/68013","pdf_url":null,"source":{"id":"https://openalex.org/S4306400205","display_name":"PolyU Institutional Research Archive (Hong Kong Polytechnic University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I14243506","host_organization_name":"Hong Kong Polytechnic University","host_organization_lineage":["https://openalex.org/I14243506"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},{"id":"pmh:oai:waseda.repo.nii.ac.jp:00042242","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2065/00058065","pdf_url":null,"source":{"id":"https://openalex.org/S4377196531","display_name":"Waseda University Repository (Waseda University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I150744194","host_organization_name":"Waseda University","host_organization_lineage":["https://openalex.org/I150744194"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},{"id":"mag:2395551797","doi":null,"is_oa":false,"landing_page_url":"http://bcmi.sjtu.edu.cn/~paclic29/proceedings/PACLIC29-1007.193.pdf","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:irdb.nii.ac.jp:00835:0003786839","doi":null,"is_oa":true,"landing_page_url":"https://waseda.repo.nii.ac.jp/records/42242","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th Pacific Asia Conference on Language, Information and Computation","raw_type":"conference paper"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W61584101","https://openalex.org/W1525201159","https://openalex.org/W1577841485","https://openalex.org/W1590411898","https://openalex.org/W1905966190","https://openalex.org/W1970381522","https://openalex.org/W1979532929","https://openalex.org/W1984022436","https://openalex.org/W1986715167","https://openalex.org/W2013292621","https://openalex.org/W2041812559","https://openalex.org/W2080211902","https://openalex.org/W2114269021","https://openalex.org/W2118251512","https://openalex.org/W2141708418","https://openalex.org/W2151401338","https://openalex.org/W2169800142","https://openalex.org/W2170931744","https://openalex.org/W2170989440","https://openalex.org/W2171913066","https://openalex.org/W2182894058","https://openalex.org/W2294566554","https://openalex.org/W3122078363","https://openalex.org/W3124878131"],"related_works":["https://openalex.org/W3088965898","https://openalex.org/W1994322930","https://openalex.org/W2041812559","https://openalex.org/W6929377418","https://openalex.org/W2734045434","https://openalex.org/W2127452865","https://openalex.org/W1905966190","https://openalex.org/W2530749689","https://openalex.org/W4404066463","https://openalex.org/W2573736132"],"abstract_inverted_index":{"In":[0,136],"this":[1,123,183],"paper,":[2],"we":[3,20,144,170],"conducted":[4],"semantic":[5,165,179],"trans-parency":[6,166,180],"rating":[7,23,41,126],"experiments":[8,54,132],"using":[9],"both":[10,37,81,97],"the":[11,16,22,52,67,83,87,92,100,105,108,112,120,125,140,157,162],"traditional":[12],"laboratory-based":[13,68,121],"method":[14],"and":[15,43,59,152,182],"crowdsourcing-based":[17,60,113],"method.":[18],"Then":[19],"com-pared":[21],"data":[24,42,47,58,147],"obtained":[25,128],"from":[26],"these":[27,130],"two":[28,53,131,146],"experiments.":[29],"We":[30,73,159],"observed":[31,75],"very":[32],"strong":[33],"cor-relation":[34],"coefficients":[35],"for":[36],"overall":[38],"seman-tic":[39],"transparency":[40,46],"constituent":[44],"se-mantic":[45],"(rho>":[48],"0:9)":[49],"which":[50],"means":[51],"may":[55,184],"yield":[56],"com-parable":[57],"experi-ment":[61],"is":[62,115],"a":[63,76,175,187],"feasible":[64],"alternative":[65],"to":[66,138,155],"experiment":[69,114],"in":[70,80,129],"linguistic":[71],"studies.":[72],"also":[74,160],"scale":[77,85,94,102,109],"shrinkage":[78,110],"phenomenon":[79],"experiments:":[82],"actual":[84,101],"of":[86,99,111,119,164,190],"rat-ing":[88],"results":[89,127,141],"cannot":[90],"cover":[91],"ideal":[93],"[0;":[95],"1],":[96],"ends":[98],"shrink":[103],"towards":[104],"center.":[106],"However,":[107],"stronger":[116],"than":[117],"that":[118,172],"experiment,":[122],"makes":[124],"not":[133],"directly":[134,142],"comparable.":[135],"order":[137],"make":[139],"compara-ble,":[143],"explored":[145],"transformation":[148,151],"al-gorithms,":[149],"z-score":[150],"adjusted":[153],"normalization":[154],"unify":[156],"scales.":[158],"in-vestigated":[161],"uncertainty":[163],"judgment":[167],"among":[168],"raters,":[169],"found":[171],"it":[173],"had":[174],"regular":[176],"relation":[177],"with":[178],"magnitude":[181],"further":[185],"reveal":[186],"general":[188],"cognitivemechanism":[189],"human":[191],"judg-ment.":[192],"1":[193]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}