{"id":"https://openalex.org/W2530749689","doi":"https://doi.org/10.1145/2935752","title":"Replacing Mechanical Turkers? Challenges in the Evaluation of Models with Semantic Properties","display_name":"Replacing Mechanical Turkers? Challenges in the Evaluation of Models with Semantic Properties","publication_year":2016,"publication_date":"2016-10-12","ids":{"openalex":"https://openalex.org/W2530749689","doi":"https://doi.org/10.1145/2935752","mag":"2530749689"},"language":"en","primary_location":{"id":"doi:10.1145/2935752","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2935752","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002709735","display_name":"Fred Morstatter","orcid":"https://orcid.org/0000-0002-0247-4328"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fred Morstatter","raw_affiliation_strings":["School of Computing, Informatics, and Decision Systems Engineering Arizona State University Tempe, Arizona 85281"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing, Informatics, and Decision Systems Engineering Arizona State University Tempe, Arizona 85281","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100338946","display_name":"Huan Liu","orcid":"https://orcid.org/0000-0002-3264-7904"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huan Liu","raw_affiliation_strings":["School of Computing, Informatics, and Decision Systems Engineering Arizona State University Tempe, Arizona 85281"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing, Informatics, and Decision Systems Engineering Arizona State University Tempe, Arizona 85281","institution_ids":["https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I55732556"],"apc_list":null,"apc_paid":null,"fwci":0.4331,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.79468377,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"7","issue":"4","first_page":"1","last_page":"4"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.7841061353683472},{"id":"https://openalex.org/keywords/informatics","display_name":"Informatics","score":0.7181239724159241},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6450797319412231},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.4896147847175598},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36150410771369934},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.35621219873428345},{"id":"https://openalex.org/keywords/library-science","display_name":"Library science","score":0.3423963189125061},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.15095192193984985},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.06943026185035706}],"concepts":[{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.7841061353683472},{"id":"https://openalex.org/C191630685","wikidata":"https://www.wikidata.org/wiki/Q4027615","display_name":"Informatics","level":2,"score":0.7181239724159241},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6450797319412231},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4896147847175598},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36150410771369934},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.35621219873428345},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.3423963189125061},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.15095192193984985},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.06943026185035706},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2935752","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2935752","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.46000000834465027,"id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G8921800242","display_name":null,"funder_award_id":"N000141410095","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W1880262756","https://openalex.org/W1989894105","https://openalex.org/W2055988035","https://openalex.org/W2095655043","https://openalex.org/W2159426623","https://openalex.org/W2517941244","https://openalex.org/W2950133940"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W4317642157","https://openalex.org/W2363440148","https://openalex.org/W4239542068","https://openalex.org/W1974749013","https://openalex.org/W134931362","https://openalex.org/W2317510431","https://openalex.org/W1990811139","https://openalex.org/W2352401039","https://openalex.org/W2181265275"],"abstract_inverted_index":{"research-article":[0],"Public":[1],"Access":[2],"Share":[3],"on":[4,142],"Replacing":[5],"Mechanical":[6],"Turkers?":[7],"Challenges":[8],"in":[9,151],"the":[10,143],"Evaluation":[11],"of":[12,21,35,53,67,86],"Models":[13],"with":[14],"Semantic":[15],"Properties":[16],"Authors:":[17],"Fred":[18],"Morstatter":[19],"School":[20,34,52,66],"Computing,":[22,36,54,68],"Informatics,":[23,37,55,69],"and":[24,38,56,70,88,119],"Decision":[25,39,57,71],"Systems":[26,40,58,72],"Engineering":[27,41,59,73],"Arizona":[28,32,42,46,60,64,74,78],"State":[29,43,61,75],"University":[30,44,62,76],"Tempe,":[31,45,63,77],"85281":[33,65],"85281View":[47,79],"Profile":[48,80],",":[49],"Huan":[50],"Liu":[51],"Authors":[81],"Info":[82],"&":[83],"Claims":[84],"Journal":[85],"Data":[87],"Information":[89],"QualityVolume":[90],"7Issue":[91],"4October":[92],"2016":[93],"Article":[94],"No.:":[95],"15pp":[96],"1\u20134https://doi.org/10.1145/2935752Published:12":[97],"October":[98],"2016Publication":[99],"History":[100],"0citation320DownloadsMetricsTotal":[101],"Citations0Total":[102],"Downloads320Last":[103],"12":[104],"Months14Last":[105],"6":[106],"weeks1":[107],"Get":[108],"Citation":[109,111,148],"AlertsNew":[110,147],"Alert":[112],"added!This":[113],"alert":[114,139],"has":[115,134],"been":[116,135],"successfully":[117],"added":[118],"will":[120,124],"be":[121,125],"sent":[122],"to:You":[123],"notified":[126],"whenever":[127],"a":[128,160],"record":[129],"that":[130],"you":[131],"have":[132],"chosen":[133],"cited.To":[136],"manage":[137],"your":[138,153],"preferences,":[140],"click":[141],"button":[144],"below.Manage":[145],"my":[146],"Alert!Please":[149],"log":[150],"to":[152,156,158],"account":[154],"Save":[155],"BinderSave":[157],"BinderCreate":[159],"New":[161],"BinderNameCancelCreateExport":[162],"CitationPublisher":[163],"SiteeReaderPDF":[164]},"counts_by_year":[{"year":2017,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}