{"id":"https://openalex.org/W2055988035","doi":"https://doi.org/10.1145/2700171.2791028","title":"Text, Topics, and Turkers","display_name":"Text, Topics, and Turkers","publication_year":2015,"publication_date":"2015-01-01","ids":{"openalex":"https://openalex.org/W2055988035","doi":"https://doi.org/10.1145/2700171.2791028","mag":"2055988035"},"language":"en","primary_location":{"id":"doi:10.1145/2700171.2791028","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2700171.2791028","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2791028&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM Conference on Hypertext & Social Media - HT '15","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=2791028&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002709735","display_name":"Fred Morstatter","orcid":"https://orcid.org/0000-0002-0247-4328"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fred Morstatter","raw_affiliation_strings":["Arizona State University, Tempe, AZ, USA","Arizona State University , Tempe , AZ , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arizona State University, Tempe, AZ, USA","institution_ids":["https://openalex.org/I55732556"]},{"raw_affiliation_string":"Arizona State University , Tempe , AZ , USA","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030571651","display_name":"J\u00fcrgen Pfeffer","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"J\u00fcrgen Pfeffer","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA","Carnegie-Mellon University, Pittsburgh, Pa., USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, Pa., USA#TAB#","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044689056","display_name":"Katja Mayer","orcid":"https://orcid.org/0000-0003-1184-595X"},"institutions":[{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Katja Mayer","raw_affiliation_strings":["University of Vienna, Vienna, Austria","University of Vienna, Vienna,\u2003Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"University of Vienna, Vienna,\u2003Austria","institution_ids":["https://openalex.org/I129774422"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100338946","display_name":"Huan Liu","orcid":"https://orcid.org/0000-0002-3264-7904"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huan Liu","raw_affiliation_strings":["Arizona State University, Tempe, AZ, USA","Arizona State University , Tempe , AZ , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arizona State University, Tempe, AZ, USA","institution_ids":["https://openalex.org/I55732556"]},{"raw_affiliation_string":"Arizona State University , Tempe , AZ , USA","institution_ids":["https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"123","last_page":"131"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.9072745442390442},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7896220684051514},{"id":"https://openalex.org/keywords/topic-model","display_name":"Topic model","score":0.7292146682739258},{"id":"https://openalex.org/keywords/coherence","display_name":"Coherence (philosophical gambling strategy)","score":0.6136361360549927},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5905749201774597},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.5887326002120972},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5730977654457092},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5636509656906128},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5027334690093994},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45754262804985046},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4566355049610138},{"id":"https://openalex.org/keywords/meaning","display_name":"Meaning (existential)","score":0.44681409001350403},{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.4466457962989807},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.41071659326553345},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.1964164674282074},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.16115331649780273},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08214154839515686}],"concepts":[{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.9072745442390442},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7896220684051514},{"id":"https://openalex.org/C171686336","wikidata":"https://www.wikidata.org/wiki/Q3532085","display_name":"Topic model","level":2,"score":0.7292146682739258},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.6136361360549927},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5905749201774597},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.5887326002120972},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5730977654457092},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5636509656906128},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5027334690093994},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45754262804985046},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4566355049610138},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.44681409001350403},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.4466457962989807},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41071659326553345},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.1964164674282074},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.16115331649780273},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08214154839515686},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2700171.2791028","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2700171.2791028","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2791028&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM Conference on Hypertext & Social Media - HT '15","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/2700171.2791028","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2700171.2791028","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2791028&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM Conference on Hypertext & Social Media - HT '15","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5099999904632568,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G8921800242","display_name":null,"funder_award_id":"N000141410095","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2055988035.pdf","grobid_xml":"https://content.openalex.org/works/W2055988035.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W107619411","https://openalex.org/W1590495275","https://openalex.org/W1845748792","https://openalex.org/W1880262756","https://openalex.org/W1970381522","https://openalex.org/W2001082470","https://openalex.org/W2038043464","https://openalex.org/W2047240862","https://openalex.org/W2078483536","https://openalex.org/W2127860643","https://openalex.org/W2130339025","https://openalex.org/W2141367839","https://openalex.org/W2142889507","https://openalex.org/W2144100511","https://openalex.org/W2146950091","https://openalex.org/W2147946282","https://openalex.org/W2149510050","https://openalex.org/W2151967815","https://openalex.org/W2158266063","https://openalex.org/W2159426623","https://openalex.org/W2244191099","https://openalex.org/W2250463706","https://openalex.org/W2250533720","https://openalex.org/W2251582277","https://openalex.org/W2277420157","https://openalex.org/W2329283878","https://openalex.org/W2803437449","https://openalex.org/W2949731435","https://openalex.org/W2963593575","https://openalex.org/W4400327440","https://openalex.org/W6639619044","https://openalex.org/W6679482899","https://openalex.org/W6682044806"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2365213443","https://openalex.org/W1553232534","https://openalex.org/W2802540510","https://openalex.org/W2171319841","https://openalex.org/W2164265388","https://openalex.org/W2045032240","https://openalex.org/W2370554703","https://openalex.org/W2018189020","https://openalex.org/W2561436554"],"abstract_inverted_index":{"Topic":[0,75,163],"modeling":[1,61],"is":[2],"an":[3,130],"important":[4],"tool":[5],"in":[6,33,157,166],"social":[7],"media":[8],"analysis,":[9],"allowing":[10],"researchers":[11],"to":[12,35,115],"quickly":[13],"understand":[14,52],"large":[15],"text":[16],"corpora":[17],"by":[18,58,73],"investigating":[19,88],"the":[20,26,37,40,43,53,68,82,96,100,103,116,123,127,139,155,158,162,175],"topics":[21,41,56,94,104,124],"underlying":[22,95,117,131],"them.":[23],"One":[24],"of":[25,29,39,45,55,70,84,102,141,154,168],"fundamental":[27],"problems":[28],"topic":[30,60,86,109,149,159],"models":[31],"lies":[32],"how":[34,112,121,137],"assess":[36],"quality":[38,83,101],"from":[42,105],"perspective":[44],"human":[46],"interpretability.":[47],"How":[48],"well":[49,138,172],"can":[50],"humans":[51],"meaning":[54],"generated":[57],"statistical":[59],"algorithms?":[62],"In":[63],"this":[64,71],"work":[65],"we":[66],"advance":[67],"study":[69],"question":[72],"introducing":[74],"Consensus:":[76],"a":[77,85],"new":[78],"measure":[79],"that":[80],"calculates":[81],"through":[87],"its":[89],"consensus":[90],"with":[91],"some":[92],"known":[93],"data.":[97],"We":[98,133],"view":[99],"three":[106],"perspectives:":[107],"1)":[108],"interpretability,":[110],"2)":[111],"documents":[113],"relate":[114],"topics,":[118],"and":[119],"3)":[120],"interpretable":[122],"are":[125],"when":[126],"corpus":[128],"has":[129],"categorization.":[132],"provide":[134],"insights":[135],"into":[136],"results":[140],"Mechanical":[142],"Turk":[143],"match":[144],"automated":[145],"methods":[146],"for":[147],"calculating":[148],"quality.":[150],"The":[151],"probability":[152],"distribution":[153],"words":[156],"best":[160,176],"fit":[161],"Coherence":[164],"measure,":[165],"terms":[167],"both":[169],"correlation":[170],"as":[171,173],"finding":[174],"topics.":[177]},"counts_by_year":[{"year":2016,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}