{"id":"https://openalex.org/W1932269785","doi":"https://doi.org/10.1145/2740908.2745401","title":"Got Many Labels?","display_name":"Got Many Labels?","publication_year":2015,"publication_date":"2015-05-18","ids":{"openalex":"https://openalex.org/W1932269785","doi":"https://doi.org/10.1145/2740908.2745401","mag":"1932269785"},"language":"en","primary_location":{"id":"doi:10.1145/2740908.2745401","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2740908.2745401","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/2740908.2745401","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th International Conference on World Wide Web","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/2740908.2745401","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102899038","display_name":"Shuo Chang","orcid":"https://orcid.org/0009-0004-2345-1186"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shuo Chang","raw_affiliation_strings":["University of Minnesota, Minneapolis, MN, USA",", University of Minnesota, Minneapolis, MN, USA"],"affiliations":[{"raw_affiliation_string":"University of Minnesota, Minneapolis, MN, USA","institution_ids":["https://openalex.org/I130238516"]},{"raw_affiliation_string":", University of Minnesota, Minneapolis, MN, USA","institution_ids":["https://openalex.org/I130238516"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058646158","display_name":"Peng Dai","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peng Dai","raw_affiliation_strings":["Google Research, Mountain View, CA, USA","Google Research, Mountain View, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Google Research, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, Mountain View, CA, USA#TAB#","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033428202","display_name":"Jilin Chen","orcid":"https://orcid.org/0000-0002-3359-0938"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jilin Chen","raw_affiliation_strings":["Google Research, Mountain View, CA, USA","Google Research, Mountain View, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Google Research, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, Mountain View, CA, USA#TAB#","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028125399","display_name":"Ed H.","orcid":"https://orcid.org/0000-0003-3230-5338"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ed H. Chi","raw_affiliation_strings":["Google Research, Mountain View, CA, USA","Google Research, Mountain View, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Google Research, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, Mountain View, CA, USA#TAB#","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5102899038"],"corresponding_institution_ids":["https://openalex.org/I130238516"],"apc_list":null,"apc_paid":null,"fwci":3.5197,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.93003572,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"397","last_page":"406"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8850904703140259},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8813102841377258},{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.7058305740356445},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.637152910232544},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6360308527946472},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.5960903763771057},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5804591178894043},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5719732642173767},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44872915744781494},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.36246442794799805},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33746129274368286}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8850904703140259},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8813102841377258},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.7058305740356445},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.637152910232544},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6360308527946472},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5960903763771057},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5804591178894043},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5719732642173767},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44872915744781494},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.36246442794799805},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33746129274368286},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2740908.2745401","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2740908.2745401","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/2740908.2745401","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th International Conference on World Wide Web","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/2740908.2745401","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2740908.2745401","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/2740908.2745401","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th International Conference on World Wide Web","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5199999809265137,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1932269785.pdf","grobid_xml":"https://content.openalex.org/works/W1932269785.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W21006490","https://openalex.org/W1597663074","https://openalex.org/W1617082848","https://openalex.org/W2026969101","https://openalex.org/W2067344105","https://openalex.org/W2101234009","https://openalex.org/W2108630796","https://openalex.org/W2111032703","https://openalex.org/W2121002600","https://openalex.org/W2137918516","https://openalex.org/W2137958601","https://openalex.org/W2145111356","https://openalex.org/W2149586224","https://openalex.org/W2150510224","https://openalex.org/W2151401338","https://openalex.org/W2153848201","https://openalex.org/W2156336347","https://openalex.org/W2997591727","https://openalex.org/W6636677605","https://openalex.org/W6680441624"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W319434848","https://openalex.org/W1676177962","https://openalex.org/W4377238720","https://openalex.org/W2998679789","https://openalex.org/W1996655143","https://openalex.org/W2466330544","https://openalex.org/W2897694452","https://openalex.org/W3094927943","https://openalex.org/W2100046112"],"abstract_inverted_index":{"Online":[0],"search":[1,63,194],"and":[2,39,64,90,131,195],"item":[3,196],"recommendation":[4],"systems":[5],"are":[6,34],"often":[7,35],"based":[8,142],"on":[9,143],"being":[10],"able":[11],"to":[12,74,123,137,183,186],"correctly":[13],"label":[14],"items":[15],"with":[16,27],"topical":[17,20],"keywords.":[18],"Typically,":[19],"labelers":[21],"analyze":[22],"the":[23,28,43,98,112,125,165,174,190],"main":[24,44,166],"text":[25,101],"associated":[26],"item,":[29],"but":[30],"social":[31,49,61,187],"media":[32,50,62,188],"posts":[33,51],"multimedia":[36],"in":[37,80],"nature":[38],"contain":[40],"contents":[41],"beyond":[42],"text.":[45],"Topic":[46],"labeling":[47],"for":[48,58,77,94,111],"is":[52],"therefore":[53],"an":[54],"important":[55],"open":[56],"problem":[57,76],"supporting":[59],"effective":[60],"recommendation.":[65,197],"In":[66],"this":[67,75],"work,":[68],"we":[69,82,119,154],"present":[70],"a":[71,84,95,149],"novel":[72],"solution":[73],"Google+":[78],"posts,":[79,189],"which":[81],"integrated":[83],"number":[85],"of":[86,97,115,127,173],"different":[87,116,139],"entity":[88,129,140],"extractors":[89],"annotators,":[91,130],"each":[92],"responsible":[93],"part":[96],"post":[99],"(e.g.":[100],"body,":[102],"embedded":[103],"picture,":[104],"video,":[105],"or":[106],"web":[107],"link).":[108],"To":[109],"account":[110],"varying":[113],"quality":[114],"annotator":[117],"outputs,":[118],"first":[120],"utilized":[121],"crowdsourcing":[122],"measure":[124],"accuracy":[126],"individual":[128,175],"then":[132],"used":[133],"supervised":[134],"machine":[135],"learning":[136],"combine":[138],"annotators":[141],"their":[144,184],"relative":[145],"accuracy.":[146],"Evaluating":[147],"using":[148],"ground":[150],"truth":[151],"data":[152],"set,":[153],"found":[155],"that":[156],"our":[157],"approach":[158],"substantially":[159],"outperforms":[160],"topic":[161,180],"labels":[162,181],"obtained":[163],"from":[164],"text,":[167],"as":[168,170],"well":[169],"naive":[171],"combinations":[172],"annotators.":[176],"By":[177],"accurately":[178],"applying":[179],"according":[182],"relevance":[185],"results":[191],"enables":[192],"better":[193]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":3},{"year":2016,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2016-06-24T00:00:00"}