{"id":"https://openalex.org/W2740579382","doi":"https://doi.org/10.18653/v1/p17-1028","title":"Aggregating and Predicting Sequence Labels from Crowd Annotations","display_name":"Aggregating and Predicting Sequence Labels from Crowd Annotations","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2740579382","doi":"https://doi.org/10.18653/v1/p17-1028","mag":"2740579382","pmid":"https://pubmed.ncbi.nlm.nih.gov/29093611"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p17-1028","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p17-1028","pdf_url":"https://www.aclweb.org/anthology/P17-1028.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th Annual Meeting of the Association for\n          Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P17-1028.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067463039","display_name":"An Nguyen","orcid":null},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"An Thanh Nguyen","raw_affiliation_strings":["University of Texas at Austin"],"affiliations":[{"raw_affiliation_string":"University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036790226","display_name":"Byron Wallace","orcid":"https://orcid.org/0000-0003-2409-7735"},"institutions":[{"id":"https://openalex.org/I87182695","display_name":"Universidad del Noreste","ror":"https://ror.org/02ahky613","country_code":"MX","type":"education","lineage":["https://openalex.org/I87182695"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Byron Wallace","raw_affiliation_strings":["Northeastern University"],"affiliations":[{"raw_affiliation_string":"Northeastern University","institution_ids":["https://openalex.org/I87182695"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021057186","display_name":"Junyi Jessy Li","orcid":"https://orcid.org/0000-0002-2550-5262"},"institutions":[{"id":"https://openalex.org/I36788626","display_name":"California University of Pennsylvania","ror":"https://ror.org/01spssf70","country_code":"US","type":"education","lineage":["https://openalex.org/I36788626"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junyi Jessy Li","raw_affiliation_strings":["University of Pennsylvania","#N#               * University of Pennsylvania"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania","institution_ids":["https://openalex.org/I36788626"]},{"raw_affiliation_string":"#N#               * University of Pennsylvania","institution_ids":["https://openalex.org/I36788626","https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032571629","display_name":"Ani Nenkova","orcid":"https://orcid.org/0000-0002-5825-7875"},"institutions":[{"id":"https://openalex.org/I36788626","display_name":"California University of Pennsylvania","ror":"https://ror.org/01spssf70","country_code":"US","type":"education","lineage":["https://openalex.org/I36788626"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ani Nenkova","raw_affiliation_strings":["University of Pennsylvania","#N#               * University of Pennsylvania"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania","institution_ids":["https://openalex.org/I36788626"]},{"raw_affiliation_string":"#N#               * University of Pennsylvania","institution_ids":["https://openalex.org/I36788626","https://openalex.org/I79576946"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023397430","display_name":"Matthew Lease","orcid":"https://orcid.org/0000-0002-0056-2834"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Lease","raw_affiliation_strings":["University of Texas at Austin"],"affiliations":[{"raw_affiliation_string":"University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5067463039"],"corresponding_institution_ids":["https://openalex.org/I86519309"],"apc_list":null,"apc_paid":null,"fwci":7.4752,"has_fulltext":true,"cited_by_count":89,"citation_normalized_percentile":{"value":0.97672739,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":"2017","issue":null,"first_page":"299","last_page":"309"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8393490314483643},{"id":"https://openalex.org/keywords/hidden-markov-model","display_name":"Hidden Markov model","score":0.6161324977874756},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5872745513916016},{"id":"https://openalex.org/keywords/sequence-labeling","display_name":"Sequence labeling","score":0.5336101055145264},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5312219858169556},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.5302072763442993},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.5111141800880432},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4959438741207123},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.49251827597618103},{"id":"https://openalex.org/keywords/named-entity-recognition","display_name":"Named-entity recognition","score":0.46370962262153625},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.42622992396354675},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.34534478187561035}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8393490314483643},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.6161324977874756},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5872745513916016},{"id":"https://openalex.org/C35639132","wikidata":"https://www.wikidata.org/wiki/Q7452468","display_name":"Sequence labeling","level":3,"score":0.5336101055145264},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5312219858169556},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5302072763442993},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5111141800880432},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4959438741207123},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.49251827597618103},{"id":"https://openalex.org/C2779135771","wikidata":"https://www.wikidata.org/wiki/Q403574","display_name":"Named-entity recognition","level":3,"score":0.46370962262153625},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42622992396354675},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.34534478187561035},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/p17-1028","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p17-1028","pdf_url":"https://www.aclweb.org/anthology/P17-1028.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th Annual Meeting of the Association for\n          Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmid:29093611","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/29093611","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the conference. Association for Computational Linguistics. Meeting","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:5662012","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/5662012","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc Conf Assoc Comput Linguist Meet","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.18653/v1/p17-1028","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p17-1028","pdf_url":"https://www.aclweb.org/anthology/P17-1028.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th Annual Meeting of the Association for\n          Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3497591915","display_name":null,"funder_award_id":"(NIH)","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G5921281487","display_name":null,"funder_award_id":"number","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6100344857","display_name":null,"funder_award_id":"UH2CA203711","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8701689675","display_name":null,"funder_award_id":"1253413","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337351","display_name":"National Cancer Institute","ror":"https://ror.org/040gcmg81"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2740579382.pdf","grobid_xml":"https://content.openalex.org/works/W2740579382.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W6869908","https://openalex.org/W9014458","https://openalex.org/W49437421","https://openalex.org/W114517082","https://openalex.org/W191327111","https://openalex.org/W1459599406","https://openalex.org/W1522548332","https://openalex.org/W1543648998","https://openalex.org/W1570013475","https://openalex.org/W1832693441","https://openalex.org/W1970381522","https://openalex.org/W2045993505","https://openalex.org/W2049633694","https://openalex.org/W2055601365","https://openalex.org/W2064675550","https://openalex.org/W2099873701","https://openalex.org/W2105594594","https://openalex.org/W2111722073","https://openalex.org/W2112511942","https://openalex.org/W2115979064","https://openalex.org/W2129345386","https://openalex.org/W2129494713","https://openalex.org/W2131864341","https://openalex.org/W2134305421","https://openalex.org/W2141649520","https://openalex.org/W2144578941","https://openalex.org/W2145111356","https://openalex.org/W2147880316","https://openalex.org/W2158899491","https://openalex.org/W2163918411","https://openalex.org/W2250244475","https://openalex.org/W2251818274","https://openalex.org/W2296107862","https://openalex.org/W2296283641","https://openalex.org/W2394968026","https://openalex.org/W2484269232","https://openalex.org/W2572658681","https://openalex.org/W2766736793","https://openalex.org/W2952087486","https://openalex.org/W2952230511","https://openalex.org/W2963083845","https://openalex.org/W4234117503"],"related_works":["https://openalex.org/W2535098683","https://openalex.org/W189110383","https://openalex.org/W2589080577","https://openalex.org/W3161409692","https://openalex.org/W2530283981","https://openalex.org/W4321523623","https://openalex.org/W4287197350","https://openalex.org/W3036779180","https://openalex.org/W3047727388","https://openalex.org/W2790360088"],"abstract_inverted_index":{"Despite":[0],"sequences":[1,59,75],"being":[2],"core":[3],"to":[4,11,36],"NLP,":[5],"scant":[6],"work":[7],"has":[8],"considered":[9],"how":[10],"handle":[12],"noisy":[13],"sequence":[14],"labels":[15,35],"from":[16,110],"multiple":[17],"annotators":[18],"for":[19,53],"the":[20],"same":[21],"text.":[22,62],"Given":[23],"such":[24],"annotations,":[25],"we":[26,65,79],"consider":[27],"two":[28,96],"complementary":[29],"tasks:":[30],"(1)":[31],"aggregating":[32],"sequential":[33],"crowd":[34,48],"infer":[37],"a":[38,54,67,81,91],"best":[39],"single":[40],"set":[41],"of":[42,93],"consensus":[43],"annotations;":[44],"and":[45,99,107,122],"(2)":[46],"using":[47,84],"annotations":[49],"as":[50],"training":[51],"data":[52,123],"model":[55],"that":[56],"can":[57],"predict":[58,74],"in":[60,76,104],"unannotated":[61,77],"For":[63],"aggregation,":[64],"propose":[66,80],"novel":[68],"Hidden":[69],"Markov":[70],"Model":[71],"variant.":[72],"To":[73],"text,":[78],"neural":[82],"approach":[83],"Long":[85],"Short":[86],"Term":[87],"Memory.":[88],"We":[89],"evaluate":[90],"suite":[92],"methods":[94],"across":[95],"different":[97],"applications":[98],"text":[100],"genres:":[101],"Named-Entity":[102],"Recognition":[103],"news":[105],"articles":[106],"Information":[108],"Extraction":[109],"biomedical":[111],"abstracts.":[112],"Results":[113],"show":[114],"improvement":[115],"over":[116],"strong":[117],"baselines.":[118],"Our":[119],"source":[120],"code":[121],"are":[124],"available":[125],"online.":[126]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":17},{"year":2021,"cited_by_count":20},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":18},{"year":2018,"cited_by_count":6}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}