{"id":"https://openalex.org/W2952395386","doi":"https://doi.org/10.18653/v1/p19-1077","title":"Crowdsourcing and Aggregating Nested Markable Annotations","display_name":"Crowdsourcing and Aggregating Nested Markable Annotations","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2952395386","doi":"https://doi.org/10.18653/v1/p19-1077","mag":"2952395386"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1077","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1077","pdf_url":"https://www.aclweb.org/anthology/P19-1077.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1077.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015462921","display_name":"Chris Madge","orcid":"https://orcid.org/0009-0007-4937-8572"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Chris Madge","raw_affiliation_strings":["Queen Mary University"],"affiliations":[{"raw_affiliation_string":"Queen Mary University","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003328793","display_name":"Juntao Yu","orcid":"https://orcid.org/0000-0001-7971-9154"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Juntao Yu","raw_affiliation_strings":["Queen Mary University"],"affiliations":[{"raw_affiliation_string":"Queen Mary University","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067137759","display_name":"Jon Chamberlain","orcid":"https://orcid.org/0000-0002-6947-8964"},"institutions":[{"id":"https://openalex.org/I110002522","display_name":"University of Essex","ror":"https://ror.org/02nkf1q06","country_code":"GB","type":"education","lineage":["https://openalex.org/I110002522"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jon Chamberlain","raw_affiliation_strings":["University of Essex"],"affiliations":[{"raw_affiliation_string":"University of Essex","institution_ids":["https://openalex.org/I110002522"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014534985","display_name":"Udo Kruschwitz","orcid":"https://orcid.org/0000-0002-5503-0341"},"institutions":[{"id":"https://openalex.org/I110002522","display_name":"University of Essex","ror":"https://ror.org/02nkf1q06","country_code":"GB","type":"education","lineage":["https://openalex.org/I110002522"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Udo Kruschwitz","raw_affiliation_strings":["University of Essex"],"affiliations":[{"raw_affiliation_string":"University of Essex","institution_ids":["https://openalex.org/I110002522"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019552564","display_name":"Silviu Paun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Silviu Paun","raw_affiliation_strings":["Queen Mary University"],"affiliations":[{"raw_affiliation_string":"Queen Mary University","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047065550","display_name":"Massimo Poesio","orcid":"https://orcid.org/0000-0001-8469-2072"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Massimo Poesio","raw_affiliation_strings":["Queen Mary University"],"affiliations":[{"raw_affiliation_string":"Queen Mary University","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5015462921"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7224,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.78402097,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"797","last_page":"807"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/coreference","display_name":"Coreference","score":0.9181641340255737},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8540948033332825},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7898399829864502},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6949058771133423},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6549109220504761},{"id":"https://openalex.org/keywords/noun-phrase","display_name":"Noun phrase","score":0.6370788812637329},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6189113259315491},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.6139659881591797},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6015720367431641},{"id":"https://openalex.org/keywords/identifier","display_name":"Identifier","score":0.5846524238586426},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5583634376525879},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5542440414428711},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.49568426609039307},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.457839697599411},{"id":"https://openalex.org/keywords/named-entity-recognition","display_name":"Named-entity recognition","score":0.42927026748657227},{"id":"https://openalex.org/keywords/resolution","display_name":"Resolution (logic)","score":0.3890531659126282},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.34801971912384033},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.22816774249076843},{"id":"https://openalex.org/keywords/noun","display_name":"Noun","score":0.19239792227745056}],"concepts":[{"id":"https://openalex.org/C28076734","wikidata":"https://www.wikidata.org/wiki/Q63087","display_name":"Coreference","level":3,"score":0.9181641340255737},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8540948033332825},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7898399829864502},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6949058771133423},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6549109220504761},{"id":"https://openalex.org/C153962237","wikidata":"https://www.wikidata.org/wiki/Q1401131","display_name":"Noun phrase","level":3,"score":0.6370788812637329},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6189113259315491},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.6139659881591797},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6015720367431641},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.5846524238586426},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5583634376525879},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5542440414428711},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.49568426609039307},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.457839697599411},{"id":"https://openalex.org/C2779135771","wikidata":"https://www.wikidata.org/wiki/Q403574","display_name":"Named-entity recognition","level":3,"score":0.42927026748657227},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.3890531659126282},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.34801971912384033},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.22816774249076843},{"id":"https://openalex.org/C121934690","wikidata":"https://www.wikidata.org/wiki/Q1084","display_name":"Noun","level":2,"score":0.19239792227745056},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/p19-1077","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1077","pdf_url":"https://www.aclweb.org/anthology/P19-1077.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.essex.ac.uk:25793","is_oa":true,"landing_page_url":null,"pdf_url":"http://repository.essex.ac.uk/25793/1/Madge2019Crowdsourcing.pdf","source":{"id":"https://openalex.org/S4306401236","display_name":"Open Access at Essex (University of Essex)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I110002522","host_organization_name":"University of Essex","host_organization_lineage":["https://openalex.org/I110002522"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1077","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1077","pdf_url":"https://www.aclweb.org/anthology/P19-1077.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.4300000071525574,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2952395386.pdf","grobid_xml":"https://content.openalex.org/works/W2952395386.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W6869908","https://openalex.org/W9014458","https://openalex.org/W31002508","https://openalex.org/W122626481","https://openalex.org/W148348866","https://openalex.org/W1489737627","https://openalex.org/W1541975828","https://openalex.org/W1543648998","https://openalex.org/W1632114991","https://openalex.org/W1918413956","https://openalex.org/W1970381522","https://openalex.org/W1971218560","https://openalex.org/W2035683813","https://openalex.org/W2073338313","https://openalex.org/W2112511942","https://openalex.org/W2123442489","https://openalex.org/W2129657639","https://openalex.org/W2141282920","https://openalex.org/W2144348168","https://openalex.org/W2155069789","https://openalex.org/W2158847908","https://openalex.org/W2164831389","https://openalex.org/W2250738489","https://openalex.org/W2250741443","https://openalex.org/W2251054028","https://openalex.org/W2251236846","https://openalex.org/W2251818274","https://openalex.org/W2251863714","https://openalex.org/W2272333457","https://openalex.org/W2295951612","https://openalex.org/W2296283641","https://openalex.org/W2476913205","https://openalex.org/W2479707183","https://openalex.org/W2575233155","https://openalex.org/W2615593621","https://openalex.org/W2625800120","https://openalex.org/W2740579382","https://openalex.org/W2764324430","https://openalex.org/W2899689163","https://openalex.org/W2944226280","https://openalex.org/W2946809241","https://openalex.org/W2963087868","https://openalex.org/W3022195534"],"related_works":["https://openalex.org/W2139373276","https://openalex.org/W3032998312","https://openalex.org/W2164455818","https://openalex.org/W2171828761","https://openalex.org/W1523977890","https://openalex.org/W1547546052","https://openalex.org/W2380927837","https://openalex.org/W2023791625","https://openalex.org/W2145020046","https://openalex.org/W2023244356"],"abstract_inverted_index":{"One":[0,115],"of":[1,12,78,89,92,116,120,146,161],"the":[2,10,13,23,117,127,136,142,147],"key":[3,118],"steps":[4],"in":[5,28,83,129],"language":[6],"resource":[7],"creation":[8],"is":[9,123,135],"identification":[11],"text":[14],"segments":[15],"to":[16,126,158],"be":[17],"annotated,":[18],"or":[19],"markablesin":[20],"our":[21,121],"case,":[22],"(potentially":[24],"nested)":[25],"noun":[26],"phrases":[27],"coreference":[29,44,139],"resolution":[30],"(or":[31],"mentions).":[32],"In":[33],"this":[34],"paper,":[35],"we":[36],"present":[37],"a":[38,55,61,76,100,159],"method":[39,66],"for":[40,43],"identifying":[41],"markables":[42,131],"annotation":[45,63,163],"that":[46],"combines":[47],"high-performance":[48],"automatic":[49,103],"markable":[50,149],"detectors":[51,150],"with":[52,54,99,138],"checking":[53],"Game-With-A-Purpose":[56],"(GWAP)":[57],"and":[58,73,81,106,144,154],"aggregation":[59],"using":[60],"Bayesian":[62],"model.":[64],"The":[65],"was":[67],"evaluated":[68],"both":[69],"on":[70,86],"news":[71],"data":[72,74],"from":[75],"variety":[77,160],"other":[79,162],"genres":[80],"results":[82],"an":[84,111],"improvement":[85],"F":[87],"1":[88],"mention":[90,104,113],"boundaries":[91],"over":[93,110],"seven":[94],"percentage":[95],"points":[96,109],"when":[97],"compared":[98],"state-of-the-art,":[101],"domain-independent":[102],"detector,":[105],"almost":[107],"three":[108],"in-domain":[112],"detector.":[114],"contributions":[119],"proposal":[122],"its":[124],"applicability":[125],"case":[128,137],"which":[130],"are":[132,151,155],"nested,":[133],"as":[134],"markables;":[140],"but":[141],"GWAP":[143],"several":[145],"proposed":[148],"task-and":[152],"language-independent":[153],"thus":[156],"applicable":[157],"scenarios.":[164]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":3}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}