{"id":"https://openalex.org/W2759357274","doi":"https://doi.org/10.18653/v1/d17-1205","title":"CROWD-IN-THE-LOOP: A Hybrid Approach for Annotating Semantic Roles","display_name":"CROWD-IN-THE-LOOP: A Hybrid Approach for Annotating Semantic Roles","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2759357274","doi":"https://doi.org/10.18653/v1/d17-1205","mag":"2759357274"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d17-1205","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1205","pdf_url":"https://www.aclweb.org/anthology/D17-1205.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D17-1205.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100329692","display_name":"Chenguang Wang","orcid":"https://orcid.org/0000-0002-7896-2360"},"institutions":[{"id":"https://openalex.org/I4210085935","display_name":"IBM Research - Almaden","ror":"https://ror.org/005w8dd04","country_code":"US","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210085935","https://openalex.org/I4210114115"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenguang Wang","raw_affiliation_strings":["IBM Research -Almaden \u2021 Zalando Research, Berlin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research -Almaden \u2021 Zalando Research, Berlin","institution_ids":["https://openalex.org/I4210085935"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032877157","display_name":"Alan Akbik","orcid":null},"institutions":[{"id":"https://openalex.org/I4210085935","display_name":"IBM Research - Almaden","ror":"https://ror.org/005w8dd04","country_code":"US","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210085935","https://openalex.org/I4210114115"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alan Akbik","raw_affiliation_strings":["IBM Research -Almaden \u2021 Zalando Research, Berlin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research -Almaden \u2021 Zalando Research, Berlin","institution_ids":["https://openalex.org/I4210085935"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050310059","display_name":"Laura Chiticariu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210085935","display_name":"IBM Research - Almaden","ror":"https://ror.org/005w8dd04","country_code":"US","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210085935","https://openalex.org/I4210114115"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"laura chiticariu","raw_affiliation_strings":["IBM Research -Almaden \u2021 Zalando Research, Berlin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research -Almaden \u2021 Zalando Research, Berlin","institution_ids":["https://openalex.org/I4210085935"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102944075","display_name":"Yunyao Li","orcid":"https://orcid.org/0009-0002-0814-4634"},"institutions":[{"id":"https://openalex.org/I4210085935","display_name":"IBM Research - Almaden","ror":"https://ror.org/005w8dd04","country_code":"US","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210085935","https://openalex.org/I4210114115"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yunyao Li","raw_affiliation_strings":["IBM Research -Almaden \u2021 Zalando Research, Berlin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research -Almaden \u2021 Zalando Research, Berlin","institution_ids":["https://openalex.org/I4210085935"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100676785","display_name":"Fei Xia","orcid":"https://orcid.org/0000-0003-4343-1444"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fei Xia","raw_affiliation_strings":["Department of Linguistics, University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Linguistics, University of Washington","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053418697","display_name":"Anbang Xu","orcid":"https://orcid.org/0009-0005-9707-7817"},"institutions":[{"id":"https://openalex.org/I4210085935","display_name":"IBM Research - Almaden","ror":"https://ror.org/005w8dd04","country_code":"US","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210085935","https://openalex.org/I4210114115"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anbang Xu","raw_affiliation_strings":["IBM Research -Almaden \u2021 Zalando Research, Berlin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research -Almaden \u2021 Zalando Research, Berlin","institution_ids":["https://openalex.org/I4210085935"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.5615,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.90271547,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1913","last_page":"1922"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9858999848365784,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8674443364143372},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8038158416748047},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.7774671912193298},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.772192120552063},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6815847754478455},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5923416614532471},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5890728831291199},{"id":"https://openalex.org/keywords/semantic-role-labeling","display_name":"Semantic role labeling","score":0.4732438027858734},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46842730045318604},{"id":"https://openalex.org/keywords/labeled-data","display_name":"Labeled data","score":0.4561885595321655},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.44425255060195923},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.39706844091415405},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38714805245399475},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.21477651596069336},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.16517066955566406},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.06952443718910217}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8674443364143372},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8038158416748047},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.7774671912193298},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.772192120552063},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6815847754478455},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5923416614532471},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5890728831291199},{"id":"https://openalex.org/C67277372","wikidata":"https://www.wikidata.org/wiki/Q7449085","display_name":"Semantic role labeling","level":3,"score":0.4732438027858734},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46842730045318604},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.4561885595321655},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44425255060195923},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.39706844091415405},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38714805245399475},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.21477651596069336},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.16517066955566406},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.06952443718910217},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/d17-1205","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1205","pdf_url":"https://www.aclweb.org/anthology/D17-1205.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/d17-1205","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1205","pdf_url":"https://www.aclweb.org/anthology/D17-1205.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.4300000071525574}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2759357274.pdf","grobid_xml":"https://content.openalex.org/works/W2759357274.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W11155487","https://openalex.org/W1522548332","https://openalex.org/W1867048546","https://openalex.org/W1970381522","https://openalex.org/W2029097927","https://openalex.org/W2075397336","https://openalex.org/W2088911157","https://openalex.org/W2092045293","https://openalex.org/W2115792525","https://openalex.org/W2138445383","https://openalex.org/W2140384587","https://openalex.org/W2158847908","https://openalex.org/W2167187514","https://openalex.org/W2168144930","https://openalex.org/W2251045406","https://openalex.org/W2251199578","https://openalex.org/W2251386628","https://openalex.org/W2251928326","https://openalex.org/W2417677256","https://openalex.org/W2559038528","https://openalex.org/W2563997644","https://openalex.org/W2566063862","https://openalex.org/W2574749251","https://openalex.org/W2727587971","https://openalex.org/W2740365665","https://openalex.org/W2950314731","https://openalex.org/W4213168938"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2962926863","https://openalex.org/W2889433666","https://openalex.org/W4289670492","https://openalex.org/W1982477054"],"abstract_inverted_index":{"Crowdsourcing":[0],"has":[1],"proven":[2],"to":[3,25,48,93,103,109],"be":[4,49],"an":[5],"effective":[6],"method":[7],"for":[8,12,30,79,123],"generating":[9],"labeled":[10],"data":[11,29],"a":[13,46,68,84,91],"range":[14],"of":[15,22,71,134],"NLP":[16],"tasks.":[17],"However,":[18],"multiple":[19],"recent":[20],"attempts":[21],"using":[23],"crowdsourcing":[24],"generate":[26],"gold-labeled":[27],"training":[28],"semantic":[31],"role":[32],"labeling":[33,73],"(SRL)":[34],"reported":[35],"only":[36],"modest":[37],"results,":[38],"indicating":[39],"that":[40,57,116],"SRL":[41,60,72,136],"is":[42,75],"perhaps":[43],"too":[44],"difficult":[45,95],"task":[47,101],"effectively":[50],"crowdsourced.":[51],"In":[52],"this":[53],"paper,":[54],"we":[55,89],"postulate":[56],"while":[58],"producing":[59,135],"annotation":[61,96,137],"does":[62],"require":[63],"expert":[64],"involvement":[65],"in":[66,76,87,141],"general,":[67],"large":[69],"subset":[70],"tasks":[74,97],"fact":[77],"appropriate":[78],"the":[80,117,121,132],"crowd.":[81],"We":[82],"present":[83],"novel":[85],"workflow":[86],"which":[88],"employ":[90],"classifier":[92],"identify":[94],"and":[98,128],"route":[99],"each":[100],"either":[102],"experts":[104,124],"or":[105],"crowd":[106],"workers":[107],"according":[108],"their":[110],"difficulties.":[111],"Our":[112],"experimental":[113],"evaluation":[114],"shows":[115],"proposed":[118],"approach":[119],"reduces":[120,131],"workload":[122],"by":[125],"over":[126],"two-thirds,":[127],"thus":[129],"significantly":[130],"cost":[133],"at":[138],"little":[139],"loss":[140],"quality.":[142]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":3}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}