{"id":"https://openalex.org/W2251477123","doi":"https://doi.org/10.18653/v1/w15-2614","title":"Effectively Crowdsourcing Radiology Report Annotations","display_name":"Effectively Crowdsourcing Radiology Report Annotations","publication_year":2015,"publication_date":"2015-01-01","ids":{"openalex":"https://openalex.org/W2251477123","doi":"https://doi.org/10.18653/v1/w15-2614","mag":"2251477123"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w15-2614","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w15-2614","pdf_url":"https://www.aclweb.org/anthology/W15-2614.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Sixth International Workshop on Health Text Mining and Information Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W15-2614.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113775732","display_name":"Anne Cocos","orcid":null},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Anne Cocos","raw_affiliation_strings":["Computer and Information Sciences Department, University of Pennsylvania"],"affiliations":[{"raw_affiliation_string":"Computer and Information Sciences Department, University of Pennsylvania","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053822814","display_name":"Aaron J. Masino","orcid":"https://orcid.org/0000-0002-2684-0548"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaron Masino","raw_affiliation_strings":["Computer and Information Sciences Department, University of Pennsylvania"],"affiliations":[{"raw_affiliation_string":"Computer and Information Sciences Department, University of Pennsylvania","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043567153","display_name":"Ting Qian","orcid":"https://orcid.org/0000-0001-6113-0341"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ting Qian","raw_affiliation_strings":["Computer and Information Sciences Department, University of Pennsylvania"],"affiliations":[{"raw_affiliation_string":"Computer and Information Sciences Department, University of Pennsylvania","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053850863","display_name":"Ellie Pavlick","orcid":"https://orcid.org/0000-0002-7155-5420"},"institutions":[{"id":"https://openalex.org/I1335321130","display_name":"Children's Hospital of Philadelphia","ror":"https://ror.org/01z7r7q48","country_code":"US","type":"funder","lineage":["https://openalex.org/I1335321130"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ellie Pavlick","raw_affiliation_strings":["Department of Biomedical and Health Informatics, The Children's Hospital of Philadelphia"],"affiliations":[{"raw_affiliation_string":"Department of Biomedical and Health Informatics, The Children's Hospital of Philadelphia","institution_ids":["https://openalex.org/I1335321130"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068508539","display_name":"Chris Callison-Burch","orcid":"https://orcid.org/0000-0001-8196-1943"},"institutions":[{"id":"https://openalex.org/I1335321130","display_name":"Children's Hospital of Philadelphia","ror":"https://ror.org/01z7r7q48","country_code":"US","type":"funder","lineage":["https://openalex.org/I1335321130"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Callison-Burch","raw_affiliation_strings":["Department of Biomedical and Health Informatics, The Children's Hospital of Philadelphia"],"affiliations":[{"raw_affiliation_string":"Department of Biomedical and Health Informatics, The Children's Hospital of Philadelphia","institution_ids":["https://openalex.org/I1335321130"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5113775732"],"corresponding_institution_ids":["https://openalex.org/I79576946"],"apc_list":null,"apc_paid":null,"fwci":2.0113,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.8987634,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"109","last_page":"114"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9652924537658691},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8108032941818237},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6573579907417297},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5511741042137146},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.5241549015045166},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5080814361572266},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.49970340728759766},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.44212955236434937},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.42979371547698975},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4219020903110504},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.30645671486854553}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9652924537658691},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8108032941818237},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6573579907417297},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5511741042137146},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5241549015045166},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5080814361572266},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49970340728759766},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.44212955236434937},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.42979371547698975},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4219020903110504},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.30645671486854553},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/w15-2614","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w15-2614","pdf_url":"https://www.aclweb.org/anthology/W15-2614.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Sixth International Workshop on Health Text Mining and Information Analysis","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/w15-2614","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w15-2614","pdf_url":"https://www.aclweb.org/anthology/W15-2614.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Sixth International Workshop on Health Text Mining and Information Analysis","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2251477123.pdf","grobid_xml":"https://content.openalex.org/works/W2251477123.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1541268242","https://openalex.org/W1826790618","https://openalex.org/W1969605785","https://openalex.org/W1970381522","https://openalex.org/W2050611670","https://openalex.org/W2054593810","https://openalex.org/W2056584528","https://openalex.org/W2064677871","https://openalex.org/W2073338313","https://openalex.org/W2083355945","https://openalex.org/W2087426400","https://openalex.org/W2093044076","https://openalex.org/W2123442489","https://openalex.org/W2139531859","https://openalex.org/W2158880898","https://openalex.org/W2164777277","https://openalex.org/W2251687211","https://openalex.org/W2251939518","https://openalex.org/W3103797191"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W2369835347","https://openalex.org/W3098003361"],"abstract_inverted_index":{"Crowdsourcing":[0],"platforms":[1],"are":[2,20],"a":[3,31],"popular":[4],"choice":[5],"for":[6],"researchers":[7],"to":[8,39,53],"gather":[9],"text":[10],"annotations":[11,19],"quickly":[12],"at":[13],"scale.":[14],"We":[15,63],"investigate":[16],"whether":[17],"crowdsourced":[18,50],"useful":[21],"when":[22],"the":[23,40,49,59,66,69],"labeling":[24],"task":[25],"requires":[26],"medical":[27],"domain":[28],"knowledge.":[29],"Comparing":[30],"sentence":[32],"classification":[33],"model":[34,42,71],"trained":[35,43],"with":[36,82],"expert-annotated":[37],"sentences":[38],"same":[41],"on":[44],"crowd-labeled":[45],"sentences,":[46],"we":[47],"find":[48],"training":[51],"data":[52],"be":[54],"just":[55],"as":[56,58],"effective":[57],"manually":[60],"produced":[61],"dataset.":[62],"can":[64],"improve":[65],"accuracy":[67],"of":[68],"crowd-fueled":[70],"without":[72],"collecting":[73],"further":[74],"labels":[75,80],"by":[76],"filtering":[77],"out":[78],"worker":[79],"applied":[81],"low":[83],"confidence.":[84]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}