{"id":"https://openalex.org/W3133569056","doi":"https://doi.org/10.1145/3442188.3445909","title":"A Pilot Study in Surveying Clinical Judgments to Evaluate Radiology Report Generation","display_name":"A Pilot Study in Surveying Clinical Judgments to Evaluate Radiology Report Generation","publication_year":2021,"publication_date":"2021-02-25","ids":{"openalex":"https://openalex.org/W3133569056","doi":"https://doi.org/10.1145/3442188.3445909","mag":"3133569056"},"language":"en","primary_location":{"id":"doi:10.1145/3442188.3445909","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442188.3445909","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3442188.3445909","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3442188.3445909","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006156483","display_name":"William Boag","orcid":"https://orcid.org/0000-0002-1485-5806"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"William Boag","raw_affiliation_strings":["MIT, USA"],"affiliations":[{"raw_affiliation_string":"MIT, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045331755","display_name":"Hassan Kan\u00e9","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hassan Kan\u00e9","raw_affiliation_strings":["WL Research, USA"],"affiliations":[{"raw_affiliation_string":"WL Research, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084318906","display_name":"Saumya Rawat","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Saumya Rawat","raw_affiliation_strings":["MIT, USA"],"affiliations":[{"raw_affiliation_string":"MIT, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013600043","display_name":"Jesse Wei","orcid":null},"institutions":[{"id":"https://openalex.org/I1316535847","display_name":"Beth Israel Deaconess Medical Center","ror":"https://ror.org/04drvxt59","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1316535847"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jesse Wei","raw_affiliation_strings":["Beth Israel Deaconess Medical Center, Department of Radiology, USA"],"affiliations":[{"raw_affiliation_string":"Beth Israel Deaconess Medical Center, Department of Radiology, USA","institution_ids":["https://openalex.org/I1316535847"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087071972","display_name":"Alexander Goehler","orcid":"https://orcid.org/0000-0002-6237-2032"},"institutions":[{"id":"https://openalex.org/I1316535847","display_name":"Beth Israel Deaconess Medical Center","ror":"https://ror.org/04drvxt59","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1316535847"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander Goehler","raw_affiliation_strings":["Beth Israel Deaconess Medical Center, Department of Radiology, USA"],"affiliations":[{"raw_affiliation_string":"Beth Israel Deaconess Medical Center, Department of Radiology, USA","institution_ids":["https://openalex.org/I1316535847"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5006156483"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.42,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.67750191,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":"269","issue":null,"first_page":"458","last_page":"465"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7517017126083374},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6658855676651001},{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.6435502767562866},{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.6241599917411804},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.6154661774635315},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.5752273201942444},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.555591344833374},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5303688049316406},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5283212065696716},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5276461839675903},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5047739744186401},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.46529874205589294},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.45388326048851013},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4317072629928589},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.41230276226997375},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3706611096858978},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.24917879700660706},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.15121009945869446},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08400008082389832},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.08342820405960083}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7517017126083374},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6658855676651001},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.6435502767562866},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.6241599917411804},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.6154661774635315},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.5752273201942444},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.555591344833374},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5303688049316406},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5283212065696716},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5276461839675903},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5047739744186401},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.46529874205589294},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.45388326048851013},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4317072629928589},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.41230276226997375},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3706611096858978},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.24917879700660706},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.15121009945869446},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08400008082389832},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.08342820405960083},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3442188.3445909","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442188.3445909","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3442188.3445909","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3442188.3445909","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442188.3445909","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3442188.3445909","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.699999988079071}],"awards":[{"id":"https://openalex.org/G3526151021","display_name":null,"funder_award_id":"1122374","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5635854791","display_name":null,"funder_award_id":"112237","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7297461854","display_name":null,"funder_award_id":"122374","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3133569056.pdf","grobid_xml":"https://content.openalex.org/works/W3133569056.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1489525520","https://openalex.org/W1509228788","https://openalex.org/W1593271688","https://openalex.org/W1602368597","https://openalex.org/W1956340063","https://openalex.org/W1967390364","https://openalex.org/W2101105183","https://openalex.org/W2123301721","https://openalex.org/W2152772232","https://openalex.org/W2250790822","https://openalex.org/W2250875036","https://openalex.org/W2334763311","https://openalex.org/W2512434509","https://openalex.org/W2600463316","https://openalex.org/W2756675635","https://openalex.org/W2760738985","https://openalex.org/W2777186991","https://openalex.org/W2806532810","https://openalex.org/W2891266422","https://openalex.org/W2895763047","https://openalex.org/W2903376039","https://openalex.org/W2918257301","https://openalex.org/W2929309437","https://openalex.org/W2944016032","https://openalex.org/W2947771965","https://openalex.org/W2963466845","https://openalex.org/W2970986500","https://openalex.org/W3000014100","https://openalex.org/W3087927280","https://openalex.org/W3096833762","https://openalex.org/W3099561884"],"related_works":["https://openalex.org/W4210416330","https://openalex.org/W2775506363","https://openalex.org/W3088136942","https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4290852288","https://openalex.org/W2949362007","https://openalex.org/W4388893791","https://openalex.org/W4283207562","https://openalex.org/W4284703357"],"abstract_inverted_index":{"The":[0,48,120],"recent":[1],"release":[2],"of":[3,12,44,50,115],"many":[4],"Chest":[5],"X-Ray":[6],"datasets":[7],"has":[8,21,53],"prompted":[9],"a":[10,40,103,108,133,145],"lot":[11],"interest":[13],"in":[14,106],"radiology":[15,118],"report":[16],"generation.":[17],"To":[18,90],"date,":[19],"this":[20],"been":[22,54],"framed":[23],"as":[24,36,46,67],"an":[25,33,116],"image":[26,35],"captioning":[27],"task,":[28],"where":[29,85],"the":[30,60,73,82,113],"machine":[31,97],"takes":[32],"RGB":[34],"input":[37],"and":[38,70,128],"generates":[39],"2-3":[41],"sentence":[42],"summary":[43],"findings":[45],"output.":[47],"quality":[49,114],"these":[51],"reports":[52],"canonically":[55],"measured":[56],"using":[57],"metrics":[58,75],"from":[59],"NLP":[61],"community":[62],"for":[63,81,102,111],"language":[64],"generation":[65],"such":[66],"Machine":[68],"Translation":[69],"Summarization.":[71],"However,":[72],"evaluation":[74,148],"(e.g.":[76],"BLEU,":[77],"CIDEr)":[78],"are":[79],"inappropriate":[80],"medical":[83],"domain,":[84],"clinical":[86],"correctness":[87],"is":[88,139],"critical.":[89],"address":[91],"this,":[92],"our":[93],"team":[94],"brought":[95],"together":[96],"learning":[98],"experts":[99],"with":[100],"radiologists":[101],"pilot":[104],"study":[105,136],"co-designing":[107],"better":[109],"metric":[110],"evaluating":[112],"algorithmically-generated":[117],"report.":[119],"interdisciplinary":[121],"collaborative":[122],"process":[123],"involved":[124],"multiple":[125],"interviews,":[126],"outreach,":[127],"preliminary":[129],"annotation":[130],"to":[131,143],"design":[132],"larger":[134],"scale":[135],"-":[137,142],"which":[138],"now":[140],"underway":[141],"build":[144],"more":[146],"meaningful":[147],"tool.":[149]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}