{"id":"https://openalex.org/W3100083965","doi":"https://doi.org/10.1145/3388440.3412413","title":"A Qualitative Evaluation of Language Models on Automatic Question-Answering for COVID-19","display_name":"A Qualitative Evaluation of Language Models on Automatic Question-Answering for COVID-19","publication_year":2020,"publication_date":"2020-09-21","ids":{"openalex":"https://openalex.org/W3100083965","doi":"https://doi.org/10.1145/3388440.3412413","mag":"3100083965","pmid":"https://pubmed.ncbi.nlm.nih.gov/32601600"},"language":"en","primary_location":{"id":"doi:10.1145/3388440.3412413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3388440.3412413","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th ACM International Conference on Bioinformatics, Computational Biology and Health Informatics","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048086803","display_name":"David Oniani","orcid":"https://orcid.org/0000-0002-9221-3059"},"institutions":[{"id":"https://openalex.org/I4210125099","display_name":"Mayo Clinic in Arizona","ror":"https://ror.org/03jp40720","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1330342723","https://openalex.org/I4210125099"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Oniani","raw_affiliation_strings":["Mayo Clinic, Kern Center for the Science of Health Care Delivery, Rochester, MN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mayo Clinic, Kern Center for the Science of Health Care Delivery, Rochester, MN, USA","institution_ids":["https://openalex.org/I4210125099"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080116611","display_name":"Yanshan Wang","orcid":"https://orcid.org/0000-0003-4433-7839"},"institutions":[{"id":"https://openalex.org/I4210146710","display_name":"Mayo Clinic in Florida","ror":"https://ror.org/03zzw1w08","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1330342723","https://openalex.org/I4210146710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yanshan Wang","raw_affiliation_strings":["Mayo Clinic, Division of Digital Health Sciences, Rochester, MN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mayo Clinic, Division of Digital Health Sciences, Rochester, MN, USA","institution_ids":["https://openalex.org/I4210146710"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9842000007629395,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7729155421257019},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5883949398994446},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5693032145500183},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5197193622589111},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.5152392983436584},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5042587518692017},{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.46449780464172363},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4638526439666748},{"id":"https://openalex.org/keywords/coronavirus-disease-2019","display_name":"Coronavirus disease 2019 (COVID-19)","score":0.4416252672672272},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.42513319849967957},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.42027798295021057},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4191405475139618},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.37236320972442627},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2890227437019348},{"id":"https://openalex.org/keywords/infectious-disease","display_name":"Infectious disease (medical specialty)","score":0.10921633243560791},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.09141430258750916}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7729155421257019},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5883949398994446},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5693032145500183},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5197193622589111},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5152392983436584},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5042587518692017},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.46449780464172363},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4638526439666748},{"id":"https://openalex.org/C3008058167","wikidata":"https://www.wikidata.org/wiki/Q84263196","display_name":"Coronavirus disease 2019 (COVID-19)","level":4,"score":0.4416252672672272},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.42513319849967957},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42027798295021057},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4191405475139618},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37236320972442627},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2890227437019348},{"id":"https://openalex.org/C524204448","wikidata":"https://www.wikidata.org/wiki/Q788926","display_name":"Infectious disease (medical specialty)","level":3,"score":0.10921633243560791},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.09141430258750916},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2779134260","wikidata":"https://www.wikidata.org/wiki/Q12136","display_name":"Disease","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3388440.3412413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3388440.3412413","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th ACM International Conference on Bioinformatics, Computational Biology and Health Informatics","raw_type":"proceedings-article"},{"id":"pmid:32601600","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/32601600","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ArXiv","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:7316096","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7316096","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ArXiv","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1145/3388440.3412413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3388440.3412413","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th ACM International Conference on Bioinformatics, Computational Biology and Health Informatics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.550000011920929,"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being"}],"awards":[{"id":"https://openalex.org/G792078797","display_name":null,"funder_award_id":"R01LM11934","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"}],"funders":[{"id":"https://openalex.org/F4320306143","display_name":"Mayo Clinic","ror":"https://ror.org/02qp3tb03"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3100083965.pdf","grobid_xml":"https://content.openalex.org/works/W3100083965.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W2101105183","https://openalex.org/W2154652894","https://openalex.org/W2608377846","https://openalex.org/W2794557536","https://openalex.org/W2880875857","https://openalex.org/W2911489562","https://openalex.org/W2944400536","https://openalex.org/W2949233314","https://openalex.org/W2949888546","https://openalex.org/W2950813464","https://openalex.org/W2954936631","https://openalex.org/W2962739339","https://openalex.org/W2967766298","https://openalex.org/W2975429091","https://openalex.org/W2981123712","https://openalex.org/W2984284833","https://openalex.org/W3008957926","https://openalex.org/W3015197879","https://openalex.org/W3015556068","https://openalex.org/W3024009241","https://openalex.org/W3037026762","https://openalex.org/W3067551478","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W3204607391","https://openalex.org/W2964413124","https://openalex.org/W4388937922","https://openalex.org/W3157284875","https://openalex.org/W2259406085","https://openalex.org/W3113264705","https://openalex.org/W2099715052","https://openalex.org/W4226247999","https://openalex.org/W4213176082","https://openalex.org/W2187398150"],"abstract_inverted_index":{"COVID-19":[0,35,171,193],"(2019":[1],"Novel":[2],"Coronavirus)":[3],"has":[4,16,36,120,325],"resulted":[5],"in":[6,105,113,140,249,279,312,329],"an":[7],"ongoing":[8],"pandemic":[9],"and":[10,23,30,51,62,67,90,95,138,151,172,184,238,245,270,277,300,341],"as":[11],"of":[12,78,87,108,116,205,307],"26":[13],"July":[14],"2020,":[15],"caused":[17],"more":[18],"than":[19],"15.7":[20],"million":[21],"cases":[22],"over":[24],"640,000":[25],"deaths.":[26],"The":[27],"highly":[28],"dynamic":[29],"rapidly":[31],"evolving":[32],"situation":[33],"with":[34,149],"made":[37,301],"it":[38,119,190,314],"difficult":[39],"to":[40,57,75,92,123,144,160,170,188,201,243,262,296,309,338],"access":[41],"accurate,":[42],"on-demand":[43],"information":[44,147],"regarding":[45],"the":[46,76,103,106,114,141,146,175,180,192,203,206,250,253,264,287],"disease.":[47],"Online":[48],"communities,":[49],"forums,":[50],"social":[52],"media":[53],"provide":[54],"potential":[55],"venues":[56],"search":[58,93],"for":[59,165,233,319],"relevant":[60,88,247],"questions":[61,66,89,97,168,340],"answers,":[63],"or":[64,317],"post":[65],"seek":[68],"answers":[69],"from":[70,225,231],"other":[71],"members.":[72],"However,":[73,132],"due":[74],"nature":[77],"such":[79,133],"sites,":[80],"there":[81],"are":[82,98,135],"always":[83],"a":[84,162,291,332],"limited":[85],"number":[86],"responses":[91,337],"from,":[94],"posted":[96],"rarely":[99,136],"answered":[100],"immediately.":[101],"With":[102],"advancements":[104],"field":[107],"natural":[109],"language":[110,117,163,182],"processing,":[111],"particularly":[112],"domain":[115],"models,":[118],"become":[121],"possible":[122],"design":[124],"chatbots":[125],"that":[126,268,334],"can":[127],"automatically":[128,166],"answer":[129],"consumer":[130],"questions.":[131],"models":[134],"applied":[137,185,210],"evaluated":[139],"healthcare":[142,153],"domain,":[143],"meet":[145],"needs":[148],"accurate":[150],"up-to-date":[152],"data.":[154],"In":[155,199,252],"this":[156],"paper,":[157],"we":[158,209,257,289],"propose":[159],"apply":[161],"model":[164,183],"answering":[167],"related":[169],"qualitatively":[173],"evaluate":[174],"generated":[176,207],"responses.":[177,251,265],"We":[178,266],"utilized":[179],"GPT-2":[181],"transfer":[186],"learning":[187],"retrain":[189],"on":[191,272,286],"Open":[194],"Research":[195],"Dataset":[196],"(CORD-19)":[197],"corpus.":[198],"order":[200],"improve":[202],"quality":[204],"responses,":[208],"4":[211],"different":[212],"approaches,":[213],"namely":[214],"tf-idf":[215,276],"(Term":[216],"Frequency":[217],"-":[218],"Inverse":[219],"Document":[220],"Frequency),":[221],"Bidirectional":[222,228],"Encoder":[223,229,241],"Representations":[224,230],"Transformers":[226,232],"(BERT),":[227],"Biomedical":[234],"Text":[235],"Mining":[236],"(BioBERT),":[237],"Universal":[239],"Sentence":[240],"(USE)":[242],"filter":[244],"retain":[246],"sentences":[248],"performance":[254],"evaluation":[255],"step,":[256],"asked":[258],"two":[259],"medical":[260],"experts":[261],"rate":[263],"found":[267],"BERT":[269],"BioBERT,":[271],"average,":[273],"outperform":[274],"both":[275,330],"USE":[278],"relevance-based":[280],"sentence":[281],"filtering":[282],"tasks.":[283],"Additionally,":[284],"based":[285],"chatbot,":[288],"created":[290],"user-friendly":[292],"interactive":[293],"web":[294],"application":[295],"be":[297],"hosted":[298],"online":[299],"its":[302],"source":[303],"code":[304],"available":[305],"free":[306],"charge":[308],"anyone":[310],"interested":[311],"running":[313],"locally,":[315],"online,":[316],"just":[318],"experimental":[320],"purposes.":[321],"Overall,":[322],"our":[323],"work":[324],"yielded":[326],"significant":[327],"results":[328],"designing":[331],"chatbot":[333],"produces":[335],"high-quality":[336],"COVID-19-related":[339],"comparing":[342],"several":[343],"embedding":[344],"generation":[345],"techniques.":[346]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":10},{"year":2020,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}