{"id":"https://openalex.org/W3100083965","doi":"https://doi.org/10.1145/3388440.3412413","title":"A Qualitative Evaluation of Language Models on Automatic Question-Answering for COVID-19","display_name":"A Qualitative Evaluation of Language Models on Automatic Question-Answering for COVID-19","publication_year":2020,"publication_date":"2020-09-21","ids":{"openalex":"https://openalex.org/W3100083965","doi":"https://doi.org/10.1145/3388440.3412413","mag":"3100083965","pmid":"https://pubmed.ncbi.nlm.nih.gov/32601600"},"language":"en","primary_location":{"id":"doi:10.1145/3388440.3412413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3388440.3412413","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th ACM International Conference on Bioinformatics, Computational Biology and Health Informatics","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048086803","display_name":"David Oniani","orcid":"https://orcid.org/0000-0002-9221-3059"},"institutions":[{"id":"https://openalex.org/I4210125099","display_name":"Mayo Clinic in Arizona","ror":"https://ror.org/03jp40720","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1330342723","https://openalex.org/I4210125099"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"David Oniani","raw_affiliation_strings":["Mayo Clinic, Kern Center for the Science of Health Care Delivery, Rochester, MN, USA"],"affiliations":[{"raw_affiliation_string":"Mayo Clinic, Kern Center for the Science of Health Care Delivery, Rochester, MN, USA","institution_ids":["https://openalex.org/I4210125099"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080116611","display_name":"Yanshan Wang","orcid":"https://orcid.org/0000-0003-4433-7839"},"institutions":[{"id":"https://openalex.org/I4210146710","display_name":"Mayo Clinic in Florida","ror":"https://ror.org/03zzw1w08","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1330342723","https://openalex.org/I4210146710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yanshan Wang","raw_affiliation_strings":["Mayo Clinic, Division of Digital Health Sciences, Rochester, MN, USA"],"affiliations":[{"raw_affiliation_string":"Mayo Clinic, Division of Digital Health Sciences, Rochester, MN, USA","institution_ids":["https://openalex.org/I4210146710"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5048086803"],"corresponding_institution_ids":["https://openalex.org/I4210125099"],"apc_list":null,"apc_paid":null,"fwci":3.8057,"has_fulltext":false,"cited_by_count":36,"citation_normalized_percentile":{"value":0.94613841,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9842000007629395,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7729155421257019},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5883949398994446},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5693032145500183},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5197193622589111},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.5152392983436584},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5042587518692017},{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.46449780464172363},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4638526439666748},{"id":"https://openalex.org/keywords/coronavirus-disease-2019","display_name":"Coronavirus disease 2019 (COVID-19)","score":0.4416252672672272},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.42513319849967957},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.42027798295021057},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4191405475139618},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.37236320972442627},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2890227437019348},{"id":"https://openalex.org/keywords/infectious-disease","display_name":"Infectious disease (medical specialty)","score":0.10921633243560791},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.09141430258750916}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7729155421257019},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5883949398994446},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5693032145500183},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5197193622589111},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5152392983436584},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5042587518692017},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.46449780464172363},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4638526439666748},{"id":"https://openalex.org/C3008058167","wikidata":"https://www.wikidata.org/wiki/Q84263196","display_name":"Coronavirus disease 2019 (COVID-19)","level":4,"score":0.4416252672672272},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.42513319849967957},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42027798295021057},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4191405475139618},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37236320972442627},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2890227437019348},{"id":"https://openalex.org/C524204448","wikidata":"https://www.wikidata.org/wiki/Q788926","display_name":"Infectious disease (medical specialty)","level":3,"score":0.10921633243560791},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.09141430258750916},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2779134260","wikidata":"https://www.wikidata.org/wiki/Q12136","display_name":"Disease","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3388440.3412413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3388440.3412413","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th ACM International Conference on Bioinformatics, Computational Biology and Health Informatics","raw_type":"proceedings-article"},{"id":"pmid:32601600","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/32601600","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ArXiv","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:7316096","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7316096","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ArXiv","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1145/3388440.3412413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3388440.3412413","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3388440.3412413","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th ACM International Conference on Bioinformatics, Computational Biology and Health Informatics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Good health and well-being","id":"https://metadata.un.org/sdg/3","score":0.550000011920929}],"awards":[{"id":"https://openalex.org/G792078797","display_name":null,"funder_award_id":"R01LM11934","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"}],"funders":[{"id":"https://openalex.org/F4320306143","display_name":"Mayo Clinic","ror":"https://ror.org/02qp3tb03"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3100083965.pdf","grobid_xml":"https://content.openalex.org/works/W3100083965.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W2101105183","https://openalex.org/W2154652894","https://openalex.org/W2608377846","https://openalex.org/W2794557536","https://openalex.org/W2880875857","https://openalex.org/W2911489562","https://openalex.org/W2944400536","https://openalex.org/W2949233314","https://openalex.org/W2949888546","https://openalex.org/W2950813464","https://openalex.org/W2954936631","https://openalex.org/W2962739339","https://openalex.org/W2967766298","https://openalex.org/W2975429091","https://openalex.org/W2981123712","https://openalex.org/W2984284833","https://openalex.org/W3008957926","https://openalex.org/W3015197879","https://openalex.org/W3015556068","https://openalex.org/W3024009241","https://openalex.org/W3037026762","https://openalex.org/W3067551478","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W3204607391","https://openalex.org/W2964413124","https://openalex.org/W4388937922","https://openalex.org/W3157284875","https://openalex.org/W2259406085","https://openalex.org/W3113264705","https://openalex.org/W2099715052","https://openalex.org/W4226247999","https://openalex.org/W4213176082","https://openalex.org/W2187398150"],"abstract_inverted_index":{"COVID-19":[0,32,168,190],"has":[1,13,33,117],"resulted":[2],"in":[3,102,110,137,223,253],"an":[4],"ongoing":[5],"pandemic":[6],"and":[7,20,27,48,59,64,87,92,135,148,169,181,215,219,244,251],"as":[8],"of":[9,75,84,105,113,202],"12":[10],"June":[11],"2020,":[12],"caused":[14],"more":[15],"than":[16],"7.4":[17],"million":[18],"cases":[19],"over":[21],"418,000":[22],"deaths.":[23],"The":[24],"highly":[25],"dynamic":[26],"rapidly":[28],"evolving":[29],"situation":[30],"with":[31,146],"made":[34],"it":[35,116,187],"difficult":[36],"to":[37,54,72,89,120,141,157,167,185,198,217,236,270],"access":[38],"accurate,":[39],"on-demand":[40],"information":[41,144],"regarding":[42],"the":[43,73,100,103,111,138,143,172,177,189,200,203,224,227,238,261],"disease.":[44],"Online":[45],"communities,":[46],"forums,":[47],"social":[49],"media":[50],"provide":[51],"potential":[52],"venues":[53],"search":[55,90],"for":[56,162],"relevant":[57,85,221],"questions":[58,63,86,94,165],"answers,":[60],"or":[61],"post":[62],"seek":[65],"answers":[66],"from":[67],"other":[68],"members.":[69],"However,":[70,129],"due":[71],"nature":[74],"such":[76,130],"sites,":[77],"there":[78],"are":[79,95,132],"always":[80],"a":[81,159,265],"limited":[82],"number":[83],"responses":[88],"from,":[91],"posted":[93],"rarely":[96,133],"answered":[97],"immediately.":[98],"With":[99],"advancements":[101],"field":[104],"natural":[106],"language":[107,114,160,179],"processing,":[108],"particularly":[109],"domain":[112],"models,":[115],"become":[118],"possible":[119],"design":[121],"chatbots":[122],"that":[123,242],"can":[124],"automatically":[125,163],"answer":[126],"consumer":[127],"questions.":[128],"models":[131],"applied":[134,182,207],"evaluated":[136],"healthcare":[139,150],"domain,":[140],"meet":[142],"needs":[145],"accurate":[147],"up-to-date":[149],"data.":[151],"In":[152,196,226],"this":[153],"paper,":[154],"we":[155,206,231,263],"propose":[156],"apply":[158],"model":[161,180],"answering":[164],"related":[166],"qualitatively":[170],"evaluate":[171],"generated":[173,204],"responses.":[174,225,239],"We":[175,240],"utilized":[176],"GPT-2":[178],"transfer":[183],"learning":[184],"retrain":[186],"on":[188,246,260],"Open":[191],"Research":[192],"Dataset":[193],"(CORD-19)":[194],"corpus.":[195],"order":[197],"improve":[199],"quality":[201],"responses,":[205],"4":[208],"different":[209],"approaches,":[210],"namely":[211],"tf-idf,":[212],"BERT,":[213],"BioBERT,":[214,245],"USE":[216,252],"filter":[218],"retain":[220],"sentences":[222],"performance":[228],"evaluation":[229],"step,":[230],"asked":[232],"two":[233],"medical":[234],"experts":[235],"rate":[237],"found":[241],"BERT":[243],"average,":[247],"outperform":[248],"both":[249],"tf-idf":[250],"relevance-based":[254],"sentence":[255],"filtering":[256],"tasks.":[257],"Additionally,":[258],"based":[259],"chatbot,":[262],"created":[264],"user-friendly":[266],"interactive":[267],"web":[268],"application":[269],"be":[271],"hosted":[272],"online.":[273]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":10},{"year":2020,"cited_by_count":2}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}