{"id":"https://openalex.org/W3046031915","doi":"https://doi.org/10.1093/jamia/ocaa271","title":"An evaluation of two commercial deep learning-based information retrieval systems for COVID-19 literature","display_name":"An evaluation of two commercial deep learning-based information retrieval systems for COVID-19 literature","publication_year":2020,"publication_date":"2020-11-13","ids":{"openalex":"https://openalex.org/W3046031915","doi":"https://doi.org/10.1093/jamia/ocaa271","mag":"3046031915","pmid":"https://pubmed.ncbi.nlm.nih.gov/33197268"},"language":"en","primary_location":{"id":"doi:10.1093/jamia/ocaa271","is_oa":true,"landing_page_url":"https://doi.org/10.1093/jamia/ocaa271","pdf_url":"https://academic.oup.com/jamia/article-pdf/28/1/132/35885578/ocaa271.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite","pubmed"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://academic.oup.com/jamia/article-pdf/28/1/132/35885578/ocaa271.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055267862","display_name":"Sarvesh Soni","orcid":"https://orcid.org/0000-0003-1704-1039"},"institutions":[{"id":"https://openalex.org/I919571938","display_name":"The University of Texas Health Science Center at Houston","ror":"https://ror.org/03gds6c39","country_code":"US","type":"education","lineage":["https://openalex.org/I919571938"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sarvesh Soni","raw_affiliation_strings":["School of Biomedical Informatics, The University of Texas Health Science Center at Houston, Houston, Texas 77030, USA"],"affiliations":[{"raw_affiliation_string":"School of Biomedical Informatics, The University of Texas Health Science Center at Houston, Houston, Texas 77030, USA","institution_ids":["https://openalex.org/I919571938"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046709245","display_name":"Kirk Roberts","orcid":"https://orcid.org/0000-0001-6525-5213"},"institutions":[{"id":"https://openalex.org/I1337852284","display_name":"Texas Medical Center","ror":"https://ror.org/00dqsbj20","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1337852284"]},{"id":"https://openalex.org/I919571938","display_name":"The University of Texas Health Science Center at Houston","ror":"https://ror.org/03gds6c39","country_code":"US","type":"education","lineage":["https://openalex.org/I919571938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kirk Roberts","raw_affiliation_strings":["School of Biomedical Informatics, The University of Texas Health Science Center at Houston, Houston, Texas 77030, USA","Texas Medical Center"],"affiliations":[{"raw_affiliation_string":"School of Biomedical Informatics, The University of Texas Health Science Center at Houston, Houston, Texas 77030, USA","institution_ids":["https://openalex.org/I919571938"]},{"raw_affiliation_string":"Texas Medical Center","institution_ids":["https://openalex.org/I1337852284"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5055267862"],"corresponding_institution_ids":["https://openalex.org/I919571938"],"apc_list":{"value":3967,"currency":"USD","value_usd":3967},"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.08525514,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"28","issue":"1","first_page":"132","last_page":"137"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6979284882545471},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6782216429710388},{"id":"https://openalex.org/keywords/unicode","display_name":"Unicode","score":0.5602406859397888},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.538715660572052},{"id":"https://openalex.org/keywords/coronavirus-disease-2019","display_name":"Coronavirus disease 2019 (COVID-19)","score":0.5293066501617432},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.5198867321014404},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.4730333983898163},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.41729187965393066},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3666132092475891},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.1853714883327484},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.10375523567199707},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.10182300209999084}],"concepts":[{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6979284882545471},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6782216429710388},{"id":"https://openalex.org/C500551929","wikidata":"https://www.wikidata.org/wiki/Q8819","display_name":"Unicode","level":2,"score":0.5602406859397888},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.538715660572052},{"id":"https://openalex.org/C3008058167","wikidata":"https://www.wikidata.org/wiki/Q84263196","display_name":"Coronavirus disease 2019 (COVID-19)","level":4,"score":0.5293066501617432},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.5198867321014404},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.4730333983898163},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.41729187965393066},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3666132092475891},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1853714883327484},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.10375523567199707},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.10182300209999084},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C2779134260","wikidata":"https://www.wikidata.org/wiki/Q12136","display_name":"Disease","level":2,"score":0.0},{"id":"https://openalex.org/C524204448","wikidata":"https://www.wikidata.org/wiki/Q788926","display_name":"Infectious disease (medical specialty)","level":3,"score":0.0}],"mesh":[{"descriptor_ui":"D000077321","descriptor_name":"Deep Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000077321","descriptor_name":"Deep Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000077321","descriptor_name":"Deep Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000086382","descriptor_name":"COVID-19","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000086382","descriptor_name":"COVID-19","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000086382","descriptor_name":"COVID-19","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000086402","descriptor_name":"SARS-CoV-2","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000086402","descriptor_name":"SARS-CoV-2","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000086402","descriptor_name":"SARS-CoV-2","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D007256","descriptor_name":"Information Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016247","descriptor_name":"Information Storage and Retrieval","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D016247","descriptor_name":"Information Storage and Retrieval","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D016247","descriptor_name":"Information Storage and Retrieval","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D057386","descriptor_name":"Search Engine","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D057386","descriptor_name":"Search Engine","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D057386","descriptor_name":"Search Engine","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":6,"locations":[{"id":"doi:10.1093/jamia/ocaa271","is_oa":true,"landing_page_url":"https://doi.org/10.1093/jamia/ocaa271","pdf_url":"https://academic.oup.com/jamia/article-pdf/28/1/132/35885578/ocaa271.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},{"id":"pmid:33197268","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33197268","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association : JAMIA","raw_type":null},{"id":"pmh:oai:arXiv.org:2007.03106","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.03106","pdf_url":"https://arxiv.org/pdf/2007.03106","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3046031915","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2007.03106.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:7717324","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7717324","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"J Am Med Inform Assoc","raw_type":"Text"},{"id":"doi:10.48550/arxiv.2007.03106","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2007.03106","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.1093/jamia/ocaa271","is_oa":true,"landing_page_url":"https://doi.org/10.1093/jamia/ocaa271","pdf_url":"https://academic.oup.com/jamia/article-pdf/28/1/132/35885578/ocaa271.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Good health and well-being","score":0.5600000023841858,"id":"https://metadata.un.org/sdg/3"}],"awards":[{"id":"https://openalex.org/G7606886254","display_name":null,"funder_award_id":"OIA-1937136","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G938040996","display_name":"Convergence Accelerator Phase I (RAISE): Knowledge Open Network Queries for Research (KONQUER)","funder_award_id":"1937136","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3046031915.pdf","grobid_xml":"https://content.openalex.org/works/W3046031915.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1759973002","https://openalex.org/W2911489562","https://openalex.org/W2963341956","https://openalex.org/W2963716420","https://openalex.org/W2970771982","https://openalex.org/W2993873509","https://openalex.org/W3003190814","https://openalex.org/W3007523830","https://openalex.org/W3020786614","https://openalex.org/W3021052948","https://openalex.org/W3023585645","https://openalex.org/W3030163527","https://openalex.org/W3030919230","https://openalex.org/W3031200717","https://openalex.org/W3035547741","https://openalex.org/W3128581554","https://openalex.org/W3172093261","https://openalex.org/W6758293299","https://openalex.org/W6777914315"],"related_works":["https://openalex.org/W3105502836","https://openalex.org/W2888144270","https://openalex.org/W2017982691","https://openalex.org/W2611221023","https://openalex.org/W1883184246","https://openalex.org/W2131321260","https://openalex.org/W2420775993","https://openalex.org/W2735295675","https://openalex.org/W2538203441","https://openalex.org/W3211006841","https://openalex.org/W1537427129","https://openalex.org/W1419101303","https://openalex.org/W2601972157","https://openalex.org/W1579404951","https://openalex.org/W2417308520","https://openalex.org/W2133838739","https://openalex.org/W2556353531","https://openalex.org/W2556731824","https://openalex.org/W2105276905","https://openalex.org/W2115682534"],"abstract_inverted_index":{"The":[0],"COVID-19":[1,21,88],"pandemic":[2],"has":[3,134],"resulted":[4],"in":[5,37,92,98,109,138],"a":[6,115],"tremendous":[7],"need":[8],"for":[9,20,87,136,148],"access":[10],"to":[11,17,26,68,80,106,113],"the":[12,46,71,99,110,123],"latest":[13],"scientific":[14],"information,":[15],"leading":[16,67],"both":[18],"corpora":[19],"literature":[22],"and":[23,90,143],"search":[24,32,48,57,85,125,141,146],"engines":[25,58,86,126,142,147],"query":[27],"such":[28],"data.":[29],"While":[30],"most":[31],"engine":[33],"research":[34],"is":[35,52],"performed":[36,103],"academia":[38],"with":[39,94],"rigorous":[40],"evaluation,":[41],"major":[42],"commercial":[43,55,84,124],"companies":[44],"dominate":[45],"web":[47],"market.":[49],"Thus,":[50],"it":[51],"expected":[53],"that":[54],"pandemic-specific":[56],"will":[59],"gain":[60],"much":[61],"higher":[62],"traction":[63],"than":[64],"academic":[65,95],"alternatives,":[66],"questions":[69],"about":[70],"empirical":[72],"performance":[73],"of":[74,118],"these":[75],"tools.":[76],"This":[77,133],"paper":[78],"seeks":[79],"empirically":[81],"evaluate":[82],"two":[83],"(Google":[89],"Amazon)":[91],"comparison":[93,117],"prototypes":[96],"evaluated":[97,130],"TREC-COVID":[100],"task.":[101],"We":[102,121],"several":[104],"steps":[105],"reduce":[107],"bias":[108],"manual":[111],"judgments":[112],"ensure":[114],"fair":[116],"all":[119],"systems.":[120],"find":[122],"sizably":[127],"underperformed":[128],"those":[129],"under":[131],"TREC-COVID.":[132],"implications":[135],"trust":[137],"popular":[139],"health":[140,150],"developing":[144],"biomedical":[145],"future":[149],"crises.":[151]},"counts_by_year":[],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}