{"id":"https://openalex.org/W4398865217","doi":"https://doi.org/10.48550/arxiv.2405.14766","title":"Evaluating Large Language Models for Public Health Classification and Extraction Tasks","display_name":"Evaluating Large Language Models for Public Health Classification and Extraction Tasks","publication_year":2024,"publication_date":"2024-05-23","ids":{"openalex":"https://openalex.org/W4398865217","doi":"https://doi.org/10.48550/arxiv.2405.14766"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.14766","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.14766","pdf_url":"https://arxiv.org/pdf/2405.14766","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.14766","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086067760","display_name":"Joshua D. Harris","orcid":"https://orcid.org/0000-0002-7606-2473"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Harris, Joshua","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066876409","display_name":"Timothy Laurence","orcid":"https://orcid.org/0000-0001-5474-621X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laurence, Timothy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071419811","display_name":"Leo Loman","orcid":"https://orcid.org/0000-0002-1163-3810"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Loman, Leo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098823943","display_name":"Fan Grayson","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Grayson, Fan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114168334","display_name":"T. F. Nonnenmacher","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nonnenmacher, Toby","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109176640","display_name":"Harry J. Long","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Long, Harry","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098823944","display_name":"Loes WalsGriffith","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"WalsGriffith, Loes","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048740340","display_name":"Amy Douglas","orcid":"https://orcid.org/0000-0001-9270-1549"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Douglas, Amy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076192139","display_name":"Holly Fountain","orcid":"https://orcid.org/0009-0007-4852-630X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fountain, Holly","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048311862","display_name":"S. Georgiou","orcid":"https://orcid.org/0000-0001-5096-0067"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Georgiou, Stelios","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057470790","display_name":"Jo Hardstaff","orcid":"https://orcid.org/0000-0003-3345-209X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hardstaff, Jo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018747082","display_name":"Kathryn L. Hopkins","orcid":"https://orcid.org/0000-0003-2809-3686"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hopkins, Kathryn","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053026454","display_name":"Y-Ling Chi","orcid":"https://orcid.org/0000-0002-6902-0566"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chi, Y-Ling","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098823942","display_name":"Galena Kuyumdzhieva","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kuyumdzhieva, Galena","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044720478","display_name":"Lesley Larkin","orcid":"https://orcid.org/0000-0003-3699-9699"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Larkin, Lesley","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068037160","display_name":"Samuel Collins","orcid":"https://orcid.org/0000-0003-4721-0040"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Collins, Samuel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061289550","display_name":"Hamish Mohammed","orcid":"https://orcid.org/0000-0002-2060-7286"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mohammed, Hamish","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042746881","display_name":"Thomas Finnie","orcid":"https://orcid.org/0000-0001-5962-4211"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Finnie, Thomas","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053517322","display_name":"Luke Hounsome","orcid":"https://orcid.org/0000-0001-5231-1394"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hounsome, Luke","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091885439","display_name":"Steven Riley","orcid":"https://orcid.org/0000-0001-7904-4804"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Borowitz, Michael","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Riley, Steven","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Riley, Steven","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":21,"corresponding_author_ids":["https://openalex.org/A5086067760"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8597999811172485,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8597999811172485,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6239297389984131},{"id":"https://openalex.org/keywords/extraction","display_name":"Extraction (chemistry)","score":0.5357256531715393},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.513118326663971},{"id":"https://openalex.org/keywords/public-health","display_name":"Public health","score":0.4300517439842224},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4222198724746704},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.13195353746414185},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.08787095546722412},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.08028903603553772}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6239297389984131},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.5357256531715393},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.513118326663971},{"id":"https://openalex.org/C138816342","wikidata":"https://www.wikidata.org/wiki/Q189603","display_name":"Public health","level":2,"score":0.4300517439842224},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4222198724746704},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.13195353746414185},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.08787095546722412},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.08028903603553772},{"id":"https://openalex.org/C159110408","wikidata":"https://www.wikidata.org/wiki/Q121176","display_name":"Nursing","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.14766","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.14766","pdf_url":"https://arxiv.org/pdf/2405.14766","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.14766","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.14766","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.14766","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.14766","pdf_url":"https://arxiv.org/pdf/2405.14766","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4398865217.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Advances":[0],"in":[1,11],"Large":[2],"Language":[3],"Models":[4],"(LLMs)":[5],"have":[6],"led":[7],"to":[8,14,59,166,188],"significant":[9,112],"interest":[10],"their":[12],"potential":[13],"support":[15,200],"human":[16],"experts":[17,187],"across":[18,84,114],"a":[19,148,192],"range":[20],"of":[21,33,44,150,195],"domains,":[22],"including":[23],"public":[24,36,73,185,201],"health.":[25],"In":[26],"this":[27],"work":[28],"we":[29,153,174],"present":[30],"automated":[31],"evaluations":[32],"LLMs":[34,61,80,119,134,179],"for":[35,62,184],"health":[37,67,74,186,202],"tasks":[38,86,106,115],"involving":[39],"the":[40,96,101],"classification":[41],"and":[42,72,158,162,199,205],"extraction":[43],"free":[45,196],"text.":[46],"We":[47,76,91,110],"combine":[48],"six":[49],"externally":[50],"annotated":[51,57],"datasets":[52,58],"with":[53,116],"seven":[54],"new":[55],"internally":[56],"evaluate":[60,77,155],"processing":[63],"text":[64,197],"related":[65],"to:":[66],"burden,":[68],"epidemiological":[69],"risk":[70],"factors,":[71],"interventions.":[75,206],"eleven":[78],"open-weight":[79,118],"(7-123":[81],"billion":[82],"parameters)":[83],"all":[85,117,133],"using":[87],"zero-shot":[88],"in-context":[89],"learning.":[90],"find":[92,163,175],"that":[93,178],"Llama-3.3-70B-Instruct":[94],"is":[95],"highest":[97],"performing":[98],"model,":[99],"achieving":[100],"best":[102],"results":[103,165,173],"on":[104,124,140,170],"8/16":[105],"(using":[107],"micro-F1":[108,123,139],"scores).":[109],"see":[111],"variation":[113],"scoring":[120],"below":[121],"60%":[122],"some":[125],"challenging":[126],"tasks,":[127,152],"such":[128,142],"as":[129,143],"Contact":[130],"Classification,":[131],"while":[132],"achieve":[135],"greater":[136],"than":[137],"80%":[138],"others,":[141],"GI":[144],"Illness":[145],"Classification.":[146],"For":[147],"subset":[149],"11":[151],"also":[154],"three":[156],"GPT-4":[157],"GPT-4o":[159],"series":[160],"models":[161],"comparable":[164],"Llama-3.3-70B-Instruct.":[167],"Overall,":[168],"based":[169],"these":[171],"initial":[172],"promising":[176],"signs":[177],"may":[180],"be":[181],"useful":[182],"tools":[183],"extract":[189],"information":[190],"from":[191],"wide":[193],"variety":[194],"sources,":[198],"surveillance,":[203],"research,":[204]},"counts_by_year":[{"year":2025,"cited_by_count":5}],"updated_date":"2026-03-12T08:34:05.389933","created_date":"2025-10-10T00:00:00"}