{"id":"https://openalex.org/W4410022944","doi":"https://doi.org/10.1088/2632-2153/add3bb","title":"LLM4Mat-bench: benchmarking large language models for materials property prediction","display_name":"LLM4Mat-bench: benchmarking large language models for materials property prediction","publication_year":2025,"publication_date":"2025-05-02","ids":{"openalex":"https://openalex.org/W4410022944","doi":"https://doi.org/10.1088/2632-2153/add3bb"},"language":"en","primary_location":{"id":"doi:10.1088/2632-2153/add3bb","is_oa":true,"landing_page_url":"https://doi.org/10.1088/2632-2153/add3bb","pdf_url":"https://iopscience.iop.org/article/10.1088/2632-2153/add3bb/pdf","source":{"id":"https://openalex.org/S4210200687","display_name":"Machine Learning Science and Technology","issn_l":"2632-2153","issn":["2632-2153"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320083","host_organization_name":"IOP Publishing","host_organization_lineage":["https://openalex.org/P4310320083","https://openalex.org/P4310311669"],"host_organization_lineage_names":["IOP Publishing","Institute of Physics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning: Science and Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://iopscience.iop.org/article/10.1088/2632-2153/add3bb/pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024826347","display_name":"Andre Niyongabo Rubungo","orcid":"https://orcid.org/0000-0003-3608-2039"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andre Niyongabo Rubungo","raw_affiliation_strings":["Department of Computer Science, Princeton University, Princeton, NJ, United States of America","Vertaix, Princeton, NJ, United States of America"],"raw_orcid":"https://orcid.org/0000-0003-3608-2039","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Princeton University, Princeton, NJ, United States of America","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Vertaix, Princeton, NJ, United States of America","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055766485","display_name":"Kangming Li","orcid":"https://orcid.org/0000-0003-4471-8527"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I2803004491","display_name":"Structural Genomics Consortium","ror":"https://ror.org/04jzps455","country_code":"CA","type":"nonprofit","lineage":["https://openalex.org/I2803004491"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Kangming Li","raw_affiliation_strings":["Acceleration Consortium, University of Toronto, Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-4471-8527","affiliations":[{"raw_affiliation_string":"Acceleration Consortium, University of Toronto, Toronto, Canada","institution_ids":["https://openalex.org/I2803004491","https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073635313","display_name":"Jason Hattrick\u2010Simpers","orcid":"https://orcid.org/0000-0003-2937-3188"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I2803004491","display_name":"Structural Genomics Consortium","ror":"https://ror.org/04jzps455","country_code":"CA","type":"nonprofit","lineage":["https://openalex.org/I2803004491"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]},{"id":"https://openalex.org/I4210149858","display_name":"Schwartz/Reisman Emergency Medicine Institute","ror":"https://ror.org/03sc84089","country_code":"CA","type":"nonprofit","lineage":["https://openalex.org/I4210149858"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jason Hattrick-Simpers","raw_affiliation_strings":["Acceleration Consortium, University of Toronto, Toronto, Canada","Department of Materials Science and Engineering, University of Toronto, Toronto, Canada","Schwartz Reisman Institute for Technology and Society, Toronto, Canada","Vector Institute for Artificial Intelligence, Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-2937-3188","affiliations":[{"raw_affiliation_string":"Acceleration Consortium, University of Toronto, Toronto, Canada","institution_ids":["https://openalex.org/I2803004491","https://openalex.org/I185261750"]},{"raw_affiliation_string":"Department of Materials Science and Engineering, University of Toronto, Toronto, Canada","institution_ids":["https://openalex.org/I185261750"]},{"raw_affiliation_string":"Schwartz Reisman Institute for Technology and Society, Toronto, Canada","institution_ids":["https://openalex.org/I4210149858"]},{"raw_affiliation_string":"Vector Institute for Artificial Intelligence, Toronto, Canada","institution_ids":["https://openalex.org/I4210127509"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063448291","display_name":"Adji Bousso Dieng","orcid":"https://orcid.org/0000-0001-5687-3554"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Adji Bousso Dieng","raw_affiliation_strings":["Department of Computer Science, Princeton University, Princeton, NJ, United States of America","Vertaix, Princeton, NJ, United States of America"],"raw_orcid":"https://orcid.org/0000-0001-5687-3554","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Princeton University, Princeton, NJ, United States of America","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Vertaix, Princeton, NJ, United States of America","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5063448291"],"corresponding_institution_ids":["https://openalex.org/I20089843"],"apc_list":{"value":1600,"currency":"GBP","value_usd":1962},"apc_paid":{"value":1600,"currency":"GBP","value_usd":1962},"fwci":4.6369,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.95540721,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"6","issue":"2","first_page":"020501","last_page":"020501"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9787999987602234,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9787999987602234,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11159","display_name":"Manufacturing Process and Optimization","score":0.9190999865531921,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7655985355377197},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.6469458937644958},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.47690606117248535},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3521852493286133},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.10226291418075562},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.049239277839660645}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7655985355377197},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.6469458937644958},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47690606117248535},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3521852493286133},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.10226291418075562},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.049239277839660645},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1088/2632-2153/add3bb","is_oa":true,"landing_page_url":"https://doi.org/10.1088/2632-2153/add3bb","pdf_url":"https://iopscience.iop.org/article/10.1088/2632-2153/add3bb/pdf","source":{"id":"https://openalex.org/S4210200687","display_name":"Machine Learning Science and Technology","issn_l":"2632-2153","issn":["2632-2153"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320083","host_organization_name":"IOP Publishing","host_organization_lineage":["https://openalex.org/P4310320083","https://openalex.org/P4310311669"],"host_organization_lineage_names":["IOP Publishing","Institute of Physics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning: Science and Technology","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:23bc7cb61fbe4c0490152242dd1f6bca","is_oa":true,"landing_page_url":"https://doaj.org/article/23bc7cb61fbe4c0490152242dd1f6bca","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Machine Learning: Science and Technology, Vol 6, Iss 2, p 020501 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1088/2632-2153/add3bb","is_oa":true,"landing_page_url":"https://doi.org/10.1088/2632-2153/add3bb","pdf_url":"https://iopscience.iop.org/article/10.1088/2632-2153/add3bb/pdf","source":{"id":"https://openalex.org/S4210200687","display_name":"Machine Learning Science and Technology","issn_l":"2632-2153","issn":["2632-2153"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320083","host_organization_name":"IOP Publishing","host_organization_lineage":["https://openalex.org/P4310320083","https://openalex.org/P4310311669"],"host_organization_lineage_names":["IOP Publishing","Institute of Physics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning: Science and Technology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2044452311","display_name":"HDR Institute: Institute for Data Driven Dynamical Design","funder_award_id":"2118201","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5565249710","display_name":null,"funder_award_id":"#2118201","funder_id":"https://openalex.org/F4320337377","funder_display_name":"Office of Advanced Cyberinfrastructure"},{"id":"https://openalex.org/G8188805235","display_name":null,"funder_award_id":"2118201","funder_id":"https://openalex.org/F4320337377","funder_display_name":"Office of Advanced Cyberinfrastructure"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337377","display_name":"Office of Advanced Cyberinfrastructure","ror":"https://ror.org/04nh1dc89"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4410022944.pdf","grobid_xml":"https://content.openalex.org/works/W4410022944.grobid-xml"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W1992985800","https://openalex.org/W2070942956","https://openalex.org/W2100716186","https://openalex.org/W2278970271","https://openalex.org/W2586209903","https://openalex.org/W2733963201","https://openalex.org/W2766856748","https://openalex.org/W2795510410","https://openalex.org/W2963341956","https://openalex.org/W2963587345","https://openalex.org/W3023937119","https://openalex.org/W3143460494","https://openalex.org/W3202900079","https://openalex.org/W3212512279","https://openalex.org/W4248414713","https://openalex.org/W4280570022","https://openalex.org/W4327564965","https://openalex.org/W4365509359","https://openalex.org/W4385484300","https://openalex.org/W4388945356","https://openalex.org/W4389132751","https://openalex.org/W4393038846","https://openalex.org/W4393212539","https://openalex.org/W4396706898","https://openalex.org/W4400074175","https://openalex.org/W4405114187","https://openalex.org/W4406284830","https://openalex.org/W6769627184","https://openalex.org/W6810047384","https://openalex.org/W6839378501","https://openalex.org/W6853104154","https://openalex.org/W6853107892","https://openalex.org/W6854274159","https://openalex.org/W6854866820","https://openalex.org/W6855991174","https://openalex.org/W6856498177","https://openalex.org/W6857430821","https://openalex.org/W6858023062","https://openalex.org/W6860715045","https://openalex.org/W6861374585","https://openalex.org/W6862183416","https://openalex.org/W6862264409","https://openalex.org/W6862397664","https://openalex.org/W6870737912"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699"],"abstract_inverted_index":{"Abstract":[0],"Large":[1],"language":[2],"models":[3,104,149],"(LLMs)":[4],"are":[5],"increasingly":[6],"being":[7],"used":[8],"in":[9,45,59,93,140,154],"materials":[10,25,66,141,155],"science.":[11],"However,":[12],"little":[13],"attention":[14],"has":[15],"been":[16],"given":[17],"to":[18,37,102,118],"benchmarking":[19],"and":[20,69,81,90,110,112,115,130,143,150,162],"standardized":[21],"evaluation":[22],"for":[23,39,95,146],"LLM-based":[24],"property":[26,121,156],"prediction,":[27],"which":[28],"hinders":[29],"progress.":[30],"We":[31,99],"present":[32],"LLM4Mat-Bench,":[33],"the":[34,41,47,120,135,144],"largest":[35],"benchmark":[36],"date":[38],"evaluating":[40],"performance":[42],"of":[43,49,124,137],"LLMs":[44,139,153],"predicting":[46],"properties":[48],"crystalline":[50],"materials.":[51],"LLM4Mat-Bench":[52,73,101],"contains":[53],"about":[54],"1.9":[55],"M":[56],"crystal":[57,78,82],"structures":[58],"total,":[60],"collected":[61],"from":[62],"10":[63],"publicly":[64],"available":[65],"data":[67],"sources,":[68],"45":[70],"distinct":[71],"properties.":[72],"features":[74],"different":[75,106],"input":[76],"modalities:":[77],"composition,":[79],"CIF,":[80],"text":[83],"description,":[84],"with":[85,105],"4.7":[86],"M,":[87,89],"615.5":[88],"3.1B":[91],"tokens":[92],"total":[94],"each":[96],"modality,":[97],"respectively.":[98],"use":[100],"fine-tune":[103],"sizes,":[107],"including":[108,127],"LLM-Prop":[109],"MatBERT,":[111],"provide":[113],"zero-shot":[114],"few-shot":[116],"prompts":[117],"evaluate":[119],"prediction":[122,157],"capabilities":[123],"LLM-chat-like":[125],"models,":[126],"Llama,":[128],"Gemma,":[129],"Mistral.":[131],"The":[132,160],"results":[133],"highlight":[134],"challenges":[136],"general-purpose":[138],"science":[142],"need":[145],"task-specific":[147,151],"predictive":[148],"instruction-tuned":[152],"7":[158,159],"Benchmark":[161],"code":[163],"can":[164],"be":[165],"found":[166],"at:":[167],"https://github.com/vertaix/LLM4Mat-Bench":[168],".":[169,170]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":7}],"updated_date":"2026-06-24T13:16:06.693445","created_date":"2025-10-10T00:00:00"}