{"id":"https://openalex.org/W7141076528","doi":"https://doi.org/10.1021/acs.jcim.5c02646","title":"Automatic Generation of a Mechanical Properties Question-Answering Data Set for Language Model Benchmarking: A Comparative Study of BERT, XLNet, and LLaMA Models","display_name":"Automatic Generation of a Mechanical Properties Question-Answering Data Set for Language Model Benchmarking: A Comparative Study of BERT, XLNet, and LLaMA Models","publication_year":2026,"publication_date":"2026-03-27","ids":{"openalex":"https://openalex.org/W7141076528","doi":"https://doi.org/10.1021/acs.jcim.5c02646","pmid":"https://pubmed.ncbi.nlm.nih.gov/41889011"},"language":"en","primary_location":{"id":"doi:10.1021/acs.jcim.5c02646","is_oa":true,"landing_page_url":"https://doi.org/10.1021/acs.jcim.5c02646","pdf_url":null,"source":{"id":"https://openalex.org/S167262187","display_name":"Journal of Chemical Information and Modeling","issn_l":"1549-9596","issn":["1549-9596","1549-960X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320006","host_organization_name":"American Chemical Society","host_organization_lineage":["https://openalex.org/P4310320006"],"host_organization_lineage_names":["American Chemical Society"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Chemical Information and Modeling","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1021/acs.jcim.5c02646","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130759159","display_name":"Minglei Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210096386","display_name":"Bridge University","ror":"https://ror.org/00cbm0437","country_code":"SS","type":"education","lineage":["https://openalex.org/I4210096386"]}],"countries":["GB","SS"],"is_corresponding":false,"raw_author_name":"Minglei Zhang","raw_affiliation_strings":["Ray Dolby Centre, Cavendish Laboratory, Department of Physics, University of Cambridge, J. J. Thomson Avenue, Cambridge CB3 0US. U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ray Dolby Centre, Cavendish Laboratory, Department of Physics, University of Cambridge, J. J. Thomson Avenue, Cambridge CB3 0US. U.K","institution_ids":["https://openalex.org/I4210096386","https://openalex.org/I241749"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068607578","display_name":"Jacqueline M. Cole","orcid":"https://orcid.org/0000-0002-1552-8743"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210096386","display_name":"Bridge University","ror":"https://ror.org/00cbm0437","country_code":"SS","type":"education","lineage":["https://openalex.org/I4210096386"]}],"countries":["GB","SS"],"is_corresponding":true,"raw_author_name":"Jacqueline M. Cole","raw_affiliation_strings":["Ray Dolby Centre, Cavendish Laboratory, Department of Physics, University of Cambridge, J. J. Thomson Avenue, Cambridge CB3 0US. U.K"],"raw_orcid":"https://orcid.org/0000-0002-1552-8743","affiliations":[{"raw_affiliation_string":"Ray Dolby Centre, Cavendish Laboratory, Department of Physics, University of Cambridge, J. J. Thomson Avenue, Cambridge CB3 0US. U.K","institution_ids":["https://openalex.org/I4210096386","https://openalex.org/I241749"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5068607578"],"corresponding_institution_ids":["https://openalex.org/I241749","https://openalex.org/I4210096386"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.66550631,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"66","issue":"7","first_page":"3840","last_page":"3857"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9366999864578247,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9366999864578247,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.013799999840557575,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.006099999882280827,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5763000249862671},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5751000046730042},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5597000122070312},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.5284000039100647},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.5260999798774719},{"id":"https://openalex.org/keywords/f1-score","display_name":"F1 score","score":0.486299991607666},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4747999906539917},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.38589999079704285},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.3400000035762787}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.758400022983551},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5763000249862671},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5751000046730042},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5597000122070312},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.5284000039100647},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.5260999798774719},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5242999792098999},{"id":"https://openalex.org/C148524875","wikidata":"https://www.wikidata.org/wiki/Q6975395","display_name":"F1 score","level":2,"score":0.486299991607666},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4747999906539917},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4564000070095062},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.38589999079704285},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3596000075340271},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.3400000035762787},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33970001339912415},{"id":"https://openalex.org/C55037315","wikidata":"https://www.wikidata.org/wiki/Q5421151","display_name":"Experimental data","level":2,"score":0.3276999890804291},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.32749998569488525},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.31619998812675476},{"id":"https://openalex.org/C2776434776","wikidata":"https://www.wikidata.org/wiki/Q19246213","display_name":"Domain adaptation","level":3,"score":0.31189998984336853},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.2985999882221222},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2867000102996826},{"id":"https://openalex.org/C2779280203","wikidata":"https://www.wikidata.org/wiki/Q17121211","display_name":"Small data","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.257099986076355},{"id":"https://openalex.org/C2779346075","wikidata":"https://www.wikidata.org/wiki/Q7268763","display_name":"Quality Score","level":3,"score":0.25619998574256897},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.25440001487731934}],"mesh":[{"descriptor_ui":"D001331","descriptor_name":"Automation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001331","descriptor_name":"Automation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007802","descriptor_name":"Language","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D007802","descriptor_name":"Language","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D019985","descriptor_name":"Benchmarking","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019985","descriptor_name":"Benchmarking","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D055595","descriptor_name":"Mechanical Phenomena","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D055595","descriptor_name":"Mechanical Phenomena","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D057225","descriptor_name":"Data Mining","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D057225","descriptor_name":"Data Mining","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true}],"locations_count":4,"locations":[{"id":"doi:10.1021/acs.jcim.5c02646","is_oa":true,"landing_page_url":"https://doi.org/10.1021/acs.jcim.5c02646","pdf_url":null,"source":{"id":"https://openalex.org/S167262187","display_name":"Journal of Chemical Information and Modeling","issn_l":"1549-9596","issn":["1549-9596","1549-960X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320006","host_organization_name":"American Chemical Society","host_organization_lineage":["https://openalex.org/P4310320006"],"host_organization_lineage_names":["American Chemical Society"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Chemical Information and Modeling","raw_type":"journal-article"},{"id":"pmid:41889011","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41889011","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of chemical information and modeling","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:13080984","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13080984/","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13080984/pdf/ci5c02646.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"J Chem Inf Model","raw_type":"Text"},{"id":"pmh:oai:www.repository.cam.ac.uk:1810/401613","is_oa":true,"landing_page_url":"https://www.repository.cam.ac.uk/handle/1810/401613","pdf_url":"https://www.repository.cam.ac.uk/bitstreams/6eddfdb3-1a3a-4c44-94af-b96b859bf5b3/download","source":{"id":"https://openalex.org/S4306401777","display_name":"Apollo (University of Cambridge)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I241749","host_organization_name":"University of Cambridge","host_organization_lineage":["https://openalex.org/I241749"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":{"id":"doi:10.1021/acs.jcim.5c02646","is_oa":true,"landing_page_url":"https://doi.org/10.1021/acs.jcim.5c02646","pdf_url":null,"source":{"id":"https://openalex.org/S167262187","display_name":"Journal of Chemical Information and Modeling","issn_l":"1549-9596","issn":["1549-9596","1549-960X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320006","host_organization_name":"American Chemical Society","host_organization_lineage":["https://openalex.org/P4310320006"],"host_organization_lineage_names":["American Chemical Society"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Chemical Information and Modeling","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.651009202003479,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G4517999","display_name":"AI for Chemistry: AIchemy","funder_award_id":"EP/Y028775/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G612342310","display_name":"AI for Chemistry: AIchemy","funder_award_id":"EP/Y028759/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W2342249984","https://openalex.org/W2427527485","https://openalex.org/W2523785361","https://openalex.org/W2525778437","https://openalex.org/W2965373594","https://openalex.org/W2968923792","https://openalex.org/W2972219719","https://openalex.org/W2983309655","https://openalex.org/W3006937107","https://openalex.org/W3011411500","https://openalex.org/W3014487746","https://openalex.org/W3033187248","https://openalex.org/W3081168214","https://openalex.org/W3200122731","https://openalex.org/W3201869313","https://openalex.org/W4214535912","https://openalex.org/W4224051134","https://openalex.org/W4229443452","https://openalex.org/W4286285970","https://openalex.org/W4288407534","https://openalex.org/W4289677850","https://openalex.org/W4294782983","https://openalex.org/W4307306401","https://openalex.org/W4308760226","https://openalex.org/W4320713584","https://openalex.org/W4327913228","https://openalex.org/W4379229569","https://openalex.org/W4388747944","https://openalex.org/W4390011017","https://openalex.org/W4391670863","https://openalex.org/W4391836235","https://openalex.org/W4391846075","https://openalex.org/W4397004462","https://openalex.org/W4399180325","https://openalex.org/W4401306886","https://openalex.org/W4401722919","https://openalex.org/W4404330874","https://openalex.org/W4404641999","https://openalex.org/W4405323124","https://openalex.org/W4405355411","https://openalex.org/W4405599065","https://openalex.org/W4405766390","https://openalex.org/W4407026132","https://openalex.org/W4407892736","https://openalex.org/W4408196005","https://openalex.org/W4408802928","https://openalex.org/W4408968357","https://openalex.org/W4410374120","https://openalex.org/W4411149009","https://openalex.org/W4412468087","https://openalex.org/W4413046406","https://openalex.org/W4414855631","https://openalex.org/W4416414513","https://openalex.org/W4416445028","https://openalex.org/W4416982455","https://openalex.org/W4417080162","https://openalex.org/W6910633587"],"related_works":[],"abstract_inverted_index":{"Contextualized":[0],"language":[1,258],"models":[2,175,221,259],"offer":[3],"new":[4],"opportunities":[5],"for":[6,166,254,260],"mining":[7],"materials-science":[8],"information":[9],"from":[10,50,79],"literature,":[11],"but":[12],"progress":[13],"is":[14],"limited":[15],"by":[16,29],"the":[17,54,80,170,205,218,233],"absence":[18],"of":[19,35,38,89,104,199,215,257],"domain-specific":[20,67,173],"question-answering":[21],"(QA)":[22],"data":[23,33,106,154,242],"sets.":[24],"This":[25],"study":[26],"addresses":[27],"this":[28,105,167],"introducing":[30],"MechQA,":[31],"a":[32,71,140],"set":[34,107,155,243],"202,068":[36],"pairs":[37,165],"questions":[39],"and":[40,100,132,139,161,181,201],"answers":[41],"about":[42],"mechanical":[43,87],"properties":[44,88],"that":[45,64,239],"have":[46],"been":[47],"automatically":[48,73],"distilled":[49],"125,967":[51],"articles":[52],"in":[53],"literature.":[55,82],"Unlike":[56],"small":[57],"manually":[58],"curated":[59],"QA":[60,164,241],"benchmarks":[61],"or":[62],"approaches":[63],"rely":[65],"on":[66],"pretraining,":[68],"MechQA":[69,121,153],"provides":[70,249],"large-scale,":[72],"generated":[74],"training":[75,160],"resource":[76],"derived":[77],"directly":[78],"primary":[81],"It":[83],"covers":[84],"five":[85],"fundamental":[86],"materials:":[90],"ultimate":[91],"tensile":[92],"strength,":[93,95,97],"yield":[94],"fracture":[96],"Young's":[98],"modulus,":[99],"ductility.":[101],"Manual":[102],"evaluation":[103],"confirmed":[108],"its":[109],"high":[110],"quality":[111],"(precision":[112],"83.76%,":[113],"recall":[114],"89.09%,":[115],"F1":[116,182,211],"score":[117,183],"86.34%).":[118],"We":[119],"apply":[120],"to":[122,232],"fine-tune":[123],"three":[124],"representative":[125],"transformer":[126],"models:":[127],"two":[128,219],"extractive":[129,174,220],"models,":[130],"BERT-base":[131],"XLNet-base,":[133],"each":[134],"with":[135,144,193,212,246],"110":[136],"M":[137],"parameters,":[138],"generative":[141],"LLaMA-3.1-Instruct":[142],"model":[143,207],"8B":[145],"parameters":[146],"fine-tuned":[147],"using":[148],"low-rank":[149],"adaptation":[150,256],"(LoRA).":[151],"The":[152],"was":[156],"partitioned":[157],"into":[158],"181,722":[159],"20,346":[162],"validation":[163,171],"application.":[168],"On":[169],"set,":[172],"achieve":[176],"strong":[177],"Exact":[178],"Match":[179],"(EM)":[180],"performance":[184,224],"(BERT:":[185],"78.03%":[186],"EM/84.50%":[187],"F1;":[188],"XLNet:":[189],"78.21%":[190],"EM/84.70%":[191],"F1)":[192],"improved":[194],"expected":[195],"calibration":[196],"error":[197],"(ECE)":[198],"7.98%":[200],"6.25%,":[202],"respectively,":[203],"while":[204],"LLaMA-domain":[206],"achieves":[208],"80.48%":[209],"EM/86.25%":[210],"an":[213,250],"ECE":[214],"8.08%.":[216],"Notably,":[217],"exhibit":[222],"competitive":[223],"despite":[225],"their":[226],"significantly":[227],"smaller":[228],"parameter":[229],"size":[230],"compared":[231],"LLaMA":[234],"model.":[235],"These":[236],"results":[237],"demonstrate":[238],"automatic":[240],"generation,":[244],"coupled":[245],"targeted":[247],"fine-tuning,":[248],"effective":[251],"data-centric":[252],"method":[253],"domain":[255],"materials":[261],"science.":[262]},"counts_by_year":[],"updated_date":"2026-04-14T06:02:45.956762","created_date":"2026-03-28T00:00:00"}