{"id":"https://openalex.org/W7126038861","doi":"https://doi.org/10.1109/bibm66473.2025.11357115","title":"PBD: A Manually Curated Full-Chain Benchmark Dataset for Evaluating LLMs on ACMG PS3/BS3 Functional Evidence Acquisition","display_name":"PBD: A Manually Curated Full-Chain Benchmark Dataset for Evaluating LLMs on ACMG PS3/BS3 Functional Evidence Acquisition","publication_year":2025,"publication_date":"2025-12-15","ids":{"openalex":"https://openalex.org/W7126038861","doi":"https://doi.org/10.1109/bibm66473.2025.11357115"},"language":null,"primary_location":{"id":"doi:10.1109/bibm66473.2025.11357115","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bibm66473.2025.11357115","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089592676","display_name":"Xinpan Yuan","orcid":"https://orcid.org/0000-0001-9509-0755"},"institutions":[{"id":"https://openalex.org/I49934816","display_name":"Hunan University of Technology","ror":"https://ror.org/04j3vr751","country_code":"CN","type":"education","lineage":["https://openalex.org/I49934816"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xinpan Yuan","raw_affiliation_strings":["Intelligence Hunan, University of Technology,Hunan Provincial Engineering Technology, Research Center for Industrial Data,Zhuzhou,Hunan,China"],"affiliations":[{"raw_affiliation_string":"Intelligence Hunan, University of Technology,Hunan Provincial Engineering Technology, Research Center for Industrial Data,Zhuzhou,Hunan,China","institution_ids":["https://openalex.org/I49934816"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123410930","display_name":"Bozhao Li","orcid":null},"institutions":[{"id":"https://openalex.org/I49934816","display_name":"Hunan University of Technology","ror":"https://ror.org/04j3vr751","country_code":"CN","type":"education","lineage":["https://openalex.org/I49934816"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bozhao Li","raw_affiliation_strings":["School of Computer Science and Artificial Intelligence, Hunan University of Technology,Zhuzhou,Hunan,China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Artificial Intelligence, Hunan University of Technology,Zhuzhou,Hunan,China","institution_ids":["https://openalex.org/I49934816"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025580085","display_name":"Chenbin Liu","orcid":"https://orcid.org/0000-0002-2037-5630"},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]},{"id":"https://openalex.org/I4210159865","display_name":"Xiangya Hospital Central South University","ror":"https://ror.org/05c1yfj14","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210159865"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenbin Liu","raw_affiliation_strings":["Xiangya Hospital, Central South University,Changsha,Hunan,China"],"affiliations":[{"raw_affiliation_string":"Xiangya Hospital, Central South University,Changsha,Hunan,China","institution_ids":["https://openalex.org/I4210159865","https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124181170","display_name":"Xinxue Li","orcid":null},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]},{"id":"https://openalex.org/I4210159865","display_name":"Xiangya Hospital Central South University","ror":"https://ror.org/05c1yfj14","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210159865"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinxue Li","raw_affiliation_strings":["Xiangya Hospital, Central South University,Changsha,Hunan,China"],"affiliations":[{"raw_affiliation_string":"Xiangya Hospital, Central South University,Changsha,Hunan,China","institution_ids":["https://openalex.org/I4210159865","https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038781609","display_name":"Liujie Hua","orcid":"https://orcid.org/0000-0002-2756-1540"},"institutions":[{"id":"https://openalex.org/I49934816","display_name":"Hunan University of Technology","ror":"https://ror.org/04j3vr751","country_code":"CN","type":"education","lineage":["https://openalex.org/I49934816"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liujie Hua","raw_affiliation_strings":["School of Computer Science and Artificial Intelligence, Hunan University of Technology,Zhuzhou,Hunan,China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Artificial Intelligence, Hunan University of Technology,Zhuzhou,Hunan,China","institution_ids":["https://openalex.org/I49934816"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124290887","display_name":"Jinchen Li","orcid":null},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]},{"id":"https://openalex.org/I4210126592","display_name":"National Clinical Research","ror":"https://ror.org/03ddkvp86","country_code":"US","type":"facility","lineage":["https://openalex.org/I4210126592"]},{"id":"https://openalex.org/I4210159865","display_name":"Xiangya Hospital Central South University","ror":"https://ror.org/05c1yfj14","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210159865"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Jinchen Li","raw_affiliation_strings":["South University,National Clinical Research Center for Geriatric Disorders, Xiangya Hospital, Central,Changsha,Hunan,China"],"affiliations":[{"raw_affiliation_string":"South University,National Clinical Research Center for Geriatric Disorders, Xiangya Hospital, Central,Changsha,Hunan,China","institution_ids":["https://openalex.org/I4210126592","https://openalex.org/I4210159865","https://openalex.org/I139660479"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124218503","display_name":"Lin Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I39555362","display_name":"University of Warwick","ror":"https://ror.org/01a77tt86","country_code":"GB","type":"education","lineage":["https://openalex.org/I39555362"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lin Wu","raw_affiliation_strings":["University of Warwick,Coventry,United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Warwick,Coventry,United Kingdom","institution_ids":["https://openalex.org/I39555362"]}]},{"author_position":"last","author":{"id":null,"display_name":"Guihu Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I139660479","display_name":"Central South University","ror":"https://ror.org/00f1zfq44","country_code":"CN","type":"education","lineage":["https://openalex.org/I139660479"]},{"id":"https://openalex.org/I4210126592","display_name":"National Clinical Research","ror":"https://ror.org/03ddkvp86","country_code":"US","type":"facility","lineage":["https://openalex.org/I4210126592"]},{"id":"https://openalex.org/I4210159865","display_name":"Xiangya Hospital Central South University","ror":"https://ror.org/05c1yfj14","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210159865"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Guihu Zhao","raw_affiliation_strings":["South University,National Clinical Research Center for Geriatric Disorders, Xiangya Hospital, Central,Changsha,Hunan,China"],"affiliations":[{"raw_affiliation_string":"South University,National Clinical Research Center for Geriatric Disorders, Xiangya Hospital, Central,Changsha,Hunan,China","institution_ids":["https://openalex.org/I4210126592","https://openalex.org/I4210159865","https://openalex.org/I139660479"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5089592676"],"corresponding_institution_ids":["https://openalex.org/I49934816"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.78605451,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1937","last_page":"1940"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11642","display_name":"Genomics and Rare Diseases","score":0.9745000004768372,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11642","display_name":"Genomics and Rare Diseases","score":0.9745000004768372,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10261","display_name":"Genetic Associations and Epidemiology","score":0.01360000018030405,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.003800000064074993,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6811000108718872},{"id":"https://openalex.org/keywords/standardization","display_name":"Standardization","score":0.6051999926567078},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.4871000051498413},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.4309000074863434},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.39570000767707825},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.3944000005722046},{"id":"https://openalex.org/keywords/genomics","display_name":"Genomics","score":0.3646000027656555}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6891999840736389},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6811000108718872},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.6051999926567078},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.521399974822998},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4871000051498413},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4607999920845032},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43459999561309814},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4309000074863434},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.39570000767707825},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.3944000005722046},{"id":"https://openalex.org/C189206191","wikidata":"https://www.wikidata.org/wiki/Q222046","display_name":"Genomics","level":4,"score":0.3646000027656555},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3440000116825104},{"id":"https://openalex.org/C161078062","wikidata":"https://www.wikidata.org/wiki/Q1068690","display_name":"Functional genomics","level":5,"score":0.3301999866962433},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3221000134944916},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.30880001187324524},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2930999994277954},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.2912999987602234},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.2800000011920929}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bibm66473.2025.11357115","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bibm66473.2025.11357115","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2051978340","https://openalex.org/W2174602966","https://openalex.org/W2911489562","https://openalex.org/W3005785247","https://openalex.org/W4388622514","https://openalex.org/W4389555243","https://openalex.org/W4402193847","https://openalex.org/W4403425919","https://openalex.org/W4404282917","https://openalex.org/W4404536308","https://openalex.org/W4405822444","https://openalex.org/W4406036876","https://openalex.org/W4407204088","https://openalex.org/W4408272644","https://openalex.org/W4409175345","https://openalex.org/W4409235391","https://openalex.org/W4411057363","https://openalex.org/W4412525388","https://openalex.org/W4414349334"],"related_works":[],"abstract_inverted_index":{"The":[0],"PS3":[1],"(Pathogenic":[2],"Functional":[3,8],"Evidence)":[4,9],"and":[5,18,33,95,119,143,146,186,192,207],"BS3":[6],"(Benign":[7],"criteria":[10],"in":[11,127,174],"the":[12,39,64,73,80,133,190],"American":[13],"College":[14],"of":[15,42,44,66,194],"Medical":[16],"Genetics":[17],"Genomics":[19],"(ACMG)":[20],"guidelines":[21],"are":[22,200],"critical":[23],"for":[24,54,153,184],"genetic":[25],"variant":[26,177,195],"classification.":[27],"However,":[28],"manual":[29],"evaluation":[30,151],"is":[31,61],"time-consuming":[32],"prone":[34],"to":[35,101,123,209],"inter-laboratory":[36],"inconsistencies,":[37],"limiting":[38],"clinical":[40],"interpretation":[41],"Variants":[43],"Uncertain":[45],"Significance":[46],"(VUS).":[47],"Large":[48],"Language":[49],"Models":[50],"(LLMs)":[51],"offer":[52],"potential":[53,173],"automated":[55,150,176],"assessment,":[56],"but":[57],"their":[58,96],"performance":[59],"validation":[60],"hindered":[62],"by":[63],"lack":[65],"standardized,":[67],"high-quality":[68],"datasets.":[69],"This":[70],"study":[71],"introduces":[72],"PS3/BS3":[74,97],"Full-Chain":[75],"Evidence":[76],"Benchmark":[77],"Dataset":[78],"(PBD),":[79],"first":[81],"manually":[82],"curated":[83],"dataset":[84,134],"comprising":[85],"77":[86],"peer-reviewed":[87],"publications,":[88],"covering":[89],"266":[90],"cDNA":[91],"variants":[92],"(including":[93],"duplicates)":[94],"rating":[98],"results.":[99],"Adhering":[100],"ClinGen":[102],"Sequence":[103],"Variant":[104],"Interpretation":[105],"(SVI)":[106],"standards,":[107],"PBD":[108,179],"includes":[109],"structured,":[110],"comprehensive":[111],"evidence":[112,120,129],"chains":[113],"spanning":[114],"genes,":[115],"diseases,":[116],"variants,":[117],"experiments,":[118],"ratings,":[121],"designed":[122],"evaluate":[124],"LLM":[125],"capabilities":[126],"functional":[128],"extraction.":[130],"We":[131],"detail":[132],"construction":[135],"process,":[136],"including":[137],"literature":[138],"screening,":[139],"data":[140,206],"extraction,":[141],"standardization,":[142],"quality":[144],"control,":[145],"developed":[147],"a":[148,181],"Python-based":[149],"pipeline":[152],"reproducible,":[154],"standardized":[155],"analysis.":[156],"Experiments":[157],"using":[158],"DeepSeek":[159],"models":[160],"<tex":[161],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[162],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">$(1.5":[163],"\\mathrm{b}":[164,167],"/":[165,168],"7":[166],"14":[169],"\\mathrm{b})$</tex>":[170],"demonstrate":[171],"PBD's":[172],"supporting":[175],"interpretation.":[178],"provides":[180],"vital":[182],"resource":[183],"bioinformatics":[185],"precision":[187],"medicine,":[188],"facilitating":[189],"development":[191],"standardization":[193],"classification":[196],"tools.":[197],"Data":[198],"examples":[199],"available":[201],"at":[202],"https://github.com/User8588/PBD,":[203],"with":[204],"full":[205],"code":[208],"be":[210],"released":[211],"upon":[212],"paper":[213],"acceptance.":[214]},"counts_by_year":[],"updated_date":"2026-03-25T23:56:10.502304","created_date":"2026-01-30T00:00:00"}