{"id":"https://openalex.org/W4399802217","doi":"https://doi.org/10.1186/s12859-024-05834-2","title":"Leveraging permutation testing to assess confidence in positive-unlabeled learning applied to high-dimensional biological datasets","display_name":"Leveraging permutation testing to assess confidence in positive-unlabeled learning applied to high-dimensional biological datasets","publication_year":2024,"publication_date":"2024-06-19","ids":{"openalex":"https://openalex.org/W4399802217","doi":"https://doi.org/10.1186/s12859-024-05834-2","pmid":"https://pubmed.ncbi.nlm.nih.gov/38898392"},"language":"en","primary_location":{"id":"doi:10.1186/s12859-024-05834-2","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12859-024-05834-2","pdf_url":null,"source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1186/s12859-024-05834-2","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083593443","display_name":"Shiwei Xu","orcid":"https://orcid.org/0000-0002-1520-2145"},"institutions":[{"id":"https://openalex.org/I107672454","display_name":"Dartmouth College","ror":"https://ror.org/049s0rh22","country_code":"US","type":"education","lineage":["https://openalex.org/I107672454"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shiwei Xu","raw_affiliation_strings":["Quantitative Biomedical Sciences Program, Dartmouth College, Hanover, NH, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Quantitative Biomedical Sciences Program, Dartmouth College, Hanover, NH, USA","institution_ids":["https://openalex.org/I107672454"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071990559","display_name":"Margaret E. Ackerman","orcid":"https://orcid.org/0000-0002-4253-3476"},"institutions":[{"id":"https://openalex.org/I107672454","display_name":"Dartmouth College","ror":"https://ror.org/049s0rh22","country_code":"US","type":"education","lineage":["https://openalex.org/I107672454"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Margaret E. Ackerman","raw_affiliation_strings":["Department of Microbiology and Immunology, Geisel School of Medicine at Dartmouth, Dartmouth College, Hanover, NH, USA. margaret.e.ackerman@dartmouth.edu","Quantitative Biomedical Sciences Program, Dartmouth College, Hanover, NH, USA. margaret.e.ackerman@dartmouth.edu","Thayer School of Engineering, Dartmouth College, 14 Engineering Dr., Hanover, NH, 03755, USA. margaret.e.ackerman@dartmouth.edu","Quantitative Biomedical Sciences Program, Dartmouth College, Hanover, NH, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Microbiology and Immunology, Geisel School of Medicine at Dartmouth, Dartmouth College, Hanover, NH, USA. margaret.e.ackerman@dartmouth.edu","institution_ids":["https://openalex.org/I107672454"]},{"raw_affiliation_string":"Quantitative Biomedical Sciences Program, Dartmouth College, Hanover, NH, USA. margaret.e.ackerman@dartmouth.edu","institution_ids":["https://openalex.org/I107672454"]},{"raw_affiliation_string":"Thayer School of Engineering, Dartmouth College, 14 Engineering Dr., Hanover, NH, 03755, USA. margaret.e.ackerman@dartmouth.edu","institution_ids":["https://openalex.org/I107672454"]},{"raw_affiliation_string":"Quantitative Biomedical Sciences Program, Dartmouth College, Hanover, NH, USA","institution_ids":["https://openalex.org/I107672454"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5071990559"],"corresponding_institution_ids":["https://openalex.org/I107672454"],"apc_list":{"value":2890,"currency":"USD","value_usd":2890},"apc_paid":{"value":2890,"currency":"USD","value_usd":2890},"fwci":0.4601,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.70357096,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":"25","issue":"1","first_page":"218","last_page":"218"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.8127999901771545,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.8127999901771545,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.03229999914765358,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12254","display_name":"Machine Learning in Bioinformatics","score":0.012900000438094139,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.7336581945419312},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7255663871765137},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6688194274902344},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.6576552391052246},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6410032510757446},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5451800227165222},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5376917123794556},{"id":"https://openalex.org/keywords/permutation","display_name":"Permutation (music)","score":0.53091961145401},{"id":"https://openalex.org/keywords/false-positive-paradox","display_name":"False positive paradox","score":0.5086134672164917},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3774915933609009},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36234188079833984}],"concepts":[{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.7336581945419312},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7255663871765137},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6688194274902344},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.6576552391052246},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6410032510757446},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5451800227165222},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5376917123794556},{"id":"https://openalex.org/C21308566","wikidata":"https://www.wikidata.org/wiki/Q7169365","display_name":"Permutation (music)","level":2,"score":0.53091961145401},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.5086134672164917},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3774915933609009},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36234188079833984},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false}],"locations_count":4,"locations":[{"id":"doi:10.1186/s12859-024-05834-2","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12859-024-05834-2","pdf_url":null,"source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},{"id":"pmid:38898392","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38898392","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC bioinformatics","raw_type":"Journal Article"},{"id":"pmh:oai:pubmedcentral.nih.gov:11186207","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/11186207","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC11186207/pdf/12859_2024_Article_5834.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BMC Bioinformatics","raw_type":"Text"},{"id":"pmh:oai:doaj.org/article:2a77ea4295814531885684129be5d940","is_oa":true,"landing_page_url":"https://doaj.org/article/2a77ea4295814531885684129be5d940","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BMC Bioinformatics, Vol 25, Iss 1, Pp 1-17 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1186/s12859-024-05834-2","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12859-024-05834-2","pdf_url":null,"source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5099999904632568}],"awards":[{"id":"https://openalex.org/G2327455602","display_name":"New analytic approaches and endpoints in human HIV vaccine correlate studies","funder_award_id":"1r56ai165448-01a1","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G3340563442","display_name":null,"funder_award_id":"P01 AI162242","funder_id":"https://openalex.org/F4320337355","funder_display_name":"National Institute of Allergy and Infectious Diseases"},{"id":"https://openalex.org/G5943923999","display_name":null,"funder_award_id":"R56AI165448","funder_id":"https://openalex.org/F4320337355","funder_display_name":"National Institute of Allergy and Infectious Diseases"}],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337355","display_name":"National Institute of Allergy and Infectious Diseases","ror":"https://ror.org/043z4tv69"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1861993554","https://openalex.org/W1966716922","https://openalex.org/W2000288463","https://openalex.org/W2017169927","https://openalex.org/W2038492487","https://openalex.org/W2041971917","https://openalex.org/W2050871273","https://openalex.org/W2059733647","https://openalex.org/W2065281378","https://openalex.org/W2101029276","https://openalex.org/W2134510195","https://openalex.org/W2156772624","https://openalex.org/W2158485828","https://openalex.org/W2160331326","https://openalex.org/W2251470557","https://openalex.org/W2498119267","https://openalex.org/W2899867782","https://openalex.org/W3035965352","https://openalex.org/W3099878876","https://openalex.org/W3209094369","https://openalex.org/W3214095416","https://openalex.org/W4212808179","https://openalex.org/W4226096540","https://openalex.org/W4300895248","https://openalex.org/W4391483130","https://openalex.org/W6675354045"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W1557094818","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4323323165","https://openalex.org/W2745033168","https://openalex.org/W3187599676"],"abstract_inverted_index":{"BACKGROUND:":[0],"Compared":[1],"to":[2,17,48,90,112,119,143,151,188,206],"traditional":[3],"supervised":[4,198],"machine":[5,199],"learning":[6,14,219],"approaches":[7],"employing":[8],"fully":[9,197],"labeled":[10],"samples,":[11],"positive-unlabeled":[12],"(PU)":[13],"techniques":[15],"aim":[16],"classify":[18,121],"\"unlabeled\"":[19],"samples":[20,80,84,118,123],"based":[21],"on":[22],"a":[23,73,88,100,204,208,222],"smaller":[24],"proportion":[25,75],"of":[26,72,76,116,147,154,173,216],"known":[27,77],"positive":[28,64,128],"examples.":[29,175],"This":[30],"more":[31],"challenging":[32],"modeling":[33],"goal":[34],"reflects":[35],"many":[36],"real-world":[37,137],"scenarios":[38],"in":[39,196,213],"which":[40,225],"negative":[41,130],"examples":[42],"are":[43],"not":[44],"available-posing":[45],"direct":[46],"challenges":[47],"defining":[49],"prediction":[50],"accuracy":[51],"and":[52,129,136,169,182],"robustness.":[53],"While":[54],"several":[55],"studies":[56],"have":[57,67],"evaluated":[58],"predictions":[59],"learned":[60],"from":[61,81,191],"only":[62],"definitive":[63],"examples,":[65],"few":[66],"investigated":[68],"whether":[69],"correct":[70],"classification":[71],"high":[74],"positives":[78],"(KP)":[79],"among":[82,165],"unlabeled":[83,122,167],"can":[85,228],"act":[86],"as":[87],"surrogate":[89],"indicate":[91],"model":[92,155,178,226],"quality.":[93],"RESULTS:":[94],"In":[95],"this":[96],"study,":[97],"we":[98],"report":[99],"novel":[101],"methodology":[102],"combining":[103],"multiple":[104],"established":[105],"PU":[106,218],"learning-based":[107],"strategies":[108],"with":[109,170,180],"permutation":[110,201],"testing":[111,202],"evaluate":[113],"the":[114,145,148,166,214],"potential":[115],"KP":[117,174],"accurately":[120],"without":[124],"using":[125],"\"ground":[126],"truth\"":[127],"labels":[131,184],"for":[132],"validation.":[133],"Multivariate":[134],"synthetic":[135],"high-dimensional":[138],"benchmark":[139,212],"datasets":[140],"were":[141],"employed":[142],"demonstrate":[144],"suitability":[146],"proposed":[149],"pipeline":[150],"provide":[152],"evidence":[153],"robustness":[156],"across":[157],"varied":[158],"underlying":[159],"ground":[160],"truth":[161],"class":[162],"label":[163],"compositions":[164],"set":[168,207],"different":[171],"proportions":[172],"Comparisons":[176],"between":[177],"performance":[179,227],"actual":[181],"permuted":[183],"could":[185],"be":[186,229],"used":[187],"distinguish":[189],"reliable":[190],"unreliable":[192],"models.":[193],"CONCLUSIONS:":[194],"As":[195],"learning,":[200],"offers":[203],"means":[205],"baseline":[209],"\"no-information":[210],"rate\"":[211],"context":[215],"semi-supervised":[217],"inference":[220],"tasks-providing":[221],"standard":[223],"against":[224],"compared.":[230]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2025-10-10T00:00:00"}