{"id":"https://openalex.org/W4399042524","doi":"https://doi.org/10.2478/jdis-2024-0014","title":"Performance evaluation of seven multi-label classification methods on real-world patent and publication datasets","display_name":"Performance evaluation of seven multi-label classification methods on real-world patent and publication datasets","publication_year":2024,"publication_date":"2024-05-01","ids":{"openalex":"https://openalex.org/W4399042524","doi":"https://doi.org/10.2478/jdis-2024-0014"},"language":"en","primary_location":{"id":"doi:10.2478/jdis-2024-0014","is_oa":true,"landing_page_url":"https://doi.org/10.2478/jdis-2024-0014","pdf_url":"https://sciendo.com/pdf/10.2478/jdis-2024-0014","source":{"id":"https://openalex.org/S2764801193","display_name":"Journal of Data and Information Science","issn_l":"2096-157X","issn":["2096-157X","2543-683X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311940","host_organization_name":"Chinese Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310311940"],"host_organization_lineage_names":["Chinese Academy of Sciences"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Science","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://sciendo.com/pdf/10.2478/jdis-2024-0014","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060199267","display_name":"Shuo Xu","orcid":"https://orcid.org/0000-0002-8602-1819"},"institutions":[{"id":"https://openalex.org/I37796252","display_name":"Beijing University of Technology","ror":"https://ror.org/037b1pp87","country_code":"CN","type":"education","lineage":["https://openalex.org/I37796252"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuo Xu","raw_affiliation_strings":["College of Economics and Management, Beijing University of Technology , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Economics and Management, Beijing University of Technology , Beijing , China","institution_ids":["https://openalex.org/I37796252"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053991395","display_name":"Yuefu Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I37796252","display_name":"Beijing University of Technology","ror":"https://ror.org/037b1pp87","country_code":"CN","type":"education","lineage":["https://openalex.org/I37796252"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuefu Zhang","raw_affiliation_strings":["College of Economics and Management, Beijing University of Technology , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Economics and Management, Beijing University of Technology , Beijing , China","institution_ids":["https://openalex.org/I37796252"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014821698","display_name":"Xin An","orcid":"https://orcid.org/0000-0001-7413-9396"},"institutions":[{"id":"https://openalex.org/I31683504","display_name":"Beijing Forestry University","ror":"https://ror.org/04xv2pc41","country_code":"CN","type":"education","lineage":["https://openalex.org/I1327237609","https://openalex.org/I31683504","https://openalex.org/I4210127390"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xin An","raw_affiliation_strings":["School of Economics & Management, Beijing Forestry University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Economics & Management, Beijing Forestry University , Beijing , China","institution_ids":["https://openalex.org/I31683504"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052201599","display_name":"Sainan Pi","orcid":null},"institutions":[{"id":"https://openalex.org/I31683504","display_name":"Beijing Forestry University","ror":"https://ror.org/04xv2pc41","country_code":"CN","type":"education","lineage":["https://openalex.org/I1327237609","https://openalex.org/I31683504","https://openalex.org/I4210127390"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sainan Pi","raw_affiliation_strings":["School of Economics & Management, Beijing Forestry University , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Economics & Management, Beijing Forestry University , Beijing , China","institution_ids":["https://openalex.org/I31683504"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5014821698"],"corresponding_institution_ids":["https://openalex.org/I31683504"],"apc_list":null,"apc_paid":null,"fwci":1.627,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.85566635,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"9","issue":"2","first_page":"81","last_page":"103"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12254","display_name":"Machine Learning in Bioinformatics","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.951200008392334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7289377450942993},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6717315912246704},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5308884978294373},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.47922641038894653},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.4356442987918854},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4269866943359375},{"id":"https://openalex.org/keywords/macro","display_name":"Macro","score":0.423510879278183},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4203700125217438},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4016614556312561},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3948606848716736},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.34764936566352844},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10689324140548706},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.08844086527824402}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7289377450942993},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6717315912246704},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5308884978294373},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.47922641038894653},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4356442987918854},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4269866943359375},{"id":"https://openalex.org/C166955791","wikidata":"https://www.wikidata.org/wiki/Q629579","display_name":"Macro","level":2,"score":0.423510879278183},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4203700125217438},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4016614556312561},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3948606848716736},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.34764936566352844},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10689324140548706},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.08844086527824402},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.2478/jdis-2024-0014","is_oa":true,"landing_page_url":"https://doi.org/10.2478/jdis-2024-0014","pdf_url":"https://sciendo.com/pdf/10.2478/jdis-2024-0014","source":{"id":"https://openalex.org/S2764801193","display_name":"Journal of Data and Information Science","issn_l":"2096-157X","issn":["2096-157X","2543-683X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311940","host_organization_name":"Chinese Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310311940"],"host_organization_lineage_names":["Chinese Academy of Sciences"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Science","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:728d75ef4d3049ef890e2da731ee565f","is_oa":true,"landing_page_url":"https://doaj.org/article/728d75ef4d3049ef890e2da731ee565f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Data and Information Science, Vol 9, Iss 2, Pp 81-103 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.2478/jdis-2024-0014","is_oa":true,"landing_page_url":"https://doi.org/10.2478/jdis-2024-0014","pdf_url":"https://sciendo.com/pdf/10.2478/jdis-2024-0014","source":{"id":"https://openalex.org/S2764801193","display_name":"Journal of Data and Information Science","issn_l":"2096-157X","issn":["2096-157X","2543-683X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311940","host_organization_name":"Chinese Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310311940"],"host_organization_lineage_names":["Chinese Academy of Sciences"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Science","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6299999952316284,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399042524.pdf"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W54887220","https://openalex.org/W66588809","https://openalex.org/W1515481220","https://openalex.org/W1753402186","https://openalex.org/W1832693441","https://openalex.org/W1870686808","https://openalex.org/W1953606363","https://openalex.org/W1998660429","https://openalex.org/W1998839399","https://openalex.org/W1999954155","https://openalex.org/W2025047573","https://openalex.org/W2029296527","https://openalex.org/W2052684427","https://openalex.org/W2074909580","https://openalex.org/W2100743709","https://openalex.org/W2118712128","https://openalex.org/W2119466907","https://openalex.org/W2123217057","https://openalex.org/W2123459668","https://openalex.org/W2138290126","https://openalex.org/W2145827727","https://openalex.org/W2147152072","https://openalex.org/W2150102617","https://openalex.org/W2156935079","https://openalex.org/W2166912588","https://openalex.org/W2180117201","https://openalex.org/W2265846598","https://openalex.org/W2407776548","https://openalex.org/W2411155974","https://openalex.org/W2550576290","https://openalex.org/W2586665457","https://openalex.org/W2900758626","https://openalex.org/W2952641483","https://openalex.org/W2990661138","https://openalex.org/W3044968736","https://openalex.org/W3098010026","https://openalex.org/W3140910462","https://openalex.org/W4200211261","https://openalex.org/W4205184193","https://openalex.org/W4224325974","https://openalex.org/W4283369267","https://openalex.org/W4285082703","https://openalex.org/W6601668641","https://openalex.org/W6632223008","https://openalex.org/W6639619044","https://openalex.org/W6679166005"],"related_works":["https://openalex.org/W2030816003","https://openalex.org/W2378211422","https://openalex.org/W4239992647","https://openalex.org/W2150013480","https://openalex.org/W1554458299","https://openalex.org/W81423522","https://openalex.org/W1509860481","https://openalex.org/W4321353415","https://openalex.org/W2488264085","https://openalex.org/W2951187577"],"abstract_inverted_index":{"Abstract":[0],"Purpose":[1],"Many":[2],"science,":[3],"technology":[4],"and":[5,62,99,103,125,147,153,169,179,211,230,253,277,314,325,368,383],"innovation":[6],"(STI)":[7],"resources":[8],"are":[9,65,106,127,358],"attached":[10],"with":[11,27,70,112,162,193,376,400],"several":[12,45],"different":[13],"labels.":[14,317],"To":[15,135],"assign":[16],"automatically":[17],"the":[18,31,42,56,76,137,139,190,205,237,245,255,294,311,332,349,397],"resulting":[19],"labels":[20,168,382],"to":[21,83],"an":[22],"interested":[23],"instance,":[24],"many":[25],"approaches":[26,50,226],"good":[28],"performance":[29,238],"on":[30,90,130,159,189,281,290,361,373,396],"benchmark":[32,73,291],"datasets":[33,64,96,161,202,283,375],"have":[34,51,221],"been":[35,53],"proposed":[36],"for":[37,217,227],"multilabel":[38,60,109],"classification":[39,88,110,219,242,299,336,356],"task":[40],"in":[41,68,174,204,224,234,321,348],"literature.":[43],"Furthermore,":[44],"open-source":[46,214],"tools":[47,215],"implementing":[48],"these":[49,131],"also":[52,220],"developed.":[54],"However,":[55],"characteristics":[57],"of":[58,72,79,167,176,239,257,287,296,334,345,381],"real-world":[59,91,95,133,201,282,297,363],"patent":[61],"publication":[63],"not":[66],"completely":[67],"line":[69],"those":[71,288],"ones.":[74],"Therefore,":[75],"main":[77],"purpose":[78],"this":[80],"paper":[81],"is":[82,329],"evaluate":[84,136],"comprehensively":[85,128,359],"seven":[86],"multi-label":[87,218,241,298,335,355],"methods":[89,111,357],"datasets.":[92,134,364],"Design/methodology/approach":[93],"Three":[94,200],"(Biological-Sciences,":[97],"Health-Sciences,":[98],"USPTO)":[100],"from":[101],"SciGraph":[102],"USPTO":[104],"database":[105],"constructed.":[107],"Seven":[108,354],"tuned":[113],"parameters":[114],"(dependency-LDA,":[115],"ML":[116,183,390],"k":[117,121,184,391],"NN,":[118],"LabelPowerset,":[119],"RA":[120],"EL,":[122],"TextCNN,":[123],"TexRNN,":[124],"TextRCNN)":[126],"compared":[129,360],"three":[132,142,362],"performance,":[138],"study":[140],"adopts":[141],"classification-based":[143],"metrics:":[144],"Macro-F1,":[145],"Micro-F1,":[146],"Hamming":[148,180],"Loss.":[149,181],"Findings":[150],"The":[151,182,273,366,389],"TextCNN":[152,367],"TextRCNN":[154,369],"models":[155,370],"show":[156],"obvious":[157],"superiority":[158],"small-scale":[160,374],"more":[163,170,194,261,377,384,401],"complex":[164,378],"hierarchical":[165,312,379],"structure":[166,380],"balanced":[171,385],"documentlabel":[172],"distribution":[173],"terms":[175],"macro-F1,":[177],"micro-F1":[178],"NN":[185,392],"method":[186,393],"works":[187,394],"better":[188,372,395],"larger-scale":[191,398],"dataset":[192,399],"unbalanced":[195,402],"document-label":[196,386,403],"distribution.":[197,387,404],"Research":[198],"limitations":[199],"differ":[203],"following":[206],"aspects:":[207],"statement,":[208],"data":[209,228],"quality,":[210],"purposes.":[212],"Additionally,":[213],"designed":[216],"intrinsic":[222],"differences":[223],"their":[225],"processing":[229],"feature":[231],"selection,":[232],"which":[233],"turn":[235],"impacts":[236],"a":[240,343],"approach.":[243],"In":[244],"near":[246],"future,":[247],"we":[248],"will":[249,338],"enhance":[250],"experimental":[251],"precision":[252],"reinforce":[254],"validity":[256],"conclusions":[258],"by":[259,309],"employing":[260],"rigorous":[262],"control":[263],"over":[264],"variables":[265],"through":[266],"introducing":[267],"expanded":[268],"parameter":[269],"settings.":[270],"Practical":[271],"implications":[272],"observed":[274],"Macro":[275],"F1":[276,279],"Micro":[278],"scores":[280],"typically":[284],"fall":[285],"short":[286],"achieved":[289],"datasets,":[292],"underscoring":[293],"complexity":[295],"tasks.":[300],"Approaches":[301],"leveraging":[302],"deep":[303,322],"learning":[304,323],"techniques":[305],"offer":[306],"promising":[307],"solutions":[308],"accommodating":[310],"relationships":[313],"interdependencies":[315],"among":[316],"With":[318],"ongoing":[319],"enhancements":[320],"algorithms":[324],"large-scale":[326],"models,":[327],"it":[328],"expected":[330],"that":[331],"efficacy":[333],"tasks":[337],"be":[339],"significantly":[340],"improved,":[341],"reaching":[342],"level":[344],"practical":[346],"utility":[347],"foreseeable":[350],"future.":[351],"Originality/value":[352],"(1)":[353],"(2)":[365],"perform":[371],"(3)":[388]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2024-05-28T00:00:00"}