{"id":"https://openalex.org/W4375869150","doi":"https://doi.org/10.1109/icassp49357.2023.10094912","title":"A Simple Yet Effective Approach to Structured Knowledge Distillation","display_name":"A Simple Yet Effective Approach to Structured Knowledge Distillation","publication_year":2023,"publication_date":"2023-05-05","ids":{"openalex":"https://openalex.org/W4375869150","doi":"https://doi.org/10.1109/icassp49357.2023.10094912"},"language":"en","primary_location":{"id":"doi:10.1109/icassp49357.2023.10094912","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icassp49357.2023.10094912","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058918636","display_name":"Wenye Lin","orcid":"https://orcid.org/0000-0002-3338-4456"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Wenye Lin","raw_affiliation_strings":["Tencent AI Lab","Tsinghua Shenzhen International Graduate School, Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025128323","display_name":"Yangming Li","orcid":"https://orcid.org/0000-0002-9794-8054"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yangming Li","raw_affiliation_strings":["Tencent AI Lab"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005411806","display_name":"Lemao Liu","orcid":"https://orcid.org/0000-0003-3804-5768"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lemao Liu","raw_affiliation_strings":["Tencent AI Lab"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087920747","display_name":"Shuming Shi","orcid":"https://orcid.org/0000-0001-7018-0682"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuming Shi","raw_affiliation_strings":["Tencent AI Lab"],"affiliations":[{"raw_affiliation_string":"Tencent AI Lab","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022672030","display_name":"Hai-Tao Zheng","orcid":"https://orcid.org/0000-0001-5128-5649"},"institutions":[{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]},{"id":"https://openalex.org/I4210114105","display_name":"Tsinghua\u2013Berkeley Shenzhen Institute","ror":"https://ror.org/02hhwwz98","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210114105","https://openalex.org/I95457486","https://openalex.org/I99065089"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hai-Tao Zheng","raw_affiliation_strings":["Tsinghua University,Tsinghua Shenzhen International Graduate School","Pengcheng Laboratory","Tsinghua Shenzhen International Graduate School, Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University,Tsinghua Shenzhen International Graduate School","institution_ids":["https://openalex.org/I4210114105","https://openalex.org/I99065089"]},{"raw_affiliation_string":"Pengcheng Laboratory","institution_ids":["https://openalex.org/I4210136793"]},{"raw_affiliation_string":"Tsinghua Shenzhen International Graduate School, Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5058918636"],"corresponding_institution_ids":["https://openalex.org/I2250653659","https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.04150449,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"27","issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7861679792404175},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7481101155281067},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.6513979434967041},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5856760740280151},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.585065484046936},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.5661946535110474},{"id":"https://openalex.org/keywords/monte-carlo-method","display_name":"Monte Carlo method","score":0.4552047550678253},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4531373083591461},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.4466860890388489},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4105914831161499},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3539736568927765},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.35374146699905396},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1346636414527893}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7861679792404175},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7481101155281067},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6513979434967041},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5856760740280151},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.585065484046936},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.5661946535110474},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.4552047550678253},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4531373083591461},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4466860890388489},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4105914831161499},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3539736568927765},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35374146699905396},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1346636414527893},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icassp49357.2023.10094912","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icassp49357.2023.10094912","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.49000000953674316,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W1934019294","https://openalex.org/W2045993505","https://openalex.org/W2134797427","https://openalex.org/W2144578941","https://openalex.org/W2294370754","https://openalex.org/W2896457183","https://openalex.org/W2914076857","https://openalex.org/W2963736842","https://openalex.org/W2964046515","https://openalex.org/W2970454332","https://openalex.org/W2976444281","https://openalex.org/W2978017171","https://openalex.org/W2997666887","https://openalex.org/W3034906024","https://openalex.org/W3034949522","https://openalex.org/W3034999214","https://openalex.org/W3105966348","https://openalex.org/W3175265131","https://openalex.org/W3200808010","https://openalex.org/W6638523607","https://openalex.org/W6640267182","https://openalex.org/W6679909955","https://openalex.org/W6755207826","https://openalex.org/W6768086466","https://openalex.org/W6768248635","https://openalex.org/W6768851824"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W4246352526","https://openalex.org/W2121910908","https://openalex.org/W915438175"],"abstract_inverted_index":{"Structured":[0],"prediction":[1,156],"models":[2],"aim":[3],"at":[4],"solving":[5],"tasks":[6,157],"where":[7],"the":[8,89,109,139,145,149,165],"output":[9,33,125],"is":[10,25,65],"a":[11,16,92],"complex":[12],"structure,":[13],"rather":[14,107],"than":[15,76,83,108],"single":[17],"variable.":[18],"Performing":[19],"knowledge":[20,90],"distillation":[21,43],"for":[22,74,123],"such":[23],"problems":[24],"non-":[26],"trivial":[27],"due":[28],"to":[29,95,142],"their":[30,101],"exponentially":[31],"large":[32],"space.":[34],"Previous":[35],"works":[36],"address":[37],"this":[38,58,113],"problem":[39],"by":[40,98],"developing":[41],"particular":[42],"strategies":[44],"(e.g.,":[45],"dynamic":[46],"programming)":[47],"that":[48,64,136,159],"are":[49],"both":[50],"complicated":[51],"and":[52,79,130],"of":[53,148],"low":[54],"run-time":[55],"efficiency.":[56],"In":[57,112],"work,":[59],"we":[60,87,115,134],"propose":[61],"an":[62],"approach":[63,161],"much":[66],"simpler":[67],"in":[68],"its":[69,96],"formulation,":[70],"far":[71],"more":[72],"efficient":[73,131],"training":[75],"existing":[77],"methods,":[78],"even":[80],"performs":[81],"better":[82,143],"our":[84,160],"baselines.":[85],"Specifically,":[86],"transfer":[88],"from":[91],"teacher":[93,150],"model":[94,141],"student":[97,140],"locally":[99],"matching":[100],"computations":[102],"on":[103,153,173],"all":[104],"internal":[105,146],"structures":[106],"final":[110],"outputs.":[111],"manner,":[114],"avoid":[116],"time-consuming":[117],"techniques":[118],"like":[119],"Monte":[120],"Carlo":[121],"Sampling":[122],"decoding":[124],"structures,":[126],"permitting":[127],"parallel":[128],"computation":[129],"training.":[132],"Besides,":[133],"show":[135],"it":[137],"encourages":[138],"mimic":[144],"behavior":[147],"model.":[151],"Experiments":[152],"two":[154,174],"structured":[155],"demonstrate":[158],"not":[162],"only":[163],"halves":[164],"time":[166],"cost,":[167],"but":[168],"also":[169],"outperforms":[170],"previous":[171],"methods":[172],"widely":[175],"adopted":[176],"benchmark":[177],"datasets.<sup":[178],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[179],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">1":[180],"2</sup>":[181]},"counts_by_year":[],"updated_date":"2025-12-23T23:11:35.936235","created_date":"2025-10-10T00:00:00"}