{"id":"https://openalex.org/W6979282908","doi":null,"title":"Learning from Stochastic Teacher Representations Using Student-Guided Knowledge Distillation","display_name":"Learning from Stochastic Teacher Representations Using Student-Guided Knowledge Distillation","publication_year":2025,"publication_date":"2025-04-19","ids":{"openalex":"https://openalex.org/W6979282908"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2504.14307","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.14307","pdf_url":"https://arxiv.org/pdf/2504.14307","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2504.14307","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103265372","display_name":"Muhammad Haseeb Aslam","orcid":"https://orcid.org/0009-0003-3564-2310","observed_orcids":["https://orcid.org/0009-0003-3564-2310","https://orcid.org/0000-0002-8977-9457"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aslam, Muhammad Haseeb","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Martinez, Clara","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martinez, Clara","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039660800","display_name":"Marco Pedersoli","orcid":"https://orcid.org/0000-0002-7601-8640","observed_orcids":["https://orcid.org/0000-0002-7601-8640"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pedersoli, Marco","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074262951","display_name":"Alessandro L. Koerich","orcid":"https://orcid.org/0000-0001-5879-7014","observed_orcids":["https://orcid.org/0000-0001-5879-7014"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koerich, Alessandro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Etemad, Ali","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Etemad, Ali","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Granger, Eric","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Granger, Eric","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.6807000041007996,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.6807000041007996,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.2687999904155731,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.03020000085234642,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9039999842643738},{"id":"https://openalex.org/keywords/self-distillation","display_name":"self-distillation","score":0.8100000023841858},{"id":"https://openalex.org/keywords/ensemble-learning","display_name":"ensemble learning","score":0.4359999895095825},{"id":"https://openalex.org/keywords/weight-averaging","display_name":"weight averaging","score":0.9350000023841858},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.6970000267028809},{"id":"https://openalex.org/keywords/affective-computing","display_name":"affective computing","score":0.7620000243186951},{"id":"https://openalex.org/keywords/wearable-devices","display_name":"wearable devices","score":0.7289999723434448},{"id":"https://openalex.org/keywords/biosignal-classification","display_name":"biosignal classification","score":0.6399999856948853},{"id":"https://openalex.org/keywords/human-activity-recognition","display_name":"human activity recognition","score":0.6589999794960022},{"id":"https://openalex.org/keywords/resource-constrained-deployment","display_name":"resource-constrained deployment","score":0.6069999933242798}],"concepts":[{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.7080000042915344},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6955000162124634},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6581000089645386},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6539999842643738},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.6478000283241272},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6322000026702881},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.6123999953269958},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.5454000234603882},{"id":"https://openalex.org/C119898033","wikidata":"https://www.wikidata.org/wiki/Q3433888","display_name":"Ensemble forecasting","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.33169999718666077},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3003000020980835},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C2779714256","wikidata":"https://www.wikidata.org/wiki/Q25305062","display_name":"Multiple Models","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2558000087738037},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.25220000743865967}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:arXiv.org:2504.14307","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.14307","pdf_url":"https://arxiv.org/pdf/2504.14307","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2504.14307","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.14307","pdf_url":"https://arxiv.org/pdf/2504.14307","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.7567999958992004,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Advances":[0],"in":[1,98,106],"self-distillation":[2,155],"have":[3],"shown":[4],"that":[5,141,214],"when":[6,31],"knowledge":[7,175],"is":[8,34,39,159,185],"distilled":[9],"from":[10,169,202],"a":[11,14,70,152],"teacher":[12,29,38,83,127,165],"to":[13,69,118,138,167,189,240],"student":[15,24,72,179],"using":[16,129,173],"the":[17,23,28,32,37,60,90,145,191,203,206,215,225],"same":[18],"deep":[19],"learning":[20,46,243],"(DL)":[21],"architecture,":[22],"performance":[25],"can":[26,219],"surpass":[27],"particularly":[30,104],"network":[33],"overparameterized":[35],"and":[36,53,85,95,123,163,209,231,234,244],"trained":[40],"with":[41,144],"early":[42],"stopping.":[43],"Alternatively,":[44],"ensemble":[45,68,242],"also":[47],"improves":[48],"performance,":[49],"although":[50],"training,":[51],"storing,":[52],"deploying":[54],"multiple":[55,82,125],"models":[56,63,84],"becomes":[57],"impractical":[58],"as":[59,112,187],"number":[61],"of":[62,81],"grows.":[64],"Even":[65],"distilling":[66,96],"an":[67],"single":[71],"model":[73,122,226],"or":[74,108],"weight":[75,245],"averaging":[76,246],"methods":[77,222],"first":[78],"requires":[79],"training":[80,157,230],"does":[86],"not":[87],"fully":[88],"leverage":[89],"inherent":[91],"stochasticity":[92],"for":[93,161],"generating":[94,133],"diversity":[97],"DL":[99],"models.":[100],"These":[101],"constraints":[102],"are":[103,142],"prohibitive":[105],"resource-constrained":[107],"latency-sensitive":[109],"applications":[110],"such":[111],"wearable":[113],"devices.":[114],"This":[115],"paper":[116],"proposes":[117],"train":[119],"only":[120],"one":[121],"generate":[124],"diverse":[126],"representations":[128,135,140,171],"distillation-time":[130],"dropout.":[131],"However,":[132],"these":[134],"stochastically":[136],"leads":[137],"noisy":[139],"misaligned":[143],"learned":[146],"task.":[147],"To":[148],"overcome":[149],"this":[150],"problem,":[151],"novel":[153],"stochastic":[154],"(SSD)":[156],"strategy":[158],"introduced":[160],"filtering":[162],"weighting":[164],"representation":[166,180],"distill":[168],"task-relevant":[170],"only,":[172],"student-guided":[174],"distillation":[176,183,192],"(SGKD).":[177],"The":[178],"at":[181,228],"each":[182],"step":[184],"used":[186],"authority":[188],"guide":[190],"process.":[193],"Experimental":[194],"results":[195],"on":[196],"real-world":[197],"affective":[198],"computing,":[199],"wearable/biosignal":[200],"datasets":[201,212],"UCR":[204],"Archive,":[205],"HAR":[207],"dataset,":[208],"image":[210],"classification":[211],"show":[213],"proposed":[216],"SSD":[217],"method":[218],"outperform":[220],"state-of-the-art":[221,241],"without":[223],"increasing":[224],"size":[227],"both":[229],"testing":[232],"time,":[233],"incurs":[235],"negligible":[236],"computational":[237],"complexity":[238],"compared":[239],"methods.":[247]},"counts_by_year":[],"updated_date":"2026-10-13T10:08:12.033832","created_date":"2025-10-10T00:00:00"}