{"id":"https://openalex.org/W7162035251","doi":"https://doi.org/10.48550/arxiv.2605.20357","title":"Consistently Informative Soft-Label Temperature for Knowledge Distillation","display_name":"Consistently Informative Soft-Label Temperature for Knowledge Distillation","publication_year":2026,"publication_date":"2026-05-19","ids":{"openalex":"https://openalex.org/W7162035251","doi":"https://doi.org/10.48550/arxiv.2605.20357"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.20357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.20357","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091095351","display_name":"Hoang-Chau Luong","orcid":"https://orcid.org/0000-0003-2306-072X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luong, Hoang-Chau","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136615764","display_name":"Nghia Van Vo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Van Vo, Nghia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136631224","display_name":"Kaiqi Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Kaiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136625698","display_name":"Lingwei Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Lingwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.39739999175071716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.39739999175071716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.302700012922287,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.043699998408555984,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.5942000150680542},{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.5439000129699707},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5267000198364258},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.5209000110626221},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.4343999922275543}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.5942000150680542},{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.5439000129699707},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5368000268936157},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5267000198364258},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.5209000110626221},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.475600004196167},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.4343999922275543},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4293000102043152},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.40070000290870667},{"id":"https://openalex.org/C140331021","wikidata":"https://www.wikidata.org/wiki/Q1868104","display_name":"Logit","level":2,"score":0.33709999918937683},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3000999987125397}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.20357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.20357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.20357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.6809201836585999,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Knowledge":[0],"distillation":[1,144,197],"(KD)":[2],"transfers":[3],"knowledge":[4],"from":[5],"a":[6,10,23,56,175],"high-capacity":[7],"teacher":[8,28,61,92,123,131,148,169],"to":[9,121,147],"compact":[11],"student":[12,94,151],"by":[13,163,187],"matching":[14],"their":[15,101],"predictive":[16],"distributions,":[17],"with":[18,64,208],"temperature":[19,59,90],"scaling":[20],"serving":[21],"as":[22],"central":[24],"mechanism":[25],"for":[26,178],"smoothing":[27],"predictions":[29,69],"and":[30,53,73,82,93,124,150,171,190,195,205],"exposing":[31],"informative":[32,130],"\"dark":[33],"knowledge\"":[34],"beyond":[35],"the":[36,40,88,122,143,164,167,172,184],"hard":[37],"label.":[38],"However,":[39],"standard":[41,203],"fixed-temperature":[42],"design":[43,127],"is":[44,160],"inherently":[45],"sample-agnostic.":[46],"Since":[47],"samples":[48],"differ":[49],"in":[50],"logit":[51,170],"scale":[52],"learning":[54,152],"difficulty,":[55],"single":[57],"global":[58],"produces":[60,128],"soft":[62,132],"labels":[63,133],"highly":[65],"inconsistent":[66],"entropy:":[67],"some":[68],"remain":[70],"overly":[71],"sharp":[72],"provide":[74],"limited":[75],"inter-class":[76],"information,":[77],"whereas":[78],"others":[79],"become":[80],"over-smoothed":[81],"lose":[83],"class-discriminative":[84],"information.":[85],"Moreover,":[86],"sharing":[87],"same":[89],"between":[91,166],"further":[95],"imposes":[96],"rigid":[97,136],"logit-scale":[98,138],"alignment":[99],"despite":[100],"capacity":[102],"mismatch.":[103],"To":[104],"address":[105],"these":[106],"limitations,":[107],"we":[108,155],"propose":[109],"CIST":[110,182],"(Consistently":[111],"Informative":[112],"Soft-label":[113],"Temperature),":[114],"which":[115],"assigns":[116],"separate":[117],"sample-wise":[118],"adaptive":[119,179],"temperatures":[120],"student.":[125],"This":[126],"consistently":[129],"while":[134],"relaxing":[135],"teacher--student":[137],"matching.":[139],"It":[140],"also":[141],"reweights":[142],"objective":[145],"according":[146],"confidence":[149],"difficulty.":[153],"Theoretically,":[154],"show":[156,199],"that":[157],"teacher-label":[158],"entropy":[159],"largely":[161],"governed":[162],"ratio":[165],"maximum":[168],"temperature,":[173,189],"providing":[174],"principled":[176],"basis":[177],"smoothing.":[180],"Empirically,":[181],"mitigates":[183],"inconsistency":[185],"induced":[186],"fixed":[188],"experiments":[191],"on":[192],"both":[193],"vision":[194],"language":[196],"tasks":[198],"consistent":[200],"improvements":[201],"over":[202],"KD":[204],"strong":[206],"baselines":[207],"negligible":[209],"computational":[210],"overhead.":[211]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-22T00:00:00"}