{"id":"https://openalex.org/W7225457597","doi":"https://doi.org/10.5281/zenodo.24032102","title":"Residual Stream Outliers as an Automatic Temperature Representation Mechanism for LLM Output Confidence","display_name":"Residual Stream Outliers as an Automatic Temperature Representation Mechanism for LLM Output Confidence","publication_year":2026,"publication_date":"2026-11-07","ids":{"openalex":"https://openalex.org/W7225457597","doi":"https://doi.org/10.5281/zenodo.24032102"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.24032102","doi":"https://doi.org/10.5281/zenodo.24032102","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032102","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.24032102","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128828148","display_name":"Haifeng Qiu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Qiu, Haifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5128828148"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8330000042915344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8330000042915344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10199999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.03909999877214432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/conditional-entropy","display_name":"conditional entropy","score":0.8669999837875366},{"id":"https://openalex.org/keywords/temperature-scaling","display_name":"temperature scaling","score":0.6230000257492065},{"id":"https://openalex.org/keywords/rmsnorm","display_name":"RMSNorm","score":0.6759999990463257},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6800000071525574},{"id":"https://openalex.org/keywords/next-token-prediction","display_name":"next-token prediction","score":0.6510000228881836},{"id":"https://openalex.org/keywords/overconfidence","display_name":"overconfidence","score":0.5709999799728394},{"id":"https://openalex.org/keywords/autoregressive-generation","display_name":"autoregressive generation","score":0.5950000286102295},{"id":"https://openalex.org/keywords/intervention-experiments","display_name":"intervention experiments","score":0.3709999918937683},{"id":"https://openalex.org/keywords/open-source-llms","display_name":"open-source LLMs","score":0.5139999985694885}],"concepts":[{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.6868000030517578},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.6610999703407288},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5990999937057495},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5040000081062317},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4505000114440918},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39480000734329224},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.3481000065803528},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.34040001034736633},{"id":"https://openalex.org/C136695289","wikidata":"https://www.wikidata.org/wiki/Q415568","display_name":"Bitstream","level":3,"score":0.3357999920845032},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.33239999413490295},{"id":"https://openalex.org/C49020025","wikidata":"https://www.wikidata.org/wiki/Q1059099","display_name":"Chaining","level":2,"score":0.3172000050544739},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.30469998717308044},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.30000001192092896},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.28790000081062317},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C58166","wikidata":"https://www.wikidata.org/wiki/Q224821","display_name":"Fuzzy logic","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C164660894","wikidata":"https://www.wikidata.org/wiki/Q2037833","display_name":"Piecewise","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2623000144958496},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.26100000739097595},{"id":"https://openalex.org/C39613435","wikidata":"https://www.wikidata.org/wiki/Q846677","display_name":"Remainder","level":2,"score":0.25690001249313354},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2540999948978424}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.5281/zenodo.24032102","doi":"https://doi.org/10.5281/zenodo.24032102","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032102","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.24032103","doi":"https://doi.org/10.5281/zenodo.24032103","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032103","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.24047434","doi":"https://doi.org/10.5281/zenodo.24047434","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24047434","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.24032102","doi":"https://doi.org/10.5281/zenodo.24032102","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.24032102","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7166865890","https://openalex.org/W7082641262","https://openalex.org/W6884967519","https://openalex.org/W7152643876","https://openalex.org/W4415795384","https://openalex.org/W7137881767","https://openalex.org/W7127138976","https://openalex.org/W7131103984","https://openalex.org/W4416036614","https://openalex.org/W7123666280"],"abstract_inverted_index":{"The":[0,298],"standard":[1],"output":[2,153,212],"pipeline":[3],"of":[4,85,118,126,136,189,214,223,233,285,310,416],"contemporary":[5],"open-source":[6],"large":[7],"language":[8,35],"models":[9],"(LLMs)":[10],"proceeds":[11],"as":[12,401],"follows:":[13],"the":[14,34,49,66,90,114,119,124,127,133,143,148,169,175,187,211,220,224,231,245,251,271,279,283,318,330,336,359,376,380,402],"final":[15,333,381],"Transformer":[16],"layer":[17,382],"outputs":[18],"a":[19,28,44,57,314,350],"hidden":[20],"state,":[21],"which":[22],"passes":[23,355],"through":[24,357],"Final":[25,128,392],"RMSNorm,":[26],"undergoes":[27],"linear":[29],"projection":[30],"to":[31,75,97,206,326,358],"logits":[32,232,276],"via":[33],"modeling":[36],"(LM)":[37],"head,":[38,226],"and":[39,145,191,208,216,237,277,290,332,363,367],"is":[40,96,205,301],"ultimately":[41],"transformed":[42],"into":[43],"next-token":[45,86],"probability":[46],"distribution":[47],"by":[48,106,112,163],"softmax":[50],"function.":[51],"In":[52],"this":[53,62,227,407],"paper,":[54],"we":[55],"examine":[56],"frequently":[58],"misunderstood":[59],"mechanism":[60,289],"within":[61,89],"pipeline:":[63],"outliers":[64],"in":[65,186,244,294,313,329,379,387,419],"residual":[67,120],"stream":[68],"are":[69],"not":[70,103,395],"numerical":[71,83,91],"noise":[72],"that":[73,260,354],"ought":[74],"be":[76],"\u201ccleaned":[77],"up,\u201d":[78],"but":[79],"rather":[80,342],"an":[81],"active":[82],"representation":[84],"certainty":[87],"encoded":[88],"distribution.":[92],"Its":[93,202],"primary":[94],"function":[95,204],"elevate":[98],"effective":[99,149],"temperature.":[100],"Outliers":[101],"do":[102],"alter":[104],"predictions":[105],"steering":[107],"specific":[108],"semantic":[109,253,275],"directions;":[110],"instead,":[111],"inflating":[113],"root-mean-square":[115],"(RMS)":[116],"statistic":[117],"stream,":[121],"they":[122],"enlarge":[123],"denominator":[125,273],"RMSNorm.":[129],"This":[130],"globally":[131],"dampens":[132],"normalized":[134],"magnitudes":[135],"all":[137],"channels\u2014suppressing":[138],"overall":[139],"logit":[140],"scales,":[141],"flattening":[142],"distribution,":[144,281],"thereby":[146],"raising":[147],"temperature":[150,409],"while":[151,239,361],"decreasing":[152],"confidence.":[154],"These":[155],"outlierspikes":[156],"predominantly":[157,229],"originate":[158],"at":[159],"token":[160,172],"boundaries":[161],"characterized":[162],"high":[164,199],"conditional":[165],"entropy\u2014most":[166],"prominently":[167],"following":[168],"sequence-initial":[170],"<BOS>":[171],"(where":[173,194],"predicting":[174],"very":[176],"first":[177],"content":[178],"word":[179],"relies":[180],"entirely":[181],"on":[182,370],"broad":[183],"corpus-level":[184,340],"priors":[185,341],"absence":[188],"context)":[190],"sentence-final":[192],"punctuation":[193,234,344],"subsequent":[195,362],"clause-initiating":[196],"tokens":[197],"exhibit":[198],"branch":[200],"entropy).":[201],"auxiliary":[203],"concurrently":[207],"marginally":[209],"increase":[210],"probabilities":[213],"whitespace":[215],"punc-tuation":[217],"characters:":[218],"along":[219],"readout":[221],"direction":[222],"LM":[225],"channel":[228,300,338],"elevates":[230],"marks,":[235],"whitespace,":[236],"delimiters,":[238],"depressing":[240],"invalid":[241],"byte":[242],"fragments":[243],"vocabulary.":[246],"Crucially,":[247],"neither":[248],"effect":[249],"alters":[250],"underlying":[252],"preference":[254],"structure.":[255],"Furthermore,":[256],"intervention":[257],"experiments":[258],"demonstrate":[259],"artificially":[261],"clipping":[262],"or":[263,345],"ablating":[264],"these":[265],"outlier":[266,299,321,377],"channels":[267,328],"induces":[268],"spurious":[269],"overconfidence:":[270],"collapsed":[272],"amplifies":[274],"sharpens":[278],"predictive":[280],"stripping":[282],"model":[284,316],"its":[286,385],"entropy-mediated":[287],"escape":[288],"trapping":[291],"autoregressive":[292],"generation":[293],"degenerate":[295],"repetition":[296],"loops.":[297],"typically":[302],"persistent":[303],"across":[304],"intermediate":[305],"layers":[306,312,348,365],"(empirically,":[307],"54":[308],"out":[309],"64":[311],"27B-parameter":[315],"share":[317],"identical":[319],"maximum":[320],"channel).":[322],"However,":[323],"it":[324,399],"shifts":[325],"alternative":[327],"initial":[331],"layers,":[334],"where":[335],"early-layer":[337],"encodes":[339],"than":[343],"whitespace.":[346],"Early":[347],"establish":[349],"baseline":[351],"confidence":[352],"regime":[353],"directly":[356],"output,":[360],"terminal":[364],"revise":[366],"reconfigureit":[368],"based":[369],"accumulated":[371],"representations.":[372],"As":[373],"training":[374],"progresses,":[375],"magnitude":[378],"recedes":[383],"from":[384],"peak":[386],"alignment":[388],"with":[389],"loss":[390],"convergence.":[391],"RMSNorm":[393],"does":[394],"eliminate":[396],"temperature;":[397],"rather,":[398],"serves":[400],"direct":[403],"implementation":[404],"substrate":[405],"for":[406],"automatic":[408],"regulation":[410],"mechanism.":[411],"v2:":[412],"Added":[413],"Appendix:":[414],"Mechanisms":[415],"Outlier":[417],"Channels":[418],"Non-Terminal":[420],"Layers":[421]},"counts_by_year":[],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2026-11-08T00:00:00"}