{"id":"https://openalex.org/W4404347231","doi":"https://doi.org/10.48550/arxiv.2410.23771","title":"What is Wrong with Perplexity for Long-context Language Modeling?","display_name":"What is Wrong with Perplexity for Long-context Language Modeling?","publication_year":2024,"publication_date":"2024-10-31","ids":{"openalex":"https://openalex.org/W4404347231","doi":"https://doi.org/10.48550/arxiv.2410.23771"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.23771","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.23771","pdf_url":"https://arxiv.org/pdf/2410.23771","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.23771","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Fang, Lizhe","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fang, Lizhe","raw_affiliation_strings":["State Key Lab of General Artificial Intelligence , School of Intelligence Science and Technology , Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Lab of General Artificial Intelligence , School of Intelligence Science and Technology , Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115592322","display_name":"Yifei Wang","orcid":"https://orcid.org/0000-0002-9634-4574","observed_orcids":["https://orcid.org/0000-0002-9634-4574","https://orcid.org/0009-0009-3170-8208"]},"institutions":[{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Yifei","raw_affiliation_strings":["MIT CSAIL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL","institution_ids":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100603529","display_name":"Zhaoyang Liu","orcid":"https://orcid.org/0009-0006-7617-6338","observed_orcids":["https://orcid.org/0009-0006-7617-6338","https://orcid.org/0009-0007-3892-7118"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Zhaoyang","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045681869","display_name":"Chenheng Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Chenheng","raw_affiliation_strings":["State Key Lab of General Artificial Intelligence , School of Intelligence Science and Technology , Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Lab of General Artificial Intelligence , School of Intelligence Science and Technology , Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085699861","display_name":"Stefanie Jegelka","orcid":"https://orcid.org/0000-0002-6121-9474","observed_orcids":["https://orcid.org/0000-0002-6121-9474"]},"institutions":[{"id":"https://openalex.org/I4403386549","display_name":"Munich Center for Machine Learning","ror":"https://ror.org/02nfy3535","country_code":"DE","type":"education","lineage":["https://openalex.org/I4403386549","https://openalex.org/I62916508","https://openalex.org/I8204097"]},{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]},{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Jegelka, Stefanie","raw_affiliation_strings":["MIT EECS , CSAIL","TUM CIT , MCML , MDSI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT EECS , CSAIL","institution_ids":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]},{"raw_affiliation_string":"TUM CIT , MCML , MDSI","institution_ids":["https://openalex.org/I4403386549","https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101403129","display_name":"Jinyang Gao","orcid":"https://orcid.org/0009-0004-3741-4976","observed_orcids":["https://orcid.org/0009-0004-3741-4976","https://orcid.org/0000-0002-2094-3554"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Gao, Jinyang","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040297543","display_name":"Bolin Ding","orcid":"https://orcid.org/0000-0003-1535-9692","observed_orcids":["https://orcid.org/0000-0003-1535-9692"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ding, Bolin","raw_affiliation_strings":["Alibaba Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba Group","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101431030","display_name":"Yisen Wang","orcid":"https://orcid.org/0000-0002-0181-2137","observed_orcids":["https://orcid.org/0000-0002-0181-2137","https://orcid.org/0000-0002-8623-6318"]},"institutions":[{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Yisen","raw_affiliation_strings":["MIT CSAIL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL","institution_ids":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":60,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9574999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9574999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.032099999487400055,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.004100000020116568,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"perplexity","score":0.8550000190734863},{"id":"https://openalex.org/keywords/long-context-language-modeling","display_name":"long-context language modeling","score":0.949999988079071},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6830000281333923},{"id":"https://openalex.org/keywords/evaluation-indicators","display_name":"evaluation metrics","score":0.6909999847412109},{"id":"https://openalex.org/keywords/token-weighting","display_name":"token weighting","score":0.4779999852180481},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.7129999995231628},{"id":"https://openalex.org/keywords/long-context-understanding","display_name":"long-context understanding","score":0.46799999475479126}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.9930883049964905},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6408384442329407},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5876981616020203},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5232174396514893},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4803551435470581},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3542419672012329},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3447527587413788},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.19664129614830017},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.1386014223098755},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.23771","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.23771","pdf_url":"https://arxiv.org/pdf/2410.23771","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.23771","doi":"https://doi.org/10.48550/arxiv.2410.23771","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.23771","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.23771","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.23771","pdf_url":"https://arxiv.org/pdf/2410.23771","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404347231.pdf","grobid_xml":"https://content.openalex.org/works/W4404347231.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W2908510526","https://openalex.org/W3035019713","https://openalex.org/W3099417250","https://openalex.org/W3118781290","https://openalex.org/W4378505330","https://openalex.org/W4382603041","https://openalex.org/W4383473945","https://openalex.org/W4384918448","https://openalex.org/W4385573858","https://openalex.org/W4387225871","https://openalex.org/W4387561528","https://openalex.org/W4389518795","https://openalex.org/W4389524066","https://openalex.org/W4390722419","https://openalex.org/W4390723197","https://openalex.org/W4390961337","https://openalex.org/W4392617597","https://openalex.org/W4393160131","https://openalex.org/W4393736311","https://openalex.org/W4393942918","https://openalex.org/W4394778349","https://openalex.org/W4394782324","https://openalex.org/W4395474395","https://openalex.org/W4401042914","https://openalex.org/W4401042981","https://openalex.org/W4401306886","https://openalex.org/W4401660192","https://openalex.org/W4402671749","https://openalex.org/W4402671766","https://openalex.org/W4402684150","https://openalex.org/W4412945452","https://openalex.org/W4415797147","https://openalex.org/W7103578140","https://openalex.org/W7127179963"],"related_works":["https://openalex.org/W4414585948","https://openalex.org/W4396881818","https://openalex.org/W4393942918","https://openalex.org/W4403752108","https://openalex.org/W4392126139","https://openalex.org/W7167907175","https://openalex.org/W4385009323","https://openalex.org/W4404261302","https://openalex.org/W4414875734","https://openalex.org/W4386272986"],"abstract_inverted_index":{"Handling":[0],"long-context":[1,47,80,96,133,192],"inputs":[2],"is":[3,197],"crucial":[4],"for":[5,45,66,79,157,186],"large":[6],"language":[7],"models":[8,94],"(LLMs)":[9],"in":[10,95,144],"tasks":[11],"such":[12],"as":[13,36],"extended":[14,26],"conversations,":[15],"document":[16],"summarization,":[17],"and":[18,32,87,182,189],"many-shot":[19],"in-context":[20],"learning.":[21],"While":[22],"recent":[23],"approaches":[24],"have":[25],"the":[27,90,178,191],"context":[28,116],"windows":[29],"of":[30,52,93,138,180,194],"LLMs":[31],"employed":[33],"perplexity":[34],"(PPL)":[35],"a":[37,63,104,114,154],"standard":[38],"evaluation":[39],"metric,":[40],"PPL":[41,72,143,181],"has":[42,55],"proven":[43],"unreliable":[44],"assessing":[46],"capabilities.":[48],"The":[49],"underlying":[50],"cause":[51],"this":[53,59,67],"limitation":[54],"remained":[56],"unclear.":[57],"In":[58,170],"work,":[60],"we":[61,101,148],"provide":[62],"comprehensive":[64],"explanation":[65],"issue.":[68],"We":[69],"find":[70],"that":[71,107,125,159],"overlooks":[73],"key":[74,110,161],"tokens,":[75,162],"which":[76],"are":[77],"essential":[78],"understanding,":[81],"by":[82,112],"averaging":[83],"across":[84,167],"all":[85],"tokens":[86,111],"thereby":[88],"obscuring":[89],"true":[91],"performance":[92,130],"scenarios.":[97],"To":[98],"address":[99],"this,":[100],"propose":[102],"\\textbf{LongPPL},":[103],"novel":[105],"metric":[106],"focuses":[108],"on":[109,131],"employing":[113],"long-short":[115],"contrastive":[117],"method":[118],"to":[119,164],"identify":[120],"them.":[121],"Our":[122],"experiments":[123],"demonstrate":[124],"LongPPL":[126],"strongly":[127],"correlates":[128],"with":[129],"various":[132],"benchmarks":[134],"(e.g.,":[135],"Pearson":[136],"correlation":[137],"-0.96),":[139],"significantly":[140],"outperforming":[141],"traditional":[142],"predictive":[145],"accuracy.":[146],"Additionally,":[147],"introduce":[149],"\\textbf{LongCE}":[150],"(Long-context":[151],"Cross-Entropy)":[152],"loss,":[153],"re-weighting":[155],"strategy":[156],"fine-tuning":[158],"prioritizes":[160],"leading":[163],"consistent":[165],"improvements":[166],"diverse":[168],"benchmarks.":[169],"summary,":[171],"these":[172],"contributions":[173],"offer":[174],"deeper":[175],"insights":[176],"into":[177],"limitations":[179],"present":[183],"effective":[184],"solutions":[185],"accurately":[187],"evaluating":[188],"enhancing":[190],"capabilities":[193],"LLMs.":[195],"Code":[196],"available":[198],"at":[199],"https://github.com/PKU-ML/LongPPL.":[200]},"counts_by_year":[{"year":2026,"cited_by_count":28},{"year":2025,"cited_by_count":30},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-11-14T00:00:00"}