{"id":"https://openalex.org/W7169858945","doi":"https://doi.org/10.48550/arxiv.2607.16848","title":"Beyond Memory Leaderboards: Evaluating Scientific Memory as Budgeted Context Restoration","display_name":"Beyond Memory Leaderboards: Evaluating Scientific Memory as Budgeted Context Restoration","publication_year":2026,"publication_date":"2026-07-18","ids":{"openalex":"https://openalex.org/W7169858945","doi":"https://doi.org/10.48550/arxiv.2607.16848"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.16848","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.16848","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.16848","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141242106","display_name":"Maksim Sheverev","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sheverev, Maksim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141249795","display_name":"David Finkelstein","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Finkelstein, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5141243446","display_name":"Sergey Nikolenko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nikolenko, Sergey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.2094999998807907,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.2094999998807907,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.17679999768733978,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.10729999840259552,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/human-memory","display_name":"Human memory","score":0.67330002784729},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5440999865531921},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.413100004196167},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.3711000084877014},{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.35429999232292175},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3474000096321106},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.3310999870300293}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.758899986743927},{"id":"https://openalex.org/C2985957978","wikidata":"https://www.wikidata.org/wiki/Q492","display_name":"Human memory","level":3,"score":0.67330002784729},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5440999865531921},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.413100004196167},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39910000562667847},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3894999921321869},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.3711000084877014},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.35429999232292175},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3474000096321106},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3310999870300293},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3147999942302704},{"id":"https://openalex.org/C76679254","wikidata":"https://www.wikidata.org/wiki/Q5165163","display_name":"Context-dependent memory","level":4,"score":0.3100000023841858},{"id":"https://openalex.org/C119907115","wikidata":"https://www.wikidata.org/wiki/Q6815725","display_name":"Memory errors","level":3,"score":0.3091000020503998},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3052000105381012},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.28780001401901245},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C150202949","wikidata":"https://www.wikidata.org/wiki/Q107602","display_name":"Pointer (user interface)","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C76188268","wikidata":"https://www.wikidata.org/wiki/Q1783165","display_name":"Context effect","level":3,"score":0.2565000057220459},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.16848","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.16848","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.16848","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.16848","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Long-term":[0],"memory":[1,12,70,192],"is":[2,136,232],"becoming":[3],"a":[4,63,185],"core":[5],"component":[6],"of":[7,107,144,180],"LLM":[8],"agents,":[9],"but":[10,103],"most":[11,139],"benchmarks":[13],"evaluate":[14,53],"conversations":[15],"or":[16],"compact":[17],"summaries,":[18],"while":[19],"research":[20],"agents":[21],"need":[22],"to":[23,219],"restore":[24],"evidence":[25],"from":[26],"full":[27,77],"scientific":[28,191],"papers.":[29],"We":[30,52,188],"introduce":[31],"two":[32],"full-text":[33],"scientific-memory":[34],"benchmarks,":[35],"Public":[36,45],"AI":[37],"Memory":[38],"(PAIM;":[39],"81":[40],"papers,":[41,49],"66":[42],"questions)":[43],"and":[44,89,112,148,158,171,208,217,223,237],"Transformers":[46],"(PTr;":[47],"252":[48],"98":[50],"questions).":[51],"eight":[54],"memory/retrieval":[55],"systems,":[56],"including":[57],"our":[58,221],"own":[59],"proposed":[60],"Theoria,":[61],"plus":[62],"no-retrieval":[64],"baseline.":[65],"Our":[66,230],"results":[67,222],"show":[68,162],"that":[69,163,190],"leaderboards":[71],"are":[72,166,240],"not":[73],"interpretable":[74],"without":[75],"the":[76,94,118,124,133,137,152,211,238],"protocol:":[78],"ingestion":[79],"granularity,":[80],"raw-text":[81],"preservation,":[82],"retrieval":[83,85,116,128],"budget,":[84],"modality,":[86],"rubric":[87],"audit,":[88],"judge":[90],"choice":[91],"all":[92],"affect":[93],"outcome.":[95],"For":[96],"example,":[97],"on":[98,184],"PAIM":[99],"Graphiti":[100],"wins":[101],"convincingly":[102],"uses":[104],"2.6M":[105],"characters":[106],"retrieved":[108],"context":[109,199],"per":[110],"query,":[111],"after":[113],"controlling":[114],"for":[115,123,151,227],"budget":[117],"lead":[119,153],"disappears.":[120],"On":[121],"PTr,":[122],"systems":[125],"where":[126],"BM25":[127],"can":[129],"be":[130,194],"added":[131],"cleanly,":[132],"sparse-dense":[134],"hybrid":[135,142],"single":[138],"significant":[140],"intervention:":[141],"variants":[143],"Simple":[145],"RAG,":[146],"Mem0,":[147],"Theoria":[149],"tie":[150],"within":[154],"0.03":[155],"points.":[156],"Multi-judge":[157],"human":[159,174],"side-by-side":[160],"calibration":[161],"LLM-as-a-judge":[164],"rankings":[165],"consistent":[167],"across":[168],"frontier":[169],"judges":[170],"agree":[172],"with":[173,176],"evaluation,":[175],"an":[177,204],"effective":[178],"resolution":[179],"roughly":[181],"one":[182],"point":[183],"ten-point":[186],"scale.":[187],"argue":[189],"should":[193],"evaluated":[195],"as":[196,203,225],"budgeted,":[197],"modality-aware":[198],"restoration":[200],"rather":[201],"than":[202],"unconstrained":[205],"architecture":[206],"leaderboard,":[207],"we":[209],"release":[210],"datasets,":[212],"harness,":[213],"raw":[214],"outputs,":[215],"judgments,":[216],"scripts":[218],"reproduce":[220],"serve":[224],"tools":[226],"such":[228],"evaluation.":[229],"code":[231],"available":[233,241],"at":[234,242],"http://gitlab.com/quantellence/research/scientific-recall-bench":[235],",":[236],"datasets":[239],"http://huggingface.co/datasets/quantellence/srb-data":[243],".":[244]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-22T00:00:00"}