{"id":"https://openalex.org/W7225311012","doi":"https://doi.org/10.5281/zenodo.23988579","title":"What Synthetic Context-Selection Tests Measure: Policy, Evidence, and Answer Format","display_name":"What Synthetic Context-Selection Tests Measure: Policy, Evidence, and Answer Format","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225311012","doi":"https://doi.org/10.5281/zenodo.23988579"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23988579","doi":"https://doi.org/10.5281/zenodo.23988579","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23988579","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23988579","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5160839128","display_name":"Ahmad Shakir","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shakir, Ahmad","raw_affiliation_strings":["Independent Researcher"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent Researcher","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5160839128"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6820999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6820999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.08720000088214874,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.07159999758005142,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/state-tracking","display_name":"state tracking","score":0.5870000123977661},{"id":"https://openalex.org/keywords/evidence-selection","display_name":"evidence selection","score":0.5189999938011169},{"id":"https://openalex.org/keywords/answer-format","display_name":"answer format","score":0.5440000295639038},{"id":"https://openalex.org/keywords/llama-3-2-3b","display_name":"Llama-3.2-3B","score":0.7440000176429749},{"id":"https://openalex.org/keywords/attentional-gating","display_name":"attentional gating","score":0.5960000157356262},{"id":"https://openalex.org/keywords/shortcut-learning","display_name":"shortcut learning","score":0.5410000085830688},{"id":"https://openalex.org/keywords/conversational-memory","display_name":"conversational memory","score":0.7630000114440918},{"id":"https://openalex.org/keywords/state-conventions","display_name":"state conventions","score":0.35899999737739563},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.40799999237060547},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5410000085830688}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6506999731063843},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.6407999992370605},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.5827999711036682},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4401000142097473},{"id":"https://openalex.org/C117978034","wikidata":"https://www.wikidata.org/wiki/Q5422192","display_name":"Extractor","level":2,"score":0.426800012588501},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.42170000076293945},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4049000144004822},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.387800008058548},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.3199999928474426},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.31790000200271606},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.31369999051094055},{"id":"https://openalex.org/C2780608745","wikidata":"https://www.wikidata.org/wiki/Q367293","display_name":"Convention","level":2,"score":0.31200000643730164},{"id":"https://openalex.org/C2775936607","wikidata":"https://www.wikidata.org/wiki/Q466845","display_name":"Tracking (education)","level":2,"score":0.3089999854564667},{"id":"https://openalex.org/C194544171","wikidata":"https://www.wikidata.org/wiki/Q21105679","display_name":"Gating","level":2,"score":0.3021000027656555},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.2824999988079071},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27869999408721924},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.26510000228881836},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2587999999523163},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.25600001215934753},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C2776916960","wikidata":"https://www.wikidata.org/wiki/Q637156","display_name":"Warrant","level":2,"score":0.25369998812675476},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23988579","doi":"https://doi.org/10.5281/zenodo.23988579","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23988579","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23988580","doi":"https://doi.org/10.5281/zenodo.23988580","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23988580","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23988579","doi":"https://doi.org/10.5281/zenodo.23988579","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23988579","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7222964441","https://openalex.org/W7202147520","https://openalex.org/W7166863176","https://openalex.org/W4412945473","https://openalex.org/W7125102031","https://openalex.org/W4416035318","https://openalex.org/W7202310264","https://openalex.org/W7168367687","https://openalex.org/W7225104430","https://openalex.org/W7213522540"],"abstract_inverted_index":{"Context-selection":[0],"benchmarks":[1],"can":[2,205],"reward":[3],"several":[4],"different":[5,148],"capabilities:":[6],"following":[7],"a":[8,19,32,40,51,70,85,117,126,159,207,234,238],"state":[9,56,109,225],"convention,":[10],"choosing":[11],"evidence,":[12,376],"using":[13],"the":[14,99,108,192,247,283,287,301,314,320,349,352,375],"resulting":[15],"context,":[16],"and":[17,31,49,94,132,140,180,187,270,276,325,330,368,378],"producing":[18],"required":[20],"answer":[21],"format.":[22],"We":[23],"examine":[24],"these":[25,212],"distinctions":[26],"in":[27,286],"two":[28],"controlled":[29,217],"studies":[30,214,284],"supporting":[33,196,259],"shortcut":[34,209],"audit":[35,198],"of":[36,72,120],"synthetic":[37,267,332],"conversations.":[38],"In":[39,125],"prospective":[41],"150-source":[42],"study,":[43,129],"each":[44],"source":[45,323],"is":[46,171,246,341],"queried":[47],"before":[48],"after":[50],"reopening":[52],"instruction.":[53],"A":[54,103,175],"benchmark-tailored":[55],"rule":[57,83],"improves":[58],"Llama-3.2-3B":[59],"both-stage":[60],"canonical":[61],"success":[62,114],"from":[63,142,221,304],"94/150":[64],"to":[65,115],"150/150":[66],"under":[67,173],"hard":[68],"selection,":[69],"difference":[71],"37.3":[73],"percentage":[74],"points":[75,122,163],"(95%":[76,164],"paired":[77],"interval":[78,165],"[32.7,":[79],"42.0]).":[80],"The":[81,195,244,251,258,280,297,370],"same":[82,100],"with":[84,360],"simple":[86],"symbolic":[87],"extractor":[88],"also":[89],"solves":[90],"all":[91],"150":[92],"pairs,":[93],"finite":[95],"attention":[96,154,178],"gating":[97],"reaches":[98],"Llama":[101,177],"score.":[102],"fixed":[104],"post-outcome":[105],"prompt":[106],"making":[107],"convention":[110],"explicit":[111],"raises":[112],"full-context":[113],"112/150,":[116],"descriptive":[118],"gain":[119],"12.0":[121],"[6.0,":[123],"18.0].":[124],"separate":[127,293,321],"200-source":[128],"one-span":[130],"suppression":[131],"retention":[133],"errors":[134],"are":[135,291,334],"matched":[136],"by":[137],"carrier":[138],"role":[139],"change":[141],"an":[143],"oracle":[144],"map,":[145],"while":[146],"exposing":[147],"final":[149],"evidence.":[150],"Its":[151],"original":[152,193,266,331],"Qwen3-0.6B":[153],"primary":[155],"does":[156,308],"not":[157,292,309,382],"resolve":[158],"canonical-success":[160],"ordering:":[161],"1.6":[162],"[-2.5,":[166],"5.9]);":[167],"registered":[168],"inactive-value":[169],"emission":[170],"lower":[172],"suppression.":[174],"later":[176],"cell":[179],"literal-output":[181],"audits":[182],"reveal":[183],"further":[184],"task,":[185],"execution,":[186],"format":[188],"dependence":[189],"without":[190],"changing":[191],"endpoints.":[194],"CPU":[197,277],"shows":[199],"why":[200],"swapping":[201],"requested":[202],"states":[203],"alone":[204],"leave":[206],"first-mention":[208],"perfect.":[210],"Together":[211],"case":[213],"distinguish":[215],"what":[216],"selection":[218],"comparisons":[219],"establish":[220],"claims":[222],"about":[223],"learned":[224],"tracking":[226],"or":[227,313],"natural":[228],"conversational":[229],"memory.":[230],"They":[231],"introduce":[232],"neither":[233],"general":[235],"selector":[236],"nor":[237],"pooled":[239],"effect":[240],"across":[241],"studies.":[242],"Materials:":[243],"PDF":[245],"consolidated":[248],"lead":[249,288],"preprint.":[250],"manuscript-source":[252],"ZIP":[253,260],"contains":[254,261],"its":[255],"LaTeX":[256],"sources.":[257],"three":[262],"full":[263],"study":[264],"reports,":[265],"datasets,":[268],"raw":[269],"cleaned":[271],"predictions,":[272],"analysis":[273,339],"code,":[274],"provenance,":[275],"replay":[278],"instructions.":[279],"reports":[281],"document":[282],"included":[285],"article;":[289],"they":[290],"archival":[294],"publications.":[295],"Reproduction:":[296],"supplied":[298],"package":[299,350],"reproduces":[300],"reported":[302],"analyses":[303],"saved":[305],"predictions.":[306],"It":[307],"contain":[310],"model":[311],"weights":[312],"complete":[315],"inference":[316,322],"implementation.":[317],"INFERENCE_REPRODUCTION.md":[318],"specifies":[319],"versions":[324],"commands.":[326],"Licensing:":[327],"Manuscript":[328],"text":[329],"data":[333],"CC":[335],"BY":[336],"4.0.":[337],"Original":[338],"code":[340],"MIT.":[342],"Third-party":[343],"materials":[344],"retain":[345],"their":[346],"own":[347],"terms;":[348],"includes":[351],"relevant":[353],"notices.":[354],"Research":[355],"assistance:":[356],"Language-model":[357],"tools":[358],"assisted":[359],"research":[361],"design,":[362],"implementation,":[363],"literature":[364],"review,":[365],"numerical":[366],"checking,":[367],"writing.":[369],"author":[371],"retains":[372],"responsibility":[373],"for":[374],"interpretation,":[377],"submission.":[379],"Status:":[380],"Preprint;":[381],"peer":[383],"reviewed.":[384]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-11-06T00:00:00"}