{"id":"https://openalex.org/W7171531797","doi":"https://doi.org/10.48550/arxiv.2607.22766","title":"First-Order In-Context Shapley: Scalable Data Auditing for LLM Alignment and Evaluation","display_name":"First-Order In-Context Shapley: Scalable Data Auditing for LLM Alignment and Evaluation","alternate_titles":[],"publication_year":2026,"publication_date":"2026-07-24","ids":{"openalex":"https://openalex.org/W7171531797","doi":"https://doi.org/10.48550/arxiv.2607.22766"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2607.22766","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2607.22766","pdf_url":"https://arxiv.org/pdf/2607.22766","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7171531797"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2607.22766","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030070547","display_name":"Yunting Song","orcid":"https://orcid.org/0000-0002-3053-1748","observed_orcids":["https://orcid.org/0000-0002-3053-1748"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Song, Yunting","raw_affiliation_strings":["Google Mountain View , CA 94043"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Mountain View , CA 94043","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143813480","display_name":"Matthew Watson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Watson, Matthew","raw_affiliation_strings":["Google Mountain View , CA 94043"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Mountain View , CA 94043","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034311510","display_name":"Moustafa Alzantot","orcid":"https://orcid.org/0000-0003-2614-9877","observed_orcids":["https://orcid.org/0000-0003-2614-9877"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alzantot, Moustafa Farid","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099230699","display_name":"Peter Grabowski","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Grabowski, Peter","raw_affiliation_strings":["Google Mountain View , CA 94043"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Mountain View , CA 94043","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5143840441","display_name":"Jun Qin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qin, Jun","raw_affiliation_strings":["Google Mountain View , CA 94043"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Mountain View , CA 94043","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.19629999995231628,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.19629999995231628,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1477999985218048,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11550000309944153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-audit","display_name":"data auditing","score":0.734000027179718},{"id":"https://openalex.org/keywords/noisy-labels","display_name":"label noise","score":0.6290000081062317},{"id":"https://openalex.org/keywords/llm-alignment","display_name":"LLM alignment","score":0.6470000147819519},{"id":"https://openalex.org/keywords/preference-data","display_name":"preference data","score":0.4440000057220459},{"id":"https://openalex.org/keywords/cooperative-games","display_name":"cooperative games","score":0.5989999771118164},{"id":"https://openalex.org/keywords/k-nearest-neighbors","display_name":"k-nearest neighbors","score":0.8199999928474426},{"id":"https://openalex.org/keywords/reward-modeling","display_name":"reward modeling","score":0.35100001096725464},{"id":"https://openalex.org/keywords/data-filtering","display_name":"data filtering","score":0.3149999976158142}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7962999939918518},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6625999808311462},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.5993000268936157},{"id":"https://openalex.org/C2776654903","wikidata":"https://www.wikidata.org/wiki/Q2601463","display_name":"SAFER","level":2,"score":0.5616000294685364},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4794999957084656},{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.44020000100135803},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4198000133037567},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.39660000801086426},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.3743000030517578},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3499999940395355},{"id":"https://openalex.org/C186027771","wikidata":"https://www.wikidata.org/wiki/Q4008379","display_name":"Valuation (finance)","level":2,"score":0.3359000086784363},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.3138999938964844},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.3118000030517578},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.3098999857902527},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.30489999055862427},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C111065885","wikidata":"https://www.wikidata.org/wiki/Q1189053","display_name":"Fuzz testing","level":3,"score":0.2921000123023987},{"id":"https://openalex.org/C187191949","wikidata":"https://www.wikidata.org/wiki/Q1138496","display_name":"Profiling (computer programming)","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C2778180026","wikidata":"https://www.wikidata.org/wiki/Q18378163","display_name":"Semantic heterogeneity","level":4,"score":0.27799999713897705},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.27559998631477356},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.26269999146461487},{"id":"https://openalex.org/C90312973","wikidata":"https://www.wikidata.org/wiki/Q7449052","display_name":"Semantic data model","level":2,"score":0.2581999897956848},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2515999972820282}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2607.22766","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2607.22766","pdf_url":"https://arxiv.org/pdf/2607.22766","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2607.22766","doi":"https://doi.org/10.48550/arxiv.2607.22766","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.22766","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2607.22766","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2607.22766","pdf_url":"https://arxiv.org/pdf/2607.22766","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7171531797.pdf","grobid_xml":"https://content.openalex.org/works/W7171531797.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W2939984132","https://openalex.org/W2963469388","https://openalex.org/W2970773487","https://openalex.org/W3177765786","https://openalex.org/W4223908421","https://openalex.org/W4327811112","https://openalex.org/W4381586770","https://openalex.org/W4384648625","https://openalex.org/W4390041933","https://openalex.org/W4402670856","https://openalex.org/W4402671800","https://openalex.org/W4410475038","https://openalex.org/W4412888711","https://openalex.org/W4414855219","https://openalex.org/W4415799187","https://openalex.org/W4417252869","https://openalex.org/W4417531280","https://openalex.org/W6947991961","https://openalex.org/W7133189643","https://openalex.org/W7133208539","https://openalex.org/W7133211372","https://openalex.org/W7133239970","https://openalex.org/W7154947098","https://openalex.org/W7196953739"],"related_works":["https://openalex.org/W4417531280","https://openalex.org/W7135407605","https://openalex.org/W4402671689","https://openalex.org/W4416034300","https://openalex.org/W4404781861","https://openalex.org/W4416033888","https://openalex.org/W7165165314","https://openalex.org/W4412888711","https://openalex.org/W7154500951","https://openalex.org/W7166846714"],"abstract_inverted_index":{"Post-training":[0],"alignment":[1],"of":[2,74,99,141,209],"Large":[3],"Language":[4],"Models":[5],"is":[6],"increasingly":[7],"bottlenecked":[8],"by":[9,224],"data":[10],"quality:":[11],"preference":[12],"and":[13,21,27,46,126,152,162,186,212,227],"instruction-tuning":[14],"corpora":[15],"accumulate":[16],"label":[17,199],"noise,":[18],"safety":[19],"inversions,":[20,231],"inter-annotator":[22],"contradictions":[23,158,170],"that":[24,69,159],"deduplication":[25],"misses":[26],"full-corpus":[28],"LLM":[29,136,234],"judging":[30],"cannot":[31],"afford.":[32],"While":[33],"Data":[34],"Shapley":[35,96],"offers":[36],"a":[37,110,127],"principled":[38],"valuation":[39,67],"framework,":[40],"exact":[41],"computation":[42],"requires":[43],"intractable":[44],"retraining,":[45],"embedding":[47],"surrogates":[48],"like":[49],"KNN-Shapley":[50],"fail":[51],"to":[52,139],"separate":[53],"high-":[54],"from":[55],"low-utility":[56],"demonstrations":[57],"($\u03c1=":[58],"0.0003$":[59],"with":[60,92,205],"generative":[61],"utility).":[62],"We":[63],"introduce":[64],"an":[65,75],"inference-only":[66],"pipeline":[68],"computes":[70],"the":[71,172,195,210],"first-order":[72],"term":[73],"in-context":[76,95],"cooperative":[77],"game":[78],"over":[79],"semantic":[80],"$k$-NN":[81],"neighborhoods":[82],"in":[83,147,183,190],"$O(k)$":[84],"forward":[85],"passes":[86],"without":[87,233],"parameter":[88],"updates,":[89],"correlating":[90],"positively":[91],"multi-shot":[93],"Monte-Carlo":[94],"on":[97,150,154,229],"100%":[98],"targets":[100],"at":[101],"19x":[102],"less":[103],"compute.":[104],"Projecting":[105],"pairwise":[106],"log-likelihood":[107],"shifts":[108],"onto":[109],"directed":[111],"influence":[112],"graph":[113],"characterizes":[114],"each":[115],"record":[116],"via":[117],"three":[118],"signals:":[119],"outgoing":[120],"Exemplifying":[121],"Advantage,":[122,125],"incoming":[123],"Exemplified":[124],"topological":[128],"One-of-a-Kind":[129],"Score":[130],"protecting":[131],"rare":[132],"prompts.":[133],"Pre-filtering":[134],"restricts":[135],"twin":[137],"arbitration":[138],"1%-5%":[140],"records":[142,179,220],"while":[143],"achieving":[144],"4.5x":[145],"enrichment":[146],"confirmed":[148,169],"errors":[149,200],"HelpSteer2":[151],"2.2x":[153],"Anthropic":[155],"HH-RLHF,":[156],"catching":[157],"zero-shot":[160],"likelihood":[161],"margin":[163],"filters":[164],"miss.":[165],"During":[166],"DPO":[167,214],"training,":[168],"oppose":[171],"update":[173],"far":[174],"more":[175],"often":[176],"than":[177],"clean":[178],"(56.7%":[180],"vs.":[181],"37.0%":[182],"LoRA":[184],"gradients)":[185],"stay":[187],"below":[188],"chance":[189],"reward":[191,222],"accuracy.":[192],"Finally,":[193],"auditing":[194],"HH-RLHF":[196],"benchmark":[197],"exposes":[198],"where":[201],"aligned":[202],"models":[203],"disagree":[204],"human":[206],"labels":[207],"73%-83%":[208],"time,":[211],"retraining":[213],"after":[215],"repairing":[216],"or":[217],"filtering":[218],"flagged":[219],"improves":[221],"accuracy":[223],"+0.5%-0.6%":[225],"overall":[226],"+3.1%-3.5%":[228],"repaired":[230],"even":[232],"arbitration.":[235]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7171531797","counts_by_year":[],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2026-07-29T00:00:00"}
