{"id":"https://openalex.org/W7203670470","doi":"https://doi.org/10.48550/arxiv.2608.16253","title":"Second-Order Response Laws for LLM Judges: Debiased Estimation of Prompt Instability","display_name":"Second-Order Response Laws for LLM Judges: Debiased Estimation of Prompt Instability","publication_year":2026,"publication_date":"2026-08-17","ids":{"openalex":"https://openalex.org/W7203670470","doi":"https://doi.org/10.48550/arxiv.2608.16253"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.16253","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.16253","pdf_url":"https://arxiv.org/pdf/2608.16253","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.16253","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5144809042","display_name":"Pengbin Feng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Feng, Pengbin","raw_affiliation_strings":["Amazon","University of Southern California"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"University of Southern California","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147435284","display_name":"Chunlei Meng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meng, Chunlei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029896002","display_name":"Daozheng Qu","orcid":"https://orcid.org/0000-0002-1950-2620","observed_orcids":["https://orcid.org/0000-0002-1950-2620"]},"institutions":[{"id":"https://openalex.org/I63072094","display_name":"University of Portsmouth","ror":"https://ror.org/03ykbk197","country_code":"GB","type":"education","lineage":["https://openalex.org/I63072094"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Qu, Daozheng","raw_affiliation_strings":["University of Portsmouth"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Portsmouth","institution_ids":["https://openalex.org/I63072094"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147394402","display_name":"Zhilin Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Zhilin","raw_affiliation_strings":["Amazon"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147379541","display_name":"Haoran Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Liu, Haoran","raw_affiliation_strings":["The Chinese University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5120424828","display_name":"Jiekai Wu","orcid":"https://orcid.org/0009-0008-1046-995X","observed_orcids":["https://orcid.org/0009-0008-1046-995X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Jiekai","raw_affiliation_strings":["Independent Researcher"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent Researcher","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.19869999587535858,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.19869999587535858,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.10639999806880951,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.094200000166893,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.9179999828338623},{"id":"https://openalex.org/keywords/unbiased-estimation","display_name":"unbiased estimation","score":0.5370000004768372},{"id":"https://openalex.org/keywords/sampling-noise","display_name":"sampling noise","score":0.8349999785423279},{"id":"https://openalex.org/keywords/prompt-robustness","display_name":"prompt robustness","score":0.44999998807907104},{"id":"https://openalex.org/keywords/qwen","display_name":"Qwen","score":0.48500001430511475},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.6019999980926514}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.7702000141143799},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5633000135421753},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.532800018787384},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5202000141143799},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.513700008392334},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.5099999904632568},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4675000011920929},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4345000088214874},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3862000107765198},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3314000070095062},{"id":"https://openalex.org/C200941418","wikidata":"https://www.wikidata.org/wiki/Q273508","display_name":"Inflation (cosmology)","level":2,"score":0.3133000135421753},{"id":"https://openalex.org/C75917345","wikidata":"https://www.wikidata.org/wiki/Q2725298","display_name":"Sampling bias","level":3,"score":0.31209999322891235},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.3098999857902527},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.30869999527931213},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.2897000014781952},{"id":"https://openalex.org/C41426520","wikidata":"https://www.wikidata.org/wiki/Q1192065","display_name":"Point estimation","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C100253034","wikidata":"https://www.wikidata.org/wiki/Q196372","display_name":"Systematic error","level":2,"score":0.2574999928474426},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2567000091075897},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.2554999887943268},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.2549000084400177}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.16253","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.16253","pdf_url":"https://arxiv.org/pdf/2608.16253","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.16253","doi":"https://doi.org/10.48550/arxiv.2608.16253","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.16253","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.16253","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.16253","pdf_url":"https://arxiv.org/pdf/2608.16253","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203670470.pdf","grobid_xml":"https://content.openalex.org/works/W7203670470.grobid-xml"},"referenced_works_count":6,"referenced_works":["https://openalex.org/W4390528697","https://openalex.org/W7133332553","https://openalex.org/W7138511917","https://openalex.org/W7153979703","https://openalex.org/W7154217415","https://openalex.org/W7156948682"],"related_works":["https://openalex.org/W7214756950","https://openalex.org/W7158423527","https://openalex.org/W7164803637","https://openalex.org/W7223313015","https://openalex.org/W4415798641","https://openalex.org/W7160858152","https://openalex.org/W7121321095","https://openalex.org/W7167897301","https://openalex.org/W7133332553","https://openalex.org/W4415338432"],"abstract_inverted_index":{"LLM":[0],"judges":[1],"are":[2,151],"often":[3],"evaluated":[4],"with":[5,84,162],"a":[6,11,30,42,54,59,109,134,144],"single":[7],"prompt":[8,31,56,63,98],"and":[9,95,105,120,133,182],"only":[10],"few":[12],"repeated":[13],"calls.":[14],"When":[15],"their":[16],"verdicts":[17],"vary,":[18],"it":[19,80],"remains":[20],"unclear":[21],"whether":[22],"the":[23,46,68,101,113,139,163],"variation":[24],"comes":[25],"from":[26,100,192],"sampling":[27],"noise":[28,83],"within":[29],"or":[32],"systematic":[33],"differences":[34],"across":[35,173],"prompts.":[36],"We":[37,87],"formalize":[38],"this":[39],"distinction":[40],"using":[41],"second-order":[43],"response":[44],"law:":[45],"distribution":[47],"of":[48,62],"prompt-conditioned":[49],"verdict":[50],"distributions":[51],"induced":[52],"by":[53],"declared":[55,96],"policy.":[57],"For":[58],"quadratic":[60],"measure":[61],"instability,":[64],"we":[65],"show":[66],"that":[67],"usual":[69],"plug-in":[70,160],"estimator":[71],"is":[72],"biased":[73],"upward":[74],"at":[75,166],"finite":[76],"repeat":[77,168],"budgets":[78],"because":[79],"confounds":[81],"within-prompt":[82],"between-prompt":[85],"variation.":[86],"derive":[88],"unbiased":[89],"estimators":[90],"for":[91],"both":[92],"sampled":[93],"prompts":[94],"fixed":[97],"censuses":[99],"difference":[102],"between":[103],"within-":[104],"across-prompt":[106],"agreement.":[107],"Under":[108],"crossed":[110],"prompt-by-answer-order":[111],"design,":[112],"same":[114],"framework":[115],"separates":[116],"prompt,":[117],"order,":[118],"interaction,":[119],"residual":[121],"call":[122],"variation,":[123],"while":[124],"retaining":[125],"invalid":[126],"completed":[127],"outputs":[128],"as":[129],"outcomes.":[130],"Known-law":[131],"simulations":[132],"byte-identical":[135],"live":[136],"null":[137],"recover":[138],"predicted":[140],"finite-$R$":[141],"inflation.":[142],"In":[143],"matched":[145,171],"Qwen":[146],"study,":[147],"corrected":[148],"low-repeat":[149],"estimates":[150],"closer":[152],"to":[153],"an":[154],"independently":[155],"acquired":[156],"$R=16$":[157],"reference":[158],"than":[159],"estimates,":[161],"largest":[164],"gains":[165],"small":[167],"budgets.":[169],"A":[170],"panel":[172],"four":[174],"frozen":[175],"judge":[176],"configurations":[177],"exhibits":[178],"configuration-specific":[179],"inflation":[180],"magnitudes":[181],"component":[183],"profiles.":[184],"Prompt":[185],"robustness":[186],"can":[187],"therefore":[188],"be":[189],"estimated":[190],"separately":[191],"finite-call":[193],"noise.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-19T00:00:00"}