{"id":"https://openalex.org/W7204591025","doi":"https://doi.org/10.48550/arxiv.2608.27309","title":"Difference-in-Differences on a Censored Rating Scale Can Manufacture an Effect: Evidence from a Pre-Registered LLM-Judge Audit","display_name":"Difference-in-Differences on a Censored Rating Scale Can Manufacture an Effect: Evidence from a Pre-Registered LLM-Judge Audit","publication_year":2026,"publication_date":"2026-08-27","ids":{"openalex":"https://openalex.org/W7204591025","doi":"https://doi.org/10.48550/arxiv.2608.27309"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.27309","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27309","pdf_url":"https://arxiv.org/pdf/2608.27309","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.27309","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148573206","display_name":"Shuyi Fan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fan, Shuyi","raw_affiliation_strings":["Columbia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004815495","display_name":"Boyuan Deng","orcid":"https://orcid.org/0000-0003-4218-3493","observed_orcids":["https://orcid.org/0000-0003-4218-3493","https://orcid.org/0000-0002-2263-5790"]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deng, Boyuan","raw_affiliation_strings":["Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148507853","display_name":"Mengyu Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Mengyu","raw_affiliation_strings":["The University of Chicago"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113021185","display_name":"Xinghong Xie","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xie, Xinhong","raw_affiliation_strings":["The Pennsylvania State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Pennsylvania State University","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100657810","display_name":"Chenyang Li","orcid":"https://orcid.org/0000-0002-3431-8573","observed_orcids":["https://orcid.org/0000-0002-3431-8573"]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Chenyang","raw_affiliation_strings":["Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148473017","display_name":"Hongyang Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhang, Hongyang","raw_affiliation_strings":["The Hong Kong Polytechnic University , Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong Polytechnic University , Hong Kong","institution_ids":["https://openalex.org/I14243506"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.6689000129699707,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.6689000129699707,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12970","display_name":"Names, Identity, and Discrimination Research","score":0.061000000685453415,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.05040000006556511,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/difference-in-differences","display_name":"difference-in-differences","score":0.8050000071525574},{"id":"https://openalex.org/keywords/productive-struggle","display_name":"productive struggle","score":0.515999972820282},{"id":"https://openalex.org/keywords/ai-bias-detection","display_name":"AI bias detection","score":0.3840000033378601},{"id":"https://openalex.org/keywords/spurious-interactions","display_name":"spurious interactions","score":0.39800000190734863}],"concepts":[{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.6425999999046326},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.6419000029563904},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.6402999758720398},{"id":"https://openalex.org/C93226319","wikidata":"https://www.wikidata.org/wiki/Q193137","display_name":"Differential (mechanical device)","level":2,"score":0.6353999972343445},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5701000094413757},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.5390999913215637},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.526199996471405},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.5078999996185303},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.47609999775886536},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.46650001406669617},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.4226999878883362},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3873000144958496},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.3790999948978424},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.37869998812675476},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.36959999799728394},{"id":"https://openalex.org/C181447626","wikidata":"https://www.wikidata.org/wiki/Q1224359","display_name":"Differential item functioning","level":4,"score":0.35089999437332153},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.3228999972343445},{"id":"https://openalex.org/C40423286","wikidata":"https://www.wikidata.org/wiki/Q284172","display_name":"Selection bias","level":2,"score":0.28929999470710754},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.287200003862381},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.28679999709129333},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.27959999442100525},{"id":"https://openalex.org/C3019921246","wikidata":"https://www.wikidata.org/wiki/Q104853310","display_name":"Differential effects","level":2,"score":0.27720001339912415},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.2563999891281128},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.25519999861717224}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.27309","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27309","pdf_url":"https://arxiv.org/pdf/2608.27309","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.27309","doi":"https://doi.org/10.48550/arxiv.2608.27309","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.27309","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.27309","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.27309","pdf_url":"https://arxiv.org/pdf/2608.27309","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204591025.pdf","grobid_xml":"https://content.openalex.org/works/W7204591025.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W3088264597","https://openalex.org/W4280504808","https://openalex.org/W4389261532","https://openalex.org/W4405766242","https://openalex.org/W4417077852","https://openalex.org/W7119510601","https://openalex.org/W7154216550","https://openalex.org/W7172438718"],"related_works":["https://openalex.org/W7167228722","https://openalex.org/W7204894163","https://openalex.org/W7211962755","https://openalex.org/W7167897301","https://openalex.org/W7119510601","https://openalex.org/W7220698693","https://openalex.org/W7168236322","https://openalex.org/W7214646924","https://openalex.org/W7168099345","https://openalex.org/W7204606853"],"abstract_inverted_index":{"LLM-judge":[0],"audits":[1],"assess":[2],"bias":[3],"by":[4],"comparing":[5],"ratings":[6,44],"across":[7,26],"matched":[8],"conditions.":[9],"Difference-in-differences":[10],"designs":[11],"take":[12],"the":[13,59,106,129,134,148],"rating":[14],"difference":[15],"between":[16],"two":[17],"candidate":[18],"responses":[19,53],"to":[20,51,74,123],"each":[21],"item":[22],"and":[23,133],"compare":[24],"it":[25,63],"a":[27,47,68,75,89],"manipulated":[28],"attribute.":[29],"Building":[30],"on":[31,34,95,105,147],"prior":[32],"work":[33],"nonlinear":[35],"difference-in-differences,":[36],"we":[37],"show":[38],"in":[39],"closed":[40],"form":[41],"that":[42],"when":[43],"are":[45],"bounded,":[46],"severity":[48,131],"shift":[49,132],"common":[50],"both":[52],"produces":[54],"an":[55,86],"observed":[56,130,139],"interaction":[57,103,127,140],"whenever":[58],"scale":[60,135],"bounds":[61],"attenuate":[62],"unequally.":[64],"We":[65],"then":[66],"report":[67],"pre-registered":[69],"audit":[70],"of":[71,88,125],"990":[72],"calls":[73],"frozen":[76],"pedagogy":[77],"judge.":[78],"The":[79,98,138],"registered":[80],"primary":[81],"analysis":[82],"did":[83],"not":[84,143],"detect":[85],"effect":[87],"stated":[90],"novice":[91],"or":[92],"advanced":[93],"profile":[94],"scaffolding":[96],"preference.":[97],"only":[99],"nominally":[100],"significant":[101],"secondary":[102],"was":[104],"productive-struggle":[107],"sub-score":[108],"($+0.378$":[109],"points;":[110],"$p":[111],"=":[112],"0.002$).":[113],"A":[114],"post-hoc":[115],"construction":[116],"with":[117],"zero":[118],"differential":[119,145],"preference":[120,146],"reproduced":[121],"79":[122],"85%":[124],"this":[126],"using":[128],"floor":[136],"alone.":[137],"therefore":[141],"does":[142],"identify":[144],"latent":[149],"scale.":[150]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-29T00:00:00"}