{"id":"https://openalex.org/W7127099165","doi":"https://doi.org/10.1371/journal.pone.0339920","title":"Human-anchored longitudinal comparison of generative AI with a bias-calibrated LLM-as-judge","display_name":"Human-anchored longitudinal comparison of generative AI with a bias-calibrated LLM-as-judge","publication_year":2026,"publication_date":"2026-02-02","ids":{"openalex":"https://openalex.org/W7127099165","doi":"https://doi.org/10.1371/journal.pone.0339920","pmid":"https://pubmed.ncbi.nlm.nih.gov/41628148","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/12863567"},"language":"en","primary_location":{"id":"doi:10.1371/journal.pone.0339920","doi":"https://doi.org/10.1371/journal.pone.0339920","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0339920","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0339920&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLOS One","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","datacite","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0339920&type=printable","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119356608","display_name":"Thomas Louis Wiese","orcid":"https://orcid.org/0000-0002-0233-5029","observed_orcids":["https://orcid.org/0000-0002-0233-5029"]},"institutions":[{"id":"https://openalex.org/I88081825","display_name":"Empire State University","ror":"https://ror.org/03sg2bq51","country_code":"US","type":"education","lineage":["https://openalex.org/I88081825"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Thomas Wiese","raw_affiliation_strings":["SUNY Empire State University, New York, United States of America"],"raw_orcid":"https://orcid.org/0000-0002-0233-5029","affiliations":[{"raw_affiliation_string":"SUNY Empire State University, New York, United States of America","institution_ids":["https://openalex.org/I88081825"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5119356608"],"corresponding_institution_ids":["https://openalex.org/I88081825"],"apc_list":{"value":2382,"currency":"USD","value_usd":2382},"apc_paid":{"value":2382,"currency":"USD","value_usd":2382},"fwci":2.7902,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.90016259,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"21","issue":"2","first_page":"e0339920","last_page":"e0339920"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.6836000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.6836000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0949999988079071,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.023900000378489494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7120000123977661},{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.8500000238418579},{"id":"https://openalex.org/keywords/bias-calibration","display_name":"bias calibration","score":0.9559999704360962},{"id":"https://openalex.org/keywords/reproducible-evaluation","display_name":"reproducible evaluation","score":0.621999979019165},{"id":"https://openalex.org/keywords/model-drift-detection","display_name":"model drift detection","score":0.5569999814033508},{"id":"https://openalex.org/keywords/change-point-detection","display_name":"change point detection","score":0.7630000114440918},{"id":"https://openalex.org/keywords/safety-metrics","display_name":"safety metrics","score":0.5289999842643738},{"id":"https://openalex.org/keywords/bradley-terry-model","display_name":"Bradley-Terry model","score":0.7089999914169312}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.6682000160217285},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6514000296592712},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.6189000010490417},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5515000224113464},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.5508999824523926},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47099998593330383},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4702000021934509},{"id":"https://openalex.org/C2777895361","wikidata":"https://www.wikidata.org/wiki/Q1758614","display_name":"Longitudinal study","level":2,"score":0.46790000796318054},{"id":"https://openalex.org/C2994165510","wikidata":"https://www.wikidata.org/wiki/Q131160","display_name":"Political correctness","level":3,"score":0.4074000120162964},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.40720000863075256},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3937999904155731},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.3675999939441681},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2985000014305115},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2824000120162964},{"id":"https://openalex.org/C3020672099","wikidata":"https://www.wikidata.org/wiki/Q857354","display_name":"Longitudinal data","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C2987082051","wikidata":"https://www.wikidata.org/wiki/Q223642","display_name":"Human interaction","level":2,"score":0.2615000009536743},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.25920000672340393},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.25699999928474426}],"mesh":[],"locations_count":8,"locations":[{"id":"doi:10.1371/journal.pone.0339920","doi":"https://doi.org/10.1371/journal.pone.0339920","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0339920","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0339920&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLOS One","raw_type":"journal-article"},{"id":"pmh:oai:europepmc.org:11671084","doi":null,"is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/12863567","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Text"},{"id":"pmid:41628148","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41628148","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PloS one","raw_type":"Journal Article"},{"id":"pmh:oai:pubmedcentral.nih.gov:12863567","doi":null,"is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC12863567/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLoS One","raw_type":"Text"},{"id":"pmh:oai:doaj.org/article:48c3805ce31b4cb3a9dd9fb64d1c8765","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/48c3805ce31b4cb3a9dd9fb64d1c8765","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"PLoS ONE, Vol 21, Iss 2, p e0339920 (2026)","raw_type":"article"},{"id":"pmh:oai:doaj.org/article:bf1c6a22c606468897ee5d4b7ff6f68c","doi":null,"is_oa":false,"landing_page_url":"https://doaj.org/article/bf1c6a22c606468897ee5d4b7ff6f68c","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"PLoS ONE, Vol 21, Iss 2 (2026)","raw_type":"article"},{"id":"pmh:oai:share.osf.io:229fcf2d-620e-4176-bb81-c4942d3f6751","doi":null,"is_oa":false,"landing_page_url":"https://osf.io/j34uc","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Project"},{"id":"doi:10.17605/osf.io/j34uc","doi":"https://doi.org/10.17605/osf.io/j34uc","is_oa":true,"landing_page_url":"https://doi.org/10.17605/osf.io/j34uc","pdf_url":null,"source":{"id":"https://openalex.org/S7407050956","display_name":"Open Science Framework","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.1371/journal.pone.0339920","doi":"https://doi.org/10.1371/journal.pone.0339920","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0339920","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0339920&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLOS One","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7127099165.pdf","grobid_xml":"https://content.openalex.org/works/W7127099165.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W2078273711","https://openalex.org/W2754517384","https://openalex.org/W4315781064","https://openalex.org/W4376632242","https://openalex.org/W4387446802","https://openalex.org/W4401043438","https://openalex.org/W4407832526","https://openalex.org/W4408568363","https://openalex.org/W4408611389","https://openalex.org/W4409883671","https://openalex.org/W4412491123"],"related_works":["https://openalex.org/W7164882581","https://openalex.org/W7134176676","https://openalex.org/W7164992558","https://openalex.org/W7211962755","https://openalex.org/W7164388337","https://openalex.org/W7111041601","https://openalex.org/W4400104877","https://openalex.org/W7154591857","https://openalex.org/W7121321095","https://openalex.org/W7168262832"],"abstract_inverted_index":{"Service":[0],"LLMs":[1],"evolve":[2],"without":[3],"public":[4],"changelogs,":[5],"complicating":[6],"reproducible":[7],"evaluation.":[8],"We":[9],"present":[10],"a":[11,27,44,54],"preregistered":[12],"human-anchored":[13],"longitudinal":[14],"study":[15],"that":[16],"tracks":[17],"three":[18],"major":[19],"model":[20],"families":[21],"over":[22],"ten":[23],"weekly":[24,52],"waves":[25],"using":[26],"fixed":[28],"prompt":[29],"bank":[30],"(N":[31],"=":[32,93],"240)":[33],"across":[34],"six":[35],"domains.":[36],"Blinded":[37],"human":[38],"raters":[39],"provided":[40,120],"correctness":[41],"judgments,":[42],"and":[43,59,82,116],"bias-calibrated":[45],"LLM-as-judge":[46],"produced":[47],"secondary":[48],"pairwise":[49],"preferences":[50],"corrected":[51],"via":[53],"Bradley-Terry":[55],"model.":[56],"Mixed-effects":[57],"modeling":[58],"change-point":[60],"detection":[61],"(PELT":[62],"with":[63,90,101],"MBIC":[64],"penalty)":[65],"identified":[66],"significant":[67],"service":[68],"drift":[69,102],"patterns.":[70],"Results":[71],"show":[72],"divergent":[73],"stability":[74],"trajectories":[75],"among":[76],"models:":[77],"one":[78,80,83],"stable,":[79],"improving,":[81],"degrading":[84],"mid-study.":[85],"Judge":[86],"calibration":[87],"increased":[88],"agreement":[89],"humans":[91],"(\u03c4":[92],"0.59-0.68)":[94],"while":[95],"reducing":[96],"volatility.":[97],"Safety":[98],"metrics":[99],"co-varied":[100],"events,":[103],"suggesting":[104],"behavioral":[105],"shifts":[106],"rather":[107],"than":[108],"confirmed":[109],"causal":[110],"changes.":[111],"All":[112],"data,":[113],"prompts,":[114],"rubrics,":[115],"parameter":[117],"configurations":[118],"are":[119],"in":[121],"supporting":[122],"files":[123],"S1-S6.":[124]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-13T13:42:25.865203","created_date":"2026-01-21T00:00:00"}