{"id":"https://openalex.org/W7120101547","doi":"https://doi.org/10.48550/arxiv.2601.05114","title":"Evaluative Fingerprints: Stable and Systematic Differences in LLM Evaluator Behavior","display_name":"Evaluative Fingerprints: Stable and Systematic Differences in LLM Evaluator Behavior","publication_year":2026,"publication_date":"2026-01-08","ids":{"openalex":"https://openalex.org/W7120101547","doi":"https://doi.org/10.48550/arxiv.2601.05114"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2601.05114","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2601.05114","pdf_url":"https://arxiv.org/pdf/2601.05114","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2601.05114","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122420062","display_name":"Wajid Nasser","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Nasser, Wajid","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5122420062"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.5884000062942505,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.5884000062942505,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.02419999986886978,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.023000000044703484,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.8629999756813049},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.7179999947547913},{"id":"https://openalex.org/keywords/inter-judge-agreement","display_name":"inter-judge agreement","score":0.9580000042915344},{"id":"https://openalex.org/keywords/rubric-based-assessment","display_name":"rubric-based assessment","score":0.5230000019073486},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.5669999718666077},{"id":"https://openalex.org/keywords/gpt-5","display_name":"GPT-5","score":0.9399999976158142},{"id":"https://openalex.org/keywords/video-quality-assessment","display_name":"video quality assessment","score":0.4189999997615814},{"id":"https://openalex.org/keywords/evidence-evaluation","display_name":"evidence evaluation","score":0.34200000762939453},{"id":"https://openalex.org/keywords/measurement-invariance","display_name":"measurement invariance","score":0.3840000033378601}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.6140999794006348},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.6013000011444092},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.48539999127388},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.4065000116825104},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.392300009727478},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.39089998602867126},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.37059998512268066},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3677000105381012},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3644999861717224},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3449999988079071},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.33809998631477356},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.3328999876976013},{"id":"https://openalex.org/C2986577269","wikidata":"https://www.wikidata.org/wiki/Q11306265","display_name":"Random noise","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.31139999628067017},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C63428231","wikidata":"https://www.wikidata.org/wiki/Q9043","display_name":"Norwegian","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.28290000557899475},{"id":"https://openalex.org/C2983787585","wikidata":"https://www.wikidata.org/wiki/Q93586","display_name":"Feature matching","level":3,"score":0.2824000120162964},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C106347477","wikidata":"https://www.wikidata.org/wiki/Q5384228","display_name":"Equating","level":3,"score":0.262800008058548},{"id":"https://openalex.org/C159447121","wikidata":"https://www.wikidata.org/wiki/Q490535","display_name":"Response bias","level":2,"score":0.2529999911785126},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.25209999084472656},{"id":"https://openalex.org/C79585631","wikidata":"https://www.wikidata.org/wiki/Q431498","display_name":"Confirmation bias","level":2,"score":0.25099998712539673}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2601.05114","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2601.05114","pdf_url":"https://arxiv.org/pdf/2601.05114","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2601.05114","doi":"https://doi.org/10.48550/arxiv.2601.05114","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.05114","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2601.05114","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2601.05114","pdf_url":"https://arxiv.org/pdf/2601.05114","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, justice, and strong institutions","score":0.6474394798278809}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7120101547.pdf","grobid_xml":"https://content.openalex.org/works/W7120101547.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM-as-judge":[0],"systems":[1],"promise":[2],"scalable,":[3],"consistent":[4,20],"evaluation.":[5],"We":[6,105,148],"find":[7],"the":[8,92,108],"opposite:":[9],"judges":[10,27,50,111,178],"are":[11,19,100,122,179,188],"consistent,":[12],"but":[13],"not":[14,180],"with":[15,21,76,86,102],"each":[16,192],"other;":[17],"they":[18,125],"themselves.":[22],"Across":[23],"3,240":[24],"evaluations":[25],"(9":[26],"x":[28,32,35],"120":[29],"unique":[30],"video":[31],"pack":[33],"items":[34],"3":[36],"independent":[37],"runs),":[38],"inter-judge":[39],"agreement":[40],"is":[41,94,175],"near-zero":[42],"(Krippendorff's":[43],"\u03b1":[44],"=":[45],"0.042).":[46],"On":[47],"two":[48],"dimensions,":[49],"disagree":[51],"more":[52],"than":[53],"random":[54],"noise":[55],"would":[56],"predict":[57],"(\u03b1":[58],"<":[59],"0).":[60],"Yet":[61],"this":[62,107],"disagreement":[63,120],"isn't":[64],"chaos;":[65],"it's":[66],"structured.":[67],"A":[68],"classifier":[69],"identifies":[70],"which":[71],"judge":[72,130],"produced":[73],"an":[74,138],"evaluation":[75],"77.1%":[77],"accuracy":[78],"from":[79],"rubric":[80],"scores":[81,202],"alone,":[82],"rising":[83],"to":[84,209],"89.9%":[85],"disposition":[87],"features.":[88],"Within":[89],"model":[90],"families,":[91],"signal":[93],"even":[95],"stronger:":[96],"GPT-4.1":[97],"and":[98,161,170],"GPT-5.2":[99],"distinguishable":[101],"99.6%":[103],"accuracy.":[104],"call":[106],"reliability":[109],"paradox:":[110],"cannot":[112],"agree":[113],"on":[114],"what":[115],"constitutes":[116],"quality,":[117],"yet":[118],"their":[119,201],"patterns":[121],"so":[123],"stable":[124,134],"function":[126],"as":[127],"fingerprints.":[128],"Each":[129],"implements":[131],"a":[132,184,204],"distinct,":[133],"theory":[135,197],"of":[136,198],"quality:":[137],"\"evaluative":[139],"disposition\"":[140],"that":[141,207],"shapes":[142],"how":[143],"it":[144],"interprets":[145],"any":[146],"rubric.":[147],"characterize":[149],"these":[150],"dispositions":[151],"along":[152],"multiple":[153],"axes:":[154],"harshness/leniency,":[155],"dimension":[156],"emphasis,":[157],"within-judge":[158],"stability":[159],"(ICC),":[160],"evidence":[162],"behavior":[163],"(receipt":[164],"validity,":[165],"semantic":[166],"linkage":[167],"via":[168],"NLI,":[169],"shotgun":[171],"index).":[172],"The":[173],"implication":[174],"stark:":[176],"LLM":[177],"interchangeable":[181],"instruments":[182],"measuring":[183],"shared":[185],"construct.":[186],"They":[187],"distinct":[189],"measurement":[190],"devices,":[191],"encoding":[193],"its":[194],"own":[195],"implicit":[196],"quality.":[199],"Averaging":[200],"produces":[203],"synthetic":[205],"verdict":[206],"corresponds":[208],"no":[210],"judge's":[211],"actual":[212],"values.":[213]},"counts_by_year":[],"updated_date":"2026-09-25T08:18:29.962427","created_date":"2026-01-10T00:00:00"}