{"id":"https://openalex.org/W7214908319","doi":"https://doi.org/10.48550/arxiv.2609.36086","title":"PADM\u00c9: Preference Alignment Data Synthesis for Meta-Evaluation of LM Agent Evaluators","display_name":"PADM\u00c9: Preference Alignment Data Synthesis for Meta-Evaluation of LM Agent Evaluators","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214908319","doi":"https://doi.org/10.48550/arxiv.2609.36086"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.36086","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36086","pdf_url":"https://arxiv.org/pdf/2609.36086","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.36086","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153683655","display_name":"Cheng Chang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148436239","display_name":"Yining Mao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mao, Yining","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5041658523","display_name":"Peng Qi","orcid":"https://orcid.org/0000-0003-0514-9464","observed_orcids":["https://orcid.org/0000-0003-0514-9464","https://orcid.org/0000-0001-9378-3355"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Peng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2849000096321106,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2849000096321106,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.266400009393692,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.04879999905824661,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/meta-evaluation","display_name":"meta-evaluation","score":0.9350000023841858},{"id":"https://openalex.org/keywords/preference-alignment","display_name":"preference alignment","score":0.8230000138282776},{"id":"https://openalex.org/keywords/data-synthesis","display_name":"data synthesis","score":0.8330000042915344},{"id":"https://openalex.org/keywords/agentic-behavior","display_name":"agentic behavior","score":0.7390000224113464},{"id":"https://openalex.org/keywords/human-judgment-alignment","display_name":"human judgment alignment","score":0.45399999618530273},{"id":"https://openalex.org/keywords/grading-criteria","display_name":"grading criteria","score":0.44600000977516174}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7081000208854675},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6875},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.6047000288963318},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5692999958992004},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5342000126838684},{"id":"https://openalex.org/C2993724205","wikidata":"https://www.wikidata.org/wiki/Q315","display_name":"Human language","level":2,"score":0.40380001068115234},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.3619999885559082},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3465000092983246},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.3172000050544739},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3034000098705292},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.29030001163482666},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28600001335144043},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.27549999952316284},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C55037315","wikidata":"https://www.wikidata.org/wiki/Q5421151","display_name":"Experimental data","level":2,"score":0.25369998812675476},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.2517000138759613},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.36086","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36086","pdf_url":"https://arxiv.org/pdf/2609.36086","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.36086","doi":"https://doi.org/10.48550/arxiv.2609.36086","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.36086","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.36086","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.36086","pdf_url":"https://arxiv.org/pdf/2609.36086","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214908319.pdf","grobid_xml":"https://content.openalex.org/works/W7214908319.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W2013784666","https://openalex.org/W2033442452","https://openalex.org/W2741691725","https://openalex.org/W2793081411","https://openalex.org/W4387596586","https://openalex.org/W4389519254","https://openalex.org/W4402671584","https://openalex.org/W4403334020","https://openalex.org/W4404782412","https://openalex.org/W4404783034","https://openalex.org/W4406975150","https://openalex.org/W4411119262","https://openalex.org/W4415130727","https://openalex.org/W7120088840","https://openalex.org/W7141148890","https://openalex.org/W7161850149","https://openalex.org/W7164021909","https://openalex.org/W7166831353","https://openalex.org/W7166852156"],"related_works":["https://openalex.org/W7165643980","https://openalex.org/W4412638471","https://openalex.org/W4411120350","https://openalex.org/W6911352883","https://openalex.org/W4401042942","https://openalex.org/W4392223819","https://openalex.org/W7166849922","https://openalex.org/W4391421129","https://openalex.org/W4416035637","https://openalex.org/W4393247764"],"abstract_inverted_index":{"Language":[0],"models":[1,177],"are":[2],"frequently":[3],"employed":[4],"to":[5,52,168],"evaluate":[6],"other":[7,195],"language":[8,116],"models.":[9],"An":[10],"LM":[11,57,80],"evaluator":[12,81],"scoring":[13,49,188],"agentic":[14,110,146],"behaviors":[15],"across":[16,144],"multiple":[17],"criteria":[18],"is":[19,41,46,50],"valuable,":[20],"provided":[21],"that":[22,103,159],"its":[23],"decisions":[24],"align":[25],"with":[26,163,178],"human":[27,44,78,120,164],"judgment.":[28],"We":[29,64,131],"call":[30],"the":[31,60,182],"problem":[32],"of":[33,62,68,83,135,141],"evaluating":[34],"this":[35],"alignment":[36],"Meta-Evaluation.":[37],"Tackling":[38],"it":[39],"directly":[40],"difficult:":[42],"collecting":[43],"data":[45,100,108],"expensive,":[47],"absolute":[48],"hard":[51],"align,":[53],"and":[54,79,124,137,148,187,191],"using":[55],"an":[56],"meta-evaluator":[58],"recurses":[59],"question":[61],"trustworthiness.":[63],"adopt":[65],"a":[66,71,84,99,127,133,139,155,171],"reformulation":[67],"meta-evaluation":[69,107],"as":[70],"preference":[72],"judgment":[73,165],"problem:":[74],"rather":[75],"than":[76],"comparing":[77],"scores":[82],"trajectory,":[85],"we":[86,96],"ask":[87],"whether":[88],"their":[89],"implied":[90],"preferences":[91],"align.":[92],"Building":[93],"on":[94,154],"this,":[95],"introduce":[97],"PADM\u00c9,":[98],"synthesis":[101],"method":[102],"generates":[104],"reliable":[105],"criterion-based":[106],"for":[109],"settings.":[111],"PADM\u00c9":[112,136,160],"uses":[113],"only":[114],"small":[115],"models,":[117],"requires":[118],"no":[119],"involvement":[121],"during":[122],"evaluations,":[123],"operates":[125],"under":[126],"low":[128],"computational":[129],"budget.":[130],"build":[132],"prototype":[134],"synthesize":[138],"dataset":[140,180],"1,000":[142],"samples":[143],"four":[145],"domains":[147],"three":[149],"evaluation":[150,185],"criteria.":[151],"Human":[152],"validation":[153],"150-sample":[156],"subset":[157],"demonstrates":[158,181],"improves":[161],"agreement":[162],"from":[166],"73%":[167],"85%":[169],"over":[170],"naive":[172],"baseline.":[173],"Meta-evaluating":[174],"25":[175],"common":[176],"our":[179],"correlations":[183],"between":[184],"performance":[186],"granularity,":[189],"leniency,":[190],"model":[192],"size,":[193],"among":[194],"factors.":[196]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-01T00:00:00"}