{"id":"https://openalex.org/W7204908307","doi":"https://doi.org/10.48550/arxiv.2608.31076","title":"Learning to Evaluate Before Improving: Automatic Rubric Induction for Automatic Research Agents","display_name":"Learning to Evaluate Before Improving: Automatic Rubric Induction for Automatic Research Agents","publication_year":2026,"publication_date":"2026-08-31","ids":{"openalex":"https://openalex.org/W7204908307","doi":"https://doi.org/10.48550/arxiv.2608.31076"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.31076","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.31076","pdf_url":"https://arxiv.org/pdf/2608.31076","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.31076","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148856466","display_name":"Xuehai Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Xuehai","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101137392","display_name":"Haowei Qin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qin, Haowei","raw_affiliation_strings":["University of Electronic Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124050353","display_name":"Tongxin Liu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Tongxin","raw_affiliation_strings":["Beijing University of Posts and Telecommunications"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148833594","display_name":"Junkai Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55712492","display_name":"Zhejiang University of Technology","ror":"https://ror.org/02djqfd08","country_code":"CN","type":"education","lineage":["https://openalex.org/I55712492"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Junkai","raw_affiliation_strings":["Zhejiang University of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University of Technology","institution_ids":["https://openalex.org/I55712492"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082272714","display_name":"Xu BuQiang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu, Buqiang","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144670865","display_name":"Jintian Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Jintian","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147427058","display_name":"Yijun Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, Yijun","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138488885","display_name":"Zirui Xue","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xue, Zirui","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148936686","display_name":"Shumin Deng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Deng, Shumin","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.4894999861717224,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.4894999861717224,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.11240000277757645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.06449999660253525,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/iterative-revision","display_name":"iterative revision","score":0.5960000157356262},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.43799999356269836}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.8780999779701233},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.8036999702453613},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7210000157356262},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.5149999856948853},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4417000114917755},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4275999963283539},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.40799999237060547},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38519999384880066},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.328900009393692},{"id":"https://openalex.org/C2779982483","wikidata":"https://www.wikidata.org/wiki/Q6094420","display_name":"Iterative refinement","level":2,"score":0.319599986076355},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.26179999113082886},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C74072328","wikidata":"https://www.wikidata.org/wiki/Q1142726","display_name":"Intelligent agent","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25600001215934753},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2502000033855438}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.31076","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.31076","pdf_url":"https://arxiv.org/pdf/2608.31076","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.31076","doi":"https://doi.org/10.48550/arxiv.2608.31076","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.31076","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.31076","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.31076","pdf_url":"https://arxiv.org/pdf/2608.31076","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204908307.pdf","grobid_xml":"https://content.openalex.org/works/W7204908307.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7101416451","https://openalex.org/W7164023999","https://openalex.org/W7203726719","https://openalex.org/W7214780466","https://openalex.org/W7167595757","https://openalex.org/W7220679195","https://openalex.org/W7220675183","https://openalex.org/W7221634798","https://openalex.org/W7164043397","https://openalex.org/W7158365662"],"abstract_inverted_index":{"Autonomous":[0],"scientific":[1,9,99,235],"research":[2,22,77,145,236],"agents":[3,43],"are":[4,55],"increasingly":[5],"applied":[6],"to":[7,36,82],"end-to-end":[8],"workflows,":[10],"including":[11],"literature":[12,105],"review,":[13],"data":[14],"analysis,":[15],"experimentation,":[16],"and":[17,32,79,106,109,113,122,130,139,147,175,229],"report":[18,146],"generation.":[19],"However,":[20],"open-ended":[21],"tasks":[23],"often":[24],"do":[25],"not":[26],"clearly":[27],"specify":[28],"the":[29,38,62,144,171,214],"analyses,":[30,47],"methods,":[31,50],"success":[33],"criteria":[34,138],"required":[35],"complete":[37],"task.":[39],"As":[40],"a":[41,72,183,188],"result,":[42],"may":[44],"miss":[45],"important":[46],"use":[48],"inappropriate":[49],"or":[51,212],"draw":[52],"conclusions":[53],"that":[54,70,223],"insufficiently":[56],"supported":[57],"by":[58],"evidence.":[59],"To":[60],"address":[61],"problem,":[63],"we":[64],"present":[65],"AutoSciRub,":[66],"an":[67,94,160,200,227],"evaluation-first":[68,224],"framework":[69],"induces":[71],"task-specific":[73],"executable":[74],"rubric":[75,118],"before":[76],"execution,":[78,84],"uses":[80],"it":[81],"guide":[83],"criterion-level":[85],"verification":[86,135],"as":[87,89],"well":[88],"iterative":[90],"revision.":[91],"AutoSciRub":[92,153,197],"decomposes":[93],"underspecified":[95],"instruction":[96],"into":[97],"atomic":[98],"goals,":[100],"grounds":[101],"them":[102],"in":[103],"relevant":[104],"task-visible":[107],"data,":[108],"synthesizes":[110],"specific,":[111],"actionable,":[112],"verifiable":[114],"criteria.":[115],"The":[116],"resulting":[117],"makes":[119],"implicit":[120],"experimental":[121],"evidential":[123],"requirements":[124],"explicit,":[125],"providing":[126],"guidance":[127,225],"for":[128,233],"experiments":[129],"analyses.":[131],"During":[132],"revision,":[133],"rubric-guided":[134],"identifies":[136],"unmet":[137],"enables":[140],"targeted":[141],"refinement":[142],"of":[143,163,193,203,216],"its":[148],"supporting":[149],"artifacts.":[150],"On":[151,187],"ResearchClawBench,":[152],"consistently":[154],"improves":[155],"all":[156],"tested":[157],"configurations,":[158],"with":[159],"average":[161,201],"gain":[162],"2.08":[164],"points":[165,177,205],"across":[166,178,206],"three":[167,179,207],"backbone":[168],"LLMs":[169],"under":[170],"fixed":[172,184],"Codex":[173],"harness":[174],"2.95":[176],"agent":[180,208],"harnesses":[181],"using":[182],"DeepSeek-V4-Flash":[185],"backbone.":[186],"randomly":[189],"sampled":[190],"20-task":[191],"subset":[192],"AstaBench":[194],"E2E":[195],"Discovery,":[196],"further":[198],"achieves":[199],"improvement":[202],"16.8":[204],"harnesses,":[209],"while":[210],"maintaining":[211],"increasing":[213],"number":[215],"successfully":[217],"completed":[218],"tasks.":[219],"These":[220],"results":[221],"demonstrate":[222],"provides":[226],"effective":[228],"generalizable":[230],"control":[231],"mechanism":[232],"autonomous":[234],"(Code:":[237],"https://github.com/zjunlp/AutoSciRub).":[238]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-02T00:00:00"}