{"id":"https://openalex.org/W7163890116","doi":"https://doi.org/10.48550/arxiv.2606.07462","title":"Act As a Real Researcher: A Suite of Benchmarks Evaluating Frontier LLMs and Agentic Harnesses in Research Lifecycle","display_name":"Act As a Real Researcher: A Suite of Benchmarks Evaluating Frontier LLMs and Agentic Harnesses in Research Lifecycle","publication_year":2026,"publication_date":"2026-06-05","ids":{"openalex":"https://openalex.org/W7163890116","doi":"https://doi.org/10.48550/arxiv.2606.07462"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.07462","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07462","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.07462","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138131415","display_name":"Jiayu Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jiayu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125589490","display_name":"Weijiang Lv","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lv, Weijiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138139805","display_name":"Bowen Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138165149","display_name":"Jing Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Jing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023014184","display_name":"Jiayi Song","orcid":"https://orcid.org/0000-0002-8724-6884"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Jiayi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138137450","display_name":"Lingyu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Lingyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100638727","display_name":"Linfu Xue","orcid":"https://orcid.org/0000-0002-7479-8647"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue, Lanxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121272087","display_name":"Luodi Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Luodi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116013973","display_name":"Zepeng Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xin, Zepeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138119204","display_name":"Kaiyu Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Kaiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138183573","display_name":"Xiangyong Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Xiangyong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.14329999685287476,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.14329999685287476,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.11159999668598175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.07440000027418137,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.7649999856948853},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6017000079154968},{"id":"https://openalex.org/keywords/frontier","display_name":"Frontier","score":0.5863000154495239},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.48590001463890076},{"id":"https://openalex.org/keywords/research-program","display_name":"Research program","score":0.39910000562667847},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.39160001277923584},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.31380000710487366}],"concepts":[{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.7649999856948853},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6017000079154968},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.5863000154495239},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5601000189781189},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5282999873161316},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.48590001463890076},{"id":"https://openalex.org/C2778035492","wikidata":"https://www.wikidata.org/wiki/Q7315176","display_name":"Research program","level":2,"score":0.39910000562667847},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.39160001277923584},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.35580000281333923},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3400999903678894},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.31380000710487366},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3131999969482422},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.29339998960494995},{"id":"https://openalex.org/C13687954","wikidata":"https://www.wikidata.org/wiki/Q4826847","display_name":"Autonomous agent","level":2,"score":0.29260000586509705},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.2736000120639801},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2660999894142151},{"id":"https://openalex.org/C2778464652","wikidata":"https://www.wikidata.org/wiki/Q309849","display_name":"Open research","level":2,"score":0.2621000111103058},{"id":"https://openalex.org/C2780938662","wikidata":"https://www.wikidata.org/wiki/Q973710","display_name":"Tying","level":2,"score":0.25949999690055847},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.25679999589920044},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.2540000081062317},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2515000104904175}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.07462","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07462","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.07462","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07462","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0,69,112],"foundation":[1],"models":[2,129],"advance":[3],"and":[4,20,46,94,130],"agent":[5],"scaffolding":[6],"becomes":[7],"increasingly":[8],"sophisticated,":[9],"agents":[10,52,88],"have":[11],"demonstrated":[12],"remarkable":[13],"proficiency":[14],"in":[15,41,101,120],"complex,":[16],"long-horizon":[17],"coding":[18],"tasks":[19],"even":[21,135],"autonomous":[22,32],"experiment":[23],"execution.":[24],"Despite":[25],"their":[26],"evolution":[27],"from":[28],"research":[29,33,44,103,173],"assistants":[30],"into":[31],"agents,":[34],"these":[35],"systems":[36],"still":[37],"exhibit":[38],"significant":[39],"limitations":[40],"field":[42],"sensitivity,":[43],"ethics,":[45],"nuanced":[47,95],"scientific":[48],"judgment.":[49],"Consequently,":[50],"frontier":[51,128],"remain":[53],"unable":[54],"to":[55,158],"fully":[56],"replace":[57],"human":[58,99,160],"researchers.":[59,161],"To":[60],"bridge":[61],"this":[62,106,121],"gap,":[63],"we":[64,108],"conceptualize":[65],"the":[66,91,117,136],"AARR":[67,84],"(Act":[68,111],"a":[70,113],"Real":[71,114],"Researcher)":[72],"benchmark":[73,119],"series.":[74,122],"Unlike":[75],"existing":[76],"benchmarks":[77],"that":[78,97,134,155,165],"primarily":[79],"assess":[80],"macro-level":[81],"execution":[82],"capabilities,":[83],"focuses":[85],"on":[86],"whether":[87],"can":[89],"emulate":[90],"professionalism,":[92],"thoroughness,":[93],"reasoning":[96],"characterize":[98],"researchers":[100],"granular":[102],"scenarios.":[104],"In":[105],"work,":[107],"propose":[109],"AARRI-Bench":[110],"Research":[115],"Intern),":[116],"first":[118],"We":[123],"conduct":[124],"extensive":[125],"experiments":[126],"across":[127],"agentic":[131],"systems,":[132],"revealing":[133],"best-performing":[137],"configuration":[138],"(Mini-SWE-Agent":[139],"with":[140],"Claude":[141],"Opus":[142],"4.7)":[143],"achieves":[144],"only":[145],"68.3\\%":[146],"success":[147],"rate,":[148],"frequently":[149],"overlooking":[150],"subtle":[151],"yet":[152],"critical":[153],"details":[154],"are":[156],"obvious":[157],"real":[159],"Our":[162,180],"results":[163],"indicate":[164],"developing":[166],"researcher-like":[167],"AI":[168],"requires":[169],"further":[170],"exploration":[171],"of":[172],"behavior,":[174],"rather":[175],"than":[176],"merely":[177],"complex":[178],"scaffolding.":[179],"data":[181],"is":[182],"released":[183],"at":[184],"https://github.com/AARR-bench/AARRI-bench.":[185]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-09T00:00:00"}