{"id":"https://openalex.org/W7138901614","doi":"https://doi.org/10.48550/arxiv.2603.16124","title":"SWE-QA-Pro: A Representative Benchmark and Scalable Training Recipe for Repository-Level Code Understanding","display_name":"SWE-QA-Pro: A Representative Benchmark and Scalable Training Recipe for Repository-Level Code Understanding","publication_year":2026,"publication_date":"2026-03-17","ids":{"openalex":"https://openalex.org/W7138901614","doi":"https://doi.org/10.48550/arxiv.2603.16124"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.16124","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16124","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.16124","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129818549","display_name":"Songcheng Cai","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Cai, Songcheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129898481","display_name":"Zhiheng Lyu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lyu, Zhiheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130013261","display_name":"Yuansheng Ni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ni, Yuansheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Xiangchao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xiangchao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130103183","display_name":"Baichuan Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Baichuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111208122","display_name":"Shenzhe Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Shenzhe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130108094","display_name":"Yi Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Yi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130138824","display_name":"Haozhe Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haozhe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129500749","display_name":"Chi Ruan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ruan, Chi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017370617","display_name":"Benjamin Schneider","orcid":"https://orcid.org/0000-0003-4876-7756"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schneider, Benjamin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114752575","display_name":"Weixu Zhang","orcid":"https://orcid.org/0009-0003-2000-7891"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Weixu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130097240","display_name":"Xiang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071453701","display_name":"Andy Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Andy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029130689","display_name":"Yuyu Zhang","orcid":"https://orcid.org/0000-0002-2460-8323"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130119678","display_name":"Ping Nie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nie, Ping","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129952540","display_name":"Wenhu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Wenhu","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":16,"corresponding_author_ids":["https://openalex.org/A5129818549"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8531000018119812,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8531000018119812,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.03240000084042549,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.008799999952316284,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/codebase","display_name":"Codebase","score":0.6080999970436096},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6000000238418579},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5788999795913696},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.5277000069618225},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.4560999870300293},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4526999890804291},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.42750000953674316},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.40619999170303345},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3862000107765198},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.3666999936103821}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8073999881744385},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6245999932289124},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.6080999970436096},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6000000238418579},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5788999795913696},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.5277000069618225},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5091999769210815},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.4560999870300293},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4526999890804291},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.42750000953674316},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.40619999170303345},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3862000107765198},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3666999936103821},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.35679998993873596},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.35370001196861267},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.3513999879360199},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.34299999475479126},{"id":"https://openalex.org/C2778671685","wikidata":"https://www.wikidata.org/wiki/Q219239","display_name":"Recipe","level":2,"score":0.33239999413490295},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3255999982357025},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3248000144958496},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.3244999945163727},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.31049999594688416},{"id":"https://openalex.org/C164155591","wikidata":"https://www.wikidata.org/wiki/Q2067766","display_name":"Satisfiability modulo theories","level":2,"score":0.29280000925064087},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29260000586509705},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.28929999470710754},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.2793999910354614},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C2776493592","wikidata":"https://www.wikidata.org/wiki/Q5158717","display_name":"Succinctness","level":2,"score":0.27410000562667847},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.26579999923706055},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.26030001044273376},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.25850000977516174},{"id":"https://openalex.org/C155911762","wikidata":"https://www.wikidata.org/wiki/Q422321","display_name":"Blueprint","level":2,"score":0.25600001215934753},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.16124","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16124","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.16124","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16124","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Agentic":[0],"repository-level":[1],"code":[2],"understanding":[3],"is":[4],"essential":[5],"for":[6,100,119],"automating":[7],"complex":[8,121],"software":[9],"engineering":[10],"tasks,":[11],"yet":[12],"the":[13,22,105,114,178,186,192],"field":[14],"lacks":[15],"reliable":[16],"benchmarks.":[17],"Existing":[18],"evaluations":[19],"often":[20],"overlook":[21],"long":[23],"tail":[24],"topics":[25],"and":[26,69,158,175,191],"rely":[27],"on":[28,173],"popular":[29],"repositories":[30,53],"where":[31,89],"Large":[32],"Language":[33],"Models":[34],"(LLMs)":[35],"can":[36],"cheat":[37],"via":[38,61],"memorized":[39],"knowledge.":[40],"To":[41],"address":[42],"this,":[43],"we":[44,123],"introduce":[45],"SWE-QA-Pro,":[46],"a":[47,71,87,97,125,132,161],"benchmark":[48],"constructed":[49],"from":[50,143],"diverse,":[51],"long-tail":[52],"with":[54,165],"executable":[55],"environments.":[56],"We":[57],"enforce":[58],"topical":[59],"balance":[60],"issue-driven":[62],"clustering":[63],"to":[64,112,153,180],"cover":[65],"under-represented":[66],"task":[67],"types":[68],"apply":[70],"rigorous":[72],"difficulty":[73],"calibration":[74],"process:":[75],"questions":[76],"solvable":[77],"by":[78,140,170],"direct-answer":[79],"baselines":[80],"are":[81],"filtered":[82],"out.":[83],"This":[84,147],"results":[85],"in":[86],"dataset":[88],"agentic":[90,108,196],"workflows":[91],"significantly":[92],"outperform":[93],"direct":[94],"answering":[95],"(e.g.,":[96],"~13-point":[98],"gap":[99,179],"Claude":[101],"Sonnet":[102],"4.5),":[103],"confirming":[104],"necessity":[106],"of":[107,116,188,194],"codebase":[109],"exploration.":[110],"Furthermore,":[111],"tackle":[113],"scarcity":[115],"training":[117,134,197],"data":[118,128],"such":[120],"behaviors,":[122],"propose":[124],"scalable":[126],"synthetic":[127],"pipeline":[129],"that":[130],"powers":[131],"two-stage":[133],"recipe:":[135],"Supervised":[136],"Fine-Tuning":[137],"(SFT)":[138],"followed":[139],"Reinforcement":[141],"Learning":[142],"AI":[144],"Feedback":[145],"(RLAIF).":[146],"approach":[148],"allows":[149],"small":[150],"open":[151],"models":[152],"learn":[154],"efficient":[155],"tool":[156],"usage":[157],"reasoning.":[159],"Empirically,":[160],"Qwen3-8B":[162],"model":[163],"trained":[164],"our":[166,189,195],"recipe":[167],"surpasses":[168],"GPT-4o":[169],"2.3":[171],"points":[172],"SWE-QA-Pro":[174],"substantially":[176],"narrows":[177],"state-of-the-art":[181],"proprietary":[182],"models,":[183],"demonstrating":[184],"both":[185],"validity":[187],"evaluation":[190],"effectiveness":[193],"workflow.":[198]},"counts_by_year":[],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2026-03-20T00:00:00"}