{"id":"https://openalex.org/W7166802094","doi":"https://doi.org/10.18653/v1/2026.acl-long.697","title":"Agentic Rubrics as Contextual Verifiers for SWE Agents","display_name":"Agentic Rubrics as Contextual Verifiers for SWE Agents","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166802094","doi":"https://doi.org/10.18653/v1/2026.acl-long.697"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.acl-long.697","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.697","pdf_url":"https://aclanthology.org/2026.acl-long.697.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-long.697.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139723393","display_name":"Mohit Raghavendra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mohit Raghavendra","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088137864","display_name":"Anisha Gunjal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anisha Gunjal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139845138","display_name":"Bing Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bing Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5079019876","display_name":"Yunzhong He","orcid":"https://orcid.org/0000-0002-5429-5372"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yunzhong He","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.79146718,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"15265","last_page":"15290"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.6820999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.6820999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.041200000792741776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11010","display_name":"Logic, Reasoning, and Knowledge","score":0.039500001817941666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.4002000093460083},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.3400999903678894},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.2987000048160553},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.29249998927116394},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.2759999930858612},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.2669999897480011}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4302999973297119},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.4002000093460083},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3937999904155731},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.3400999903678894},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2987000048160553},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2531000077724457},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.251800000667572},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2515000104904175},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2515000104904175}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-long.697","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.697","pdf_url":"https://aclanthology.org/2026.acl-long.697.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-long.697","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.697","pdf_url":"https://aclanthology.org/2026.acl-long.697.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166802094.pdf","grobid_xml":"https://content.openalex.org/works/W7166802094.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Verification":[0],"is":[1,161],"critical":[2],"for":[3,11,163,182],"improving":[4],"agents:":[5],"it":[6,94],"provides":[7],"the":[8,79,126],"reward":[9],"signal":[10,181],"Reinforcement":[12],"Learning":[13],"and":[14,52,64,87,114,178],"enables":[15],"inference-time":[16],"gains":[17],"through":[18],"Test-Time":[19],"Scaling":[20],"(TTS).Despite":[21],"its":[22],"importance,":[23],"verification":[24,180],"in":[25,61,129],"software":[26],"engineering":[27],"(SWE)":[28],"agent":[29,76],"settings":[30],"often":[31],"relies":[32],"on":[33,112,116],"code":[34],"execution,":[35],"which":[36],"can":[37],"be":[38],"difficult":[39],"to":[40,43,66,81],"scale":[41],"due":[42],"environment":[44],"setup":[45],"overhead.Scalable":[46],"alternatives":[47],"such":[48],"as":[49],"patch":[50],"classifiers":[51],"heuristic":[53],"methods":[54],"exist,":[55],"but":[56],"they":[57],"are":[58,90,141],"less":[59],"grounded":[60],"codebase":[62],"context":[63,159],"harder":[65],"interpret.To":[67],"this":[68],"end,":[69],"we":[70],"explore":[71],"Agentic":[72,105,172],"Rubrics:":[73],"an":[74,175],"expert":[75],"interacts":[77],"with":[78,118,143],"repository":[80],"create":[82],"a":[83,108,121],"context-grounded":[84],"rubric":[85,135,139],"checklist,":[86],"candidate":[88],"patches":[89],"then":[91],"scored":[92],"against":[93],"without":[95],"requiring":[96],"test":[97],"execution.On":[98],"SWE-Bench":[99],"Verified":[100],"under":[101],"parallel":[102],"TTS":[103],"evaluation,":[104],"Rubrics":[106,173],"achieve":[107],"score":[109],"of":[110],"54.2%":[111],"Qwen3-Coder-30B-A3B":[113],"40.6%":[115],"Qwen3-32B,":[117],"at":[119],"least":[120],"+3.5":[122],"percentagepoint":[123],"gain":[124],"over":[125],"strongest":[127],"baseline":[128],"our":[130],"comparison":[131],"set.We":[132],"further":[133],"analyze":[134],"behavior,":[136],"showing":[137],"that":[138,150,157,171],"scores":[140],"consistent":[142],"ground-truth":[144],"tests":[145,151],"while":[146],"also":[147],"flagging":[148],"issues":[149],"do":[152],"not":[153],"capture.Our":[154],"ablations":[155],"show":[156],"agentic":[158],"gathering":[160],"essential":[162],"producing":[164],"codebase-specific,":[165],"unambiguous":[166],"criteria.Together,":[167],"these":[168],"results":[169],"suggest":[170],"provide":[174],"efficient,":[176],"scalable,":[177],"granular":[179],"SWE":[183],"agents.":[184]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}