{"id":"https://openalex.org/W7203621245","doi":"https://doi.org/10.48550/arxiv.2608.13867","title":"Engineering Reliable Coding Agents: Evaluating and Operating the System Around the Model","display_name":"Engineering Reliable Coding Agents: Evaluating and Operating the System Around the Model","alternate_titles":[],"publication_year":2026,"publication_date":"2026-08-14","ids":{"openalex":"https://openalex.org/W7203621245","doi":"https://doi.org/10.48550/arxiv.2608.13867"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.13867","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.13867","pdf_url":"https://arxiv.org/pdf/2608.13867","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7203621245"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.13867","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5146411546","display_name":"Stephanie Jarmak","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Jarmak, Stephanie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5146411546"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"reviewed_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.6317999958992004,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.6317999958992004,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.25049999356269836,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.0632999986410141,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-coding-agents","display_name":"coding agents","score":0.9679999947547913},{"id":"https://openalex.org/keywords/reliability","display_name":"reliability","score":0.42100000381469727},{"id":"https://openalex.org/keywords/execution-environment","display_name":"execution environment","score":0.546999990940094},{"id":"https://openalex.org/keywords/retrieval","display_name":"retrieval","score":0.3529999852180481},{"id":"https://openalex.org/keywords/state-management","display_name":"state management","score":0.8740000128746033},{"id":"https://openalex.org/keywords/memory-management","display_name":"memory management","score":0.6129999756813049},{"id":"https://openalex.org/keywords/permission","display_name":"permission","score":0.6069999933242798},{"id":"https://openalex.org/keywords/resource-allocation","display_name":"resource allocation","score":0.9769999980926514},{"id":"https://openalex.org/keywords/failure-analysis","display_name":"failure analysis","score":0.44699999690055847},{"id":"https://openalex.org/keywords/observability","display_name":"observability","score":0.6899999976158142},{"id":"https://openalex.org/keywords/verification","display_name":"verification","score":0.46000000834465027},{"id":"https://openalex.org/keywords/agent-skills","display_name":"agent skills","score":0.382999986410141}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7608000040054321},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5600000023841858},{"id":"https://openalex.org/C36299963","wikidata":"https://www.wikidata.org/wiki/Q1369844","display_name":"Observability","level":2,"score":0.5149000287055969},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.4964999854564667},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.46700000762939453},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4562000036239624},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.43689998984336853},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.43619999289512634},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4237000048160553},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.41190001368522644},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3278999924659729},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.320499986410141},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3068999946117401},{"id":"https://openalex.org/C63540848","wikidata":"https://www.wikidata.org/wiki/Q3140932","display_name":"Fault tolerance","level":2,"score":0.28299999237060547},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.27709999680519104},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.27630001306533813},{"id":"https://openalex.org/C79897977","wikidata":"https://www.wikidata.org/wiki/Q5054568","display_name":"Causal chain","level":2,"score":0.26600000262260437},{"id":"https://openalex.org/C152745839","wikidata":"https://www.wikidata.org/wiki/Q5438153","display_name":"Fault detection and isolation","level":3,"score":0.25839999318122864},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.25619998574256897},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.2549999952316284}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.13867","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.13867","pdf_url":"https://arxiv.org/pdf/2608.13867","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.13867","doi":"https://doi.org/10.48550/arxiv.2608.13867","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.13867","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.13867","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.13867","pdf_url":"https://arxiv.org/pdf/2608.13867","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[{"id":"https://openalex.org/study-designs/systematic-review","display_name":"Systematic Review"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203621245.pdf","grobid_xml":"https://content.openalex.org/works/W7203621245.grobid-xml"},"referenced_works_count":191,"referenced_works":["https://openalex.org/W192446467","https://openalex.org/W1834470623","https://openalex.org/W1973004537","https://openalex.org/W1975879668","https://openalex.org/W1992479210","https://openalex.org/W2006291142","https://openalex.org/W2023718005","https://openalex.org/W2053154970","https://openalex.org/W2057322651","https://openalex.org/W2084413241","https://openalex.org/W2095958260","https://openalex.org/W2120422789","https://openalex.org/W2133943399","https://openalex.org/W2141992894","https://openalex.org/W2469513535","https://openalex.org/W2735864825","https://openalex.org/W2761479024","https://openalex.org/W2809256243","https://openalex.org/W2890801208","https://openalex.org/W2891424260","https://openalex.org/W2896510283","https://openalex.org/W2943403009","https://openalex.org/W2959586312","https://openalex.org/W2963120839","https://openalex.org/W2968986602","https://openalex.org/W2975950293","https://openalex.org/W3005700362","https://openalex.org/W3008900438","https://openalex.org/W3034439313","https://openalex.org/W3048826825","https://openalex.org/W3100501376","https://openalex.org/W3103950664","https://openalex.org/W3135261467","https://openalex.org/W3159250634","https://openalex.org/W3200421165","https://openalex.org/W3210990339","https://openalex.org/W4224919172","https://openalex.org/W4280654158","https://openalex.org/W4285247202","https://openalex.org/W4292243187","https://openalex.org/W4300455941","https://openalex.org/W4302797994","https://openalex.org/W4307001524","https://openalex.org/W4310561894","https://openalex.org/W4362659486","https://openalex.org/W4366003124","https://openalex.org/W4379548473","https://openalex.org/W4383067302","https://openalex.org/W4386889431","https://openalex.org/W4387098977","https://openalex.org/W4387321091","https://openalex.org/W4387355948","https://openalex.org/W4387561453","https://openalex.org/W4387799916","https://openalex.org/W4388858772","https://openalex.org/W4388886073","https://openalex.org/W4390480971","https://openalex.org/W4392538905","https://openalex.org/W4392822190","https://openalex.org/W4394010871","https://openalex.org/W4394591918","https://openalex.org/W4394638297","https://openalex.org/W4394709422","https://openalex.org/W4394778349","https://openalex.org/W4399364360","https://openalex.org/W4399597845","https://openalex.org/W4399836817","https://openalex.org/W4399837631","https://openalex.org/W4400195536","https://openalex.org/W4400337927","https://openalex.org/W4401208776","https://openalex.org/W4402502784","https://openalex.org/W4402671543","https://openalex.org/W4402683971","https://openalex.org/W4402860127","https://openalex.org/W4403334501","https://openalex.org/W4403363594","https://openalex.org/W4403447861","https://openalex.org/W4403564508","https://openalex.org/W4404087993","https://openalex.org/W4404345312","https://openalex.org/W4404354805","https://openalex.org/W4407176798","https://openalex.org/W4407208970","https://openalex.org/W4407231303","https://openalex.org/W4407800405","https://openalex.org/W4408075037","https://openalex.org/W4410218573","https://openalex.org/W4411119597","https://openalex.org/W4411403240","https://openalex.org/W4411449940","https://openalex.org/W4412886699","https://openalex.org/W4412944752","https://openalex.org/W4412945638","https://openalex.org/W4413267850","https://openalex.org/W4414223135","https://openalex.org/W4414688275","https://openalex.org/W4414808569","https://openalex.org/W4414813448","https://openalex.org/W4414887410","https://openalex.org/W4414971614","https://openalex.org/W4415035813","https://openalex.org/W4415101712","https://openalex.org/W4415194395","https://openalex.org/W4415435606","https://openalex.org/W4415746208","https://openalex.org/W4415797413","https://openalex.org/W4415797772","https://openalex.org/W4416034875","https://openalex.org/W4416036918","https://openalex.org/W4416037879","https://openalex.org/W4416113188","https://openalex.org/W4416268504","https://openalex.org/W4416360710","https://openalex.org/W4416366023","https://openalex.org/W4417070626","https://openalex.org/W4417086184","https://openalex.org/W4417295250","https://openalex.org/W6891850465","https://openalex.org/W6910824972","https://openalex.org/W6929618441","https://openalex.org/W6948017749","https://openalex.org/W6948372968","https://openalex.org/W6948524426","https://openalex.org/W6979261539","https://openalex.org/W7078346967","https://openalex.org/W7103751489","https://openalex.org/W7116834104","https://openalex.org/W7124227718","https://openalex.org/W7125957064","https://openalex.org/W7127276108","https://openalex.org/W7127926068","https://openalex.org/W7128378279","https://openalex.org/W7128532971","https://openalex.org/W7128608924","https://openalex.org/W7128774696","https://openalex.org/W7131388287","https://openalex.org/W7131436133","https://openalex.org/W7133198585","https://openalex.org/W7133198810","https://openalex.org/W7133230429","https://openalex.org/W7133239970","https://openalex.org/W7133303706","https://openalex.org/W7134977827","https://openalex.org/W7136044173","https://openalex.org/W7138888486","https://openalex.org/W7143298091","https://openalex.org/W7148886208","https://openalex.org/W7151869330","https://openalex.org/W7152996651","https://openalex.org/W7153230826","https://openalex.org/W7154182599","https://openalex.org/W7154461864","https://openalex.org/W7154629359","https://openalex.org/W7154687677","https://openalex.org/W7155170273","https://openalex.org/W7157305287","https://openalex.org/W7157791259","https://openalex.org/W7158100772","https://openalex.org/W7160254132","https://openalex.org/W7160387505","https://openalex.org/W7160437446","https://openalex.org/W7160908075","https://openalex.org/W7160931718","https://openalex.org/W7160979797","https://openalex.org/W7161135864","https://openalex.org/W7161274789","https://openalex.org/W7161292090","https://openalex.org/W7161951393","https://openalex.org/W7162075157","https://openalex.org/W7162286480","https://openalex.org/W7162457387","https://openalex.org/W7162468227","https://openalex.org/W7162557786","https://openalex.org/W7162791705","https://openalex.org/W7163217227","https://openalex.org/W7163225412","https://openalex.org/W7163515152","https://openalex.org/W7166863277","https://openalex.org/W7168134016","https://openalex.org/W7171331735","https://openalex.org/W7171366694","https://openalex.org/W7171383512","https://openalex.org/W7171670812","https://openalex.org/W7171950379","https://openalex.org/W7171988539","https://openalex.org/W7172019685","https://openalex.org/W7172276332","https://openalex.org/W7172403885","https://openalex.org/W7172440478","https://openalex.org/W7201914381"],"related_works":["https://openalex.org/W7221021903","https://openalex.org/W7204154703","https://openalex.org/W7214207320","https://openalex.org/W7171301459","https://openalex.org/W7221035750","https://openalex.org/W7214330245","https://openalex.org/W7203661215","https://openalex.org/W7154274794","https://openalex.org/W7172403349","https://openalex.org/W7165662610"],"abstract_inverted_index":{"AI":[0],"coding":[1,50],"agents":[2,51],"are":[3,111],"commonly":[4],"evaluated":[5],"as":[6,10,113],"models":[7],"but":[8,20],"deployed":[9],"systems.":[11],"Their":[12],"reliability":[13,143,182],"depends":[14],"not":[15],"only":[16],"on":[17,21,232,249],"model":[18,89,200],"capability,":[19],"the":[22,94,168],"harness,":[23],"execution":[24,123],"state,":[25],"retrieval,":[26,125],"memory":[27],"and":[28,34,42,48,64,80,109,164,172,181,184,208,229,234,247],"state":[29,126],"management,":[30,127],"permissions,":[31],"review":[32,218],"interfaces,":[33],"resource":[35],"allocation.":[36],"This":[37],"monograph":[38,136],"examines":[39],"those":[40],"boundaries":[41],"develops":[43],"a":[44,70,114,138,160,195],"framework":[45,161],"for":[46,162,198],"evaluating":[47],"operating":[49],"reliably.":[52],"It":[53],"synthesizes":[54],"164":[55],"scholarly":[56],"works,":[57],"100":[58],"practitioner":[59],"records,":[60,63],"29":[61],"benchmark":[62],"17":[65],"author-system":[66],"case":[67],"records":[68],"through":[69],"structured":[71,220],"multivocal":[72],"review,":[73],"targeted":[74],"update":[75],"audits,":[76],"software-engineering":[77],"coverage":[78],"analysis,":[79],"distributed-systems":[81],"evidence":[82,158,190,224],"synthesis.":[83],"Across":[84],"this":[85],"evidence,":[86],"many":[87],"apparent":[88],"failures":[90],"originate":[91],"elsewhere":[92],"in":[93,117,120,151],"system,":[95],"while":[96],"improvements":[97],"at":[98],"one":[99],"layer":[100],"often":[101],"fail":[102],"to":[103,105],"propagate":[104],"end-to-end":[106],"outcomes.":[107],"Evaluation":[108],"operation":[110],"treated":[112],"dependency":[115,163],"chain":[116],"which":[118,239,244],"weaknesses":[119],"task":[121],"construction,":[122],"environments,":[124],"verification,":[128],"or":[129],"observability":[130],"can":[131],"invalidate":[132],"downstream":[133],"conclusions.":[134],"The":[135,217,236],"contributes":[137],"versioned":[139],"catalog":[140],"of":[141],"206":[142],"records:":[144],"193":[145],"gated":[146],"practices,":[147],"including":[148],"56":[149],"developed":[150],"depth,":[152],"plus":[153],"13":[154],"research":[155],"leads;":[156],"an":[157],"ledger;":[159],"repair":[165],"asymmetry":[166],"across":[167],"agent":[169,177,187],"lifecycle;":[170],"measurements":[171],"failure":[173],"cases":[174],"from":[175,202],"operated":[176],"systems;":[178],"runnable":[179],"evaluation":[180],"protocols;":[183],"five":[185],"reusable":[186],"skills":[188],"with":[189],"maps.":[191],"Together,":[192],"these":[193],"provide":[194],"system-level":[196],"methodology":[197],"distinguishing":[199],"capability":[201],"infrastructure":[203],"effects,":[204],"designing":[205],"defensible":[206],"evaluations,":[207],"building":[209],"systems":[210],"that":[211],"recover":[212],"safely":[213],"when":[214],"components":[215],"fail.":[216],"is":[219],"rather":[221],"than":[222],"exhaustive,":[223],"strength":[225],"varies":[226],"by":[227],"topic,":[228],"results":[230],"depend":[231],"workload":[233],"configuration.":[235],"methods":[237],"record":[238],"search":[240],"lanes":[241],"were":[242],"executed,":[243],"remain":[245],"unexecuted,":[246],"limits":[248],"evidence-grading":[250],"claims.":[251]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7203621245","counts_by_year":[{"year":2026,"cited_by_count":4}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2026-08-18T00:00:00"}
