{"id":"https://openalex.org/W7202353659","doi":"https://doi.org/10.48550/arxiv.2608.11727","title":"Harness-IF: Evaluating Instruction Following Across Instruction Surfaces in Coding Agents","display_name":"Harness-IF: Evaluating Instruction Following Across Instruction Surfaces in Coding Agents","publication_year":2026,"publication_date":"2026-08-12","ids":{"openalex":"https://openalex.org/W7202353659","doi":"https://doi.org/10.48550/arxiv.2608.11727"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.11727","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.11727","pdf_url":"https://arxiv.org/pdf/2608.11727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.11727","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5147126831","display_name":"Zining Huang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huang, Zining","raw_affiliation_strings":["ByteDance Seed ,","Tsinghua University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]},{"raw_affiliation_string":"Tsinghua University ,","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092998429","display_name":"Haoran Que","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Que, Haoran","raw_affiliation_strings":["ByteDance Seed ,","Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]},{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136256120","display_name":"Hong Zeng","orcid":"https://orcid.org/0009-0006-3762-0731","observed_orcids":["https://orcid.org/0009-0006-3762-0731"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeng, Hong","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126981487","display_name":"Ge Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Ge","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147077392","display_name":"Zuo Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Zuo","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147093289","display_name":"Jin Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, Jin","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138709646","display_name":"Haodong Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Haodong","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147077988","display_name":"Zhongfei Hou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hou, Zhongfei","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147105498","display_name":"Changxin Pu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pu, Changxin","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148926172","display_name":"Shen Yan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan, Shen","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148931907","display_name":"Wenhao Huang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huang, Wenhao","raw_affiliation_strings":["ByteDance Seed ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Seed ,","institution_ids":["https://openalex.org/I4405256866"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.7045999765396118,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.7045999765396118,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.026900000870227814,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.01860000006854534,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/instruction-following","display_name":"instruction following","score":0.8399999737739563},{"id":"https://openalex.org/keywords/ai-coding-agents","display_name":"coding agents","score":0.9629999995231628},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.6079999804496765},{"id":"https://openalex.org/keywords/rule-compliance","display_name":"rule compliance","score":0.49000000953674316},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.5170000195503235},{"id":"https://openalex.org/keywords/agent-evaluation-benchmarks","display_name":"agent evaluation benchmarks","score":0.3400000035762787}],"concepts":[{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.7631999850273132},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.724399983882904},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5378000140190125},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4584999978542328},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45579999685287476},{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.41600000858306885},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.40860000252723694},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.39890000224113464},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3874000012874603},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3431999981403351},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.33649998903274536},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.32760000228881836},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.32120001316070557},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.31369999051094055}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.11727","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.11727","pdf_url":"https://arxiv.org/pdf/2608.11727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.11727","doi":"https://doi.org/10.48550/arxiv.2608.11727","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.11727","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.11727","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.11727","pdf_url":"https://arxiv.org/pdf/2608.11727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.7265007495880127,"display_name":"Peace, justice, and strong institutions","id":"https://metadata.un.org/sdg/16"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7202353659.pdf","grobid_xml":"https://content.openalex.org/works/W7202353659.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7124450990","https://openalex.org/W7154403075","https://openalex.org/W7213284899","https://openalex.org/W7171537577","https://openalex.org/W7204603595","https://openalex.org/W7223477500","https://openalex.org/W7221841517","https://openalex.org/W7148262333","https://openalex.org/W7223021097","https://openalex.org/W7172433353"],"abstract_inverted_index":{"When":[0],"a":[1,5,47,59,72,138,150,175],"coding":[2,55],"agent":[3,74],"obeys":[4],"rule,":[6],"it":[7],"may":[8],"simply":[9],"have":[10],"been":[11],"going":[12],"to":[13,129],"do":[14],"that":[15],"anyway.":[16],"Existing":[17],"instruction-following":[18],"benchmarks":[19,33],"cannot":[20],"tell":[21],"the":[22,28,68,100,135,156],"difference:":[23],"they":[24],"concentrate":[25],"rules":[26,44,63,89],"in":[27],"user":[29,191],"turn,":[30],"while":[31],"coding-agent":[32],"emphasize":[34],"final":[35],"task":[36],"success.":[37],"We":[38],"introduce":[39,82],"Harness-IF,":[40],"which":[41,86],"scores":[42,87,145],"operational":[43],"one":[45],"at":[46],"time":[48],"from":[49,58,79],"execution":[50],"evidence:":[51],"60":[52],"realistic":[53],"multi-turn":[54],"items":[56],"drawn":[57],"642-rule":[60],"library,":[61],"256":[62],"receiving":[64],"verdicts,":[65],"placed":[66],"on":[67,124,170],"five":[69],"configurable":[70],"surfaces":[71],"deployed":[73],"reads.":[75],"To":[76],"separate":[77,172],"compliance":[78,148],"coincidence":[80],"we":[81],"Against-Prior":[83],"Accuracy":[84],"(AP-Acc),":[85],"only":[88],"labeled":[90],"as":[91],"opposing":[92],"unprompted":[93],"defaults,":[94],"observed":[95],"by":[96,127,149],"re-running":[97],"tasks":[98],"with":[99,141,185],"rule":[101],"withheld":[102],"across":[103],"nine":[104,171],"probe":[105],"builds":[106,173],"and":[107,117,134,160,190,196],"curated":[108],"otherwise.":[109],"Across":[110],"12":[111],"frontier":[112],"models,":[113],"accuracy":[114],"spans":[115],"72.1-85.9%":[116],"AP-Acc":[118],"66.1-78.6%;":[119],"every":[120],"model":[121],"is":[122],"worse":[123],"against-prior":[125],"rules,":[126],"3.6":[128],"7.4":[130],"points":[131],"(mean":[132],"5.81),":[133],"direction":[136],"survives":[137],"common-support":[139],"analysis":[140],"item-clustered":[142],"intervals.":[143],"Aggregate":[144],"therefore":[146],"overstate":[147],"model-specific":[151],"margin:":[152],"prior":[153],"control":[154],"leaves":[155],"top":[157],"build":[158],"unchanged":[159],"exchanges":[161],"three":[162],"adjacent":[163],"rank":[164],"pairs.":[165],"A":[166],"counterbalanced":[167],"conflict":[168],"pilot":[169],"adds":[174],"second":[176],"result:":[177],"pooled":[178],"precedence":[179],"does":[180],"not":[181],"follow":[182],"prompt":[183],"depth,":[184],"system":[186],"prompts,":[187],"project":[188],"files,":[189],"instructions":[192],"ahead":[193],"of":[194],"tool":[195],"skill":[197],"descriptions.":[198]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-08-14T00:00:00"}