{"id":"https://openalex.org/W4416238405","doi":"https://doi.org/10.1145/3786181.3788708","title":"Continuous Benchmark Generation for Evaluating Enterprise-scale LLM Agents","display_name":"Continuous Benchmark Generation for Evaluating Enterprise-scale LLM Agents","publication_year":2026,"publication_date":"2026-04-12","ids":{"openalex":"https://openalex.org/W4416238405","doi":"https://doi.org/10.1145/3786181.3788708"},"language":null,"primary_location":{"id":"doi:10.1145/3786181.3788708","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786181.3788708","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Large Language Models For Code","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3786181.3788708","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061115099","display_name":"Divyanshu Saxena","orcid":"https://orcid.org/0000-0002-7568-0624"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Divyanshu Saxena","raw_affiliation_strings":["UT Austin, Austin, Texas, USA"],"raw_orcid":"https://orcid.org/0000-0002-7568-0624","affiliations":[{"raw_affiliation_string":"UT Austin, Austin, Texas, USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Rishikesh Maurya","orcid":"https://orcid.org/0009-0004-4496-0203"},"institutions":[{"id":"https://openalex.org/I4210124949","display_name":"Microsoft Research (India)","ror":"https://ror.org/02w7f3w92","country_code":"IN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210124949"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Rishikesh Maurya","raw_affiliation_strings":["Microsoft, Bengaluru, India"],"raw_orcid":"https://orcid.org/0009-0004-4496-0203","affiliations":[{"raw_affiliation_string":"Microsoft, Bengaluru, India","institution_ids":["https://openalex.org/I4210124949"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069402619","display_name":"Xiaohong Ou","orcid":"https://orcid.org/0000-0003-4762-7698"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaoxuan Ou","raw_affiliation_strings":["Microsoft, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0000-6667-4321","affiliations":[{"raw_affiliation_string":"Microsoft, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017379860","display_name":"Gagan Somashekar","orcid":"https://orcid.org/0000-0001-6949-8685"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gagan Somashekar","raw_affiliation_strings":["Microsoft, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0001-6949-8685","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032355009","display_name":"Shachee Mishra Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I4210162141","display_name":"Microsoft (India)","ror":"https://ror.org/04ww0w091","country_code":"IN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210162141"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Shachee Mishra Gupta","raw_affiliation_strings":["Microsoft, Noida, India"],"raw_orcid":"https://orcid.org/0000-0003-1332-690X","affiliations":[{"raw_affiliation_string":"Microsoft, Noida, India","institution_ids":["https://openalex.org/I4210162141"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025679453","display_name":"Arun Iyer","orcid":null},"institutions":[{"id":"https://openalex.org/I4210124949","display_name":"Microsoft Research (India)","ror":"https://ror.org/02w7f3w92","country_code":"IN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210124949"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Arun Iyer","raw_affiliation_strings":["Microsoft, Bengaluru, India"],"raw_orcid":"https://orcid.org/0000-0001-7377-7599","affiliations":[{"raw_affiliation_string":"Microsoft, Bengaluru, India","institution_ids":["https://openalex.org/I4210124949"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081699461","display_name":"Yu Kang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu Kang","raw_affiliation_strings":["Microsoft, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0004-1735-5876","affiliations":[{"raw_affiliation_string":"Microsoft, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101967802","display_name":"Chetan Bansal","orcid":"https://orcid.org/0000-0003-0102-8139"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chetan Bansal","raw_affiliation_strings":["Microsoft, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0003-0102-8139","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035329776","display_name":"Aditya Akella","orcid":"https://orcid.org/0000-0002-5920-170X"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aditya Akella","raw_affiliation_strings":["UT Austin, Austin, Texas, USA"],"raw_orcid":"https://orcid.org/0000-0002-5920-170X","affiliations":[{"raw_affiliation_string":"UT Austin, Austin, Texas, USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070722259","display_name":"Saravan Rajmohan","orcid":"https://orcid.org/0000-0002-2019-213X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saravan Rajmohan","raw_affiliation_strings":["Microsoft, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0002-2019-213X","affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.01840741,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.298799991607666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.298799991607666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12203","display_name":"Mobile Agent-Based Network Management","score":0.06459999829530716,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.04969999939203262,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.718500018119812},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.578000009059906},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.4999000132083893},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.4934999942779541},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.49230000376701355},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.36500000953674316}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7301999926567078},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.718500018119812},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.578000009059906},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4999000132083893},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4934999942779541},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.49230000376701355},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3610999882221222},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.35269999504089355},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.34619998931884766},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.33880001306533813},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.289900004863739},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.27410000562667847},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2648000121116638}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3786181.3788708","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786181.3788708","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Large Language Models For Code","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2511.10049","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.10049","pdf_url":"https://arxiv.org/pdf/2511.10049","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2511.10049","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.10049","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3786181.3788708","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786181.3788708","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Large Language Models For Code","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2721915658","display_name":"National Science Foundation Expeditions in Computing: Learning Directed Operating System -- A Clean-Slate Paradigm for Operating Systems Design and Implementation","funder_award_id":"2326576","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"rapid":[1,148],"adoption":[2],"of":[3,22,31,69,85,97,135],"AI":[4,23,87],"agents":[5,24],"across":[6],"domains":[7],"has":[8],"made":[9],"systematic":[10],"evaluation":[11,36,84,145],"crucial":[12],"for":[13,38,43,51,93],"ensuring":[14],"their":[15],"usefulness":[16],"and":[17,33,56,60,81,123,153],"successful":[18],"production":[19],"deployment.":[20],"Evaluation":[21],"typically":[25],"involves":[26],"using":[27],"a":[28,67,94,107,132,143],"fixed":[29],"set":[30],"benchmarks":[32,48,76,129],"computing":[34],"multiple":[35],"metrics":[37],"the":[39,75,78,120],"agent.":[40],"While":[41],"sufficient":[42],"simple":[44],"coding":[45],"tasks,":[46],"these":[47],"fall":[49],"short":[50],"enterprise-scale":[52],"agents,":[53],"where":[54,117],"services":[55],"requirements":[57,79],"evolve":[58,74],"continuously":[59],"ground-truth":[61],"examples":[62],"are":[63],"sparse.":[64],"We":[65,89],"propose":[66],"process":[68,140],"benchmark":[70],"generation":[71],"that":[72],"helps":[73],"as":[77],"change":[80],"perform":[82],"robust":[83],"evolving":[86],"agents.":[88],"instantiate":[90],"this":[91,139],"approach":[92,112],"case":[95],"study":[96],"service":[98],"migration":[99],"from":[100,130],"one":[101],"deployment":[102],"platform":[103],"to":[104,127],"another":[105],"at":[106],"large":[108],"public":[109],"enterprise.":[110],"Our":[111],"relies":[113],"on":[114,150],"semi-structured":[115],"documents":[116],"developers":[118],"express":[119],"high-level":[121],"intent,":[122],"uses":[124],"state-of-the-art":[125],"LLMs":[126],"generate":[128],"just":[131],"small":[133],"number":[134],"such":[136],"documents.":[137],"Overall,":[138],"results":[141],"in":[142],"maintainable":[144],"framework,":[146],"enabling":[147],"feedback":[149],"agent":[151],"performance":[152],"facilitating":[154],"targeted":[155],"improvements.":[156]},"counts_by_year":[],"updated_date":"2026-07-09T05:49:46.723101","created_date":"2025-11-15T00:00:00"}