{"id":"https://openalex.org/W7214799133","doi":"https://doi.org/10.48550/arxiv.2609.31906","title":"EmailBench: A Benchmark for Evaluating LLM Agents on Enterprise Email and Productivity Tasks","display_name":"EmailBench: A Benchmark for Evaluating LLM Agents on Enterprise Email and Productivity Tasks","publication_year":2026,"publication_date":"2026-09-25","ids":{"openalex":"https://openalex.org/W7214799133","doi":"https://doi.org/10.48550/arxiv.2609.31906"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.31906","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.31906","pdf_url":"https://arxiv.org/pdf/2609.31906","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.31906","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153500351","display_name":"Mukul Singh","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Singh, Mukul","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018302659","display_name":"Mansi Uniyal","orcid":"https://orcid.org/0009-0009-8102-5421","observed_orcids":["https://orcid.org/0009-0009-8102-5421"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Uniyal, Mansi","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071527691","display_name":"Devin Devlin","orcid":"https://orcid.org/0009-0005-2207-673X","observed_orcids":["https://orcid.org/0009-0005-2207-673X"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devlin, Devin","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101518985","display_name":"Wen Xie","orcid":"https://orcid.org/0000-0002-0274-4981","observed_orcids":["https://orcid.org/0000-0002-0274-4981"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xie, Wen","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153518024","display_name":"Big Thadawasin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thadawasin, Big","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030878165","display_name":"Ritam Dutt","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dutt, Ritam","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000544794","display_name":"Vivian W.-M. Lai","orcid":"https://orcid.org/0000-0003-3165-6136","observed_orcids":["https://orcid.org/0000-0003-3165-6136"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lai, Vivian","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027309886","display_name":"Hyeonsu B Kang","orcid":"https://orcid.org/0000-0002-1990-2050","observed_orcids":["https://orcid.org/0000-0002-1990-2050"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kang, Hyeonsu B.","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.29440000653266907,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.29440000653266907,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.20749999582767487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.11379999667406082,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.7400000095367432},{"id":"https://openalex.org/keywords/enterprise-email","display_name":"enterprise email","score":0.8529999852180481},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.41600000858306885},{"id":"https://openalex.org/keywords/enron-corpus","display_name":"Enron corpus","score":0.8130000233650208},{"id":"https://openalex.org/keywords/tool-evaluation","display_name":"tool evaluation","score":0.41600000858306885},{"id":"https://openalex.org/keywords/rubrics","display_name":"rubrics","score":0.5950000286102295},{"id":"https://openalex.org/keywords/temporal-reasoning","display_name":"temporal reasoning","score":0.6549999713897705},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"information retrieval","score":0.6380000114440918},{"id":"https://openalex.org/keywords/task-completion","display_name":"task completion","score":0.30399999022483826}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8690999746322632},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.8331999778747559},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7692000269889832},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6802999973297119},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6721000075340271},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.6617000102996826},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.4993000030517578},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4912000000476837},{"id":"https://openalex.org/C204983608","wikidata":"https://www.wikidata.org/wiki/Q2111958","display_name":"Productivity","level":2,"score":0.4375},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.40290001034736633},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.39410001039505005},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.36559998989105225},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.34779998660087585},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.33869999647140503},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.30799999833106995},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.2890999913215637},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2637999951839447},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.26159998774528503},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2531000077724457}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.31906","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.31906","pdf_url":"https://arxiv.org/pdf/2609.31906","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.31906","doi":"https://doi.org/10.48550/arxiv.2609.31906","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.31906","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.31906","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.31906","pdf_url":"https://arxiv.org/pdf/2609.31906","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214799133.pdf","grobid_xml":"https://content.openalex.org/works/W7214799133.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7223099241","https://openalex.org/W7165674583","https://openalex.org/W7151854062","https://openalex.org/W7171566560","https://openalex.org/W7128767013","https://openalex.org/W4401201100","https://openalex.org/W4416870030","https://openalex.org/W4396653533","https://openalex.org/W4416036585","https://openalex.org/W7155014488"],"abstract_inverted_index":{"Enterprise":[0],"email":[1,26,39,52],"agents":[2],"must":[3],"combine":[4],"information":[5],"retrieval,":[6],"structured":[7],"state":[8],"changes,":[9],"temporal":[10],"reasoning,":[11],"and":[12,40,63,159],"multi-step":[13],"coordination.":[14],"Recent":[15],"agent":[16],"benchmarks":[17],"include":[18],"productivity":[19,41],"tasks,":[20],"but":[21],"few":[22],"center":[23],"on":[24,98],"typed":[25,51],"workflows":[27],"in":[28],"a":[29,35,50,58,64,99,146],"self-contained":[30,147],"environment.":[31],"We":[32,93],"introduce":[33],"EmailBench,":[34],"benchmark":[36,48],"of":[37,109,113],"206":[38],"scenarios":[42,110],"across":[43,128],"16":[44],"task":[45,129,142],"categories.":[46,130],"The":[47,103],"couples":[49],"API":[53,121],"specification":[54],"with":[55,89,123,153],"provider-neutral":[56],"naming,":[57],"deterministic":[59],"synthetic":[60],"Enron-inspired":[61],"corpus,":[62],"scenario":[65],"suite":[66],"whose":[67],"topic":[68],"selection":[69],"was":[70],"informed":[71],"by":[72],"aggregate":[73],"task-intent":[74],"telemetry":[75],"from":[76],"an":[77,119],"interactive":[78],"prototype.":[79],"Its":[80],"hybrid":[81],"evaluation":[82,161],"protocol":[83],"combines":[84],"258":[85],"executable":[86],"static":[87],"assertions":[88],"211":[90],"LLM":[91],"rubrics.":[92],"evaluate":[94],"eight":[95],"LM":[96],"configurations":[97],"fixed":[100],"single-user":[101],"corpus.":[102],"best-performing":[104],"configuration":[105],"passes":[106],"only":[107],"33.5%":[108],"despite":[111],"99.7%":[112],"its":[114],"tool":[115,136,155],"calls":[116],"completing":[117],"without":[118],"observed":[120],"failure,":[122],"pass":[124],"rates":[125],"varying":[126],"substantially":[127],"This":[131],"gap":[132],"shows":[133],"that":[134],"valid":[135],"execution":[137],"is":[138],"not":[139],"equivalent":[140],"to":[141],"completion.":[143],"EmailBench":[144],"provides":[145],"environment":[148],"for":[149],"end-to-end":[150],"email-agent":[151],"evaluation,":[152],"broader":[154],"coverage,":[156],"multi-persona":[157],"testing,":[158],"repeated-run":[160],"as":[162],"future":[163],"work":[164],"areas.":[165]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-30T00:00:00"}