{"id":"https://openalex.org/W4401201100","doi":"https://doi.org/10.48550/arxiv.2407.19056","title":"OfficeBench: Benchmarking Language Agents across Multiple Applications for Office Automation","display_name":"OfficeBench: Benchmarking Language Agents across Multiple Applications for Office Automation","publication_year":2024,"publication_date":"2024-07-26","ids":{"openalex":"https://openalex.org/W4401201100","doi":"https://doi.org/10.48550/arxiv.2407.19056"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.19056","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19056","pdf_url":"https://arxiv.org/pdf/2407.19056","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.19056","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114406806","display_name":"Zilong Wang","orcid":"https://orcid.org/0009-0001-0016-7973","observed_orcids":["https://orcid.org/0009-0001-0016-7973"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zilong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109756737","display_name":"Yuedong Cui","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cui, Yuedong","raw_affiliation_strings":["UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UCLA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109297770","display_name":"Li Zhong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhong, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102965451","display_name":"Zimin Zhang","orcid":"https://orcid.org/0000-0003-4417-7249","observed_orcids":["https://orcid.org/0000-0003-4417-7249"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Zimin","raw_affiliation_strings":["UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UCLA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110195053","display_name":"Da Yin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yin, Da","raw_affiliation_strings":["UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UCLA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013041264","display_name":"Bill Yuchen Lin","orcid":"https://orcid.org/0000-0002-1149-0186","observed_orcids":["https://orcid.org/0000-0002-1149-0186"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lin, Bill Yuchen","raw_affiliation_strings":["Allen Institute for AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5039500313","display_name":"Jingbo Shang","orcid":"https://orcid.org/0000-0002-7249-4404","observed_orcids":["https://orcid.org/0000-0002-7249-4404"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shang, Jingbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":87,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.20829999446868896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10100000351667404,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/office-automation","display_name":"office automation","score":0.8889999985694885},{"id":"https://openalex.org/keywords/language-agents","display_name":"language agents","score":0.7559999823570251},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.6439999938011169},{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.8159999847412109},{"id":"https://openalex.org/keywords/long-horizon-planning","display_name":"long-horizon planning","score":0.871999979019165},{"id":"https://openalex.org/keywords/application-switching","display_name":"application switching","score":0.7440000176429749},{"id":"https://openalex.org/keywords/hallucinations","display_name":"hallucinations","score":0.4880000054836273}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9420087933540344},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.6202443242073059},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5552654266357422},{"id":"https://openalex.org/C2779518306","wikidata":"https://www.wikidata.org/wiki/Q2734036","display_name":"Office automation","level":2,"score":0.45025670528411865},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3946508467197418},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2821519672870636},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.27128487825393677},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.11192291975021362},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.06761160492897034},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.06471818685531616}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.19056","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19056","pdf_url":"https://arxiv.org/pdf/2407.19056","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.19056","doi":"https://doi.org/10.48550/arxiv.2407.19056","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.19056","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.19056","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19056","pdf_url":"https://arxiv.org/pdf/2407.19056","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401201100.pdf","grobid_xml":"https://content.openalex.org/works/W4401201100.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W3126325318","https://openalex.org/W3210904070","https://openalex.org/W3212748493","https://openalex.org/W4221161476","https://openalex.org/W4327810158","https://openalex.org/W4378942217","https://openalex.org/W4385682136","https://openalex.org/W4386942705","https://openalex.org/W4387225871","https://openalex.org/W4390041933","https://openalex.org/W4392679398","https://openalex.org/W4402667024","https://openalex.org/W4402670687","https://openalex.org/W4415797149","https://openalex.org/W4415797693","https://openalex.org/W7006598167"],"related_works":["https://openalex.org/W4416870030","https://openalex.org/W7164201473","https://openalex.org/W7171884725","https://openalex.org/W4405627213","https://openalex.org/W7167257467","https://openalex.org/W7214799133","https://openalex.org/W4396653533","https://openalex.org/W7117963021","https://openalex.org/W7154311626","https://openalex.org/W7151854062"],"abstract_inverted_index":{"Office":[0],"automation":[1,30,68],"significantly":[2],"enhances":[3],"human":[4,155],"productivity":[5],"by":[6,161],"automatically":[7],"finishing":[8],"routine":[9],"tasks":[10,39,79],"in":[11,20,47,80,97,144,182],"the":[12,15,23,28,48,65,114,118,134,154],"workflow.":[13,119],"Beyond":[14],"basic":[16],"information":[17,45],"extraction":[18],"studied":[19],"much":[21],"of":[22,57,64,117,138],"prior":[24],"document":[25],"AI":[26],"literature,":[27],"office":[29,38,49,67,78,82,146,163,198],"research":[31],"should":[32],"be":[33],"extended":[34],"to":[35,42,76,88,173],"more":[36],"realistic":[37,81],"which":[40,187],"require":[41],"integrate":[43],"various":[44],"sources":[46],"system":[50],"and":[51,101,157,176],"produce":[52],"outputs":[53],"through":[54],"a":[55,98,107,141],"series":[56],"decision-making":[58],"processes.":[59],"We":[60,165],"introduce":[61],"OfficeBench,":[62],"one":[63],"first":[66],"benchmarks":[69],"for":[70,192,197],"evaluating":[71],"current":[72],"LLM":[73,86],"agents'":[74],"capability":[75],"address":[77],"workflows.":[83,164],"OfficeBench":[84],"requires":[85],"agents":[87],"perform":[89],"feasible":[90],"long-horizon":[91],"planning,":[92],"proficiently":[93],"switch":[94],"between":[95,184],"applications":[96],"timely":[99],"manner,":[100],"accurately":[102],"ground":[103],"their":[104],"actions":[105],"within":[106],"large":[108],"combined":[109],"action":[110],"space,":[111],"based":[112],"on":[113,125],"contextual":[115],"demands":[116],"Applying":[120],"our":[121],"customized":[122],"evaluation":[123],"methods":[124],"each":[126],"task,":[127],"we":[128],"find":[129],"that":[130,168],"GPT-4":[131],"Omni":[132],"achieves":[133],"highest":[135],"pass":[136],"rate":[137],"47.00%,":[139],"demonstrating":[140],"decent":[142],"performance":[143,156],"handling":[145],"tasks.":[147],"However,":[148],"this":[149],"is":[150],"still":[151],"far":[152],"below":[153],"accuracy":[158],"standards":[159],"required":[160],"real-world":[162],"further":[166],"observe":[167],"most":[169],"issues":[170],"are":[171],"related":[172],"operation":[174],"redundancy":[175],"hallucinations,":[177],"as":[178,180],"well":[179],"limitations":[181],"switching":[183],"multiple":[185],"applications,":[186],"may":[188],"provide":[189],"valuable":[190],"insights":[191],"developing":[193],"effective":[194],"agent":[195],"frameworks":[196],"automation.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":68},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-08-01T00:00:00"}