{"id":"https://openalex.org/W4414581779","doi":"https://doi.org/10.48550/arxiv.2505.18878","title":"CRMArena-Pro: Holistic Assessment of LLM Agents Across Diverse Business Scenarios and Interactions","display_name":"CRMArena-Pro: Holistic Assessment of LLM Agents Across Diverse Business Scenarios and Interactions","publication_year":2025,"publication_date":"2025-05-24","ids":{"openalex":"https://openalex.org/W4414581779","doi":"https://doi.org/10.48550/arxiv.2505.18878"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2505.18878","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.18878","pdf_url":"https://arxiv.org/pdf/2505.18878","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.18878","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017859556","display_name":"Kung-Hsiang Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Kung-Hsiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042929220","display_name":"Akshara Prabhakar","orcid":"https://orcid.org/0009-0001-7274-8734"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Prabhakar, Akshara","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119753695","display_name":"Onkar Thorat","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thorat, Onkar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004155361","display_name":"Divyansh Agarwal","orcid":"https://orcid.org/0000-0003-0404-6877"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agarwal, Divyansh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113067484","display_name":"Prafulla Kumar Choubey","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choubey, Prafulla Kumar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055486743","display_name":"Yixin Mao","orcid":"https://orcid.org/0000-0002-3140-2569"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mao, Yixin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042646536","display_name":"Silvio Savarese","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Savarese, Silvio","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032046813","display_name":"Caiming Xiong","orcid":"https://orcid.org/0000-0003-0349-8628"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiong, Caiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5066791810","display_name":"Chien-Sheng Wu","orcid":"https://orcid.org/0000-0002-5598-5324"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Chien-Sheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.891700029373169,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.891700029373169,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.8453999757766724,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.7628999948501587,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/confidentiality","display_name":"Confidentiality","score":0.7134000062942505},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6334999799728394},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6134999990463257},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.583899974822998},{"id":"https://openalex.org/keywords/transformative-learning","display_name":"Transformative learning","score":0.4982999861240387},{"id":"https://openalex.org/keywords/scarcity","display_name":"Scarcity","score":0.45190000534057617},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.38269999623298645}],"concepts":[{"id":"https://openalex.org/C71745522","wikidata":"https://www.wikidata.org/wiki/Q2476929","display_name":"Confidentiality","level":2,"score":0.7134000062942505},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6334999799728394},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6134999990463257},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.583899974822998},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5759000182151794},{"id":"https://openalex.org/C70587473","wikidata":"https://www.wikidata.org/wiki/Q7834111","display_name":"Transformative learning","level":2,"score":0.4982999861240387},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.48890000581741333},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.45570001006126404},{"id":"https://openalex.org/C109747225","wikidata":"https://www.wikidata.org/wiki/Q815758","display_name":"Scarcity","level":2,"score":0.45190000534057617},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.4226999878883362},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.38269999623298645},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.3776000142097473},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32100000977516174},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3133000135421753},{"id":"https://openalex.org/C199776023","wikidata":"https://www.wikidata.org/wiki/Q202875","display_name":"Negotiation","level":2,"score":0.30820000171661377},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C85345410","wikidata":"https://www.wikidata.org/wiki/Q851587","display_name":"Business process","level":3,"score":0.265500009059906},{"id":"https://openalex.org/C2780440489","wikidata":"https://www.wikidata.org/wiki/Q5227278","display_name":"Data-driven","level":2,"score":0.2648000121116638},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2572000026702881},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.2565000057220459},{"id":"https://openalex.org/C22467394","wikidata":"https://www.wikidata.org/wiki/Q849359","display_name":"Multidisciplinary approach","level":2,"score":0.25529998540878296},{"id":"https://openalex.org/C108170787","wikidata":"https://www.wikidata.org/wiki/Q3951828","display_name":"Agency (philosophy)","level":2,"score":0.25369998812675476}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2505.18878","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.18878","pdf_url":"https://arxiv.org/pdf/2505.18878","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.18878","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.18878","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.18878","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.18878","pdf_url":"https://arxiv.org/pdf/2505.18878","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"While":[0,127],"AI":[1],"agents":[2,62,108,135,148],"hold":[3],"transformative":[4],"potential":[5],"in":[6,30,63,124,183],"business,":[7],"effective":[8],"performance":[9,118],"benchmarking":[10],"is":[11],"hindered":[12],"by":[13,96],"the":[14,179],"scarcity":[15],"of":[16,40,60],"public,":[17],"realistic":[18,58],"business":[19,42,142],"data":[20],"on":[21,69,115],"widely":[22],"used":[23],"platforms.":[24],"Existing":[25],"benchmarks":[26],"often":[27,161],"lack":[28],"fidelity":[29],"their":[31],"environments,":[32],"data,":[33],"and":[34,44,78,81,87,99,175,188],"agent-user":[35],"interactions,":[36],"with":[37,71,117],"limited":[38],"coverage":[39],"diverse":[41,64,97],"scenarios":[43],"industries.":[45],"To":[46],"address":[47],"these":[48],"gaps,":[49],"we":[50],"introduce":[51],"CRMArena-Pro,":[52,116],"a":[53,168],"novel":[54],"benchmark":[55],"for":[56,84,133,181],"holistic,":[57],"assessment":[59],"LLM":[61,107,173],"professional":[65],"settings.":[66,126],"CRMArena-Pro":[67],"expands":[68],"CRMArena":[70],"nineteen":[72],"expert-validated":[73],"tasks":[74],"across":[75],"sales,":[76],"service,":[77],"'configure,":[79],"price,":[80],"quote'":[82],"processes,":[83],"both":[85],"Business-to-Business":[86],"Business-to-Customer":[88],"scenarios.":[89],"It":[90],"distinctively":[91],"incorporates":[92],"multi-turn":[93,125,184],"interactions":[94],"guided":[95],"personas":[98],"robust":[100],"confidentiality":[101,152,186],"awareness":[102],"assessments.":[103],"Experiments":[104],"reveal":[105],"leading":[106],"achieve":[109],"only":[110],"around":[111],"58%":[112],"single-turn":[113,138],"success":[114],"dropping":[119],"significantly":[120],"to":[121],"approximately":[122],"35%":[123],"Workflow":[128],"Execution":[129],"proves":[130],"more":[131],"tractable":[132],"top":[134],"(over":[136],"83%":[137],"success),":[139],"other":[140],"evaluated":[141],"skills":[143],"present":[144],"greater":[145],"challenges.":[146],"Furthermore,":[147],"exhibit":[149],"near-zero":[150],"inherent":[151],"awareness;":[153],"though":[154],"targeted":[155],"prompting":[156],"can":[157],"improve":[158],"this,":[159],"it":[160],"compromises":[162],"task":[163],"performance.":[164],"These":[165],"findings":[166],"highlight":[167],"substantial":[169],"gap":[170],"between":[171],"current":[172],"capabilities":[174],"enterprise":[176],"demands,":[177],"underscoring":[178],"need":[180],"advancements":[182],"reasoning,":[185],"adherence,":[187],"versatile":[189],"skill":[190],"acquisition.":[191]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}