{"id":"https://openalex.org/W4416037216","doi":"https://doi.org/10.18653/v1/2025.emnlp-industry.106","title":"GEMMAS: Graph-based Evaluation Metrics for Multi Agent Systems","display_name":"GEMMAS: Graph-based Evaluation Metrics for Multi Agent Systems","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416037216","doi":"https://doi.org/10.18653/v1/2025.emnlp-industry.106"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.emnlp-industry.106","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.emnlp-industry.106","pdf_url":"https://aclanthology.org/2025.emnlp-industry.106.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: Industry Track","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.emnlp-industry.106.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Jisoo Lee","orcid":null},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jisoo Lee","raw_affiliation_strings":["Seoul National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Seoul National University","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Raeyoung Chang","orcid":null},"institutions":[{"id":"https://openalex.org/I148751991","display_name":"Sogang University","ror":"https://ror.org/056tn4839","country_code":"KR","type":"education","lineage":["https://openalex.org/I148751991"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Raeyoung Chang","raw_affiliation_strings":["Sogang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sogang University","institution_ids":["https://openalex.org/I148751991"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dongwook Kwon","orcid":null},"institutions":[{"id":"https://openalex.org/I161024014","display_name":"Kwangwoon University","ror":"https://ror.org/02e9zc863","country_code":"KR","type":"education","lineage":["https://openalex.org/I161024014"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Dongwook Kwon","raw_affiliation_strings":["Kwangwoon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kwangwoon University","institution_ids":["https://openalex.org/I161024014"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100750289","display_name":"Harmanpreet Singh","orcid":"https://orcid.org/0000-0001-8676-4375"},"institutions":[{"id":"https://openalex.org/I148751991","display_name":"Sogang University","ror":"https://ror.org/056tn4839","country_code":"KR","type":"education","lineage":["https://openalex.org/I148751991"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Harmanpreet Singh","raw_affiliation_strings":["Sogang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sogang University","institution_ids":["https://openalex.org/I148751991"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101775529","display_name":"Nikhil Verma","orcid":"https://orcid.org/0000-0002-8849-891X"},"institutions":[{"id":"https://openalex.org/I2818286","display_name":"LG (United States)","ror":"https://ror.org/02b948n83","country_code":"US","type":"company","lineage":["https://openalex.org/I2818286","https://openalex.org/I4210131320"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nikhil Verma","raw_affiliation_strings":["LG Electronics , Toronto AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LG Electronics , Toronto AI Lab","institution_ids":["https://openalex.org/I2818286"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.5435,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.94200472,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1522","last_page":"1532"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.16419999301433563,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.16419999301433563,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.10980000346899033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.053300000727176666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-agent-system","display_name":"Multi-agent system","score":0.38420000672340393},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.2759999930858612},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.25839999318122864},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.2533000111579895},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.24899999797344208}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5758000016212463},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.38420000672340393},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31769999861717224},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.289000004529953},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2533000111579895},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.24899999797344208},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.24779999256134033},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.23690000176429749}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.emnlp-industry.106","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.emnlp-industry.106","pdf_url":"https://aclanthology.org/2025.emnlp-industry.106.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: Industry Track","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.emnlp-industry.106","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.emnlp-industry.106","pdf_url":"https://aclanthology.org/2025.emnlp-industry.106.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: Industry Track","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1248055536","display_name":null,"funder_award_id":"RS-2022-00143911","funder_id":"https://openalex.org/F4320328359","funder_display_name":"Ministry of Science and ICT, South Korea"},{"id":"https://openalex.org/G4700831490","display_name":null,"funder_award_id":"2022-","funder_id":"https://openalex.org/F4320335489","funder_display_name":"Institute for Information and Communications Technology Promotion"}],"funders":[{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"},{"id":"https://openalex.org/F4320329803","display_name":"LG Electronics","ror":"https://ror.org/03ddh2c27"},{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416037216.pdf","grobid_xml":"https://content.openalex.org/works/W4416037216.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multi-agent":[0],"systems":[1,101],"built":[2],"on":[3,10,18,98],"language":[4],"models":[5],"have":[6],"shown":[7],"strong":[8],"performance":[9,134],"collaborative":[11,148],"reasoning":[12,35,88],"tasks.However,":[13],"existing":[14],"evaluations":[15],"focus":[16],"only":[17,103],"the":[19,22,48,137],"correctness":[20],"of":[21,139],"final":[23],"output,":[24],"overlooking":[25],"how":[26],"inefficient":[27],"communication":[28],"and":[29,36,80,95,114,135,146],"poor":[30],"coordination":[31],"contribute":[32],"to":[33,73,85],"redundant":[34,87],"higher":[37],"computational":[38],"costs.We":[39],"introduce":[40],"GEMMAS,":[41],"a":[42,57,104],"graph-based":[43],"evaluation":[44],"framework":[45],"that":[46,126],"analyzes":[47],"internal":[49,122],"collaboration":[50,62],"process":[51],"by":[52,110],"modeling":[53],"agent":[54],"interactions":[55],"as":[56],"directed":[58],"acyclic":[59],"graph.To":[60],"capture":[61],"quality,":[63],"we":[64],"propose":[65],"two":[66],"process-level":[67,140],"metrics:":[68],"Information":[69],"Diversity":[70],"Score":[71],"(IDS)":[72],"measure":[74],"semantic":[75],"variation":[76,120],"in":[77,107,112,116,121,142],"inter-agent":[78],"messages,":[79],"Unnecessary":[81],"Path":[82],"Ratio":[83],"(UPR)":[84],"quantify":[86],"paths.We":[89],"evaluate":[90],"GEMMAS":[91],"across":[92],"five":[93],"benchmarks":[94],"highlight":[96,136],"results":[97],"GSM8K,":[99],"where":[100],"with":[102],"2.1%":[105],"difference":[106],"accuracy":[108],"differ":[109],"12.8%":[111],"IDS":[113],"80%":[115],"UPR,":[117],"revealing":[118],"substantial":[119],"collaboration.These":[123],"findings":[124],"demonstrate":[125],"outcome-only":[127],"metrics":[128],"are":[129],"insufficient":[130],"for":[131],"evaluating":[132],"multi-agent":[133],"importance":[138],"diagnostics":[141],"designing":[143],"more":[144],"interpretable":[145],"resource-efficient":[147],"AI":[149],"systems.":[150]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-11-08T00:00:00"}