{"id":"https://openalex.org/W4414818151","doi":"https://doi.org/10.48550/arxiv.2510.01659","title":"MDSEval: A Meta-Evaluation Benchmark for Multimodal Dialogue Summarization","display_name":"MDSEval: A Meta-Evaluation Benchmark for Multimodal Dialogue Summarization","publication_year":2025,"publication_date":"2025-10-02","ids":{"openalex":"https://openalex.org/W4414818151","doi":"https://doi.org/10.48550/arxiv.2510.01659"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.01659","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.01659","pdf_url":"https://arxiv.org/pdf/2510.01659","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.01659","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014118666","display_name":"Yinhong Liu","orcid":"https://orcid.org/0009-0006-6723-0558","observed_orcids":["https://orcid.org/0009-0006-6723-0558","https://orcid.org/0009-0006-2294-8371"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]},{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Liu, Yinhong","raw_affiliation_strings":["AWS AI Labs","Language Technology Lab , University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]},{"raw_affiliation_string":"Language Technology Lab , University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062082438","display_name":"Jianfeng He","orcid":"https://orcid.org/0009-0007-2421-757X","observed_orcids":["https://orcid.org/0009-0007-2421-757X","https://orcid.org/0000-0001-6995-2980"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"He, Jianfeng","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100341891","display_name":"Hang Su","orcid":"https://orcid.org/0000-0002-6877-6783","observed_orcids":["https://orcid.org/0000-0002-6877-6783"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Su, Hang","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059256506","display_name":"Ruixue Lian","orcid":"https://orcid.org/0000-0002-3919-3204","observed_orcids":["https://orcid.org/0000-0002-3919-3204"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lian, Ruixue","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116269520","display_name":"Yi Nian","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nian, Yi","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015729447","display_name":"Jake W. Vincent","orcid":"https://orcid.org/0000-0002-4794-4370","observed_orcids":["https://orcid.org/0000-0002-4794-4370"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vincent, Jake","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080301608","display_name":"Srikanth Vishnubhotla","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vishnubhotla, Srikanth","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018704921","display_name":"Robinson Piramuthu","orcid":"https://orcid.org/0000-0002-1767-8382","observed_orcids":["https://orcid.org/0000-0002-1767-8382"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Piramuthu, Robinson","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070002963","display_name":"Saab Mansour","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mansour, Saab","raw_affiliation_strings":["AWS AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWS AI Labs","institution_ids":["https://openalex.org/I1311688040"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.573199987411499,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.573199987411499,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.3984000086784363,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.0034000000450760126,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.671999990940094},{"id":"https://openalex.org/keywords/human-judgment","display_name":"human judgment","score":0.5759999752044678},{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.8330000042915344},{"id":"https://openalex.org/keywords/assessment-bias","display_name":"evaluation bias","score":0.9120000004768372}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8956000208854675},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8712000250816345},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8008000254631042},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.7044000029563904},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6525999903678894},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5532000064849854},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47429999709129333},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.39089998602867126},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.3833000063896179},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36959999799728394},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.2922999858856201},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.28360000252723694},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.28220000863075256},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.25589999556541443}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.01659","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.01659","pdf_url":"https://arxiv.org/pdf/2510.01659","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.01659","doi":"https://doi.org/10.48550/arxiv.2510.01659","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.01659","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.01659","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.01659","pdf_url":"https://arxiv.org/pdf/2510.01659","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414818151.pdf","grobid_xml":"https://content.openalex.org/works/W4414818151.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7131817584","https://openalex.org/W7161204972","https://openalex.org/W7127589714","https://openalex.org/W6920602854","https://openalex.org/W4401042947","https://openalex.org/W4403302385","https://openalex.org/W7197016422","https://openalex.org/W2890721473","https://openalex.org/W2997326549","https://openalex.org/W6992955394"],"abstract_inverted_index":{"Multimodal":[0],"Dialogue":[1],"Summarization":[2],"(MDS)":[3],"is":[4,91],"a":[5,36,77],"critical":[6],"task":[7],"with":[8],"wide-ranging":[9],"applications.":[10],"To":[11,69],"support":[12],"the":[13,50,92],"development":[14],"of":[15],"effective":[16],"MDS":[17],"models,":[18],"robust":[19],"automatic":[20],"evaluation":[21,99,108],"methods":[22,34],"are":[23],"essential":[24],"for":[25,54],"reducing":[26],"both":[27],"cost":[28],"and":[29,61,73,96,119],"human":[30,42,62],"effort.":[31],"However,":[32],"such":[33],"require":[35],"strong":[37],"meta-evaluation":[38,52],"benchmark":[39,53,105],"grounded":[40],"in":[41,113],"annotations.":[43],"In":[44],"this":[45],"work,":[46],"we":[47,75],"introduce":[48],"MDSEval,":[49],"first":[51,93],"MDS,":[55],"consisting":[56],"image-sharing":[57],"dialogues,":[58],"corresponding":[59],"summaries,":[60],"judgments":[63],"across":[64,87],"eight":[65],"well-defined":[66],"quality":[67,72],"aspects.":[68],"ensure":[70],"data":[71],"richfulness,":[74],"propose":[76],"novel":[78],"filtering":[79],"framework":[80],"leveraging":[81],"Mutually":[82],"Exclusive":[83],"Key":[84],"Information":[85],"(MEKI)":[86],"modalities.":[88],"Our":[89],"work":[90],"to":[94,102,122],"identify":[95],"formalize":[97],"key":[98],"dimensions":[100],"specific":[101],"MDS.":[103],"We":[104],"state-of-the-art":[106],"modal":[107],"methods,":[109],"revealing":[110],"their":[111,120],"limitations":[112],"distinguishing":[114],"summaries":[115],"from":[116],"advanced":[117],"MLLMs":[118],"susceptibility":[121],"various":[123],"bias.":[124]},"counts_by_year":[],"updated_date":"2026-10-11T09:32:39.932853","created_date":"2025-10-10T00:00:00"}