{"id":"https://openalex.org/W7151233677","doi":"https://doi.org/10.48550/arxiv.2604.04009","title":"Benchmarking and Evaluating VLMs for Software Architecture Diagram Understanding","display_name":"Benchmarking and Evaluating VLMs for Software Architecture Diagram Understanding","publication_year":2026,"publication_date":"2026-04-05","ids":{"openalex":"https://openalex.org/W7151233677","doi":"https://doi.org/10.48550/arxiv.2604.04009"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.04009","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04009","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.04009","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133133043","display_name":"Shuyin Ouyang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ouyang, Shuyin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133139814","display_name":"Jie M. Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jie M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133097232","display_name":"Jingzhi Gong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gong, Jingzhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085228874","display_name":"Gunel Jahangirova","orcid":"https://orcid.org/0000-0002-1423-1083"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jahangirova, Gunel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133107105","display_name":"Mohammad Reza Mousavi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mousavi, Mohammad Reza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120703691","display_name":"Jack Johns","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Johns, Jack","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085409570","display_name":"Beum Seuk Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Beum Seuk","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102066731","display_name":"Adam Ziolkowski","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ziolkowski, Adam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087795845","display_name":"Botond Virginas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Virginas, Botond","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5014697247","display_name":"Joost Noppen","orcid":"https://orcid.org/0009-0004-4203-1582"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Noppen, Joost","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.4381999969482422,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.4381999969482422,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10639","display_name":"Advanced Software Engineering Methodologies","score":0.18950000405311584,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.17810000479221344,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-architecture-description","display_name":"Software architecture description","score":0.5501000285148621},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.5065000057220459},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.4814000129699707},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.4767000079154968},{"id":"https://openalex.org/keywords/resource-oriented-architecture","display_name":"Resource-oriented architecture","score":0.4713999927043915},{"id":"https://openalex.org/keywords/software-design-description","display_name":"Software design description","score":0.4578999876976013},{"id":"https://openalex.org/keywords/reference-architecture","display_name":"Reference architecture","score":0.4375},{"id":"https://openalex.org/keywords/software-architecture","display_name":"Software architecture","score":0.4332999885082245},{"id":"https://openalex.org/keywords/component-based-software-engineering","display_name":"Component-based software engineering","score":0.3779999911785126}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7050999999046326},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6473000049591064},{"id":"https://openalex.org/C73219336","wikidata":"https://www.wikidata.org/wiki/Q7554254","display_name":"Software architecture description","level":5,"score":0.5501000285148621},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.5065000057220459},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.4814000129699707},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.4767000079154968},{"id":"https://openalex.org/C97382630","wikidata":"https://www.wikidata.org/wiki/Q13501132","display_name":"Resource-oriented architecture","level":5,"score":0.4713999927043915},{"id":"https://openalex.org/C56909552","wikidata":"https://www.wikidata.org/wiki/Q2297624","display_name":"Software design description","level":5,"score":0.4578999876976013},{"id":"https://openalex.org/C55356503","wikidata":"https://www.wikidata.org/wiki/Q2136675","display_name":"Reference architecture","level":4,"score":0.4375},{"id":"https://openalex.org/C35869016","wikidata":"https://www.wikidata.org/wiki/Q846636","display_name":"Software architecture","level":3,"score":0.4332999885082245},{"id":"https://openalex.org/C174683762","wikidata":"https://www.wikidata.org/wiki/Q609588","display_name":"Component-based software engineering","level":4,"score":0.3779999911785126},{"id":"https://openalex.org/C4379982","wikidata":"https://www.wikidata.org/wiki/Q1273511","display_name":"Software visualization","level":5,"score":0.3587000072002411},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3456000089645386},{"id":"https://openalex.org/C91597405","wikidata":"https://www.wikidata.org/wiki/Q446366","display_name":"System context diagram","level":3,"score":0.33570000529289246},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.32280001044273376},{"id":"https://openalex.org/C52913732","wikidata":"https://www.wikidata.org/wiki/Q857102","display_name":"Software design","level":4,"score":0.30140000581741333},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.2948000133037567},{"id":"https://openalex.org/C126383566","wikidata":"https://www.wikidata.org/wiki/Q4787220","display_name":"Architecture tradeoff analysis method","level":5,"score":0.29269999265670776},{"id":"https://openalex.org/C202105479","wikidata":"https://www.wikidata.org/wiki/Q265013","display_name":"Software evolution","level":5,"score":0.2906000018119812},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.2802000045776367},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.2800000011920929},{"id":"https://openalex.org/C5977032","wikidata":"https://www.wikidata.org/wiki/Q5289815","display_name":"Domain engineering","level":5,"score":0.27799999713897705},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.265500009059906},{"id":"https://openalex.org/C98025372","wikidata":"https://www.wikidata.org/wiki/Q477538","display_name":"Systems architecture","level":3,"score":0.2639000117778778},{"id":"https://openalex.org/C76518257","wikidata":"https://www.wikidata.org/wiki/Q271680","display_name":"Software framework","level":5,"score":0.2596000134944916},{"id":"https://openalex.org/C72280835","wikidata":"https://www.wikidata.org/wiki/Q635346","display_name":"Architectural pattern","level":5,"score":0.2567000091075897}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.04009","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04009","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.04009","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.04009","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5124689340591431,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Software":[0,65],"architecture":[1,51,73,127],"diagrams":[2,52,74,89],"are":[3],"important":[4],"design":[5],"artifacts":[6,79],"for":[7,64,132,179],"communicating":[8],"system":[9],"structure,":[10],"behavior,":[11],"and":[12,39,93,100,106,119,158,168,186],"data":[13],"organization":[14],"throughout":[15],"the":[16,41,115,135,153,169],"software":[17,31,50,77,126,173,190],"development":[18],"lifecycle.":[19],"Although":[20],"recent":[21],"progress":[22],"in":[23,155],"large":[24],"language":[25],"models":[26,46],"has":[27],"substantially":[28],"advanced":[29],"code-centric":[30],"engineering":[32,78,191],"tasks":[33,103],"such":[34],"as":[35,75],"code":[36],"generation,":[37],"testing,":[38],"maintenance,":[40],"ability":[42],"of":[43,171],"modern":[44],"vision-language":[45],"(VLMs)":[47],"to":[48],"understand":[49],"remains":[53,130],"underexplored.":[54],"To":[55],"address":[56],"this":[57],"gap,":[58],"we":[59],"present":[60],"SADU,":[61],"a":[62,163,177],"benchmark":[63],"Architecture":[66],"Diagram":[67],"Understanding":[68],"that":[69,125],"evaluates":[70],"VLMs":[71,113,167],"on":[72,182],"structured":[76,98],"rather":[80],"than":[81],"generic":[82],"images.":[83],"SADU":[84,175],"contains":[85],"154":[86],"carefully":[87],"curated":[88],"spanning":[90],"behavioral,":[91],"structural,":[92],"ER":[94],"diagrams,":[95],"paired":[96],"with":[97],"annotations":[99],"2,431":[101],"question-answer":[102],"covering":[104],"counting":[105],"retrieval":[107],"reasoning.":[108],"We":[109],"evaluate":[110],"11":[111],"state-of-the-art":[112],"from":[114],"Gemini,":[116],"Claude,":[117],"GPT,":[118],"Qwen":[120],"families.":[121],"Our":[122],"results":[123,150],"show":[124],"diagram":[128,156],"understanding":[129],"challenging":[131],"current":[133,166],"models:":[134],"best-performing":[136],"model":[137],"gemini-3-flash-preview":[138],"achieves":[139,146],"only":[140,145],"70.18\\%":[141],"accuracy,":[142],"while":[143],"gpt-4o-mini":[144],"17.77\\%":[147],"accuracy.":[148],"The":[149],"further":[151],"reveal":[152],"weaknesses":[154],"reasoning":[157],"visual":[159],"relation":[160],"grounding,":[161],"highlighting":[162],"gap":[164],"between":[165],"needs":[170],"design-stage":[172],"engineering.":[174],"provides":[176],"foundation":[178],"future":[180],"research":[181],"diagram-aware":[183],"AI":[184],"systems":[185],"more":[187],"faithful":[188],"AI-assisted":[189],"workflows.":[192]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-08T00:00:00"}