{"id":"https://openalex.org/W4413318921","doi":"https://doi.org/10.1109/jiot.2025.3600573","title":"MT-Agent: Constructing a GUI Agent via Modality Enhancement and Text-Guided Fusion","display_name":"MT-Agent: Constructing a GUI Agent via Modality Enhancement and Text-Guided Fusion","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4413318921","doi":"https://doi.org/10.1109/jiot.2025.3600573"},"language":"en","primary_location":{"id":"doi:10.1109/jiot.2025.3600573","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jiot.2025.3600573","pdf_url":null,"source":{"id":"https://openalex.org/S2480266640","display_name":"IEEE Internet of Things Journal","issn_l":"2327-4662","issn":["2327-4662","2372-2541"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Internet of Things Journal","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Jinhan Dong","orcid":"https://orcid.org/0009-0002-8575-6378"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinhan Dong","raw_affiliation_strings":["Beijing University of Posts and Telecommunications, China"],"raw_orcid":"https://orcid.org/0009-0002-8575-6378","affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039217360","display_name":"Lei Jin","orcid":"https://orcid.org/0000-0003-4855-2464"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Jin","raw_affiliation_strings":["Beijing University of Posts and Telecommunications, China"],"raw_orcid":"https://orcid.org/0000-0003-4855-2464","affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100419455","display_name":"Zhihong Zhang","orcid":"https://orcid.org/0000-0001-5227-8926"},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhihong Zhang","raw_affiliation_strings":["China Mobile, Suzhou Software Technology Co., Ltd, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile, Suzhou Software Technology Co., Ltd, China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wei Tang","orcid":null},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Tang","raw_affiliation_strings":["China Mobile, Suzhou Software Technology Co., Ltd, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile, Suzhou Software Technology Co., Ltd, China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084505306","display_name":"Runqing Zhang","orcid":"https://orcid.org/0000-0002-0503-7254"},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Runqing Zhang","raw_affiliation_strings":["China Mobile, Suzhou Software Technology Co., Ltd, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile, Suzhou Software Technology Co., Ltd, China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101792623","display_name":"Liqiang Xu","orcid":"https://orcid.org/0000-0003-0162-0626"},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liqiang Xu","raw_affiliation_strings":["China Mobile, Suzhou Software Technology Co., Ltd, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"China Mobile, Suzhou Software Technology Co., Ltd, China","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076090670","display_name":"Junliang Xing","orcid":"https://orcid.org/0000-0001-6801-0510"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junliang Xing","raw_affiliation_strings":["Tsinghua University, China"],"raw_orcid":"https://orcid.org/0000-0001-6801-0510","affiliations":[{"raw_affiliation_string":"Tsinghua University, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.1045555,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"1"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9354000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9354000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.934499979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.906499981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7415122389793396},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.637053906917572},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.5003159046173096},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.334713876247406},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3099057674407959}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7415122389793396},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.637053906917572},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.5003159046173096},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.334713876247406},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3099057674407959},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jiot.2025.3600573","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jiot.2025.3600573","pdf_url":null,"source":{"id":"https://openalex.org/S2480266640","display_name":"IEEE Internet of Things Journal","issn_l":"2327-4662","issn":["2327-4662","2372-2541"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Internet of Things Journal","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3705289226","display_name":null,"funder_award_id":"82274685","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5102826188","display_name":null,"funder_award_id":"62472046","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6857468845","display_name":null,"funder_award_id":"500422813","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W3138516171","https://openalex.org/W4402670435","https://openalex.org/W4402671607","https://openalex.org/W4402753874","https://openalex.org/W4403888362","https://openalex.org/W4404612908","https://openalex.org/W4404781885","https://openalex.org/W4404792858","https://openalex.org/W4412887860"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2385859805","https://openalex.org/W2530972254","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109"],"abstract_inverted_index":{"Graphical":[0],"User":[1],"Interfaces":[2],"(GUIs)":[3],"play":[4],"a":[5,56,156],"crucial":[6],"role":[7],"in":[8,27,78],"facilitating":[9],"user-computer":[10],"interactions,":[11],"making":[12],"them":[13],"an":[14,98,119,148],"essential":[15],"focus":[16],"of":[17,40,48,90,153,160,185],"research.":[18],"However,":[19],"current":[20],"automated":[21],"GUI":[22,113,169],"agents":[23],"face":[24],"significant":[25],"challenges":[26],"effectively":[28],"associating":[29],"task":[30],"implementations":[31],"with":[32,135],"specific":[33],"visual":[34,49,67,76,107,136],"elements,":[35],"and":[36,66,155,192,202],"the":[37,46,72,87,133,168,205],"resolution":[38],"constraints":[39],"Vision-Language":[41],"Models":[42],"(VLMs)":[43],"also":[44],"limit":[45],"richness":[47,89],"information.":[50,137],"To":[51],"this":[52],"end,":[53],"we":[54,117],"propose":[55],"novel":[57],"multi-modal":[58],"agent":[59],"named":[60],"MT-Agent,":[61],"which":[62,126],"enhances":[63],"both":[64],"textual":[65],"input":[68,91],"modalities":[69],"to":[70,74,109,131,176,198],"enable":[71],"model":[73],"perceive":[75],"elements":[77],"GUIs":[79],"more":[80],"effectively.":[81],"Specifically,":[82],"Textual":[83],"Modality":[84,103],"Enhancement":[85,104],"improves":[86],"semantic":[88],"text":[92,129],"by":[93],"capturing":[94],"task-specific":[95],"details":[96,108],"via":[97],"external":[99],"VLM,":[100],"while":[101,180],"Visual":[102],"incorporates":[105],"fine-grained":[106],"better":[110],"represent":[111],"critical":[112],"elements.":[114],"In":[115,138],"addition,":[116],"introduce":[118],"innovative":[120],"text-guided":[121],"directional":[122],"feature":[123],"fusion":[124],"mechanism,":[125],"leverages":[127],"enriched":[128],"features":[130],"guide":[132],"integration":[134],"experiments,":[139],"MT-Agent":[140,172],"demonstrated":[141],"exceptional":[142],"performance":[143,174],"on":[144,167],"AITZ":[145],"dataset,":[146],"achieving":[147],"action":[149],"type":[150],"prediction":[151,158],"accuracy":[152,159],"84.80%":[154],"step":[157],"58.07%,":[161],"surpassing":[162],"previous":[163,177],"state-of-the-art":[164,178],"models.":[165],"Furthermore,":[166],"Odyssey":[170],"benchmark,":[171],"achieves":[173],"comparable":[175],"models":[179],"using":[181],"only":[182],"about":[183],"1/20":[184],"their":[186],"trainable":[187],"parameters.":[188],"Our":[189],"codes,":[190],"demos,":[191],"relevant":[193],"data":[194],"will":[195],"be":[196],"released":[197],"facilitate":[199],"further":[200],"research":[201],"validation":[203],"within":[204],"scientific":[206],"community.":[207]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}