{"id":"https://openalex.org/W4402299909","doi":"https://doi.org/10.1109/tits.2024.3450584","title":"MS-DETR: Multispectral Pedestrian Detection Transformer With Loosely Coupled Fusion and Modality-Balanced Optimization","display_name":"MS-DETR: Multispectral Pedestrian Detection Transformer With Loosely Coupled Fusion and Modality-Balanced Optimization","publication_year":2024,"publication_date":"2024-09-06","ids":{"openalex":"https://openalex.org/W4402299909","doi":"https://doi.org/10.1109/tits.2024.3450584"},"language":"en","primary_location":{"id":"doi:10.1109/tits.2024.3450584","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2024.3450584","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089584286","display_name":"Yinghui Xing","orcid":"https://orcid.org/0000-0001-6021-8261"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yinghui Xing","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0001-6021-8261","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shuo Yang","orcid":"https://orcid.org/0009-0008-7171-105X"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuo Yang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0009-0008-7171-105X","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082259804","display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-4152-5295"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Song Wang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033395835","display_name":"Shizhou Zhang","orcid":"https://orcid.org/0000-0003-3429-576X"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shizhou Zhang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0003-3429-576X","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025755839","display_name":"Guoqiang Liang","orcid":"https://orcid.org/0000-0002-8710-5520"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guoqiang Liang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0002-8710-5520","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040327019","display_name":"Xiuwei Zhang","orcid":"https://orcid.org/0000-0001-7230-1476"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiuwei Zhang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0001-7230-1476","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028235866","display_name":"Yanning Zhang","orcid":"https://orcid.org/0000-0002-2977-8057"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanning Zhang","raw_affiliation_strings":["School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0002-2977-8057","affiliations":[{"raw_affiliation_string":"School of Computer Science, Northwestern Polytechnical University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I17145004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I17145004"],"apc_list":null,"apc_paid":null,"fwci":4.644,"has_fulltext":false,"cited_by_count":28,"citation_normalized_percentile":{"value":0.96389325,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"25","issue":"12","first_page":"20628","last_page":"20642"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9754999876022339,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9754999876022339,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10689","display_name":"Remote-Sensing Image Classification","score":0.9696999788284302,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12389","display_name":"Infrared Target Detection Methodologies","score":0.9580000042915344,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multispectral-image","display_name":"Multispectral image","score":0.770914614200592},{"id":"https://openalex.org/keywords/pedestrian-detection","display_name":"Pedestrian detection","score":0.5923992395401001},{"id":"https://openalex.org/keywords/sensor-fusion","display_name":"Sensor fusion","score":0.5884762406349182},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5421984195709229},{"id":"https://openalex.org/keywords/pedestrian","display_name":"Pedestrian","score":0.5112717151641846},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.5070376396179199},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.4563901424407959},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4272681474685669},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.41263464093208313},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3332290053367615},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.32526957988739014},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.20378148555755615},{"id":"https://openalex.org/keywords/transport-engineering","display_name":"Transport engineering","score":0.1799250841140747},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.07008609175682068},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.06869673728942871}],"concepts":[{"id":"https://openalex.org/C173163844","wikidata":"https://www.wikidata.org/wiki/Q1761440","display_name":"Multispectral image","level":2,"score":0.770914614200592},{"id":"https://openalex.org/C2780156472","wikidata":"https://www.wikidata.org/wiki/Q2355550","display_name":"Pedestrian detection","level":3,"score":0.5923992395401001},{"id":"https://openalex.org/C33954974","wikidata":"https://www.wikidata.org/wiki/Q486494","display_name":"Sensor fusion","level":2,"score":0.5884762406349182},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5421984195709229},{"id":"https://openalex.org/C2777113093","wikidata":"https://www.wikidata.org/wiki/Q221488","display_name":"Pedestrian","level":2,"score":0.5112717151641846},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5070376396179199},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.4563901424407959},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4272681474685669},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.41263464093208313},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3332290053367615},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32526957988739014},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.20378148555755615},{"id":"https://openalex.org/C22212356","wikidata":"https://www.wikidata.org/wiki/Q775325","display_name":"Transport engineering","level":1,"score":0.1799250841140747},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.07008609175682068},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.06869673728942871},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tits.2024.3450584","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2024.3450584","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Sustainable cities and communities","score":0.5799999833106995,"id":"https://metadata.un.org/sdg/11"}],"awards":[{"id":"https://openalex.org/G1584941972","display_name":null,"funder_award_id":"62201467","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3084392947","display_name":null,"funder_award_id":"2023M742842","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G3143984095","display_name":null,"funder_award_id":"2024JC-YBQN-0719","funder_id":"https://openalex.org/F4320336567","funder_display_name":"Natural Science Basic Research Program of Shaanxi Province"},{"id":"https://openalex.org/G4479186740","display_name":null,"funder_award_id":"62101453","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5474364752","display_name":null,"funder_award_id":"2022JC-DW-08","funder_id":"https://openalex.org/F4320336567","funder_display_name":"Natural Science Basic Research Program of Shaanxi Province"},{"id":"https://openalex.org/G7740149046","display_name":null,"funder_award_id":"2022TQ0260","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G796754823","display_name":null,"funder_award_id":"2021A1515110544","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321543","display_name":"China Postdoctoral Science Foundation","ror":"https://ror.org/0426zh255"},{"id":"https://openalex.org/F4320336567","display_name":"Natural Science Basic Research Program of Shaanxi Province","ror":null},{"id":"https://openalex.org/F4320337111","display_name":"Basic and Applied Basic Research Foundation of Guangdong Province","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1910108985","https://openalex.org/W2031454541","https://openalex.org/W2058757422","https://openalex.org/W2102605133","https://openalex.org/W2139479830","https://openalex.org/W2194775991","https://openalex.org/W2415234561","https://openalex.org/W2497039038","https://openalex.org/W2608096492","https://openalex.org/W2608295741","https://openalex.org/W2613599172","https://openalex.org/W2741620214","https://openalex.org/W2765407302","https://openalex.org/W2792925809","https://openalex.org/W2913889214","https://openalex.org/W2929607865","https://openalex.org/W2963188557","https://openalex.org/W2963404857","https://openalex.org/W2963579094","https://openalex.org/W2964027659","https://openalex.org/W2987131085","https://openalex.org/W3018757597","https://openalex.org/W3035682985","https://openalex.org/W3036931590","https://openalex.org/W3046421354","https://openalex.org/W3096609285","https://openalex.org/W3116967329","https://openalex.org/W3118570274","https://openalex.org/W3130394099","https://openalex.org/W3158128549","https://openalex.org/W3159619744","https://openalex.org/W3164576087","https://openalex.org/W3165924482","https://openalex.org/W3171583481","https://openalex.org/W3186570689","https://openalex.org/W3203003533","https://openalex.org/W3213472242","https://openalex.org/W4214627427","https://openalex.org/W4220724622","https://openalex.org/W4221138453","https://openalex.org/W4291910385","https://openalex.org/W4304080855","https://openalex.org/W4311105417","https://openalex.org/W4312639100","https://openalex.org/W4312845279","https://openalex.org/W4312925417","https://openalex.org/W4313007055","https://openalex.org/W4313136325","https://openalex.org/W4319301128","https://openalex.org/W4366378409","https://openalex.org/W4367721788","https://openalex.org/W4379528676","https://openalex.org/W4381885522","https://openalex.org/W4382657967","https://openalex.org/W4385151993","https://openalex.org/W4385801610","https://openalex.org/W4387188337","https://openalex.org/W4387968078","https://openalex.org/W6753836424","https://openalex.org/W6757817989","https://openalex.org/W6766978945","https://openalex.org/W6784094891","https://openalex.org/W6787826751","https://openalex.org/W6796814978","https://openalex.org/W6810226654","https://openalex.org/W6811230874","https://openalex.org/W6845935626","https://openalex.org/W6845958478"],"related_works":["https://openalex.org/W4318664220","https://openalex.org/W2771047279","https://openalex.org/W4388409104","https://openalex.org/W2124951708","https://openalex.org/W3132270449","https://openalex.org/W4377289091","https://openalex.org/W2972620127","https://openalex.org/W3013647784","https://openalex.org/W2981141433","https://openalex.org/W2997281059"],"abstract_inverted_index":{"Multispectral":[0],"pedestrian":[1,44,52,133],"detection":[2],"is":[3,187],"an":[4,147,165],"important":[5],"task":[6],"for":[7],"many":[8],"around-the-clock":[9],"applications,":[10],"since":[11],"the":[12,28,38,76,84,91,123,176],"visible":[13,77,154],"and":[14,34,66,75,78,112,155,159,180],"thermal":[15,79,156],"modalities":[16],"can":[17],"provide":[18],"complementary":[19],"information":[20],"especially":[21],"under":[22],"low":[23],"light":[24],"conditions.":[25],"Due":[26],"to":[27,56,136,141],"presence":[29],"of":[30,62],"two":[31,63],"modalities,":[32,129],"misalignment":[33,92],"modality":[35],"imbalance":[36],"are":[37,81],"most":[39],"significant":[40],"issues":[41],"in":[42,83],"multispectral":[43],"detection.":[45],"In":[46],"this":[47],"paper,":[48],"we":[49,96,144],"propose":[50,146],"MultiSpectral":[51],"DEtection":[53],"TRansformer":[54],"(MS-DETR)":[55],"fix":[57],"above":[58],"issues.":[59],"MS-DETR":[60,171],"consists":[61],"modality-specific":[64],"backbones":[65],"Transformer":[67,73,86],"encoders,":[68],"followed":[69],"by":[70,103],"a":[71,98],"multi-modal":[72,85,94,109],"decoder,":[74],"features":[80,110],"fused":[82],"decoder.":[87],"To":[88],"well":[89],"resist":[90],"between":[93],"images,":[95],"design":[97],"loosely":[99],"coupled":[100],"fusion":[101],"strategy":[102],"sparsely":[104],"sampling":[105],"some":[106],"keypoints":[107],"from":[108],"independently":[111],"fusing":[113],"them":[114],"with":[115],"adaptively":[116],"learned":[117],"attention":[118],"weights.":[119],"Moreover,":[120],"based":[121],"on":[122,175],"insight":[124],"that":[125],"not":[126],"only":[127],"different":[128,132,138],"but":[130],"also":[131],"instances":[134],"tend":[135],"have":[137],"confidence":[139],"scores":[140],"final":[142],"detection,":[143],"further":[145],"instance-aware":[148],"modality-balanced":[149],"optimization":[150],"strategy,":[151],"which":[152],"preserves":[153],"decoder":[157],"branches":[158],"aligns":[160],"their":[161],"predicted":[162],"slots":[163],"through":[164],"instance-wise":[166],"dynamic":[167],"loss.":[168],"Our":[169],"end-to-end":[170],"shows":[172],"superior":[173],"performance":[174],"challenging":[177],"KAIST,":[178],"CVC-14":[179],"LLVIP":[181],"benchmark":[182],"datasets.":[183],"The":[184],"source":[185],"code":[186],"available":[188],"at":[189],"<uri":[190],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[191],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">https://github.com/YinghuiXing/MS-DETR</uri>.":[192]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-16T13:24:37.021932","created_date":"2025-10-10T00:00:00"}