{"id":"https://openalex.org/W3162947368","doi":"https://doi.org/10.1109/tpami.2022.3227571","title":"How Trustworthy are Performance Evaluations for Basic Vision Tasks?","display_name":"How Trustworthy are Performance Evaluations for Basic Vision Tasks?","publication_year":2022,"publication_date":"2022-12-08","ids":{"openalex":"https://openalex.org/W3162947368","doi":"https://doi.org/10.1109/tpami.2022.3227571","mag":"3162947368","pmid":"https://pubmed.ncbi.nlm.nih.gov/37015490"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2022.3227571","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2022.3227571","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069386456","display_name":"Tran Thien Dat Nguyen","orcid":"https://orcid.org/0000-0001-9185-4009"},"institutions":[{"id":"https://openalex.org/I205640436","display_name":"Curtin University","ror":"https://ror.org/02n415q13","country_code":"AU","type":"education","lineage":["https://openalex.org/I205640436"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Tran Thien Dat Nguyen","raw_affiliation_strings":["School of Electrical Engineering, Computing and Mathematical Sciences, Curtin University, Bentley, WA, Australia"],"raw_orcid":"https://orcid.org/0000-0001-9185-4009","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Computing and Mathematical Sciences, Curtin University, Bentley, WA, Australia","institution_ids":["https://openalex.org/I205640436"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034608678","display_name":"Hamid Rezatofighi","orcid":"https://orcid.org/0000-0002-8659-8773"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Hamid Rezatofighi","raw_affiliation_strings":["Faculty of Information Technology, Monash University, Clayton, VIC, Australia"],"raw_orcid":"https://orcid.org/0000-0002-8659-8773","affiliations":[{"raw_affiliation_string":"Faculty of Information Technology, Monash University, Clayton, VIC, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072792589","display_name":"Ba\u2010Ngu Vo","orcid":"https://orcid.org/0000-0003-4202-7722"},"institutions":[{"id":"https://openalex.org/I205640436","display_name":"Curtin University","ror":"https://ror.org/02n415q13","country_code":"AU","type":"education","lineage":["https://openalex.org/I205640436"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Ba-Ngu Vo","raw_affiliation_strings":["School of Electrical Engineering, Computing and Mathematical Sciences, Curtin University, Bentley, WA, Australia"],"raw_orcid":"https://orcid.org/0000-0003-4202-7722","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Computing and Mathematical Sciences, Curtin University, Bentley, WA, Australia","institution_ids":["https://openalex.org/I205640436"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070960013","display_name":"Ba-Tuong Vo","orcid":"https://orcid.org/0000-0002-3954-238X"},"institutions":[{"id":"https://openalex.org/I205640436","display_name":"Curtin University","ror":"https://ror.org/02n415q13","country_code":"AU","type":"education","lineage":["https://openalex.org/I205640436"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Ba-Tuong Vo","raw_affiliation_strings":["School of Electrical Engineering, Computing and Mathematical Sciences, Curtin University, Bentley, WA, Australia"],"raw_orcid":"https://orcid.org/0000-0002-3954-238X","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering, Computing and Mathematical Sciences, Curtin University, Bentley, WA, Australia","institution_ids":["https://openalex.org/I205640436"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042646536","display_name":"Silvio Savarese","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Silvio Savarese","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089444805","display_name":"Ian Reid","orcid":"https://orcid.org/0000-0001-7790-6423"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Ian Reid","raw_affiliation_strings":["School of Computer Science, University of Adelaide, Adelaide, SA, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7790-6423","affiliations":[{"raw_affiliation_string":"School of Computer Science, University of Adelaide, Adelaide, SA, Australia","institution_ids":["https://openalex.org/I5681781"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.447,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.90188328,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"45","issue":"7","first_page":"8538","last_page":"8552"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.714738667011261},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.6716011762619019},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6597011089324951},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5578721165657043},{"id":"https://openalex.org/keywords/intersection","display_name":"Intersection (aeronautics)","score":0.548423171043396},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5191178321838379},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.46885642409324646},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.4596620798110962},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4509744346141815},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42953240871429443},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3662837743759155},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.11175134778022766}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.714738667011261},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.6716011762619019},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6597011089324951},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5578721165657043},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.548423171043396},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5191178321838379},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.46885642409324646},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.4596620798110962},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4509744346141815},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42953240871429443},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3662837743759155},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.11175134778022766},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2022.3227571","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2022.3227571","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:37015490","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37015490","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6277066168","display_name":"Collision Avoidance in Shipping Lanes via Intelligent Sensor Data Fusion ","funder_award_id":"LP200301507","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":85,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1521019969","https://openalex.org/W1594957066","https://openalex.org/W1772076007","https://openalex.org/W1861492603","https://openalex.org/W1985514943","https://openalex.org/W2031454541","https://openalex.org/W2031489346","https://openalex.org/W2055649245","https://openalex.org/W2087003958","https://openalex.org/W2096590953","https://openalex.org/W2097117768","https://openalex.org/W2102285609","https://openalex.org/W2122088301","https://openalex.org/W2124781496","https://openalex.org/W2125101937","https://openalex.org/W2126885789","https://openalex.org/W2142060261","https://openalex.org/W2150066425","https://openalex.org/W2155325587","https://openalex.org/W2163385949","https://openalex.org/W2183341477","https://openalex.org/W2193145675","https://openalex.org/W2194775991","https://openalex.org/W2340897893","https://openalex.org/W2412782625","https://openalex.org/W2511791013","https://openalex.org/W2527269634","https://openalex.org/W2547098537","https://openalex.org/W2549139847","https://openalex.org/W2565639579","https://openalex.org/W2573106815","https://openalex.org/W2606355385","https://openalex.org/W2612445135","https://openalex.org/W2739374836","https://openalex.org/W2765741142","https://openalex.org/W2766984662","https://openalex.org/W2781642774","https://openalex.org/W2798875622","https://openalex.org/W2820179196","https://openalex.org/W2887311111","https://openalex.org/W2889935068","https://openalex.org/W2895071559","https://openalex.org/W2897582990","https://openalex.org/W2900208617","https://openalex.org/W2900871370","https://openalex.org/W2901925479","https://openalex.org/W2902154261","https://openalex.org/W2906499972","https://openalex.org/W2920942303","https://openalex.org/W2951353299","https://openalex.org/W2962766617","https://openalex.org/W2962919973","https://openalex.org/W2963150697","https://openalex.org/W2963163009","https://openalex.org/W2963374479","https://openalex.org/W2963382789","https://openalex.org/W2963835147","https://openalex.org/W2964350391","https://openalex.org/W2967533163","https://openalex.org/W2970308037","https://openalex.org/W2981393651","https://openalex.org/W2986732333","https://openalex.org/W2999219213","https://openalex.org/W3015395304","https://openalex.org/W3086436251","https://openalex.org/W3099887740","https://openalex.org/W3104778224","https://openalex.org/W3104979525","https://openalex.org/W3106250896","https://openalex.org/W3136413417","https://openalex.org/W3216889285","https://openalex.org/W4239181501","https://openalex.org/W4246193833","https://openalex.org/W4295331127","https://openalex.org/W4297775537","https://openalex.org/W4297785339","https://openalex.org/W4300479382","https://openalex.org/W6635797942","https://openalex.org/W6714138976","https://openalex.org/W6729956949","https://openalex.org/W6737664043","https://openalex.org/W6753065084","https://openalex.org/W6754036027","https://openalex.org/W6763930870"],"related_works":["https://openalex.org/W90316445","https://openalex.org/W2348909947","https://openalex.org/W4292672442","https://openalex.org/W4327743613","https://openalex.org/W2362101859","https://openalex.org/W2791431590","https://openalex.org/W3199750033","https://openalex.org/W1978900583","https://openalex.org/W2350688482","https://openalex.org/W2053575972"],"abstract_inverted_index":{"This":[0,63],"article":[1],"examines":[2],"performance":[3,71],"evaluation":[4],"criteria":[5,112,124],"for":[6,70,79,115,130],"basic":[7],"vision":[8],"tasks":[9],"involving":[10],"sets":[11,116],"of":[12,24,34,60,68,117],"objects":[13],"namely,":[14],"object":[15],"detection,":[16],"instance-level":[17],"segmentation":[18],"and":[19,87,109],"multi-object":[20],"tracking.":[21],"The":[22],"rankings":[23],"algorithms":[25],"by":[26,105],"a":[27,61,66],"criterion":[28],"can":[29,56],"fluctuate":[30],"with":[31,90],"different":[32],"choices":[33],"parameters,":[35],"e.g.":[36],"Intersection":[37],"over":[38],"Union":[39],"(IoU)":[40],"threshold,":[41],"making":[42],"their":[43],"evaluations":[44,59],"unreliable.":[45],"More":[46],"importantly,":[47],"there":[48],"is":[49],"no":[50],"means":[51],"to":[52,77],"verify":[53],"whether":[54],"we":[55],"trust":[57],"the":[58,95,127],"criterion.":[62],"work":[64],"suggests":[65],"notion":[67],"trustworthiness":[69],"criteria,":[72,108],"which":[73],"requires":[74],"(i)":[75],"robustness":[76],"parameters":[78],"reliability,":[80],"(ii)":[81],"contextual":[82],"meaningfulness":[83],"in":[84],"sanity":[85],"tests,":[86],"(iii)":[88],"consistency":[89],"mathematical":[91],"requirements":[92,102,129],"such":[93],"as":[94],"metric":[96],"properties.":[97],"We":[98,119],"observe":[99],"that":[100],"these":[101,123],"were":[103],"overlooked":[104],"many":[106],"widely-used":[107],"explore":[110],"alternative":[111],"using":[113],"metrics":[114],"shapes.":[118],"also":[120],"assess":[121],"all":[122],"based":[125],"on":[126],"suggested":[128],"trustworthiness.":[131]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":7}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}