{"id":"https://openalex.org/W7131136686","doi":"https://doi.org/10.1109/robio66223.2025.11376337","title":"Establishing a Black-Box Benchmarking Framework for Commercial AI Object Detection Systems in Heavy Industries","display_name":"Establishing a Black-Box Benchmarking Framework for Commercial AI Object Detection Systems in Heavy Industries","publication_year":2025,"publication_date":"2025-12-03","ids":{"openalex":"https://openalex.org/W7131136686","doi":"https://doi.org/10.1109/robio66223.2025.11376337"},"language":null,"primary_location":{"id":"doi:10.1109/robio66223.2025.11376337","is_oa":false,"landing_page_url":"https://doi.org/10.1109/robio66223.2025.11376337","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Robotics and Biomimetics (ROBIO)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107608753","display_name":"Adam Scicluna","orcid":null},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Adam Scicluna","raw_affiliation_strings":["Robotics Institute, University of Technology Sydney (UTS),Faculty of Engineering and Information Technology,Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics Institute, University of Technology Sydney (UTS),Faculty of Engineering and Information Technology,Australia","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126609469","display_name":"Gavin Paul","orcid":null},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gavin Paul","raw_affiliation_strings":["Robotics Institute, University of Technology Sydney (UTS),Faculty of Engineering and Information Technology,Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics Institute, University of Technology Sydney (UTS),Faculty of Engineering and Information Technology,Australia","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064494061","display_name":"Nathan Kirchner","orcid":null},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Nathan Kirchner","raw_affiliation_strings":["Robotics Institute, University of Technology Sydney (UTS),Faculty of Engineering and Information Technology,Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robotics Institute, University of Technology Sydney (UTS),Faculty of Engineering and Information Technology,Australia","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126655420","display_name":"Alexander Wendel","orcid":null},"institutions":[{"id":"https://openalex.org/I2801607516","display_name":"Associated Press","ror":"https://ror.org/0231jxj79","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I2801607516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander Wendel","raw_affiliation_strings":["Presien,Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Presien,Australia","institution_ids":["https://openalex.org/I2801607516"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068587682","display_name":"Alexander Virgona","orcid":null},"institutions":[{"id":"https://openalex.org/I2801607516","display_name":"Associated Press","ror":"https://ror.org/0231jxj79","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I2801607516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander Virgona","raw_affiliation_strings":["Presien,Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Presien,Australia","institution_ids":["https://openalex.org/I2801607516"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"364","last_page":"369"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.19509999454021454,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.19509999454021454,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12707","display_name":"Vehicle License Plate Recognition","score":0.026900000870227814,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.024800000712275505,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.911300003528595},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.6309000253677368},{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.5996000170707703},{"id":"https://openalex.org/keywords/object-detection","display_name":"Object detection","score":0.5386999845504761},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5267000198364258},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4918000102043152}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.911300003528595},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6319000124931335},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.6309000253677368},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.5996000170707703},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.5386999845504761},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5267000198364258},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4918000102043152},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.40709999203681946},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40529999136924744},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.36149999499320984},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.3610999882221222},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.32829999923706055},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.3093999922275543},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.28999999165534973},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28130000829696655},{"id":"https://openalex.org/C152745839","wikidata":"https://www.wikidata.org/wiki/Q5438153","display_name":"Fault detection and isolation","level":3,"score":0.26510000228881836},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/robio66223.2025.11376337","is_oa":false,"landing_page_url":"https://doi.org/10.1109/robio66223.2025.11376337","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Robotics and Biomimetics (ROBIO)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320338255","display_name":"ARC Industrial Transformation Training Centre for Uniquely Australian Foods","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1861492603","https://openalex.org/W2031489346","https://openalex.org/W2115579991","https://openalex.org/W2328067583","https://openalex.org/W2756408590","https://openalex.org/W2888728157","https://openalex.org/W2946352279","https://openalex.org/W3025550719","https://openalex.org/W3094233077","https://openalex.org/W3104128335","https://openalex.org/W3156089158","https://openalex.org/W3203241482","https://openalex.org/W4205561327","https://openalex.org/W4211019953","https://openalex.org/W4224273747","https://openalex.org/W4225327430","https://openalex.org/W4318619081","https://openalex.org/W4376504466","https://openalex.org/W4381848566","https://openalex.org/W4387527822","https://openalex.org/W4388439974","https://openalex.org/W4389612032","https://openalex.org/W4396767342","https://openalex.org/W4401806686","https://openalex.org/W4402727142","https://openalex.org/W4403677451"],"related_works":[],"abstract_inverted_index":{"Commercial":[0],"AI-driven":[1],"object":[2,53,154],"detection":[3,54,106,155],"systems":[4,55,62,156],"are":[5,111],"increasingly":[6],"deployed":[7],"in":[8,56,122,157],"heavy":[9,57],"industries":[10],"to":[11,26,39,96],"enhance":[12],"safety":[13],"and":[14,37,78,91,99,108,145],"efficiency.":[15],"However,":[16],"the":[17,24,32,44],"absence":[18],"of":[19,34,67,94],"standardised":[20],"benchmarking":[21,48],"frameworks":[22],"limits":[23],"ability":[25],"objectively":[27],"assess":[28],"system":[29,109,134],"reliability,":[30],"increasing":[31],"risk":[33],"bias,":[35],"inconsistencies,":[36],"susceptibility":[38],"manipulation.":[40],"This":[41],"paper":[42],"presents":[43],"first":[45],"structured":[46],"black-box":[47],"methodology":[49,60],"for":[50,132,151],"complete":[51],"AI":[52,153],"industries.":[58],"The":[59,81,136],"evaluates":[61],"(hardware":[63],"+":[64],"software)":[65],"independently":[66],"their":[68],"internal":[69],"algorithms":[70],"through":[71,113],"video-based":[72],"testing,":[73,87],"ensuring":[74],"a":[75,114],"fair,":[76],"reproducible,":[77],"industry-relevant":[79],"assessment.":[80,135],"approach":[82],"integrates":[83],"controlled":[84],"monitor-based":[85],"experimental":[86],"automated":[88],"performance":[89],"analysis,":[90],"contextual":[92],"weighting":[93],"errors":[95],"provide":[97],"transparent":[98],"comparable":[100],"evaluations.":[101],"Intrinsic":[102],"challenges,":[103],"such":[104],"as":[105],"latency":[107],"misclassification,":[110],"examined":[112],"rigorous":[115],"testbed":[116],"design.":[117],"Results":[118],"reveal":[119],"systematic":[120],"biases":[121],"monitorbased":[123],"evaluation,":[124],"including":[125],"display":[126],"configuration":[127],"effects,":[128],"highlighting":[129],"key":[130],"considerations":[131],"reliable":[133],"framework":[137],"enables":[138],"vendor-neutral":[139],"comparisons,":[140],"improves":[141],"realworld":[142],"reliability":[143],"assessments,":[144],"provides":[146],"stakeholders":[147],"with":[148],"actionable":[149],"insights":[150],"deploying":[152],"high-risk":[158],"environments.":[159]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-02-24T00:00:00"}