{"id":"https://openalex.org/W3115793754","doi":"https://doi.org/10.1109/itsc45102.2020.9294223","title":"Identification of Spurious Labels in Machine Learning Data Sets using N-Version Validation","display_name":"Identification of Spurious Labels in Machine Learning Data Sets using N-Version Validation","publication_year":2020,"publication_date":"2020-09-20","ids":{"openalex":"https://openalex.org/W3115793754","doi":"https://doi.org/10.1109/itsc45102.2020.9294223","mag":"3115793754"},"language":"en","primary_location":{"id":"doi:10.1109/itsc45102.2020.9294223","is_oa":false,"landing_page_url":"https://doi.org/10.1109/itsc45102.2020.9294223","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2020 IEEE 23rd International Conference on Intelligent Transportation Systems (ITSC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027343585","display_name":"Malte Mues","orcid":"https://orcid.org/0000-0002-6291-9886"},"institutions":[{"id":"https://openalex.org/I200332995","display_name":"TU Dortmund University","ror":"https://ror.org/01k97gp34","country_code":"DE","type":"education","lineage":["https://openalex.org/I200332995"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Malte Mues","raw_affiliation_strings":["Dortmund University of Technology, Dortmund, Germany"],"affiliations":[{"raw_affiliation_string":"Dortmund University of Technology, Dortmund, Germany","institution_ids":["https://openalex.org/I200332995"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086203491","display_name":"Sebastian Gerard","orcid":null},"institutions":[{"id":"https://openalex.org/I200332995","display_name":"TU Dortmund University","ror":"https://ror.org/01k97gp34","country_code":"DE","type":"education","lineage":["https://openalex.org/I200332995"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sebastian Gerard","raw_affiliation_strings":["Dortmund University of Technology, Dortmund, Germany"],"affiliations":[{"raw_affiliation_string":"Dortmund University of Technology, Dortmund, Germany","institution_ids":["https://openalex.org/I200332995"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037058130","display_name":"Falk Howar","orcid":"https://orcid.org/0000-0002-9524-4459"},"institutions":[{"id":"https://openalex.org/I200332995","display_name":"TU Dortmund University","ror":"https://ror.org/01k97gp34","country_code":"DE","type":"education","lineage":["https://openalex.org/I200332995"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Falk Howar","raw_affiliation_strings":["Dortmund University of Technology, Dortmund, Germany"],"affiliations":[{"raw_affiliation_string":"Dortmund University of Technology, Dortmund, Germany","institution_ids":["https://openalex.org/I200332995"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5027343585"],"corresponding_institution_ids":["https://openalex.org/I200332995"],"apc_list":null,"apc_paid":null,"fwci":0.2651,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.64975891,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"abs 1512 3385","issue":null,"first_page":"1","last_page":"7"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8078215718269348},{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.7294872403144836},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7225964069366455},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.705609917640686},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6519750952720642},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5780181288719177},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.53106689453125},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.5124990940093994},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5056321620941162},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.4642283320426941},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4412638545036316},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.44043731689453125},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4137084484100342},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.12111732363700867}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8078215718269348},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.7294872403144836},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7225964069366455},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.705609917640686},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6519750952720642},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5780181288719177},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.53106689453125},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.5124990940093994},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5056321620941162},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.4642283320426941},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4412638545036316},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.44043731689453125},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4137084484100342},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.12111732363700867},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/itsc45102.2020.9294223","is_oa":false,"landing_page_url":"https://doi.org/10.1109/itsc45102.2020.9294223","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2020 IEEE 23rd International Conference on Intelligent Transportation Systems (ITSC)","raw_type":"proceedings-article"},{"id":"pmh:oai:fraunhofer.de:N-635194","is_oa":false,"landing_page_url":"http://publica.fraunhofer.de/documents/N-635194.html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400801","display_name":"Publikationsdatenbank der Fraunhofer-Gesellschaft (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Fraunhofer ISST","raw_type":"Conference Paper"},{"id":"pmh:oai:publica.fraunhofer.de:publica/411205","is_oa":false,"landing_page_url":"https://publica.fraunhofer.de/handle/publica/411205","pdf_url":null,"source":{"id":"https://openalex.org/S4306400318","display_name":"Fraunhofer-Publica (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5899999737739563,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1836465849","https://openalex.org/W1903029394","https://openalex.org/W1968256263","https://openalex.org/W2024453793","https://openalex.org/W2065100127","https://openalex.org/W2101938307","https://openalex.org/W2129360963","https://openalex.org/W2194775991","https://openalex.org/W2328067583","https://openalex.org/W2340897893","https://openalex.org/W2560023338","https://openalex.org/W2576845151","https://openalex.org/W2630837129","https://openalex.org/W2771470790","https://openalex.org/W2790504560","https://openalex.org/W2799352588","https://openalex.org/W2810908953","https://openalex.org/W2892614179","https://openalex.org/W2905221822","https://openalex.org/W2949117887","https://openalex.org/W2961005338","https://openalex.org/W2963041685","https://openalex.org/W2968638474","https://openalex.org/W2970971581","https://openalex.org/W4285719527","https://openalex.org/W4295312788","https://openalex.org/W6638667902","https://openalex.org/W6687483927","https://openalex.org/W6739696289","https://openalex.org/W6746476816","https://openalex.org/W6750749703","https://openalex.org/W6766176776","https://openalex.org/W6766978945"],"related_works":["https://openalex.org/W3113091479","https://openalex.org/W2162899405","https://openalex.org/W941090075","https://openalex.org/W2044987316","https://openalex.org/W3134374554","https://openalex.org/W2237480245","https://openalex.org/W2075065631","https://openalex.org/W2519167559","https://openalex.org/W4311248832","https://openalex.org/W4223555864"],"abstract_inverted_index":{"Machine":[0],"learning":[1,20],"components":[2],"are":[3],"becoming":[4],"popular":[5],"for":[6,17,29,146],"the":[7,40,46,62,93,99,109,126,136],"automotive":[8],"industry.":[9],"More":[10],"and":[11,36,76,97,111],"more":[12],"data":[13,51,67,118,130],"sets":[14],"become":[15],"available":[16],"training":[18],"machine":[19],"components.":[21],"All":[22],"of":[23,49,101,114],"them":[24],"provide":[25],"ground":[26,94],"truth":[27,95],"labels":[28],"images.":[30],"The":[31,69,132],"labeling":[32],"process":[33],"is":[34],"expensive":[35],"potentially":[37],"error-prone.":[38],"At":[39],"same":[41],"time,":[42],"label":[43,63,96,115,144],"correctness":[44],"defines":[45],"business":[47],"value":[48],"a":[50,66,81],"set.":[52,68,131],"In":[53],"this":[54,89],"paper,":[55],"we":[56],"use":[57],"N-Version":[58],"approach":[59,70,137],"to":[60],"assess":[61],"quality":[64,116],"in":[65,80],"combines":[71],"N":[72],"state-of-the-art":[73],"neural":[74],"networks":[75],"aggregates":[77],"their":[78],"results":[79],"single":[82],"verdict":[83],"using":[84],"majority":[85,90],"voting.":[86],"We":[87,120],"analyze":[88],"vote":[91],"against":[92],"compute":[98],"percentage":[100],"disagreeing":[102],"pixels":[103],"along":[104],"with":[105],"other":[106],"metrics,":[107],"enabling":[108],"automated":[110],"detailed":[112],"analysis":[113],"on":[117],"sets.":[119],"evaluate":[121],"our":[122],"methodology":[123],"by":[124],"classifying":[125],"BDD100K":[127],"drivable":[128],"area":[129],"evaluation":[133],"shows":[134],"that":[135],"identifies":[138],"misclassified":[139],"scenes":[140],"or":[141],"inconsistencies":[142],"between":[143],"semantics":[145],"similar":[147],"scenes.":[148]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}