{"id":"https://openalex.org/W4394673197","doi":"https://doi.org/10.1145/3664646.3664766","title":"An AI System Evaluation Framework for Advancing AI Safety: Terminology, Taxonomy, Lifecycle Mapping","display_name":"An AI System Evaluation Framework for Advancing AI Safety: Terminology, Taxonomy, Lifecycle Mapping","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4394673197","doi":"https://doi.org/10.1145/3664646.3664766"},"language":"en","primary_location":{"id":"doi:10.1145/3664646.3664766","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664646.3664766","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3664646.3664766","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109703664","display_name":"Boming Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"government","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]},{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Boming Xia","raw_affiliation_strings":["CSIRO's Data61, Sydney, Australia / University of New South Wales, Sydney, Australia"],"raw_orcid":"https://orcid.org/0009-0003-7385-4023","affiliations":[{"raw_affiliation_string":"CSIRO's Data61, Sydney, Australia / University of New South Wales, Sydney, Australia","institution_ids":["https://openalex.org/I42894916","https://openalex.org/I31746571","https://openalex.org/I1292875679"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100652461","display_name":"Qinghua Lu","orcid":"https://orcid.org/0000-0002-9466-1672"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"government","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]},{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Qinghua Lu","raw_affiliation_strings":["CSIRO's Data61, Sydney, Australia / University of New South Wales, Sydney, Australia"],"raw_orcid":"https://orcid.org/0000-0002-9466-1672","affiliations":[{"raw_affiliation_string":"CSIRO's Data61, Sydney, Australia / University of New South Wales, Sydney, Australia","institution_ids":["https://openalex.org/I42894916","https://openalex.org/I31746571","https://openalex.org/I1292875679"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064683660","display_name":"Liming Zhu","orcid":"https://orcid.org/0000-0001-5839-3765"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"government","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]},{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Liming Zhu","raw_affiliation_strings":["CSIRO's Data61, Sydney, Australia / University of New South Wales, Sydney, Australia"],"raw_orcid":"https://orcid.org/0000-0001-5839-3765","affiliations":[{"raw_affiliation_string":"CSIRO's Data61, Sydney, Australia / University of New South Wales, Sydney, Australia","institution_ids":["https://openalex.org/I42894916","https://openalex.org/I31746571","https://openalex.org/I1292875679"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028641941","display_name":"Zhenchang Xing","orcid":"https://orcid.org/0000-0001-7663-1421"},"institutions":[{"id":"https://openalex.org/I118347636","display_name":"Australian National University","ror":"https://ror.org/019wvm592","country_code":"AU","type":"education","lineage":["https://openalex.org/I118347636"]},{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Zhenchang Xing","raw_affiliation_strings":["CSIRO's Data61, Canberra, Australia / Australian National University, Canberra, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7663-1421","affiliations":[{"raw_affiliation_string":"CSIRO's Data61, Canberra, Australia / Australian National University, Canberra, Australia","institution_ids":["https://openalex.org/I42894916","https://openalex.org/I118347636"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.8328,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.86841919,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"74","last_page":"78"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9351999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9351999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.7240611910820007},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4435533583164215},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.08341613411903381},{"id":"https://openalex.org/keywords/ecology","display_name":"Ecology","score":0.06170934438705444}],"concepts":[{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.7240611910820007},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4435533583164215},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.08341613411903381},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.06170934438705444}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3664646.3664766","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664646.3664766","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},{"id":"pmh:oai:openresearch-repository.anu.edu.au:1885/733751677","is_oa":false,"landing_page_url":"http://www.scopus.com/inward/record.url?scp=85199903661&partnerID=8YFLogxK","pdf_url":null,"source":{"id":"https://openalex.org/S4306402539","display_name":"ANU Open Research (Australian National University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I118347636","host_organization_name":"Australian National University","host_organization_lineage":["https://openalex.org/I118347636"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"},{"id":"pmh:oai:arXiv.org:2404.05388","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.05388","pdf_url":"https://arxiv.org/pdf/2404.05388","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3664646.3664766","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3664646.3664766","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.44999998807907104,"display_name":"Responsible consumption and production","id":"https://metadata.un.org/sdg/12"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W4391913857","https://openalex.org/W2350741829","https://openalex.org/W2530322880"],"abstract_inverted_index":{"The":[0],"advent":[1],"of":[2,35],"advanced":[3],"AI":[4,36,62,78,88,95,103,113],"underscores":[5],"the":[6,33],"urgent":[7],"need":[8],"for":[9,61,87,101],"comprehensive":[10,54],"safety":[11,79],"evaluations,":[12],"necessitating":[13],"collaboration":[14],"across":[15,28,74],"communities":[16,75],"(i.e.,":[17],"AI,":[18],"software":[19],"engineering,":[20],"and":[21,26,53,98],"governance).":[22],"However,":[23],"divergent":[24],"practices":[25],"terminologies":[27],"these":[29],"communities,":[30],"combined":[31],"with":[32],"complexity":[34],"systems\u2014of":[37],"which":[38],"models":[39],"are":[40],"only":[41],"a":[42,59,82,92,109],"part\u2014and":[43],"environmental":[44],"affordances":[45],"(e.g.,":[46],"access":[47],"to":[48,71],"tools),":[49],"obstruct":[50],"effective":[51],"communication":[52,73],"evaluation.":[55],"This":[56,106],"paper":[57],"proposes":[58],"framework":[60,107],"system":[63,89,114],"evaluation":[64,115],"comprising":[65],"three":[66],"components:":[67],"1)":[68],"harmonised":[69],"terminology":[70],"facilitate":[72],"involved":[76],"in":[77],"evaluation;":[80,90],"2)":[81],"taxonomy":[83],"identifying":[84],"essential":[85],"elements":[86],"3)":[91],"mapping":[93],"between":[94],"lifecycle,":[96],"stakeholders,":[97],"requisite":[99],"evaluations":[100],"accountable":[102],"supply":[104],"chain.":[105],"catalyses":[108],"deeper":[110],"discourse":[111],"on":[112],"beyond":[116],"model-centric":[117],"approaches.":[118]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2024-04-11T00:00:00"}