{"id":"https://openalex.org/W2900501704","doi":"https://doi.org/10.1145/3243734.3278505","title":"Enabling Fair ML Evaluations for Security","display_name":"Enabling Fair ML Evaluations for Security","publication_year":2018,"publication_date":"2018-10-15","ids":{"openalex":"https://openalex.org/W2900501704","doi":"https://doi.org/10.1145/3243734.3278505","mag":"2900501704"},"language":"en","primary_location":{"id":"doi:10.1145/3243734.3278505","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3243734.3278505","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://kclpure.kcl.ac.uk/ws/files/112249343/Enabling_Fair_ML_Evaluations_PENDLEBURY_Published2018_GREEN_AAM.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090260871","display_name":"Feargus Pendlebury","orcid":"https://orcid.org/0000-0003-1140-322X"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]},{"id":"https://openalex.org/I184558857","display_name":"Royal Holloway University of London","ror":"https://ror.org/04g2vpn86","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I184558857"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Feargus Pendlebury","raw_affiliation_strings":["Royal Holloway, University of London & King's College London, Egham & London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Royal Holloway, University of London & King's College London, Egham & London, United Kingdom","institution_ids":["https://openalex.org/I184558857","https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072597369","display_name":"Fabio Pierazzi","orcid":"https://orcid.org/0000-0002-1254-1758"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]},{"id":"https://openalex.org/I184558857","display_name":"Royal Holloway University of London","ror":"https://ror.org/04g2vpn86","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I184558857"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Fabio Pierazzi","raw_affiliation_strings":["Royal Holloway, University of London & King's College London, Egham & London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Royal Holloway, University of London & King's College London, Egham & London, United Kingdom","institution_ids":["https://openalex.org/I184558857","https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046061066","display_name":"Roberto Jordaney","orcid":null},"institutions":[{"id":"https://openalex.org/I184558857","display_name":"Royal Holloway University of London","ror":"https://ror.org/04g2vpn86","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I184558857"]},{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Roberto Jordaney","raw_affiliation_strings":["Royal Holloway, University of London & King's College London, Egham & London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Royal Holloway, University of London & King's College London, Egham & London, United Kingdom","institution_ids":["https://openalex.org/I184558857","https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044468650","display_name":"Johannes Kinder","orcid":"https://orcid.org/0000-0002-8594-7839"},"institutions":[{"id":"https://openalex.org/I184558857","display_name":"Royal Holloway University of London","ror":"https://ror.org/04g2vpn86","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I184558857"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Johannes Kinder","raw_affiliation_strings":["Royal Holloway, University of London, Egham, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Royal Holloway, University of London, Egham, United Kingdom","institution_ids":["https://openalex.org/I184558857"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036908366","display_name":"Lorenzo Cavallaro","orcid":"https://orcid.org/0000-0002-3878-2680"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lorenzo Cavallaro","raw_affiliation_strings":["King's College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"King's College London, London, United Kingdom","institution_ids":["https://openalex.org/I183935753"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5090260871"],"corresponding_institution_ids":["https://openalex.org/I183935753","https://openalex.org/I184558857"],"apc_list":null,"apc_paid":null,"fwci":1.3265,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.82331903,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"2264","last_page":"2266"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8535915613174438},{"id":"https://openalex.org/keywords/malware","display_name":"Malware","score":0.7537766695022583},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.6851069331169128},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.684065580368042},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5573976039886475},{"id":"https://openalex.org/keywords/ranging","display_name":"Ranging","score":0.5481153726577759},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5420428514480591},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5404404997825623},{"id":"https://openalex.org/keywords/network-security","display_name":"Network security","score":0.4399052560329437},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.3664376139640808},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.15356886386871338},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.07428964972496033}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8535915613174438},{"id":"https://openalex.org/C541664917","wikidata":"https://www.wikidata.org/wiki/Q14001","display_name":"Malware","level":2,"score":0.7537766695022583},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.6851069331169128},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.684065580368042},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5573976039886475},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.5481153726577759},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5420428514480591},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5404404997825623},{"id":"https://openalex.org/C182590292","wikidata":"https://www.wikidata.org/wiki/Q989632","display_name":"Network security","level":2,"score":0.4399052560329437},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3664376139640808},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.15356886386871338},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.07428964972496033},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3243734.3278505","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3243734.3278505","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},{"id":"pmh:oai:kclpure.kcl.ac.uk:openaire/47c39372-d7d0-4653-b69e-6c1481cd1564","is_oa":true,"landing_page_url":"https://kclpure.kcl.ac.uk/portal/en/publications/47c39372-d7d0-4653-b69e-6c1481cd1564","pdf_url":"https://kclpure.kcl.ac.uk/ws/files/112249343/Enabling_Fair_ML_Evaluations_PENDLEBURY_Published2018_GREEN_AAM.pdf","source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Pendlebury, F, Pierazzi, F, Jordaney, R, Kinder, J & Cavallaro, L 2018, Enabling Fair ML Evaluations for Security. in Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security. Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security, pp. 2264-2266. https://doi.org/10.1145/3243734.3278505","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:pure.atira.dk:publications/47c39372-d7d0-4653-b69e-6c1481cd1564","is_oa":false,"landing_page_url":"https://kclpure.kcl.ac.uk/portal/en/publications/enabling-fair-ml-evaluations-for-security(47c39372-d7d0-4653-b69e-6c1481cd1564).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":{"id":"pmh:oai:kclpure.kcl.ac.uk:openaire/47c39372-d7d0-4653-b69e-6c1481cd1564","is_oa":true,"landing_page_url":"https://kclpure.kcl.ac.uk/portal/en/publications/47c39372-d7d0-4653-b69e-6c1481cd1564","pdf_url":"https://kclpure.kcl.ac.uk/ws/files/112249343/Enabling_Fair_ML_Evaluations_PENDLEBURY_Published2018_GREEN_AAM.pdf","source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Pendlebury, F, Pierazzi, F, Jordaney, R, Kinder, J & Cavallaro, L 2018, Enabling Fair ML Evaluations for Security. in Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security. Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security, pp. 2264-2266. https://doi.org/10.1145/3243734.3278505","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals","score":0.47999998927116394}],"awards":[{"id":"https://openalex.org/G4478209220","display_name":null,"funder_award_id":"EP/L022710/2","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G4849522399","display_name":null,"funder_award_id":"EP/P009301/1,EP/L022710/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G5653216463","display_name":"MobSec: Malware and Security in the Mobile Age","funder_award_id":"EP/L022710/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G6949464859","display_name":null,"funder_award_id":"EP/P009","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8452545418","display_name":null,"funder_award_id":"unknown","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8498677286","display_name":"Centre for Doctoral Training in Cyber Security at Royal Holloway: Renewal of the CS-CDT at RHUL","funder_award_id":"EP/P009301/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8719353587","display_name":null,"funder_award_id":"EP/P0","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2900501704.pdf","grobid_xml":"https://content.openalex.org/works/W2900501704.grobid-xml"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W258019806","https://openalex.org/W1966948031","https://openalex.org/W2031342017","https://openalex.org/W2083183119","https://openalex.org/W2118978333","https://openalex.org/W2122672392","https://openalex.org/W2156204309","https://openalex.org/W2407313496","https://openalex.org/W2464629196","https://openalex.org/W2534043454","https://openalex.org/W2594639291","https://openalex.org/W2603160474","https://openalex.org/W2753594008","https://openalex.org/W2782558714","https://openalex.org/W2884019276","https://openalex.org/W2903158431","https://openalex.org/W2964136807"],"related_works":["https://openalex.org/W2783354812","https://openalex.org/W4384112194","https://openalex.org/W2103009189","https://openalex.org/W4312958259","https://openalex.org/W4308259661","https://openalex.org/W4390813131","https://openalex.org/W2349383066","https://openalex.org/W4328132048","https://openalex.org/W1969901537","https://openalex.org/W2376202349"],"abstract_inverted_index":{"Machine":[0],"learning":[1,84],"is":[2,124],"widely":[3],"used":[4,45],"in":[5,86,114],"security":[6,88],"research":[7],"to":[8,15,27,35,46,57,78,128,136,145,151],"classify":[9],"malicious":[10,16],"activity,":[11],"ranging":[12],"from":[13,59],"malware":[14],"URLs":[17],"and":[18,41,139],"network":[19],"traffic.":[20],"However,":[21],"published":[22],"performance":[23,81,141],"numbers":[24],"often":[25],"seem":[26],"leave":[28],"little":[29],"room":[30],"for":[31,107],"improvement":[32],"and,":[33],"due":[34],"a":[36,87,91,108,115,134],"wide":[37],"range":[38],"of":[39,73,82,101,111],"datasets":[40],"configurations,":[42],"cannot":[43],"be":[44],"directly":[47],"compare":[48],"alternative":[49],"approaches;":[50],"moreover,":[51],"most":[52],"evaluations":[53],"have":[54],"been":[55],"found":[56],"suffer":[58],"experimental":[60],"bias":[61],"which":[62,149],"positively":[63],"inflates":[64],"results.":[65],"In":[66,103],"this":[67],"manuscript":[68],"we":[69],"discuss":[70],"the":[71,80,130],"implementation":[72],"Tesseract,":[74],"an":[75,98],"open-source":[76,127],"tool":[77],"evaluate":[79],"machine":[83],"classifiers":[85,113],"setting":[89],"mimicking":[90],"deployment":[92],"with":[93,133],"typical":[94],"data":[95],"feeds":[96],"over":[97],"extended":[99],"period":[100],"time.":[102],"particular,":[104],"Tesseract":[105,123],"allows":[106],"fair":[109],"comparison":[110],"different":[112],"realistic":[116],"scenario,":[117],"without":[118],"disadvantaging":[119],"any":[120],"given":[121],"classifier.":[122],"available":[125],"as":[126],"provide":[129],"academic":[131],"community":[132],"way":[135],"report":[137],"sound":[138],"comparable":[140],"results,":[142],"but":[143],"also":[144],"help":[146],"practitioners":[147],"decide":[148],"system":[150],"deploy":[152],"under":[153],"specific":[154],"budget":[155],"constraints.":[156]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}