{"id":"https://openalex.org/W2753123930","doi":"https://doi.org/10.1609/aaai.v33i01.33015425","title":"How Does Knowledge of the AUC Constrain the Set of Possible Ground-Truth Labelings?","display_name":"How Does Knowledge of the AUC Constrain the Set of Possible Ground-Truth Labelings?","publication_year":2019,"publication_date":"2019-07-17","ids":{"openalex":"https://openalex.org/W2753123930","doi":"https://doi.org/10.1609/aaai.v33i01.33015425","mag":"2753123930"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v33i01.33015425","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.33015425","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v33i01.33015425","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027788582","display_name":"Jacob Whitehill","orcid":"https://orcid.org/0000-0002-5851-312X"},"institutions":[{"id":"https://openalex.org/I107077323","display_name":"Worcester Polytechnic Institute","ror":"https://ror.org/05ejpqr48","country_code":"US","type":"education","lineage":["https://openalex.org/I107077323"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jacob Whitehill","raw_affiliation_strings":["Worcester Polytechnic Institute","Worcester polytechnic Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Worcester Polytechnic Institute","institution_ids":["https://openalex.org/I107077323"]},{"raw_affiliation_string":"Worcester polytechnic Institute","institution_ids":["https://openalex.org/I107077323"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5027788582"],"corresponding_institution_ids":["https://openalex.org/I107077323"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.002263,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":"01","first_page":"5425","last_page":"5432"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.8087193369865417},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.7174075841903687},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6146863102912903},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5779522657394409},{"id":"https://openalex.org/keywords/test-set","display_name":"Test set","score":0.570892333984375},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.5596842169761658},{"id":"https://openalex.org/keywords/binary-classification","display_name":"Binary classification","score":0.5353662371635437},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5050998330116272},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49676257371902466},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"Support vector machine","score":0.3942223787307739},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.38797858357429504},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35309988260269165},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.29590165615081787},{"id":"https://openalex.org/keywords/arithmetic","display_name":"Arithmetic","score":0.09342887997627258}],"concepts":[{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.8087193369865417},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.7174075841903687},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6146863102912903},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5779522657394409},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.570892333984375},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.5596842169761658},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.5353662371635437},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5050998330116272},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49676257371902466},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.3942223787307739},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38797858357429504},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35309988260269165},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29590165615081787},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.09342887997627258},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v33i01.33015425","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.33015425","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1709.02418","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1709.02418","pdf_url":"https://arxiv.org/pdf/1709.02418","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2753123930","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1709.02418.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1709.02418","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1709.02418","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v33i01.33015425","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v33i01.33015425","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.6200000047683716,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1970074386","https://openalex.org/W1982723861","https://openalex.org/W1987926440","https://openalex.org/W2042469398","https://openalex.org/W2048123501","https://openalex.org/W2077641783","https://openalex.org/W2111233787","https://openalex.org/W2112380340","https://openalex.org/W2132594509","https://openalex.org/W2205852953","https://openalex.org/W2911978475","https://openalex.org/W2951865872","https://openalex.org/W2952550721","https://openalex.org/W2963578006","https://openalex.org/W6676434424","https://openalex.org/W6676639149","https://openalex.org/W6758947373"],"related_works":["https://openalex.org/W2962913294","https://openalex.org/W3096140069","https://openalex.org/W2514278201","https://openalex.org/W2340502550","https://openalex.org/W2883762059","https://openalex.org/W2963023185","https://openalex.org/W3184237879","https://openalex.org/W2987822190","https://openalex.org/W3028888589","https://openalex.org/W2517118512","https://openalex.org/W1834813338","https://openalex.org/W2571630279","https://openalex.org/W2964230093","https://openalex.org/W2950151615","https://openalex.org/W1706338097","https://openalex.org/W3133989806","https://openalex.org/W2125497405","https://openalex.org/W2804546363","https://openalex.org/W3092683409","https://openalex.org/W2962997990"],"abstract_inverted_index":{"Recent":[0],"work":[1],"on":[2,33],"privacy-preserving":[3],"machine":[4],"learning":[5],"has":[6],"considered":[7],"how":[8,80,111,170],"datamining":[9],"competitions":[10],"such":[11,134],"as":[12,157],"Kaggle":[13],"could":[14],"potentially":[15],"be":[16,173],"\u201chacked\u201d,":[17],"either":[18],"intentionally":[19],"or":[20],"inadvertently,":[21],"by":[22],"using":[23],"information":[24],"from":[25,85],"an":[26,86,125],"oracle":[27,180],"that":[28],"reports":[29],"a":[30,161],"classifier\u2019s":[31],"accuracy":[32,60],"the":[34,57,63,66,76,81,94,98,104,115,130,149,185],"test":[35,105,162,187],"set":[36,106,116],"(Blum":[37],"and":[38,42,70,122,136],"Hardt":[39,41],"2015;":[40,46],"Ullman":[43],"2014;":[44],"Zheng":[45],"Whitehill":[47],"2016).":[48],"For":[49],"binary":[50],"classification":[51],"tasks":[52],"in":[53,71],"particular,":[54],"one":[55],"of":[56,79,88,100,103,118,133,151,179],"most":[58],"common":[59],"metrics":[61],"is":[62,83,165],"Area":[64],"Under":[65,97],"ROC":[67],"Curve":[68],"(AUC),":[69],"this":[72],"paper":[73],"we":[74,109,123,168],"explore":[75],"mathematical":[77],"structure":[78],"AUC":[82,107],"computed":[84],"n-vector":[87],"real-valued":[89],"\u201cguesses\u201d":[90],"with":[91,189],"respect":[92],"to":[93,128,137,182],"ground-truth":[95,120],"labels.":[96],"assumption":[99],"perfect":[101],"knowledge":[102],"c=p/q,":[108],"show":[110,169],"knowing":[112],"c":[113],"constrains":[114],"W":[117,171],"possible":[119],"labelings,":[121],"derive":[124],"algorithm":[126],"both":[127],"compute":[129],"exact":[131],"number":[132,150],"labelings":[135,153],"enumerate":[138],"efficiently":[139,174],"over":[140],"them.":[141],"We":[142],"also":[143],"provide":[144],"empirical":[145],"evidence":[146],"that,":[147],"surprisingly,":[148],"compatible":[152],"can":[154,172],"actually":[155],"decrease":[156],"n":[158],"grows,":[159],"until":[160],"set-dependent":[163],"threshold":[164],"reached.":[166],"Finally,":[167],"whittled":[175],"down,":[176],"through":[177],"pairs":[178],"queries,":[181],"infer":[183],"all":[184],"groundtruth":[186],"labels":[188],"complete":[190],"certainty.":[191]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}