{"id":"https://openalex.org/W4281784576","doi":"https://doi.org/10.48550/arxiv.2205.13532","title":"Selective Prediction via Training Dynamics","display_name":"Selective Prediction via Training Dynamics","publication_year":2022,"publication_date":"2022-05-26","ids":{"openalex":"https://openalex.org/W4281784576","doi":"https://doi.org/10.48550/arxiv.2205.13532"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2205.13532","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.13532","pdf_url":"https://arxiv.org/pdf/2205.13532","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2205.13532","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030986512","display_name":"Stephan Rabanser","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Rabanser, Stephan","raw_affiliation_strings":["University of Toronto & Vector Institute Anvith Thudi"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto & Vector Institute Anvith Thudi","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010910495","display_name":"Anvith Thudi","orcid":"https://orcid.org/0009-0005-9062-7863","observed_orcids":["https://orcid.org/0009-0005-9062-7863"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thudi, Anvith","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009409152","display_name":"Kimia Hamidieh","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Hamidieh, Kimia","raw_affiliation_strings":["Massachusetts Institute of Technology","University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]},{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044067668","display_name":"Adam Dziedzic","orcid":"https://orcid.org/0000-0001-9786-2296","observed_orcids":["https://orcid.org/0000-0001-9786-2296"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dziedzic, Adam","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Bahceci, Israfil","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Bahceci, Israfil","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sediq, Akram Bin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sediq, Akram Bin","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sokun, Hamza","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sokun, Hamza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Papernot, Nicolas","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I4210127509","display_name":"Vector Institute","ror":"https://ror.org/03kqdja62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210127509"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Papernot, Nicolas","raw_affiliation_strings":["University of Toronto & Vector Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto & Vector Institute","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I4210127509"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.551800012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.551800012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.36820000410079956,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.023800000548362732,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/selective-prediction","display_name":"selective prediction","score":0.9990000128746033},{"id":"https://openalex.org/keywords/training-dynamics","display_name":"training dynamics","score":0.765999972820282},{"id":"https://openalex.org/keywords/model-checkpoints","display_name":"model checkpoints","score":0.49300000071525574},{"id":"https://openalex.org/keywords/image-classification","display_name":"image classification","score":0.5849999785423279},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"regression analysis","score":0.765999972820282},{"id":"https://openalex.org/keywords/time-series-forecasting","display_name":"time series forecasting","score":0.7689999938011169}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7353479266166687},{"id":"https://openalex.org/C73000952","wikidata":"https://www.wikidata.org/wiki/Q17007827","display_name":"Discretization","level":2,"score":0.6261520385742188},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6250057220458984},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5833980441093445},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5604784488677979},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5546271204948425},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.529503583908081},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5148414969444275},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4892897307872772},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.47376540303230286},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.4599460959434509},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.4529392719268799},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.45288893580436707},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3603353798389435},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.120767742395401},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08793151378631592},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2205.13532","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.13532","pdf_url":"https://arxiv.org/pdf/2205.13532","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2205.13532","doi":"https://doi.org/10.48550/arxiv.2205.13532","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2205.13532","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2205.13532","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.13532","pdf_url":"https://arxiv.org/pdf/2205.13532","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309949","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4281784576.pdf","grobid_xml":"https://content.openalex.org/works/W4281784576.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2531327146","https://openalex.org/W2612983688","https://openalex.org/W2764163036","https://openalex.org/W2893463703","https://openalex.org/W2903996579","https://openalex.org/W2952278188","https://openalex.org/W2963649890","https://openalex.org/W3042879175","https://openalex.org/W3080532707","https://openalex.org/W3088647504","https://openalex.org/W3139361705","https://openalex.org/W3172093261","https://openalex.org/W3176783553","https://openalex.org/W4321854344","https://openalex.org/W4378498727","https://openalex.org/W4391212612","https://openalex.org/W4399511957","https://openalex.org/W6888914244","https://openalex.org/W6907717285","https://openalex.org/W7133214657"],"related_works":["https://openalex.org/W7101842389","https://openalex.org/W7128769081","https://openalex.org/W4391212620","https://openalex.org/W7163613663","https://openalex.org/W4292387985","https://openalex.org/W6979339784","https://openalex.org/W7138128164","https://openalex.org/W3212919788","https://openalex.org/W7221019912","https://openalex.org/W2608317659"],"abstract_inverted_index":{"Selective":[0],"Prediction":[1],"is":[2,33,149],"the":[3,34,50,54,89,109,122,137],"task":[4],"of":[5,93,111],"rejecting":[6],"inputs":[7],"a":[8,16,94,98,103],"model":[9,29,51],"would":[10],"predict":[11],"incorrectly":[12],"on.":[13],"This":[14],"involves":[15],"trade-off":[17],"between":[18],"input":[19],"space":[20],"coverage":[21],"(how":[22,31],"many":[23],"data":[24,38,130],"points":[25,131],"are":[26],"accepted)":[27],"and":[28,63,157,160,185],"utility":[30],"good":[32],"performance":[35,82],"on":[36,48,181,198],"accepted":[37],"points).":[39],"Current":[40],"methods":[41],"for":[42,154],"selective":[43,80,167,200],"prediction":[44,81,139,168,201],"typically":[45],"impose":[46],"constraints":[47],"either":[49],"architecture":[52],"or":[53],"optimization":[55],"objective;":[56],"this":[57],"inhibits":[58],"their":[59],"usage":[60],"in":[61,143],"practice":[62],"introduces":[64],"unknown":[65],"interactions":[66],"with":[67,136,165],"pre-existing":[68],"loss":[69],"functions.":[70],"In":[71,126],"contrast":[72],"to":[73],"prior":[74],"work,":[75],"we":[76,128],"show":[77],"that":[78,190],"state-of-the-art":[79,195],"can":[83,161],"be":[84,162],"attained":[85],"solely":[86],"from":[87,113],"studying":[88],"(discretized)":[90],"training":[91,120],"dynamics":[92],"model.":[95],"We":[96],"propose":[97],"general":[99],"framework":[100],"that,":[101],"given":[102],"test":[104],"input,":[105],"monitors":[106],"metrics":[107],"capturing":[108],"instability":[110],"predictions":[112],"intermediate":[114],"models":[115],"(i.e.,":[116,151],"checkpoints)":[117],"obtained":[118],"during":[119],"w.r.t.":[121],"final":[123,138],"model's":[124],"prediction.":[125],"particular,":[127],"reject":[129],"exhibiting":[132],"too":[133],"much":[134],"disagreement":[135],"at":[140],"late":[141],"stages":[142],"training.":[144],"The":[145],"proposed":[146],"rejection":[147],"mechanism":[148],"domain-agnostic":[150],"it":[152,171],"works":[153],"both":[155],"discrete":[156],"real-valued":[158],"prediction)":[159],"flexibly":[163],"combined":[164],"existing":[166],"approaches":[169],"as":[170],"does":[172],"not":[173],"require":[174],"any":[175],"train-time":[176],"modifications.":[177],"Our":[178],"experimental":[179],"evaluation":[180],"image":[182],"classification,":[183],"regression,":[184],"time":[186],"series":[187],"problems":[188],"shows":[189],"our":[191],"method":[192],"beats":[193],"past":[194],"accuracy/utility":[196],"trade-offs":[197],"typical":[199],"benchmarks.":[202]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-06-13T00:00:00"}