{"id":"https://openalex.org/W3040607201","doi":null,"title":"In the Wild: From ML Models to Pragmatic ML Systems","display_name":"In the Wild: From ML Models to Pragmatic ML Systems","publication_year":2021,"publication_date":"2021-05-04","ids":{"openalex":"https://openalex.org/W3040607201","mag":"3040607201"},"language":"en","primary_location":{"id":"mag:3040607201","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=fpew0ll7wl","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035949823","display_name":"Matthew Wallingford","orcid":"https://orcid.org/0000-0002-6797-9802","observed_orcids":["https://orcid.org/0000-0002-6797-9802"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Wallingford","raw_affiliation_strings":["University of Washington ;"],"raw_orcid":"https://orcid.org/0000-0002-6797-9802","affiliations":[{"raw_affiliation_string":"University of Washington ;","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058266558","display_name":"Aditya Kusupati","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aditya Kusupati","raw_affiliation_strings":["Department of Computer Science, University of Washington,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Washington,","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004324969","display_name":"Keivan Alizadeh-Vahid","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Keivan Alizadeh-Vahid","raw_affiliation_strings":["Univ of Washington, Seattle#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Univ of Washington, Seattle#TAB#","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018904034","display_name":"Aaron Walsman","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaron Walsman","raw_affiliation_strings":["Department of Computer Science, University of Washington,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Washington,","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018421415","display_name":"Aniruddha Kembhavi","orcid":"https://orcid.org/0000-0002-7608-7443","observed_orcids":["https://orcid.org/0000-0002-7608-7443"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aniruddha Kembhavi","raw_affiliation_strings":["ALLEN INSTITUTE FOR ARTIFICIAL INTELLIGENCE"],"raw_orcid":"https://orcid.org/0000-0002-7608-7443","affiliations":[{"raw_affiliation_string":"ALLEN INSTITUTE FOR ARTIFICIAL INTELLIGENCE","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101576595","display_name":"Ali Farhadi","orcid":"https://orcid.org/0000-0001-7249-2380","observed_orcids":["https://orcid.org/0000-0001-7249-2380"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ali Farhadi","raw_affiliation_strings":["University of Washington ;"],"raw_orcid":"https://orcid.org/0000-0001-7249-2380","affiliations":[{"raw_affiliation_string":"University of Washington ;","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.219,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.48655036,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":95},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.001500000013038516,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/continual-learning","display_name":"continual learning","score":0.6499999761581421},{"id":"https://openalex.org/keywords/few-shot-learning","display_name":"few-shot learning","score":0.878000020980835},{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.9610000252723694},{"id":"https://openalex.org/keywords/meta-training","display_name":"meta-training","score":0.5820000171661377},{"id":"https://openalex.org/keywords/open-ended-learning","display_name":"open-ended learning","score":0.4950000047683716},{"id":"https://openalex.org/keywords/distribution-shift","display_name":"distribution shift","score":0.6449999809265137},{"id":"https://openalex.org/keywords/overfitting","display_name":"overfitting","score":0.6200000047683716},{"id":"https://openalex.org/keywords/moco","display_name":"MoCo","score":0.8650000095367432}],"concepts":[{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.8103442192077637},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6925646066665649},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6886379718780518},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6606501936912537},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6211208701133728},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5894403457641602},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.5551589727401733},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5447874069213867},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.4323025941848755},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.42684391140937805},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4264587461948395},{"id":"https://openalex.org/C84976871","wikidata":"https://www.wikidata.org/wiki/Q2015673","display_name":"Openness to experience","level":2,"score":0.41812601685523987},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.156032532453537},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13944339752197266},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3040607201","doi":null,"is_oa":false,"landing_page_url":"https://openreview.net/pdf?id=fpew0ll7wl","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W2108598243","https://openalex.org/W2194775991","https://openalex.org/W2294370754","https://openalex.org/W2401823607","https://openalex.org/W2432717477","https://openalex.org/W2562986790","https://openalex.org/W2601450892","https://openalex.org/W2604763608","https://openalex.org/W2612445135","https://openalex.org/W2753160622","https://openalex.org/W2796346823","https://openalex.org/W2803672301","https://openalex.org/W2808439681","https://openalex.org/W2898290017","https://openalex.org/W2904063889","https://openalex.org/W2904849495","https://openalex.org/W2911906319","https://openalex.org/W2939582364","https://openalex.org/W2948672349","https://openalex.org/W2952053192","https://openalex.org/W2962933664","https://openalex.org/W2963943197","https://openalex.org/W2970502342","https://openalex.org/W2970971581","https://openalex.org/W2986604550","https://openalex.org/W3005273253","https://openalex.org/W3005680577","https://openalex.org/W3007684729","https://openalex.org/W3012540512","https://openalex.org/W3023371261","https://openalex.org/W3034451759","https://openalex.org/W3035180000","https://openalex.org/W3035524453","https://openalex.org/W3037853434","https://openalex.org/W3097784654","https://openalex.org/W3102616566","https://openalex.org/W3105422445","https://openalex.org/W3108975329","https://openalex.org/W3118608800","https://openalex.org/W3205249428"],"related_works":["https://openalex.org/W2902452762","https://openalex.org/W3110517527","https://openalex.org/W7220813946","https://openalex.org/W4206893455","https://openalex.org/W6973642653","https://openalex.org/W4392488358","https://openalex.org/W4311128791","https://openalex.org/W644965770","https://openalex.org/W7106788273","https://openalex.org/W7127142635"],"abstract_inverted_index":{"Enabling":[0],"robust":[1],"intelligence":[2],"in":[3,90,94,234,288],"the":[4,33,39,79,128,137,155,192,205,215,260],"wild":[5],"entails":[6],"learning":[7,16,44,116],"systems":[8,26],"that":[9,92,203,227,237,281],"offer":[10],"uninterrupted":[11],"inference":[12],"while":[13,133,172,189],"affording":[14,66],"sustained":[15],"from":[17,201],"varying":[18],"amounts":[19],"of":[20,73,84,130,136,165,195,207,247],"data":[21,150,166,187],"and":[22,35,62,69,81,117,145,154,167,183,252,265,277],"supervision.":[23],"Such":[24],"ML":[25,110],"must":[27,168],"be":[28],"able":[29],"to":[30,38,100,175,180,217,232],"cope":[31],"with":[32,185],"openness":[34],"variability":[36],"inherent":[37],"real":[40],"world.":[41],"The":[42],"machine":[43],"community":[45],"has":[46,88],"organically":[47],"broken":[48],"down":[49,245],"this":[50,76],"challenging":[51],"task":[52,262],"into":[53],"manageable":[54],"sub":[55],"tasks":[56],"such":[57,141],"as":[58,142],"supervised,":[59],"few-shot,":[60],"continual,":[61],"self-supervised":[63,253],"learning;":[64],"each":[65],"distinct":[67],"challenges":[68],"a":[70,114,160,163,235],"unique":[71],"set":[72],"methods.":[74],"Notwithstanding":[75],"remarkable":[77],"progress,":[78],"simplified":[80],"isolated":[82],"nature":[83],"these":[85],"experimental":[86,250],"setups":[87],"resulted":[89],"methods":[91,243,274,286],"excel":[93],"their":[95,248],"specific":[96],"settings,":[97],"but":[98],"struggle":[99],"generalize":[101],"beyond":[102],"them.":[103],"To":[104],"foster":[105],"research":[106],"towards":[107,219],"more":[108,220],"general":[109],"systems,":[111],"we":[112,225,269],"present":[113,198,270],"new":[115,264,272],"evaluation":[118],"framework":[119],"-":[120],"I{N}":[121],"TH{E}":[122],"WIL{D}":[123],"(NED).":[124],"NED":[125,202],"naturally":[126],"integrates":[127],"objectives":[129],"previous":[131],"frameworks":[132],"removing":[134],"many":[135],"overly":[138],"strong":[139],"assumptions":[140],"predefined":[143],"training":[144,239],"test":[146],"phases,":[147],"sufficient":[148],"labeled":[149],"for":[151,191],"every":[152],"class,":[153],"closed-world":[156],"assumption.":[157],"In":[158],"NED,":[159],"learner":[161],"faces":[162],"stream":[164],"make":[169],"sequential":[170],"predictions":[171],"choosing":[173],"how":[174],"update":[176],"itself,":[177],"adapt":[178],"quickly":[179],"novel":[181,199],"classes,":[182],"deal":[184],"changing":[186],"distributions;":[188],"optimizing":[190],"total":[193],"amount":[194],"compute.":[196],"We":[197],"insights":[200],"contradict":[204],"findings":[206],"less":[208],"realistic":[209],"or":[210],"smaller-scale":[211],"experiments":[212],"which":[213],"emphasizes":[214],"need":[216],"move":[218],"pragmatic":[221,273],"setups.":[222],"For":[223],"example,":[224],"show":[226],"meta-training":[228],"causes":[229],"larger":[230],"networks":[231],"overfit":[233],"way":[236],"supervised":[238],"does":[240],"not,":[241],"few-shot":[242],"break":[244],"outside":[246],"narrow":[249],"setting,":[251],"method":[254],"MoCo":[255],"performs":[256],"significantly":[257,282],"worse":[258],"when":[259],"downstream":[261],"contains":[263],"old":[266],"classes.":[267],"Additionally,":[268],"two":[271],"(Exemplar":[275],"Tuning":[276],"Minimum":[278],"Distance":[279],"Thresholding)":[280],"outperform":[283],"all":[284],"other":[285],"evaluated":[287],"NED.":[289]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2020-07-10T00:00:00"}