{"id":"https://openalex.org/W2953183526","doi":"https://doi.org/10.18653/v1/p19-1266","title":"Deep Dominance - How to Properly Compare Deep Neural Models","display_name":"Deep Dominance - How to Properly Compare Deep Neural Models","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2953183526","doi":"https://doi.org/10.18653/v1/p19-1266","mag":"2953183526"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1266","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1266","pdf_url":"https://www.aclweb.org/anthology/P19-1266.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1266.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089112142","display_name":"Rotem Dror","orcid":null},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":true,"raw_author_name":"Rotem Dror","raw_affiliation_strings":["Faculty of Industrial Engineering and Management, Technion, IIT"],"affiliations":[{"raw_affiliation_string":"Faculty of Industrial Engineering and Management, Technion, IIT","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050987183","display_name":"Segev Shlomov","orcid":"https://orcid.org/0000-0003-1216-8284"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Segev Shlomov","raw_affiliation_strings":["Faculty of Industrial Engineering and Management, Technion, IIT"],"affiliations":[{"raw_affiliation_string":"Faculty of Industrial Engineering and Management, Technion, IIT","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054952724","display_name":"Roi Reichart","orcid":"https://orcid.org/0000-0001-6918-0554"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Roi Reichart","raw_affiliation_strings":["Faculty of Industrial Engineering and Management, Technion, IIT"],"affiliations":[{"raw_affiliation_string":"Faculty of Industrial Engineering and Management, Technion, IIT","institution_ids":["https://openalex.org/I174306211"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5089112142"],"corresponding_institution_ids":["https://openalex.org/I174306211"],"apc_list":null,"apc_paid":null,"fwci":6.0035,"has_fulltext":true,"cited_by_count":96,"citation_normalized_percentile":{"value":0.96856234,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2773","last_page":"2785"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7495087385177612},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.7245824337005615},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.6096880435943604},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6060381531715393},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5816317200660706},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5689636468887329},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5084565877914429},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.45187562704086304},{"id":"https://openalex.org/keywords/test-set","display_name":"Test set","score":0.42789602279663086}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7495087385177612},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7245824337005615},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.6096880435943604},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6060381531715393},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5816317200660706},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5689636468887329},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5084565877914429},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.45187562704086304},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.42789602279663086},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1266","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1266","pdf_url":"https://www.aclweb.org/anthology/P19-1266.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1266","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1266","pdf_url":"https://www.aclweb.org/anthology/P19-1266.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.46000000834465027}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2953183526.pdf","grobid_xml":"https://content.openalex.org/works/W2953183526.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1533861849","https://openalex.org/W1632114991","https://openalex.org/W1972978214","https://openalex.org/W2026888312","https://openalex.org/W2062989416","https://openalex.org/W2064675550","https://openalex.org/W2095705004","https://openalex.org/W2098921539","https://openalex.org/W2107030350","https://openalex.org/W2131774270","https://openalex.org/W2134275125","https://openalex.org/W2153848201","https://openalex.org/W2157275230","https://openalex.org/W2296283641","https://openalex.org/W2552110825","https://openalex.org/W2560651656","https://openalex.org/W2604593109","https://openalex.org/W2611081871","https://openalex.org/W2738180183","https://openalex.org/W2794509261","https://openalex.org/W2798935874","https://openalex.org/W2857028992","https://openalex.org/W2951299559","https://openalex.org/W2952087486","https://openalex.org/W2962902328","https://openalex.org/W2963114681","https://openalex.org/W2963403868","https://openalex.org/W2963571341","https://openalex.org/W2963583956","https://openalex.org/W2963586744","https://openalex.org/W2963940534","https://openalex.org/W2964121744","https://openalex.org/W4254387140","https://openalex.org/W4300455941","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W3204184292","https://openalex.org/W3176564347","https://openalex.org/W1985458517","https://openalex.org/W2355833770","https://openalex.org/W3031039437","https://openalex.org/W3095877357","https://openalex.org/W183202219","https://openalex.org/W2072565696","https://openalex.org/W10861731","https://openalex.org/W2050451745"],"abstract_inverted_index":{"Comparing":[0],"between":[1,56,66,98,111],"Deep":[2],"Neural":[3],"Network":[4],"(DNN)":[5],"models":[6,25,127],"based":[7],"on":[8,11,39,61],"their":[9,35,57],"performance":[10],"unseen":[12,62],"data":[13],"is":[14,71],"crucial":[15],"for":[16,73,92,105,128],"the":[17,20,40,93,103,133,149,161],"progress":[18],"of":[19,30,120],"NLP":[21,162],"field.":[22],"However,":[23],"these":[24],"have":[26],"a":[27,54,88,106,156],"large":[28],"number":[29],"hyper-parameters":[31],"and,":[32],"being":[33],"non-convex,":[34],"convergence":[36],"point":[37],"depends":[38],"random":[41],"values":[42],"chosen":[43],"at":[44],"initialization":[45],"and":[46,113,117],"during":[47],"training.":[48],"Proper":[49],"DNN":[50,126],"comparison":[51,55,109],"hence":[52],"requires":[53],"empirical":[58],"score":[59],"distributions":[60],"data,":[63],"rather":[64],"than":[65],"single":[67],"evaluation":[68],"scores":[69],"as":[70],"standard":[72],"more":[74],"simple,":[75],"convex":[76],"models.":[77],"In":[78],"this":[79,86],"paper,":[80],"we":[81,151],"propose":[82,152],"to":[83,85,144],"adapt":[84],"problem":[87],"recently":[89],"proposed":[90,134,141],"test":[91,135,150],"Almost":[94],"Stochastic":[95],"Dominance":[96],"relation":[97],"two":[99],"distributions.":[100],"We":[101,147],"define":[102],"criteria":[104,138],"high":[107],"quality":[108],"method":[110],"DNNs,":[112],"show,":[114],"both":[115],"theoretically":[116],"through":[118],"analysis":[119],"extensive":[121],"experimental":[122],"results":[123],"with":[124],"leading":[125],"sequence":[129],"tagging":[130],"tasks,":[131],"that":[132],"meets":[136],"all":[137],"while":[139],"previously":[140],"methods":[142],"fail":[143],"do":[145],"so.":[146],"hope":[148],"here":[153],"will":[154],"set":[155],"new":[157],"working":[158],"practice":[159],"in":[160],"community.":[163],"1":[164]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":19},{"year":2023,"cited_by_count":26},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":9}],"updated_date":"2026-02-03T00:53:05.648605","created_date":"2025-10-10T00:00:00"}