{"id":"https://openalex.org/W3198147755","doi":null,"title":"WALNUT: A Benchmark on Weakly Supervised Learning for Natural Language Understanding.","display_name":"WALNUT: A Benchmark on Weakly Supervised Learning for Natural Language Understanding.","publication_year":2021,"publication_date":"2021-08-20","ids":{"openalex":"https://openalex.org/W3198147755","mag":"3198147755"},"language":"en","primary_location":{"id":"mag:3198147755","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2108.12603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://export.arxiv.org/pdf/2108.12603","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Guo\u2010qing Zheng","orcid":"https://orcid.org/0000-0003-1707-5669","observed_orcids":["https://orcid.org/0000-0003-1707-5669"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Guo\u2010qing Zheng","raw_affiliation_strings":["(Microsoft)"],"raw_orcid":"https://orcid.org/0000-0003-1707-5669","affiliations":[{"raw_affiliation_string":"(Microsoft)","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070865859","display_name":"Giannis Karamanolakis","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Giannis Karamanolakis","raw_affiliation_strings":["Columbia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058670321","display_name":"Kai Shu","orcid":"https://orcid.org/0000-0002-6043-1764","observed_orcids":["https://orcid.org/0000-0002-6043-1764"]},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kai Shu","raw_affiliation_strings":["Illinois Institute of technology"],"raw_orcid":"https://orcid.org/0000-0002-6043-1764","affiliations":[{"raw_affiliation_string":"Illinois Institute of technology","institution_ids":["https://openalex.org/I180949307"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021000040","display_name":"Ahmed Hassan Awadallah","orcid":"https://orcid.org/0000-0001-6426-3537","observed_orcids":["https://orcid.org/0000-0001-6426-3537"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed Hassan Awadallah","raw_affiliation_strings":["(Microsoft)"],"raw_orcid":"https://orcid.org/0000-0001-6426-3537","affiliations":[{"raw_affiliation_string":"(Microsoft)","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6753000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6753000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2587999999523163,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0284000001847744,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/weakly-supervised-learning","display_name":"weakly supervised learning","score":0.9279999732971191},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"natural language understanding","score":0.9900000095367432},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.8980000019073486}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7727733850479126},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.6897580027580261},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6837928295135498},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6696131825447083},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5838791728019714},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5634300112724304},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5407035946846008},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.46127912402153015},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4198795557022095},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39779791235923767},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2787749767303467},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.06790101528167725},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3198147755","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2108.12603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3198147755","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2108.12603","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1866072925","https://openalex.org/W1988584482","https://openalex.org/W2079735306","https://openalex.org/W2107598941","https://openalex.org/W2107966677","https://openalex.org/W2113459411","https://openalex.org/W2144578941","https://openalex.org/W2169099542","https://openalex.org/W2170240176","https://openalex.org/W2179519966","https://openalex.org/W2250265269","https://openalex.org/W2250539671","https://openalex.org/W2251818274","https://openalex.org/W2296283641","https://openalex.org/W2346452181","https://openalex.org/W2437771934","https://openalex.org/W2465978385","https://openalex.org/W2593560537","https://openalex.org/W2609602893","https://openalex.org/W2626778328","https://openalex.org/W2742330194","https://openalex.org/W2769041395","https://openalex.org/W2785495800","https://openalex.org/W2787560479","https://openalex.org/W2795282075","https://openalex.org/W2857028992","https://openalex.org/W2890931111","https://openalex.org/W2911424454","https://openalex.org/W2963310665","https://openalex.org/W2963341956","https://openalex.org/W2964292098","https://openalex.org/W2964317478","https://openalex.org/W2965373594","https://openalex.org/W2971015282","https://openalex.org/W2978017171","https://openalex.org/W2980282514","https://openalex.org/W2990704537","https://openalex.org/W2994976222","https://openalex.org/W2997918300","https://openalex.org/W3014563232","https://openalex.org/W3022333975","https://openalex.org/W3031781733","https://openalex.org/W3033187248","https://openalex.org/W3034563230","https://openalex.org/W3092571094","https://openalex.org/W3098903812","https://openalex.org/W3110687497","https://openalex.org/W3153866062"],"related_works":["https://openalex.org/W4308291264","https://openalex.org/W3176553463","https://openalex.org/W4385568372","https://openalex.org/W3169195805","https://openalex.org/W4385571929","https://openalex.org/W7217473491","https://openalex.org/W3153495647","https://openalex.org/W4403227743","https://openalex.org/W7126418359","https://openalex.org/W4283318968"],"abstract_inverted_index":{"Building":[0],"quality":[1],"machine":[2],"learning":[3],"models":[4],"for":[5,42,63,71,99,119,146,164,186],"natural":[6],"language":[7],"understanding":[8],"(NLU)":[9],"tasks":[10,76,105,113,117,167],"relies":[11],"heavily":[12],"on":[13,47,96,135,175],"labeled":[14,29],"data.":[15],"Weak":[16],"supervision":[17,24,41,54,69,98,145,152,163],"has":[18],"been":[19],"shown":[20],"to":[21,35,91,138,171,177],"provide":[22],"valuable":[23],"when":[25],"large":[26],"amount":[27],"of":[28,74,103,143,161],"data":[30],"is":[31,77],"unavailable":[32],"or":[33,51],"expensive":[34],"obtain.":[36],"Existing":[37],"works":[38],"studying":[39],"weak":[40,53,68,97,123,129,144,151,162,180],"NLU":[43,64,75,104,147,166],"either":[44],"mostly":[45],"focus":[46],"a":[48,61,72,87],"specific":[49],"task":[50,121],"simulate":[52],"signals":[55,70],"from":[56],"ground-truth":[57],"labels.":[58],"To":[59],"date":[60],"benchmark":[62,137,183],"with":[65,106,149],"real":[66],"world":[67],"collection":[73],"still":[78],"not":[79],"available.":[80],"In":[81],"this":[82,193],"paper,":[83],"we":[84],"propose":[85],"such":[86],"benchmark,":[88],"named":[89],"WALNUT,":[90],"advocate":[92],"and":[93,114,118,154,168,184],"facilitate":[94],"research":[95,174],"NLU.":[100],"WALNUT":[101,170],"consists":[102],"different":[107],"types,":[108],"including":[109],"both":[110],"document-level":[111],"prediction":[112,116],"token-level":[115],"each":[120],"contains":[122],"labels":[124],"generated":[125],"by":[126],"multiple":[127],"real-world":[128],"sources.":[130],"We":[131,157],"conduct":[132],"baseline":[133],"evaluations":[134],"the":[136,141,159],"systematically":[139],"test":[140],"value":[142],"tasks,":[148],"various":[150],"methods":[153],"model":[155],"architectures.":[156],"demonstrate":[158],"benefits":[160],"low-resource":[165],"expect":[169],"stimulate":[172],"further":[173],"methodologies":[176],"best":[178],"leverage":[179],"supervision.":[181],"The":[182],"code":[185],"baselines":[187],"will":[188],"be":[189],"publicly":[190],"available":[191],"at":[192],"http":[194],"URL.":[195]},"counts_by_year":[{"year":2022,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2021-09-13T00:00:00"}