{"id":"https://openalex.org/W6903660153","doi":"https://doi.org/10.1184/r1/22213705.v1","title":"Robustifying NLP with Humans in the Loop","display_name":"Robustifying NLP with Humans in the Loop","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W6903660153","doi":"https://doi.org/10.1184/r1/22213705.v1"},"language":"en","primary_location":{"id":"pmh:oai:figshare.com:article/22213705","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Kaushik, Divyansh","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Kaushik, Divyansh","raw_affiliation_strings":["Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.27879516,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5302000045776367,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5302000045776367,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.1949000060558319,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.13860000669956207,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.8148999810218811},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.7232999801635742},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6909000277519226},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.6080999970436096},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5950000286102295},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.5386999845504761},{"id":"https://openalex.org/keywords/independent-and-identically-distributed-random-variables","display_name":"Independent and identically distributed random variables","score":0.527400016784668}],"concepts":[{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.8148999810218811},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.7232999801635742},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7071999907493591},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6909000277519226},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6798999905586243},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.6080999970436096},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5950000286102295},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.5386999845504761},{"id":"https://openalex.org/C141513077","wikidata":"https://www.wikidata.org/wiki/Q378542","display_name":"Independent and identically distributed random variables","level":3,"score":0.527400016784668},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5116000175476074},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.46399998664855957},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42660000920295715},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.38449999690055847},{"id":"https://openalex.org/C10347200","wikidata":"https://www.wikidata.org/wiki/Q1960297","display_name":"Hindsight bias","level":2,"score":0.38280001282691956},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.35760000348091125},{"id":"https://openalex.org/C2775922551","wikidata":"https://www.wikidata.org/wiki/Q7135033","display_name":"Parallels","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2632000148296356}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:figshare.com:article/22213705","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},{"id":"doi:10.1184/r1/22213705.v1","is_oa":true,"landing_page_url":"https://doi.org/10.1184/r1/22213705.v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050927","display_name":"KiltHub Repository","issn_l":null,"issn":[],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/22213705","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Despite":[0],"machine":[1],"learning":[2,191],"(ML)\u2019s":[3],"many":[4],"practical":[5],"breakthroughs,":[6],"formidable":[7],"ob-stacles":[8],"obstruct":[9],"its":[10],"deployment":[11],"in":[12,56,134,171,200,222],"consequential":[13],"applications.":[14],"Modern":[15],"ML":[16],"modelshave":[17],"repeatedly":[18],"been":[19],"shown":[20],"to":[21,33,36,89,181,184],"rely":[22],"on":[23,67,92,94,121,165,176,205],"spurious":[24,188],"signals,":[25],"such":[26,147],"as":[27,54,148,209],"surface":[28],"level":[29],"texturesin":[30],"images,":[31],"and":[32,69,141,151,161,179,225],"be":[34,90,115,185],"sensitive":[35],"background":[37],"scenery,":[38],"even":[39],"when":[40],"the":[41,57,102,122,135,158,172,206,216,227],"task":[42],"addressesthe":[43],"recognition":[44],"of":[45,104,110,124,157,195,229,234],"foreground":[46],"objects.":[47],"In":[48,126],"NLP,":[49],"these":[50],"issues":[51],"have":[52],"emerged":[53],"centralconcerns":[55],"literature":[58],"onannotation":[59],"artifactsandbias.":[60],"Moreover,":[61],"while":[62],"modernML":[63],"performs":[64],"remarkably":[65],"well":[66,164],"independent":[68],"identically":[70],"distributed":[71],"(iid)":[72],"hold-out":[73],"data,":[74],"performance":[75],"often":[76],"decays":[77],"catastrophically":[78],"under":[79],"both":[80],"naturally":[81],"occurringand":[82],"adversarial":[83],"distribution":[84],"shift.":[85],"We":[86],"desire":[87],"decisions":[88],"based":[91],"qualifications,not":[93],"distant":[95],"proxies":[96],"that":[97,116],"are":[98],"spuriously":[99],"associated":[100],"with":[101],"outcome":[103,123],"interest.":[105,125],"Ar-guably":[106],"one":[107],"key":[108],"distinction":[109],"an":[111],"actual":[112],"qualification":[113],"might":[114],"it":[117],"actually":[118],"exertscausal":[119],"influence":[120],"this":[127],"thesis,":[128],"we":[129,138,174,203,210],"make":[130],"progress":[131],"towardsthese":[132],"goals:":[133],"first":[136],"part,":[137],"scrutinize":[139],"benchmarks":[140],"problem":[142],"formulation":[143],"forpopular":[144],"NLP":[145,223],"tasks,":[146],"question":[149],"answering":[150],"how":[152],"models":[153,183],"may":[154],"ignore":[155],"crucialparts":[156],"input":[159],"altogether":[160],"yet":[162],"perform":[163],"a":[166,231],"held":[167],"out":[168],"test":[169],"set;":[170],"secondpart,":[173],"focus":[175,204],"introducing":[177],"methods":[178],"datasets":[180],"train":[182],"less":[186],"relianton":[187],"correlations":[189],"by":[190,215,220],"from":[192],"several":[193],"forms":[194],"human":[196,207],"feedback":[197],"(soughtvia":[198],"crowdsourcing);":[199],"part":[201],"three":[202],"workforce":[208],"discussthe":[211],"ethical":[212],"tensions":[213],"posed":[214],"diverse":[217,232],"roles":[218],"played":[219],"crowdworkers":[221],"re-search,":[224],"discuss":[226],"implications":[228],"selecting":[230],"cohort":[233],"crowdworkerson":[235],"resulting":[236],"human-in-the-loop":[237],"feedback.":[238]},"counts_by_year":[],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}