{"id":"https://openalex.org/W2946140580","doi":"https://doi.org/10.1162/tacl_a_00279","title":"Trick Me If You Can: Human-in-the-Loop Generation of Adversarial Examples for Question Answering","display_name":"Trick Me If You Can: Human-in-the-Loop Generation of Adversarial Examples for Question Answering","publication_year":2019,"publication_date":"2019-07-26","ids":{"openalex":"https://openalex.org/W2946140580","doi":"https://doi.org/10.1162/tacl_a_00279","mag":"2946140580"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00279","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00279","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00279/1923125/tacl_a_00279.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00279/1923125/tacl_a_00279.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110564361","display_name":"Eric Wallace","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122018","display_name":"University of Maryland Extension","ror":"https://ror.org/03r8q5f36","country_code":"US","type":"education","lineage":["https://openalex.org/I4210122018"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eric Wallace","raw_affiliation_strings":["EE and UMIACS, University of Maryland","Univ. of Maryland,#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EE and UMIACS, University of Maryland","institution_ids":["https://openalex.org/I4210122018"]},{"raw_affiliation_string":"Univ. of Maryland,#TAB#","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100707159","display_name":"Pedro Rodr\u00edguez","orcid":"https://orcid.org/0000-0001-8572-0725"},"institutions":[{"id":"https://openalex.org/I4210122018","display_name":"University of Maryland Extension","ror":"https://ror.org/03r8q5f36","country_code":"US","type":"education","lineage":["https://openalex.org/I4210122018"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pedro Rodriguez","raw_affiliation_strings":["CS, UMIACS, University of Maryland","Univ. of Maryland,#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CS, UMIACS, University of Maryland","institution_ids":["https://openalex.org/I4210122018"]},{"raw_affiliation_string":"Univ. of Maryland,#TAB#","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057958949","display_name":"Shi Feng","orcid":"https://orcid.org/0000-0002-2846-7652"},"institutions":[{"id":"https://openalex.org/I4210122018","display_name":"University of Maryland Extension","ror":"https://ror.org/03r8q5f36","country_code":"US","type":"education","lineage":["https://openalex.org/I4210122018"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shi Feng","raw_affiliation_strings":["CS, UMIACS, University of Maryland","Univ. of Maryland,#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CS, UMIACS, University of Maryland","institution_ids":["https://openalex.org/I4210122018"]},{"raw_affiliation_string":"Univ. of Maryland,#TAB#","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083459020","display_name":"Ikuya Yamada","orcid":"https://orcid.org/0000-0003-2340-131X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ikuya Yamada","raw_affiliation_strings":["Studio Ousia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Studio Ousia","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081307846","display_name":"Jordan Boyd\u2010Graber","orcid":"https://orcid.org/0000-0002-7770-4431"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jordan Boyd-Graber","raw_affiliation_strings":["CS, iSchool, UMIACS, LSC, University of Maryland","Univ. of Maryland,#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CS, iSchool, UMIACS, LSC, University of Maryland","institution_ids":[]},{"raw_affiliation_string":"Univ. of Maryland,#TAB#","institution_ids":["https://openalex.org/I66946132"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3192,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.68728244,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"7","issue":null,"first_page":"387","last_page":"401"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.9490506649017334},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7336499691009521},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7311139106750488},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5286393761634827},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.48928162455558777},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.48830103874206543},{"id":"https://openalex.org/keywords/cover","display_name":"Cover (algebra)","score":0.43864360451698303},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.43555450439453125},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.35821056365966797},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08354219794273376}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.9490506649017334},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7336499691009521},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7311139106750488},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5286393761634827},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.48928162455558777},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.48830103874206543},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.43864360451698303},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.43555450439453125},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.35821056365966797},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08354219794273376},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1162/tacl_a_00279","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00279","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00279/1923125/tacl_a_00279.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1809.02701","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1809.02701","pdf_url":"https://arxiv.org/pdf/1809.02701","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2946140580","is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1809.02701","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:doaj.org/article:1dd0a9a941164ca08ee56ff3d58b3044","is_oa":true,"landing_page_url":"https://doaj.org/article/1dd0a9a941164ca08ee56ff3d58b3044","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 7, Pp 387-401 (2019)","raw_type":"article"},{"id":"doi:10.48550/arxiv.1809.02701","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1809.02701","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00279","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00279","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00279/1923125/tacl_a_00279.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.7099999785423279,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1668375095","display_name":"CAREER: Human-Computer Cooperation for Word-by-Word Question Answering","funder_award_id":"1822494","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G211012689","display_name":null,"funder_award_id":"HR0011-15-C-0113","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G5264030819","display_name":"III: Medium: Collaborative Research: Closing the User-Model Loop for Understanding Topics in Large Document Collections","funder_award_id":"1409287","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6217427624","display_name":null,"funder_award_id":"IIS-1409287","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307943","display_name":"Raytheon Company","ror":"https://ror.org/0354t7b78"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2946140580.pdf","grobid_xml":"https://content.openalex.org/works/W2946140580.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W242120643","https://openalex.org/W1964162497","https://openalex.org/W2144501137","https://openalex.org/W2157331557","https://openalex.org/W2171278097","https://openalex.org/W2250473257","https://openalex.org/W2250539671","https://openalex.org/W2282821441","https://openalex.org/W2295586832","https://openalex.org/W2427527485","https://openalex.org/W2473344385","https://openalex.org/W2528321558","https://openalex.org/W2562979205","https://openalex.org/W2600110521","https://openalex.org/W2612431505","https://openalex.org/W2657631929","https://openalex.org/W2749807327","https://openalex.org/W2765383698","https://openalex.org/W2791469229","https://openalex.org/W2791882636","https://openalex.org/W2798665661","https://openalex.org/W2798858969","https://openalex.org/W2799007037","https://openalex.org/W2799054028","https://openalex.org/W2799194071","https://openalex.org/W2893424960","https://openalex.org/W2898662126","https://openalex.org/W2906152891","https://openalex.org/W2912924812","https://openalex.org/W2919420119","https://openalex.org/W2951622387","https://openalex.org/W2962713901","https://openalex.org/W2962727366","https://openalex.org/W2962736243","https://openalex.org/W2962809918","https://openalex.org/W2962851944","https://openalex.org/W2963126845","https://openalex.org/W2963159690","https://openalex.org/W2963223515","https://openalex.org/W2963341956","https://openalex.org/W2963394326","https://openalex.org/W2963612171","https://openalex.org/W2963661177","https://openalex.org/W2963969878","https://openalex.org/W2964153729","https://openalex.org/W3099049262","https://openalex.org/W3140009346"],"related_works":["https://openalex.org/W2759063673","https://openalex.org/W2852524866","https://openalex.org/W2901989000","https://openalex.org/W3091373078","https://openalex.org/W3103326498","https://openalex.org/W3205028211","https://openalex.org/W2596832594","https://openalex.org/W99602827","https://openalex.org/W212208591","https://openalex.org/W2967994095","https://openalex.org/W2970449539","https://openalex.org/W2068793242","https://openalex.org/W2962967746","https://openalex.org/W2970536767","https://openalex.org/W2981638363","https://openalex.org/W3209619108","https://openalex.org/W2903073381","https://openalex.org/W3166444100","https://openalex.org/W2742616629","https://openalex.org/W2251987652"],"abstract_inverted_index":{"Adversarial":[0],"evaluation":[1],"stress-tests":[2],"a":[3,59],"model\u2019s":[4],"understanding":[5],"of":[6,45],"natural":[7],"language.":[8],"Because":[9],"past":[10],"approaches":[11],"expose":[12],"superficial":[13],"patterns,":[14],"the":[15,41,81],"resulting":[16,72],"adversarial":[17,29,69,96],"examples":[18],"are":[19,34,74],"limited":[20],"in":[21,111],"complexity":[22],"and":[23,91],"diversity.":[24],"We":[25,39,53],"propose":[26],"human-":[27],"in-the-loop":[28],"generation,":[30],"where":[31,65],"human":[32],"authors":[33,42],"guided":[35],"to":[36,58,85,104],"break":[37],"models.":[38,94],"aid":[40],"with":[43],"interpretations":[44],"model":[46],"predictions":[47],"through":[48],"an":[49],"interactive":[50],"user":[51],"interface.":[52],"apply":[54],"this":[55],"generation":[56],"framework":[57],"question":[60,113],"answering":[61],"task":[62],"called":[63],"Quizbowl,":[64],"trivia":[66],"enthusiasts":[67],"craft":[68],"questions.":[70],"The":[71,95],"questions":[73,82,97],"validated":[75],"via":[76],"live":[77],"human\u2013computer":[78],"matches:":[79],"Although":[80],"appear":[83],"ordinary":[84],"humans,":[86],"they":[87],"systematically":[88],"stump":[89],"neural":[90],"information":[92],"retrieval":[93],"cover":[98],"diverse":[99],"phenomena":[100],"from":[101],"multi-hop":[102],"reasoning":[103],"entity":[105],"type":[106],"distractors,":[107],"exposing":[108],"open":[109],"challenges":[110],"robust":[112],"answering.":[114]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}