{"id":"https://openalex.org/W4389519970","doi":"https://doi.org/10.18653/v1/2023.emnlp-tutorial.3","title":"Designing, Evaluating, and Learning from Humans Interacting with NLP Models","display_name":"Designing, Evaluating, and Learning from Humans Interacting with NLP Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389519970","doi":"https://doi.org/10.18653/v1/2023.emnlp-tutorial.3"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.emnlp-tutorial.3","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-tutorial.3","pdf_url":"https://aclanthology.org/2023.emnlp-tutorial.3.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2023.emnlp-tutorial.3.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004225142","display_name":"Tongshuang Wu","orcid":"https://orcid.org/0000-0003-1630-0588"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Tongshuang Wu","raw_affiliation_strings":["Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089413311","display_name":"Diyi Yang","orcid":"https://orcid.org/0000-0003-1220-3983"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Diyi Yang","raw_affiliation_strings":["Stanford University \u00b1 University of Washington"],"affiliations":[{"raw_affiliation_string":"Stanford University \u00b1 University of Washington","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047302679","display_name":"Sebastin Santy","orcid":"https://orcid.org/0000-0003-1162-0865"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sebastin Santy","raw_affiliation_strings":["Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5004225142"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":0.3491,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.67164875,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"13","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9853000044822693,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8079612851142883},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6493818759918213},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.6302952170372009},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.573357880115509},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5243126153945923},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.4627656042575836},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.45113450288772583},{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.44488388299942017},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.43049079179763794},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.42440080642700195},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4121553301811218},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.41113534569740295},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4076663553714752},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.16705286502838135}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8079612851142883},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6493818759918213},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.6302952170372009},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.573357880115509},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5243126153945923},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.4627656042575836},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45113450288772583},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.44488388299942017},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.43049079179763794},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.42440080642700195},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4121553301811218},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.41113534569740295},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4076663553714752},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.16705286502838135},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2023.emnlp-tutorial.3","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-tutorial.3","pdf_url":"https://aclanthology.org/2023.emnlp-tutorial.3.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.emnlp-tutorial.3","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-tutorial.3","pdf_url":"https://aclanthology.org/2023.emnlp-tutorial.3.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.5899999737739563,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389519970.pdf","grobid_xml":"https://content.openalex.org/works/W4389519970.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1565828035","https://openalex.org/W1986345088","https://openalex.org/W2059216172","https://openalex.org/W2130500927","https://openalex.org/W2251171258","https://openalex.org/W2404161646","https://openalex.org/W2916904544","https://openalex.org/W2966491090","https://openalex.org/W2971046749","https://openalex.org/W2997195635","https://openalex.org/W3032875465","https://openalex.org/W3035507081","https://openalex.org/W3047508744","https://openalex.org/W3116793277","https://openalex.org/W3118813946","https://openalex.org/W3128442340","https://openalex.org/W3153592532","https://openalex.org/W3153978352","https://openalex.org/W3154565472","https://openalex.org/W3160638507","https://openalex.org/W3163443091","https://openalex.org/W3174519801","https://openalex.org/W3202970975","https://openalex.org/W3204393347","https://openalex.org/W3206079177","https://openalex.org/W4220664175","https://openalex.org/W4220747294","https://openalex.org/W4225165463","https://openalex.org/W4226034056","https://openalex.org/W4226278401","https://openalex.org/W4226399820","https://openalex.org/W4231908945","https://openalex.org/W4286910674","https://openalex.org/W4287674181","https://openalex.org/W4296343374","https://openalex.org/W4297630849","https://openalex.org/W4302275696","https://openalex.org/W4385573018"],"related_works":["https://openalex.org/W2098987383","https://openalex.org/W2417260800","https://openalex.org/W1596203174","https://openalex.org/W2117933979","https://openalex.org/W2283130723","https://openalex.org/W103938586","https://openalex.org/W2104718772","https://openalex.org/W4233992201","https://openalex.org/W2980207396","https://openalex.org/W3156493709"],"abstract_inverted_index":{"The":[0],"rapid":[1],"advancement":[2],"of":[3,18,62,84,92,96,113,198],"natural":[4],"language":[5],"processing":[6],"(NLP)":[7],"research":[8],"has":[9],"led":[10],"to":[11,23,31,59,72,106,144,180,187,192,204,219,224,241],"various":[12],"applications":[13],"spanning":[14],"a":[15,82,108],"wide":[16],"range":[17],"domains":[19],"that":[20,194,212],"require":[21],"models":[22,146,195,229],"interact":[24],"with":[25,151,158,200,216],"humans":[26,55],"\u2013":[27,135,154],"e.g.,":[28],"chatbots":[29],"responding":[30],"human":[32,38,232],"inquiries,":[33],"machine":[34],"translation":[35],"systems":[36],"assisting":[37],"translators,":[39],"designers":[40],"prompting":[41],"Large":[42],"Language":[43],"Models":[44],"for":[45,119],"co-creation":[46],"or":[47,68,156],"prototyping":[48],"AI-infused":[49],"applications,":[50],"etc.":[51,102],"In":[52],"these":[53],"cases,":[54],"interaction":[56,80,210],"is":[57],"key":[58,114],"the":[60,76,90,94,130,162,231],"success":[61],"NLP":[63,145,152,220,228],"applications;":[64],"any":[65],"potential":[66],"misconceptions":[67],"differences":[69],"might":[70],"lead":[71],"error":[73],"cascades":[74],"at":[75],"subsequent":[77],"stages.":[78],"Such":[79],"involves":[81],"lot":[83],"design":[85,97,205],"choices":[86],"around":[87],"models,":[88],"e.g.":[89],"sensitivity":[91],"interfaces,":[93],"impact":[95],"choice":[98],"and":[99,110,116,138,178,209,226,245,250],"evaluation":[100],"questions,":[101],"This":[103],"tutorial":[104,125],"aims":[105],"provide":[107,213],"systematic":[109],"up-to-date":[111],"overview":[112],"considerations":[115],"effective":[117],"approaches":[118],"studying":[120],"human-NLP":[121],"model":[122],"interactions.":[123,233],"Our":[124],"will":[126,165,235,246],"focus":[127],"specifically":[128],"on":[129],"scenario":[131],"where":[132],"end":[133,214],"users":[134,215],"lay":[136],"people":[137],"domain":[139],"experts":[140],"who":[141],"have":[142],"access":[143,218],"but":[147],"are":[148,196],"less":[149],"familiar":[150],"techniques":[153],"use":[155,166,236],"collaborate":[157],"deployed":[159],"models.":[160],"Throughout":[161],"tutorial,":[163],"we":[164],"five":[167],"case":[168],"studies":[169],"(on":[170],"classifier-assisted":[171],"decision":[172],"making,":[173],"machine-aided":[174],"translation,":[175],"dialog":[176],"systems,":[177],"prompting)":[179],"cover":[181],"three":[182],"major":[183],"themes:":[184],"(1)":[185],"how":[186,203,223],"conduct":[188],"human-in-the-loop":[189],"usability":[190],"evaluations":[191],"ensure":[193],"capable":[197],"interacting":[199],"humans;":[201],"(2)":[202],"user":[206],"interfaces":[207],"(UIs)":[208],"mechanisms":[211],"easy":[217],"models;":[221],"(3)":[222],"learn":[225],"improve":[227],"through":[230],"We":[234],"best":[237],"practices":[238],"from":[239],"HCI":[240],"ground":[242],"our":[243],"discussion,":[244],"highlight":[247],"current":[248],"challenges":[249],"future":[251],"directions.":[252]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-01-23T23:20:30.427331","created_date":"2025-10-10T00:00:00"}