{"id":"https://openalex.org/W6960210614","doi":"https://doi.org/10.13016/hwza-vyz4","title":"Gathering Natural Language Processing Data Using Experts","display_name":"Gathering Natural Language Processing Data Using Experts","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W6960210614","doi":"https://doi.org/10.13016/hwza-vyz4"},"language":"en","primary_location":{"id":"doi:10.13016/hwza-vyz4","is_oa":true,"landing_page_url":"https://doi.org/10.13016/hwza-vyz4","pdf_url":null,"source":{"id":"https://openalex.org/S4306402644","display_name":"Digital Repository at the University of Maryland (University of Maryland College Park)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I66946132","host_organization_name":"University of Maryland, College Park","host_organization_lineage":["https://openalex.org/I66946132"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Collection"},"type":"dissertation","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.13016/hwza-vyz4","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Peskov, Denis","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Peskov, Denis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10733","display_name":"Wheat and Barley Genetics and Pathology","score":0.7161999940872192,"subfield":{"id":"https://openalex.org/subfields/1110","display_name":"Plant Science"},"field":{"id":"https://openalex.org/fields/11","display_name":"Agricultural and Biological Sciences"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10733","display_name":"Wheat and Barley Genetics and Pathology","score":0.7161999940872192,"subfield":{"id":"https://openalex.org/subfields/1110","display_name":"Plant Science"},"field":{"id":"https://openalex.org/fields/11","display_name":"Agricultural and Biological Sciences"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10012","display_name":"Genetic diversity and population structure","score":0.04659999907016754,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10434","display_name":"Chromosomal and Genetic Variations","score":0.04320000112056732,"subfield":{"id":"https://openalex.org/subfields/1110","display_name":"Plant Science"},"field":{"id":"https://openalex.org/fields/11","display_name":"Agricultural and Biological Sciences"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.6292999982833862},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5322999954223633},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5076000094413757},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5034000277519226},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.49779999256134033},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.4754999876022339},{"id":"https://openalex.org/keywords/pace","display_name":"Pace","score":0.47130000591278076},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.44440001249313354},{"id":"https://openalex.org/keywords/data-quality","display_name":"Data quality","score":0.4036000072956085}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8623999953269958},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.6292999982833862},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5322999954223633},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5117999911308289},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5076000094413757},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5034000277519226},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.49779999256134033},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.4754999876022339},{"id":"https://openalex.org/C2777526511","wikidata":"https://www.wikidata.org/wiki/Q691543","display_name":"Pace","level":2,"score":0.47130000591278076},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.44440001249313354},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4260999858379364},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.4036000072956085},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.3935999870300293},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.3783999979496002},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.3382999897003174},{"id":"https://openalex.org/C2778464652","wikidata":"https://www.wikidata.org/wiki/Q309849","display_name":"Open research","level":2,"score":0.2994000017642975},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.2912999987602234},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.28529998660087585},{"id":"https://openalex.org/C2993776861","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Open domain","level":3,"score":0.2831000089645386},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.27869999408721924},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2784000039100647},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.27250000834465027},{"id":"https://openalex.org/C56288433","wikidata":"https://www.wikidata.org/wiki/Q58673","display_name":"Data manipulation language","level":2,"score":0.2671000063419342},{"id":"https://openalex.org/C190954187","wikidata":"https://www.wikidata.org/wiki/Q5270587","display_name":"Dialog system","level":3,"score":0.26159998774528503},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2587999999523163}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.13016/hwza-vyz4","is_oa":true,"landing_page_url":"https://doi.org/10.13016/hwza-vyz4","pdf_url":null,"source":{"id":"https://openalex.org/S4306402644","display_name":"Digital Repository at the University of Maryland (University of Maryland College Park)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I66946132","host_organization_name":"University of Maryland, College Park","host_organization_lineage":["https://openalex.org/I66946132"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Collection"}],"best_oa_location":{"id":"doi:10.13016/hwza-vyz4","is_oa":true,"landing_page_url":"https://doi.org/10.13016/hwza-vyz4","pdf_url":null,"source":{"id":"https://openalex.org/S4306402644","display_name":"Digital Repository at the University of Maryland (University of Maryland College Park)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I66946132","host_organization_name":"University of Maryland, College Park","host_organization_lineage":["https://openalex.org/I66946132"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Collection"},"sustainable_development_goals":[{"score":0.40833383798599243,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Natural":[0],"language":[1,41,297],"processing":[2],"needs":[3],"substantial":[4],"data":[5,48,69,117,136,215,332],"to":[6,20,47,72,245,264,376,403],"make":[7],"robust":[8],"predictions.":[9],"Automatic":[10],"methods,":[11],"unspecialized":[12],"crowds,":[13],"and":[14,23,54,89,93,101,108,119,220,257,294,305,342,352,361,378],"domain":[15,33,212],"experts":[16,34,213,349],"can":[17,123,217],"be":[18],"used":[19,153],"collect":[21],"conversational":[22],"question":[24,59],"answering":[25,60],"NLP":[26,222,346],"datasets.":[27,223],"A":[28,43,143],"hybrid":[29],"solution":[30],"of":[31,110,116,127,164,275,282,339,380],"combining":[32],"with":[35,176,239,367,371,390],"the":[36,106,114,162,180,247,250,273,337,365,388,399,404],"crowd":[37,366,400],"generates":[38],"large-scale,":[39],"free-form":[40,396],"data.":[42,170,267],"low-cost,":[44],"high-output":[45],"approach":[46],"creation":[49],"is":[50,130,138,152,255,298],"automation.":[51],"We":[52,103,171,237,369],"create":[53,67,218,310],"analyze":[55],"a":[56,97,148,203,270,280,290,311,319,391],"large-scale":[57,135],"audio":[58],"dataset":[61],"through":[62,141,323],"text-to-speech":[63],"technology.":[64],"Additionally,":[65,308],"we":[66,208,225,260,309],"synthetic":[68],"from":[70],"templates":[71],"identify":[73],"limitations":[74],"in":[75,79,132,146,242,269,300,318,331,354,364],"machine":[76],"translation.":[77],"However,":[78,348],"Quizbowl":[80],"questions":[81],"are":[82,286,350],"read":[83],"at":[84,113],"an":[85,177],"unusually":[86],"fast":[87],"pace":[88],"involve":[90],"highly":[91],"technical":[92],"multi-cultural":[94],"words":[95],"causing":[96,202],"disparity":[98],"between":[99],"automation":[100,111],"reality.":[102],"conclude":[104],"that":[105,210,285,315],"cost-savings":[107],"scalability":[109],"come":[112],"cost":[115],"quality":[118,181,204],"naturalness.":[120],"Human":[121],"input":[122],"provide":[124],"this":[125,166,254],"degree":[126],"naturalness,":[128],"but":[129,179],"limited":[131,353],"scale.":[133],"Hence,":[134],"collection":[137,333],"frequently":[139],"done":[140],"crowd-sourcing.":[142],"question-rewriting":[144],"task,":[145],"which":[147,229],"long":[149],"information-gathering":[150],"conversation":[151,389],"as":[154],"source":[155],"material":[156],"for":[157,168,195,214,253,407],"many":[158],"stand-alone":[159],"questions,":[160],"shows":[161],"limitation":[163],"using":[165,211,398],"methodology":[167],"generating":[169],"automatically":[172],"prevent":[173],"unsatisfactory":[174],"submissions":[175],"interface,":[178],"control":[182,205],"process":[183],"requires":[184],"manually":[185],"reviewing":[186],"5,000":[187],"questions.":[188],"Standard":[189],"inter-annotator":[190],"agreement":[191],"metrics,":[192],"while":[193,288],"useful":[194],"annotation,":[196],"cannot":[197],"easily":[198],"evaluate":[199],"generated":[200],"data,":[201,248],"issue.":[206],"Therefore,":[207],"posit":[209],"generation":[216],"novel":[219,345],"reliable":[221,392],"First,":[224],"introduce":[226],"computational":[227],"adaptation,":[228],"adapts,":[230],"rather":[231],"than":[232],"translates,":[233],"entities":[234],"across":[235,384],"cultures.":[236],"work":[238,370],"native":[240],"speakers":[241],"two":[243],"countries":[244],"generate":[246,279,377],"since":[249],"gold":[251],"label":[252],"subjective":[256],"paramount.":[258],"Furthermore,":[259],"hire":[261],"professional":[262],"translators":[263],"assess":[265],"our":[266],"Last,":[268],"study":[271],"on":[272],"game":[274,291],"Diplomacy,":[276],"community":[277],"members":[278],"corpus":[281],"17,000":[283],"messages":[284],"self-annotated":[287],"playing":[289],"about":[292],"trust":[293],"deception.":[295],"The":[296,328],"varied":[299],"length,":[301],"tone,":[302],"vocabulary,":[303],"punctuation,":[304],"even":[306],"emojis.":[307],"real-time":[312],"self-annotation":[313],"system":[314],"annotates":[316],"deception":[317],"manner":[320],"not":[321],"possible":[322],"crowd-sourced":[324],"or":[325],"automatic":[326],"methods.":[327],"extra":[329],"effort":[330],"will":[334],"hopefully":[335],"ensure":[336],"longevity":[338],"these":[340,402],"datasets":[341],"galvanize":[343],"other":[344],"ideas.":[347],"expensive":[351],"number.":[355],"Hybrid":[356],"solutions":[357],"pair":[358],"potentially":[359],"unreliable":[360],"unverified":[362],"users":[363],"experts.":[368],"Amazon":[372,394],"customer":[373],"service":[374],"agents":[375],"annotate":[379],"goal-oriented":[381],"81,000":[382],"conversations":[383],"six":[385],"domains.":[386],"Grounding":[387],"conversationalist---the":[393],"agent---creates":[395],"conversations;":[397],"scales":[401],"size":[405],"needed":[406],"neural":[408],"networks.":[409]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}