{"id":"https://openalex.org/W3171875349","doi":"https://doi.org/10.1162/coli_a_00436","title":"Annotation Curricula to Implicitly Train Non-Expert Annotators","display_name":"Annotation Curricula to Implicitly Train Non-Expert Annotators","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W3171875349","doi":"https://doi.org/10.1162/coli_a_00436","mag":"3171875349"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00436","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00436","pdf_url":"https://direct.mit.edu/coli/article-pdf/48/2/343/2029108/coli_a_00436.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/coli/article-pdf/48/2/343/2029108/coli_a_00436.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081951157","display_name":"Ji-Ung Lee","orcid":"https://orcid.org/0000-0002-8428-2003"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Ji-Ung Lee","raw_affiliation_strings":["UKP Lab / TU Darmstadt. lee@ukp.informatik.tu-darmstadt.de"],"affiliations":[{"raw_affiliation_string":"UKP Lab / TU Darmstadt. lee@ukp.informatik.tu-darmstadt.de","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070380110","display_name":"Jan-Christoph Klie","orcid":"https://orcid.org/0000-0003-0181-6450"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Jan-Christoph Klie","raw_affiliation_strings":["UKP Lab / TU Darmstadt. klie@ukp.informatik.tu-darmstadt.de"],"affiliations":[{"raw_affiliation_string":"UKP Lab / TU Darmstadt. klie@ukp.informatik.tu-darmstadt.de","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027450194","display_name":"Iryna Gurevych","orcid":"https://orcid.org/0000-0003-2187-7621"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Iryna Gurevych","raw_affiliation_strings":["UKP Lab / TU Darmstadt. gurevych@ukp.informatik.tu-darmstadt.de"],"affiliations":[{"raw_affiliation_string":"UKP Lab / TU Darmstadt. gurevych@ukp.informatik.tu-darmstadt.de","institution_ids":["https://openalex.org/I31512782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5027450194","https://openalex.org/A5070380110","https://openalex.org/A5081951157"],"corresponding_institution_ids":["https://openalex.org/I31512782"],"apc_list":null,"apc_paid":null,"fwci":1.6842,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.86990661,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"48","issue":"2","first_page":"343","last_page":"373"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.9023526310920715},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8301535844802856},{"id":"https://openalex.org/keywords/curriculum","display_name":"Curriculum","score":0.7060133218765259},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.62155681848526},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.5937882661819458},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.5737460255622864},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5190735459327698},{"id":"https://openalex.org/keywords/paragraph","display_name":"Paragraph","score":0.4724731147289276},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4260937571525574},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.42285582423210144},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.41512390971183777},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4071022868156433},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40289583802223206},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2324087917804718}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.9023526310920715},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8301535844802856},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.7060133218765259},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.62155681848526},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.5937882661819458},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.5737460255622864},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5190735459327698},{"id":"https://openalex.org/C2777206241","wikidata":"https://www.wikidata.org/wiki/Q194431","display_name":"Paragraph","level":2,"score":0.4724731147289276},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4260937571525574},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.42285582423210144},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.41512390971183777},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4071022868156433},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40289583802223206},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2324087917804718},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/coli_a_00436","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00436","pdf_url":"https://direct.mit.edu/coli/article-pdf/48/2/343/2029108/coli_a_00436.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:tubiblio.ulb.tu-darmstadt.de:130009","is_oa":true,"landing_page_url":"https://direct.mit.edu/coli/article/48/2/343/109903/Annotation-Curricula-to-Implicitly-Train-Non","pdf_url":null,"source":{"id":"https://openalex.org/S4377196390","display_name":"TUbilio (Technical University of Darmstadt)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I31512782","host_organization_name":"Technische Universit\u00e4t Darmstadt","host_organization_lineage":["https://openalex.org/I31512782"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Artikel"},{"id":"pmh:oai:doaj.org/article:5b25a81bccc24a7e917d571292f34883","is_oa":true,"landing_page_url":"https://doaj.org/article/5b25a81bccc24a7e917d571292f34883","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 48, Iss 2 (2022)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00436","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00436","pdf_url":"https://direct.mit.edu/coli/article-pdf/48/2/343/2029108/coli_a_00436.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.44999998807907104,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3171875349.pdf","grobid_xml":"https://content.openalex.org/works/W3171875349.grobid-xml"},"referenced_works_count":135,"referenced_works":["https://openalex.org/W40549020","https://openalex.org/W136468075","https://openalex.org/W141030562","https://openalex.org/W1484084878","https://openalex.org/W1507711477","https://openalex.org/W1516184288","https://openalex.org/W1528012351","https://openalex.org/W1532558977","https://openalex.org/W1818395637","https://openalex.org/W1825499807","https://openalex.org/W1840435438","https://openalex.org/W1947674219","https://openalex.org/W1970381522","https://openalex.org/W1978633512","https://openalex.org/W1995470018","https://openalex.org/W2004022065","https://openalex.org/W2012878613","https://openalex.org/W2041282815","https://openalex.org/W2044275064","https://openalex.org/W2057102944","https://openalex.org/W2067177016","https://openalex.org/W2074790166","https://openalex.org/W2085989833","https://openalex.org/W2101234009","https://openalex.org/W2102162869","https://openalex.org/W2108682032","https://openalex.org/W2135943618","https://openalex.org/W2144245357","https://openalex.org/W2152748974","https://openalex.org/W2171671120","https://openalex.org/W2172235535","https://openalex.org/W2250941251","https://openalex.org/W2251026067","https://openalex.org/W2251387352","https://openalex.org/W2270070752","https://openalex.org/W2293720556","https://openalex.org/W2295692079","https://openalex.org/W2296073425","https://openalex.org/W2304127486","https://openalex.org/W2514166524","https://openalex.org/W2518589307","https://openalex.org/W2559655401","https://openalex.org/W2576049115","https://openalex.org/W2596585349","https://openalex.org/W2597787948","https://openalex.org/W2612690371","https://openalex.org/W2615071493","https://openalex.org/W2616721392","https://openalex.org/W2768348081","https://openalex.org/W2807405475","https://openalex.org/W2887428522","https://openalex.org/W2889470921","https://openalex.org/W2899689163","https://openalex.org/W2912280546","https://openalex.org/W2913668833","https://openalex.org/W2914236322","https://openalex.org/W2914331073","https://openalex.org/W2946085671","https://openalex.org/W2948086026","https://openalex.org/W2951048105","https://openalex.org/W2951911250","https://openalex.org/W2962739339","https://openalex.org/W2962784628","https://openalex.org/W2963341956","https://openalex.org/W2963750782","https://openalex.org/W2964282813","https://openalex.org/W2970352191","https://openalex.org/W2970442950","https://openalex.org/W2970450573","https://openalex.org/W2970641574","https://openalex.org/W2970693987","https://openalex.org/W2978017171","https://openalex.org/W2982235587","https://openalex.org/W2984472034","https://openalex.org/W2989205083","https://openalex.org/W2990138404","https://openalex.org/W2995188922","https://openalex.org/W2996908057","https://openalex.org/W2997892236","https://openalex.org/W3003196939","https://openalex.org/W3011890417","https://openalex.org/W3013854004","https://openalex.org/W3024308166","https://openalex.org/W3034238904","https://openalex.org/W3034653145","https://openalex.org/W3034775979","https://openalex.org/W3034850762","https://openalex.org/W3035433211","https://openalex.org/W3037207300","https://openalex.org/W3037554857","https://openalex.org/W3037629406","https://openalex.org/W3047398590","https://openalex.org/W3087841442","https://openalex.org/W3088669183","https://openalex.org/W3097301690","https://openalex.org/W3099585677","https://openalex.org/W3101345273","https://openalex.org/W3103154346","https://openalex.org/W3118813946","https://openalex.org/W3119810598","https://openalex.org/W3177058128","https://openalex.org/W3177173791","https://openalex.org/W3194676777","https://openalex.org/W3199139675","https://openalex.org/W3212368439","https://openalex.org/W4206723194","https://openalex.org/W4231040763","https://openalex.org/W4234188104","https://openalex.org/W4238634189","https://openalex.org/W4250944796","https://openalex.org/W6601618085","https://openalex.org/W6605564802","https://openalex.org/W6605834934","https://openalex.org/W6628826841","https://openalex.org/W6638584940","https://openalex.org/W6640559804","https://openalex.org/W6653695713","https://openalex.org/W6675206559","https://openalex.org/W6675354045","https://openalex.org/W6679390333","https://openalex.org/W6680836039","https://openalex.org/W6685206407","https://openalex.org/W6731948381","https://openalex.org/W6735374517","https://openalex.org/W6745609711","https://openalex.org/W6748634344","https://openalex.org/W6753942111","https://openalex.org/W6755207826","https://openalex.org/W6763607942","https://openalex.org/W6764588566","https://openalex.org/W6768851824","https://openalex.org/W6772370209","https://openalex.org/W6772935423","https://openalex.org/W6783368674","https://openalex.org/W6787794667"],"related_works":["https://openalex.org/W3177062893","https://openalex.org/W3125143773","https://openalex.org/W2007032764","https://openalex.org/W803550684","https://openalex.org/W2483226803","https://openalex.org/W3143937874","https://openalex.org/W4312926500","https://openalex.org/W2067280619","https://openalex.org/W4251343851","https://openalex.org/W4352977312"],"abstract_inverted_index":{"Abstract":[0],"Annotation":[1,177],"studies":[2],"often":[3],"require":[4],"annotators":[5,70],"to":[6,60,67,77,81,139,160,186,195,199],"familiarize":[7],"themselves":[8],"with":[9,132],"the":[10,16,24,32,72,141,149,167,210],"task,":[11],"its":[12],"annotation":[13,55,91,97,124,169,175,197,204],"scheme,":[14],"and":[15,28,95,103,107,202,207],"data":[17,188,208],"domain.":[18],"This":[19],"can":[20,163,180],"be":[21,78,181],"overwhelming":[22],"in":[23,36,126],"beginning,":[25],"mentally":[26],"taxing,":[27],"induce":[29],"errors":[30],"into":[31,71],"resulting":[33],"annotations;":[34],"especially":[35],"citizen":[37],"science":[38],"or":[39],"crowdsourcing":[40],"scenarios":[41],"where":[42],"domain":[43],"expertise":[44],"is":[45,66,217],"not":[46],"required.":[47],"To":[48,85,190],"alleviate":[49],"these":[50],"issues,":[51],"this":[52,88],"work":[53,89],"proposes":[54],"curricula,":[56],"a":[57,82,119,127,157,173,182],"novel":[58],"approach":[59],"implicitly":[61],"train":[62],"annotators.":[63],"The":[64,152],"goal":[65],"gradually":[68],"introduce":[69],"task":[73],"by":[74],"ordering":[75,101],"instances":[76,162],"annotated":[79],"according":[80],"learning":[83],"curriculum.":[84],"do":[86],"so,":[87],"formalizes":[90],"curricula":[92,125,178,198],"for":[93,123,145],"sentence-":[94],"paragraph-level":[96],"tasks,":[98],"defines":[99],"an":[100],"strategy,":[102],"identifies":[104],"well-performing":[105],"heuristics":[106],"interactively":[108],"trained":[109],"models":[110],"on":[111],"three":[112],"existing":[113],"English":[114,146],"datasets.":[115],"Finally,":[116],"we":[117],"provide":[118],"proof":[120],"of":[121,214],"concept":[122],"carefully":[128],"designed":[129],"user":[130,211],"study":[131,212],"40":[133],"voluntary":[134],"participants":[135],"who":[136],"are":[137],"asked":[138],"identify":[140],"most":[142],"fitting":[143],"misconception":[144],"tweets":[147],"about":[148],"Covid-19":[150],"pandemic.":[151],"results":[153],"indicate":[154],"that":[155],"using":[156],"simple":[158],"heuristic":[159],"order":[161],"already":[164],"significantly":[165],"reduce":[166],"total":[168],"time":[170],"while":[171],"preserving":[172],"high":[174],"quality.":[176],"thus":[179],"promising":[183],"research":[184],"direction":[185],"improve":[187],"collection.":[189],"facilitate":[191],"future":[192],"research\u2014for":[193],"instance,":[194],"adapt":[196],"specific":[200],"tasks":[201],"expert":[203],"scenarios\u2014all":[205],"code":[206],"from":[209],"consisting":[213],"2,400":[215],"annotations":[216],"made":[218],"available.1":[219]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1}],"updated_date":"2026-02-26T08:16:20.718346","created_date":"2025-10-10T00:00:00"}