{"id":"https://openalex.org/W4385859641","doi":"https://doi.org/10.1162/tacl_a_00586","title":"Design Choices for Crowdsourcing Implicit Discourse Relations: Revealing the Biases Introduced by Task Design","display_name":"Design Choices for Crowdsourcing Implicit Discourse Relations: Revealing the Biases Introduced by Task Design","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385859641","doi":"https://doi.org/10.1162/tacl_a_00586"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00586","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1162/tacl_a_00586","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00586/2154364/tacl_a_00586.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00586/2154364/tacl_a_00586.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034624275","display_name":"Valentina Pyatkin","orcid":null},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":true,"raw_author_name":"Valentina Pyatkin","raw_affiliation_strings":["Bar Ilan University, Ramat Gan, Israel. pyatkiv@biu.ac.il"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bar Ilan University, Ramat Gan, Israel. pyatkiv@biu.ac.il","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110213970","display_name":"Frances Yung","orcid":null},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Frances Yung","raw_affiliation_strings":["Saarland University, Saarbr\u00fccken, Germany. frances"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Saarland University, Saarbr\u00fccken, Germany. frances","institution_ids":["https://openalex.org/I91712215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081914412","display_name":"Merel Scholman","orcid":"https://orcid.org/0000-0002-0223-8464"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]},{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE","NL"],"is_corresponding":true,"raw_author_name":"Merel C. J. Scholman","raw_affiliation_strings":["Saarland University, Saarbr\u00fccken, Germany. m.c.j.scholman@coli.uni-saarland.de","Utrecht University, Utrecht, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Saarland University, Saarbr\u00fccken, Germany. m.c.j.scholman@coli.uni-saarland.de","institution_ids":["https://openalex.org/I91712215"]},{"raw_affiliation_string":"Utrecht University, Utrecht, Netherlands","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063283689","display_name":"Reut Tsarfaty","orcid":"https://orcid.org/0000-0001-6406-6568"},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":true,"raw_author_name":"Reut Tsarfaty","raw_affiliation_strings":["Bar Ilan University, Ramat Gan, Israel. reut.tsarfaty@biu.ac.il"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bar Ilan University, Ramat Gan, Israel. reut.tsarfaty@biu.ac.il","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033006575","display_name":"Ido Dagan","orcid":"https://orcid.org/0000-0003-3415-3374"},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":true,"raw_author_name":"Ido Dagan","raw_affiliation_strings":["Bar Ilan University, Ramat Gan, Israel. dagan@cs.biu.ac.il"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bar Ilan University, Ramat Gan, Israel. dagan@cs.biu.ac.il","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023605306","display_name":"Vera Demberg","orcid":"https://orcid.org/0000-0002-8834-0020"},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Vera Demberg","raw_affiliation_strings":["Saarland University, Saarbr\u00fccken, Germany. vera@coli.uni-saarland.de"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Saarland University, Saarbr\u00fccken, Germany. vera@coli.uni-saarland.de","institution_ids":["https://openalex.org/I91712215"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5023605306","https://openalex.org/A5033006575","https://openalex.org/A5034624275","https://openalex.org/A5063283689","https://openalex.org/A5081914412"],"corresponding_institution_ids":["https://openalex.org/I13955877","https://openalex.org/I193662353","https://openalex.org/I91712215"],"apc_list":null,"apc_paid":null,"fwci":0.4604,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.7283522,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"11","issue":null,"first_page":"1014","last_page":"1032"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9027591943740845},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8695264458656311},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8434977531433105},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7375327348709106},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.6905812621116638},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6725850105285645},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.611361563205719},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.5815472602844238},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5572813749313354},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.5000259876251221},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.46136680245399475},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.09905955195426941},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.07816383242607117}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9027591943740845},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8695264458656311},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8434977531433105},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7375327348709106},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.6905812621116638},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6725850105285645},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.611361563205719},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.5815472602844238},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5572813749313354},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.5000259876251221},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.46136680245399475},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09905955195426941},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.07816383242607117},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00586","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1162/tacl_a_00586","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00586/2154364/tacl_a_00586.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:dd3f71f900c741cea9d009dab0e2995e","is_oa":true,"landing_page_url":"https://doaj.org/article/dd3f71f900c741cea9d009dab0e2995e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 11 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00586","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1162/tacl_a_00586","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00586/2154364/tacl_a_00586.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4399999976158142,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1595172143","display_name":null,"funder_award_id":"501100001659","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G5511866461","display_name":null,"funder_award_id":"10.13039/501100001659","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G8568307836","display_name":null,"funder_award_id":"SFB1102","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320322252","display_name":"Israel Science Foundation","ror":"https://ror.org/04sazxf24"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385859641.pdf","grobid_xml":"https://content.openalex.org/works/W4385859641.grobid-xml"},"referenced_works_count":82,"referenced_works":["https://openalex.org/W280304298","https://openalex.org/W1495022714","https://openalex.org/W1516184288","https://openalex.org/W1581597064","https://openalex.org/W1702329404","https://openalex.org/W1840435438","https://openalex.org/W1970381522","https://openalex.org/W1985661114","https://openalex.org/W2045738181","https://openalex.org/W2053154970","https://openalex.org/W2070327778","https://openalex.org/W2082802837","https://openalex.org/W2122906199","https://openalex.org/W2123381015","https://openalex.org/W2141766660","https://openalex.org/W2147638277","https://openalex.org/W2156354361","https://openalex.org/W2186590411","https://openalex.org/W2208202786","https://openalex.org/W2252039660","https://openalex.org/W2295951612","https://openalex.org/W2514732464","https://openalex.org/W2529095243","https://openalex.org/W2531638282","https://openalex.org/W2563826943","https://openalex.org/W2609564346","https://openalex.org/W2739499354","https://openalex.org/W2741036097","https://openalex.org/W2741389354","https://openalex.org/W2795975316","https://openalex.org/W2799915114","https://openalex.org/W2904008843","https://openalex.org/W2915085887","https://openalex.org/W2931179098","https://openalex.org/W2941486823","https://openalex.org/W2949637934","https://openalex.org/W2959143276","https://openalex.org/W2963323070","https://openalex.org/W2963647696","https://openalex.org/W2963766765","https://openalex.org/W2964842699","https://openalex.org/W2970911549","https://openalex.org/W2979826702","https://openalex.org/W2985347336","https://openalex.org/W2987755095","https://openalex.org/W3005295611","https://openalex.org/W3034906811","https://openalex.org/W3034928924","https://openalex.org/W3092614566","https://openalex.org/W3098467034","https://openalex.org/W3100292568","https://openalex.org/W3100363073","https://openalex.org/W3105725765","https://openalex.org/W3106510757","https://openalex.org/W3111114607","https://openalex.org/W3153661306","https://openalex.org/W3159682551","https://openalex.org/W3168194750","https://openalex.org/W3184904351","https://openalex.org/W3196841052","https://openalex.org/W3202673127","https://openalex.org/W3213754525","https://openalex.org/W3214105842","https://openalex.org/W4200444669","https://openalex.org/W4313459382","https://openalex.org/W4412273007","https://openalex.org/W6600880057","https://openalex.org/W6610134503","https://openalex.org/W6657136032","https://openalex.org/W6676325225","https://openalex.org/W6688022499","https://openalex.org/W6691144765","https://openalex.org/W6691296735","https://openalex.org/W6732215679","https://openalex.org/W6751622830","https://openalex.org/W6754050135","https://openalex.org/W6760939382","https://openalex.org/W6779068807","https://openalex.org/W6804842067","https://openalex.org/W6888772748","https://openalex.org/W6888803745","https://openalex.org/W6944868811"],"related_works":["https://openalex.org/W2612422652","https://openalex.org/W2798706400","https://openalex.org/W4306696048","https://openalex.org/W2963726954","https://openalex.org/W3118022799","https://openalex.org/W2290964183","https://openalex.org/W1905155149","https://openalex.org/W2250638193","https://openalex.org/W1982003829","https://openalex.org/W1872130062"],"abstract_inverted_index":{"Abstract":[0],"Disagreement":[1],"in":[2],"natural":[3,45,109],"language":[4,46,110],"annotation":[5,21,93,111,143],"has":[6,35,69],"mostly":[7],"been":[8,71],"studied":[9],"from":[10],"a":[11,36,66],"perspective":[12],"of":[13,30,53,85,99,151],"biases":[14,98],"introduced":[15],"by":[16],"the":[17,20,51,78,83,97,119,141],"annotators":[18,124],"and":[19,95,128,160],"frameworks.":[22],"Here,":[23],"we":[24,59],"propose":[25],"to":[26,49,73,77],"analyze":[27],"another":[28],"source":[29],"bias\u2014task":[31],"design":[32,121],"bias,":[33],"which":[34],"particularly":[37],"strong":[38],"impact":[39],"on":[40],"crowdsourced":[41],"linguistic":[42],"annotations":[43,84],"where":[44],"is":[47],"used":[48],"elicit":[50],"interpretation":[52],"lay":[54],"annotators.":[55],"For":[56],"this":[57,149],"purpose":[58],"look":[60],"at":[61],"implicit":[62],"discourse":[63,87,131],"relation":[64,132],"annotation,":[65],"task":[67,120],"that":[68,118,129,148],"repeatedly":[70],"shown":[72],"be":[74,135,154],"difficult":[75],"due":[76],"relations\u2019":[79],"ambiguity.":[80],"We":[81,116,145],"compare":[82],"1,200":[86],"relations":[88,127],"obtained":[89],"using":[90],"two":[91],"distinct":[92],"tasks":[94,112],"quantify":[96],"both":[100],"methods":[101,107],"across":[102],"four":[103],"different":[104],"domains.":[105],"Both":[106],"are":[108],"designed":[113],"for":[114],"crowdsourcing.":[115],"show":[117],"can":[122,134],"push":[123],"towards":[125],"certain":[126],"some":[130],"senses":[133],"better":[136],"elicited":[137],"with":[138],"one":[139],"or":[140],"other":[142],"approach.":[144],"also":[146],"conclude":[147],"type":[150],"bias":[152],"should":[153],"taken":[155],"into":[156],"account":[157],"when":[158],"training":[159],"testing":[161],"models.":[162]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}