{"id":"https://openalex.org/W3194338218","doi":"https://doi.org/10.1145/3465336.3475109","title":"What Is Unclear? Computational Assessment of Task Clarity in Crowdsourcing","display_name":"What Is Unclear? Computational Assessment of Task Clarity in Crowdsourcing","publication_year":2021,"publication_date":"2021-08-25","ids":{"openalex":"https://openalex.org/W3194338218","doi":"https://doi.org/10.1145/3465336.3475109","mag":"3194338218"},"language":"en","primary_location":{"id":"doi:10.1145/3465336.3475109","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3465336.3475109","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32st ACM Conference on Hypertext and Social Media","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075931228","display_name":"Zahra Nouri","orcid":"https://orcid.org/0000-0002-7551-6242"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Zahra Nouri","raw_affiliation_strings":["Paderborn University, Paderborn, Germany"],"affiliations":[{"raw_affiliation_string":"Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038081564","display_name":"Ujwal Gadiraju","orcid":"https://orcid.org/0000-0002-6189-6539"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Ujwal Gadiraju","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022823882","display_name":"Gregor Engels","orcid":"https://orcid.org/0000-0001-5397-9548"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gregor Engels","raw_affiliation_strings":["Paderborn University, Paderborn, Germany"],"affiliations":[{"raw_affiliation_string":"Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014375244","display_name":"Henning Wachsmuth","orcid":"https://orcid.org/0000-0003-2792-621X"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Henning Wachsmuth","raw_affiliation_strings":["Paderborn University, Paderborn, Germany"],"affiliations":[{"raw_affiliation_string":"Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5075931228"],"corresponding_institution_ids":["https://openalex.org/I206945453"],"apc_list":null,"apc_paid":null,"fwci":1.6986,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.86872647,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"165","last_page":"175"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9872000217437744,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9385578632354736},{"id":"https://openalex.org/keywords/clarity","display_name":"CLARITY","score":0.7799657583236694},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6971210241317749},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6763802766799927},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3418150544166565},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.16224294900894165},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1103346049785614},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.08637747168540955},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.078622967004776}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9385578632354736},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.7799657583236694},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6971210241317749},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6763802766799927},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3418150544166565},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.16224294900894165},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1103346049785614},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.08637747168540955},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.078622967004776},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3465336.3475109","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3465336.3475109","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32st ACM Conference on Hypertext and Social Media","raw_type":"proceedings-article"},{"id":"pmh:oai:tudelft.nl:uuid:bee2b753-d5a9-4958-a673-1d17bfae896d","is_oa":false,"landing_page_url":"http://resolver.tudelft.nl/uuid:bee2b753-d5a9-4958-a673-1d17bfae896d","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4300000071525574,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W33555860","https://openalex.org/W1497983610","https://openalex.org/W1507711477","https://openalex.org/W1509228788","https://openalex.org/W2012964103","https://openalex.org/W2036367260","https://openalex.org/W2058179030","https://openalex.org/W2065049201","https://openalex.org/W2070248097","https://openalex.org/W2116915306","https://openalex.org/W2119787124","https://openalex.org/W2127008633","https://openalex.org/W2132481689","https://openalex.org/W2136222780","https://openalex.org/W2136606893","https://openalex.org/W2148479118","https://openalex.org/W2151401338","https://openalex.org/W2153770157","https://openalex.org/W2168144930","https://openalex.org/W2173800084","https://openalex.org/W2251818274","https://openalex.org/W2380756684","https://openalex.org/W2398514781","https://openalex.org/W2547436379","https://openalex.org/W2588579248","https://openalex.org/W2598654328","https://openalex.org/W2607311634","https://openalex.org/W2726137762","https://openalex.org/W2741340396","https://openalex.org/W2794219599","https://openalex.org/W2799915114","https://openalex.org/W2896881735","https://openalex.org/W2898764742","https://openalex.org/W2913650309","https://openalex.org/W2913841425","https://openalex.org/W2946474696","https://openalex.org/W2974398320","https://openalex.org/W2998679789","https://openalex.org/W3047840587","https://openalex.org/W3093239427","https://openalex.org/W3118140671","https://openalex.org/W3121257585","https://openalex.org/W3124258878","https://openalex.org/W4205337489","https://openalex.org/W4206744772","https://openalex.org/W4235819089","https://openalex.org/W4236157893","https://openalex.org/W4239083664","https://openalex.org/W4289361798"],"related_works":["https://openalex.org/W2827818489","https://openalex.org/W4309046940","https://openalex.org/W2785516643","https://openalex.org/W2756443057","https://openalex.org/W3194338218","https://openalex.org/W2726137762","https://openalex.org/W3155158955","https://openalex.org/W3128051602","https://openalex.org/W2809632469","https://openalex.org/W151193258"],"abstract_inverted_index":{"Designing":[0],"tasks":[1,175],"clearly":[2],"to":[3,24,34,47,97,101,194],"facilitate":[4],"accurate":[5],"task":[6,29,60,79,106,120,135,157,199,213],"completion":[7],"is":[8],"a":[9,123],"challenging":[10],"endeavor":[11],"for":[12,208],"requesters":[13,22,196],"on":[14,189,201],"crowdsourcing":[15,202],"platforms.":[16,203],"Prior":[17],"research":[18],"shows":[19],"that":[20,44,81,118,150,168],"inexperienced":[21],"fail":[23],"write":[25],"clear":[26],"and":[27,73,91,112,172],"complete":[28],"descriptions":[30,61,80,107,117,158,176],"which":[31,102],"directly":[32],"leads":[33],"low":[35],"quality":[36],"submissions":[37],"from":[38],"workers.":[39],"By":[40],"complementing":[41],"existing":[42],"works":[43],"have":[45],"aimed":[46],"address":[48],"this":[49,52,138],"challenge,":[50],"in":[51,59,78,84,95,105,132,156,180,197,211],"paper":[53],"we":[54,126,140],"study":[55,98],"whether":[56],"clarity":[57,76,103,130,154,200],"flaws":[58,77,104,131,155],"can":[62,108,159,206],"be":[63,109,160],"identified":[64],"automatically":[65],"using":[66],"natural":[67],"language":[68],"processing":[69],"methods.":[70],"We":[71,87,166],"identify":[72],"synthesize":[74],"seven":[75],"are":[82,177],"grounded":[83],"relevant":[85],"literature.":[86],"build":[88],"both":[89],"BERT-based":[90],"feature-based":[92],"binary":[93],"classifiers,":[94],"order":[96],"the":[99,145,153,164,169,190],"extent":[100],"computationally":[110],"assessed,":[111],"understand":[113],"textual":[114],"properties":[115,205],"of":[116,129,144,174,192],"affect":[119],"clarity.":[121,183],"Through":[122],"crowdsourced":[124],"study,":[125],"collect":[127],"annotations":[128],"1332":[133],"real":[134],"descriptions.":[136,214],"Using":[137],"dataset,":[139],"evaluate":[141],"several":[142],"configurations":[143],"classifiers.":[146,165],"Our":[147],"results":[148],"indicate":[149],"nearly":[151],"all":[152],"assessed":[161],"reasonably":[162],"by":[163],"found":[167],"content,":[170],"style,":[171],"readability":[173],"particularly":[178],"important":[179,187],"shaping":[181],"their":[182],"This":[184],"work":[185],"has":[186],"implications":[188],"design":[191],"tools":[193],"help":[195],"improving":[198,212],"Flaw-specific":[204],"provide":[207],"valuable":[209],"guidance":[210]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}