{"id":"https://openalex.org/W4360991133","doi":"https://doi.org/10.1145/3581641.3584039","title":"Supporting Requesters in Writing Clear Crowdsourcing Task Descriptions Through Computational Flaw Assessment","display_name":"Supporting Requesters in Writing Clear Crowdsourcing Task Descriptions Through Computational Flaw Assessment","publication_year":2023,"publication_date":"2023-03-27","ids":{"openalex":"https://openalex.org/W4360991133","doi":"https://doi.org/10.1145/3581641.3584039"},"language":"en","primary_location":{"id":"doi:10.1145/3581641.3584039","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3581641.3584039","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://repository.tudelft.nl/file/File_fc8effff-efcd-419a-a27b-6b9a3b94e446","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075931228","display_name":"Zahra Nouri","orcid":"https://orcid.org/0000-0002-7551-6242"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Zahra Nouri","raw_affiliation_strings":["Department of Computer Science, Paderborn University, Germany"],"raw_orcid":"https://orcid.org/0000-0002-7551-6242","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Paderborn University, Germany","institution_ids":["https://openalex.org/I206945453"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070469763","display_name":"Nikhil Prakash","orcid":"https://orcid.org/0000-0002-5976-2961"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nikhil Prakash","raw_affiliation_strings":["Khoury College of Computer Sciences, Northeastern University, United States"],"raw_orcid":"https://orcid.org/0000-0002-5976-2961","affiliations":[{"raw_affiliation_string":"Khoury College of Computer Sciences, Northeastern University, United States","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038081564","display_name":"Ujwal Gadiraju","orcid":"https://orcid.org/0000-0002-6189-6539"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Ujwal Gadiraju","raw_affiliation_strings":["Web Information Systems, Delft University of Technology, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-6189-6539","affiliations":[{"raw_affiliation_string":"Web Information Systems, Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014375244","display_name":"Henning Wachsmuth","orcid":"https://orcid.org/0000-0003-2792-621X"},"institutions":[{"id":"https://openalex.org/I114112103","display_name":"Leibniz University Hannover","ror":"https://ror.org/0304hq317","country_code":"DE","type":"education","lineage":["https://openalex.org/I114112103"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Henning Wachsmuth","raw_affiliation_strings":["Institute of Artificial Intelligence, Leibniz University Hannover, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2792-621X","affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence, Leibniz University Hannover, Germany","institution_ids":["https://openalex.org/I114112103"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.3321,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.9069982,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"737","last_page":"749"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9609000086784363,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/helpfulness","display_name":"Helpfulness","score":0.9230031967163086},{"id":"https://openalex.org/keywords/clarity","display_name":"CLARITY","score":0.8954407572746277},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8732032775878906},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8083690404891968},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7956427335739136},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5585156083106995},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5548946857452393},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3766661286354065},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3498098850250244},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07964223623275757},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.07255285978317261},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07164019346237183}],"concepts":[{"id":"https://openalex.org/C2781265381","wikidata":"https://www.wikidata.org/wiki/Q5710255","display_name":"Helpfulness","level":2,"score":0.9230031967163086},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.8954407572746277},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8732032775878906},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8083690404891968},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7956427335739136},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5585156083106995},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5548946857452393},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3766661286354065},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3498098850250244},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07964223623275757},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.07255285978317261},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07164019346237183},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3581641.3584039","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3581641.3584039","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},{"id":"pmh:oai:tudelft.nl:uuid:ad171033-1e2f-45f6-a7cd-30977abab4f8","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:ad171033-1e2f-45f6-a7cd-30977abab4f8","pdf_url":"https://repository.tudelft.nl/file/File_fc8effff-efcd-419a-a27b-6b9a3b94e446","source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"},{"id":"pmh:oai:share.osf.io:8fb93408-5e21-44ee-afc8-957f851e2d9c","is_oa":false,"landing_page_url":"https://osf.io/2uqtf","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Project"},{"id":"pmh:oai:share.osf.io:E01F9-D4E-485","is_oa":false,"landing_page_url":"http://osf.io/2uqtf/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"project"}],"best_oa_location":{"id":"pmh:oai:tudelft.nl:uuid:ad171033-1e2f-45f6-a7cd-30977abab4f8","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:ad171033-1e2f-45f6-a7cd-30977abab4f8","pdf_url":"https://repository.tudelft.nl/file/File_fc8effff-efcd-419a-a27b-6b9a3b94e446","source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"},"sustainable_development_goals":[{"score":0.5099999904632568,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4360991133.pdf","grobid_xml":"https://content.openalex.org/works/W4360991133.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1509228788","https://openalex.org/W2012964103","https://openalex.org/W2036367260","https://openalex.org/W2070248097","https://openalex.org/W2118288941","https://openalex.org/W2127008633","https://openalex.org/W2132481689","https://openalex.org/W2136222780","https://openalex.org/W2148479118","https://openalex.org/W2149684865","https://openalex.org/W2168144930","https://openalex.org/W2398514781","https://openalex.org/W2547436379","https://openalex.org/W2588579248","https://openalex.org/W2588628504","https://openalex.org/W2598654328","https://openalex.org/W2607311634","https://openalex.org/W2726137762","https://openalex.org/W2783946743","https://openalex.org/W2791546809","https://openalex.org/W2794219599","https://openalex.org/W2809806295","https://openalex.org/W2896881735","https://openalex.org/W2898764742","https://openalex.org/W2946474696","https://openalex.org/W2974398320","https://openalex.org/W2997591727","https://openalex.org/W3118140671","https://openalex.org/W3121257585","https://openalex.org/W3194338218","https://openalex.org/W4230939871"],"related_works":["https://openalex.org/W2613921548","https://openalex.org/W4285360723","https://openalex.org/W4281847990","https://openalex.org/W2002563848","https://openalex.org/W2488228222","https://openalex.org/W2570913877","https://openalex.org/W3037056935","https://openalex.org/W2934621214","https://openalex.org/W2092282862","https://openalex.org/W3196817267"],"abstract_inverted_index":{"Quality":[0],"control":[1],"is":[2,41],"an,":[3],"if":[4],"not":[5],"the,":[6],"essential":[7],"challenge":[8],"in":[9,47,64,89,206,322],"crowdsourcing.":[10],"Unsatisfactory":[11],"responses":[12],"from":[13,22,29,107],"crowd":[14,196,266],"workers":[15,267],"have":[16],"been":[17],"found":[18],"to":[19,57,80,103,110,208,218],"particularly":[20],"result":[21],"ambiguous":[23],"and":[24,83,112,151,178],"incomplete":[25],"task":[26,31,36,91,105,130,136,154,173,215,275,300,320],"descriptions,":[27,310],"often":[28],"inexperienced":[30],"requesters.":[32],"However,":[33],"creating":[34],"clear":[35],"descriptions":[37,92,106,131,174,205,276,301,321],"with":[38,195,203,291],"sufficient":[39,160],"information":[40,230],"a":[42,75,134,159,165,191],"complex":[43],"process":[44,66],"for":[45,138,251,254],"requesters":[46,59,79,171,224,247,280],"crowdsourcing":[48],"marketplaces.":[49],"In":[50,164,289],"this":[51,65,71,144],"paper,":[52],"we":[53,73,169],"investigate":[54],"the":[55,96,114,139,146,153,176,180,210,213,223,226,229,233,257,265,270,274,279,282,286,293,298,308,313,317],"extent":[56],"which":[58],"can":[60,100,148],"be":[61,101],"supported":[62],"effectively":[63],"through":[67],"computational":[68],"techniques.":[69],"To":[70],"end,":[72],"developed":[74],"tool":[76,99,177,234,283,314],"that":[77,132,269,312],"enables":[78],"iteratively":[81,149],"identify":[82],"correct":[84],"eight":[85,140],"common":[86],"clarity":[87,115,141,211,272,295,318],"flaws":[88],"their":[90],"before":[93],"deployment":[94],"on":[95,128],"platform.":[97],"The":[98,246],"used":[102],"write":[104],"scratch":[108],"or":[109,236,243],"assess":[111],"improve":[113],"of":[116,162,184,212,222,228,259,264,273,297,307,319],"prepared":[117],"descriptions.":[118,216],"It":[119],"employs":[120],"machine":[121],"learning-based":[122],"natural":[123],"language":[124],"processing":[125],"models":[126],"trained":[127],"real-world":[129],"score":[133],"given":[135],"description":[137,155],"flaws.":[142],"On":[143],"basis,":[145],"requester":[147],"revise":[150],"reassess":[152],"until":[156],"it":[157],"reaches":[158],"level":[161],"clarity.":[163],"first":[166],"user":[167,193],"study,":[168],"let":[170],"create":[172],"using":[175,281],"rate":[179,209],"tool\u2019s":[181],"different":[182],"aspects":[183],"helpfulness":[185,227],"thereafter.":[186],"We":[187],"then":[188],"carried":[189],"out":[190],"second":[192],"study":[194],"workers,":[197],"as":[198,241],"those":[199,306],"who":[200],"are":[201],"confronted":[202],"such":[204],"practice,":[207],"created":[214,277],"According":[217],"our":[219],"results,":[220],"65%":[221],"classified":[225],"provided":[231],"by":[232,278],"high":[235,238],"very":[237,244],"(only":[239],"12%":[240],"low":[242],"low).":[245],"saw":[248],"some":[249],"room":[250],"improvement":[252],"though,":[253],"example,":[255],"concerning":[256],"display":[258],"bad":[260],"examples.":[261],"Nevertheless,":[262],"76%":[263],"believe":[268],"overall":[271,323],"improves":[284],"over":[285],"initial":[287,309],"version.":[288],"line":[290],"this,":[292],"automatically-computed":[294],"scores":[296],"edited":[299],"were":[302],"generally":[303],"higher":[304],"than":[305],"indicating":[311],"reliably":[315],"predicts":[316],"terms.":[324]},"counts_by_year":[{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}