{"id":"https://openalex.org/W2896078687","doi":"https://doi.org/10.1145/3242587.3242621","title":"Crowdsourcing Similarity Judgments for Agreement Analysis in End-User Elicitation Studies","display_name":"Crowdsourcing Similarity Judgments for Agreement Analysis in End-User Elicitation Studies","publication_year":2018,"publication_date":"2018-10-11","ids":{"openalex":"https://openalex.org/W2896078687","doi":"https://doi.org/10.1145/3242587.3242621","mag":"2896078687"},"language":"en","primary_location":{"id":"doi:10.1145/3242587.3242621","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3242587.3242621","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3242587.3242621","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3242587.3242621","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064398083","display_name":"Abdullah Ali","orcid":"https://orcid.org/0000-0002-3362-7462"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Abdullah X. Ali","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062285844","display_name":"Meredith Ringel Morris","orcid":"https://orcid.org/0000-0003-1436-9223"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Meredith Ringel Morris","raw_affiliation_strings":["Microsoft Research, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Seattle, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086377685","display_name":"Jacob O. Wobbrock","orcid":"https://orcid.org/0000-0003-3675-5491"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jacob O. Wobbrock","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5064398083"],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":5.9534,"has_fulltext":true,"cited_by_count":38,"citation_normalized_percentile":{"value":0.95948654,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"177","last_page":"188"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11500","display_name":"Evacuation and Crowd Dynamics","score":0.9764000177383423,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11980","display_name":"Human Mobility and Location-Based Analysis","score":0.9757999777793884,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9185541272163391},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7645508050918579},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.7259395718574524},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.5532912611961365},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.47958460450172424},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.46008777618408203},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.35458868741989136},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34981685876846313},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.33455079793930054},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2615855932235718}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9185541272163391},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7645508050918579},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.7259395718574524},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.5532912611961365},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.47958460450172424},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.46008777618408203},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.35458868741989136},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34981685876846313},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33455079793930054},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2615855932235718},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3242587.3242621","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3242587.3242621","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3242587.3242621","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3242587.3242621","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3242587.3242621","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3242587.3242621","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1899923107","display_name":null,"funder_award_id":"IIS-170275","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3239930498","display_name":null,"funder_award_id":"IIS?1702751","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4493455584","display_name":null,"funder_award_id":"IS-1702751","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7271568891","display_name":"CHS: Medium: Improving the Accessibility of Mobile Applications by Enabling Third-Party Assessment, Repair, and Enhancement","funder_award_id":"1702751","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7669408176","display_name":null,"funder_award_id":"IIS-1702751","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2896078687.pdf","grobid_xml":"https://content.openalex.org/works/W2896078687.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W119458596","https://openalex.org/W177399310","https://openalex.org/W1573900212","https://openalex.org/W1956340063","https://openalex.org/W1965694745","https://openalex.org/W1974554406","https://openalex.org/W1978580638","https://openalex.org/W1985875030","https://openalex.org/W2015776973","https://openalex.org/W2024060531","https://openalex.org/W2025647423","https://openalex.org/W2033955170","https://openalex.org/W2045127238","https://openalex.org/W2051279338","https://openalex.org/W2058179030","https://openalex.org/W2062526841","https://openalex.org/W2065228210","https://openalex.org/W2091858563","https://openalex.org/W2100697272","https://openalex.org/W2113305605","https://openalex.org/W2120396827","https://openalex.org/W2121044470","https://openalex.org/W2127008633","https://openalex.org/W2136691781","https://openalex.org/W2142119621","https://openalex.org/W2161304134","https://openalex.org/W2167372977","https://openalex.org/W2171610853","https://openalex.org/W2290102723","https://openalex.org/W2395981059","https://openalex.org/W2497475504","https://openalex.org/W2623293810","https://openalex.org/W2788969155","https://openalex.org/W2796156077","https://openalex.org/W2807449890","https://openalex.org/W2914959486","https://openalex.org/W2951342632","https://openalex.org/W2963775347","https://openalex.org/W4230939871"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"End-user":[0],"elicitation":[1,36,62,68],"studies":[2,37,63],"are":[3],"a":[4,24,75,86,127],"popular":[5],"design":[6],"method,":[7],"but":[8],"their":[9],"data":[10],"require":[11],"substantial":[12],"time":[13],"and":[14,42,60,125],"effort":[15],"to":[16,30,48,64,95,133],"analyze.":[17],"In":[18,46],"this":[19],"paper,":[20],"we":[21,52],"present":[22],"Crowdsensus,":[23],"crowd-powered":[25],"tool":[26],"that":[27],"enables":[28],"researchers":[29,56],"efficiently":[31],"analyze":[32,65],"the":[33,112,117,130],"results":[34,119],"of":[35,70,88,99,122],"using":[38,136,144],"subjective":[39],"human":[40],"judgment":[41],"automatic":[43],"clustering":[44],"algorithms.":[45],"addition":[47],"our":[49],"own":[50],"analysis,":[51],"asked":[53],"six":[54],"expert":[55],"with":[57,78],"experience":[58],"running":[59],"analyzing":[61],"an":[66],"end-user":[67],"dataset":[69],"10":[71],"functions":[72,124],"for":[73,85,120],"operating":[74],"web-browser,":[76],"each":[77],"43":[79],"voice":[80,90],"commands":[81,103],"elicited":[82],"from":[83,104],"end-users":[84],"total":[87],"430":[89,102],"commands.":[91],"We":[92],"used":[93],"Crowdsensus":[94,137],"gather":[96],"similarity":[97],"judgments":[98],"these":[100],"same":[101,118],"410":[105],"online":[106],"crowd":[107,110],"workers.":[108],"The":[109],"outperformed":[111],"experts":[113,131],"by":[114],"arriving":[115],"at":[116],"seven":[121],"eight":[123],"resolving":[126],"function":[128],"where":[129],"failed":[132],"agree.":[134],"Also,":[135],"was":[138],"about":[139],"four":[140],"times":[141],"faster":[142],"than":[143],"experts.":[145]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":7}],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2025-10-10T00:00:00"}