{"id":"https://openalex.org/W4366851110","doi":"https://doi.org/10.1145/3539618.3591694","title":"Hear Me Out: A Study on the Use of the Voice Modality for Crowdsourced Relevance Assessments","display_name":"Hear Me Out: A Study on the Use of the Voice Modality for Crowdsourced Relevance Assessments","publication_year":2023,"publication_date":"2023-07-18","ids":{"openalex":"https://openalex.org/W4366851110","doi":"https://doi.org/10.1145/3539618.3591694"},"language":"en","primary_location":{"id":"doi:10.1145/3539618.3591694","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3591694","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591694","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591694","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019952416","display_name":"Nirmal Roy","orcid":"https://orcid.org/0000-0003-0860-5269"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Nirmal Roy","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-0860-5269","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088069213","display_name":"Agathe Balayn","orcid":"https://orcid.org/0000-0003-2725-5305"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Agathe Balayn","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-2725-5305","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070722958","display_name":"David Maxwell","orcid":"https://orcid.org/0000-0002-8914-2448"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"David Maxwell","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-8914-2448","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084915697","display_name":"Claudia Hauff","orcid":"https://orcid.org/0000-0001-9879-6470"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Claudia Hauff","raw_affiliation_strings":["Spotify & Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-9879-6470","affiliations":[{"raw_affiliation_string":"Spotify & Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"718","last_page":"728"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9829000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8278723359107971},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8071210384368896},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.7812234163284302},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.669703483581543},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6426819562911987},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5555046796798706},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5139338374137878},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.49210232496261597},{"id":"https://openalex.org/keywords/presentation","display_name":"Presentation (obstetrics)","score":0.463348925113678},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.41334208846092224},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.388694167137146},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2644307017326355},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.26168012619018555},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.09629863500595093},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07271334528923035}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8278723359107971},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8071210384368896},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.7812234163284302},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.669703483581543},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6426819562911987},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5555046796798706},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5139338374137878},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.49210232496261597},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.463348925113678},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41334208846092224},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.388694167137146},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2644307017326355},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.26168012619018555},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.09629863500595093},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07271334528923035},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3539618.3591694","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3591694","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591694","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2304.10881","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.10881","pdf_url":"https://arxiv.org/pdf/2304.10881","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:tudelft.nl:uuid:ec734213-f60a-4b53-8397-be8dd9044e08","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:ec734213-f60a-4b53-8397-be8dd9044e08","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"}],"best_oa_location":{"id":"doi:10.1145/3539618.3591694","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3591694","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591694","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8227748103","display_name":"SearchX: Integrating search and sensemaking into large-scale open online  learning","funder_award_id":"639.022.722","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366851110.pdf","grobid_xml":"https://content.openalex.org/works/W4366851110.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W8870360","https://openalex.org/W18726577","https://openalex.org/W81814971","https://openalex.org/W91851626","https://openalex.org/W153133746","https://openalex.org/W266671442","https://openalex.org/W1065334109","https://openalex.org/W1734538896","https://openalex.org/W1965647483","https://openalex.org/W1986177399","https://openalex.org/W1988711048","https://openalex.org/W1997544838","https://openalex.org/W1999047234","https://openalex.org/W2026969101","https://openalex.org/W2031322820","https://openalex.org/W2044361482","https://openalex.org/W2087484885","https://openalex.org/W2089350162","https://openalex.org/W2090375332","https://openalex.org/W2100507040","https://openalex.org/W2111993661","https://openalex.org/W2117092977","https://openalex.org/W2118932991","https://openalex.org/W2120357670","https://openalex.org/W2130955505","https://openalex.org/W2141766660","https://openalex.org/W2157289187","https://openalex.org/W2262972580","https://openalex.org/W2400428418","https://openalex.org/W2569307494","https://openalex.org/W2569446996","https://openalex.org/W2585292421","https://openalex.org/W2597866956","https://openalex.org/W2616605651","https://openalex.org/W2759323897","https://openalex.org/W2801816216","https://openalex.org/W2913017407","https://openalex.org/W2913254100","https://openalex.org/W2913964908","https://openalex.org/W2946144951","https://openalex.org/W3006484790","https://openalex.org/W3022682265","https://openalex.org/W3088965898","https://openalex.org/W3122197089","https://openalex.org/W3188970555","https://openalex.org/W3200118725","https://openalex.org/W4205987607","https://openalex.org/W4231402390","https://openalex.org/W4255321213","https://openalex.org/W4256250826","https://openalex.org/W4285417481","https://openalex.org/W6600367688"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W1497983610","https://openalex.org/W1980158193"],"abstract_inverted_index":{"The":[0],"creation":[1],"of":[2,44,57,72,96,148,172],"relevance":[3,71,95,162,220],"assessments":[4],"by":[5],"human":[6],"assessors":[7,32,67,173],"(often":[8],"nowadays":[9],"crowdworkers)":[10],"is":[11,64],"a":[12,45,76,81,88,215],"vital":[13],"step":[14],"when":[15],"building":[16],"IR":[17],"test":[18],"collections.":[19],"Prior":[20],"works":[21],"have":[22,37],"investigated":[23],"assessor":[24,50],"quality":[25,186],"&":[26],"behaviour,":[27],"and":[28,52,98,132,168],"tooling":[29],"to":[30,68,108,160,174,217],"support":[31],"in":[33,111,125,164,187],"their":[34,126],"task.":[35],"We":[36,79,117],"few":[38],"insights":[39],"though":[40],"into":[41],"the":[42,55,70,94,103,112,130,165,170,184,188,211],"impact":[43],"document's":[46],"presentation":[47],"modality":[48,213],"on":[49,87],"efficiency":[51],"effectiveness.":[53],"Given":[54],"rise":[56],"voice-based":[58,77],"interfaces,":[59],"we":[60,207],"investigate":[61],"whether":[62],"it":[63,140,153],"feasible":[65],"for":[66],"judge":[69],"text":[73,113,131],"documents":[74,147],"via":[75],"interface.":[78],"ran":[80],"user":[82],"study":[83],"(n":[84],"=":[85],"49)":[86],"crowdsourcing":[89],"platform":[90],"where":[91],"participants":[92,121],"judged":[93],"short":[97],"long":[99],"documents-":[100],"sampled":[101],"from":[102,222],"TREC":[104],"Deep":[105],"Learning":[106],"corpus-presented":[107],"them":[109],"either":[110],"or":[114],"voice":[115,133,166,189,212],"modality.":[116],"found":[118],"that:":[119],"(i)":[120],"are":[122,178,194],"equally":[123],"accurate":[124,197],"judgements":[127,163],"across":[128],"both":[129],"modality;":[134],"(ii)":[135],"with":[136,191,200],"increased":[137],"document":[138],"length":[139,149],"takes":[141,154],"partic-":[142],"ipants":[143],"significantly":[144,195],"longer":[145],"(for":[146],">":[150],"120":[151],"words":[152],"almost":[155],"twice":[156],"as":[157,214],"much":[158],"time)":[159],"make":[161],"condition;":[167],"(iii)":[169],"ability":[171],"ignore":[175],"stimuli":[176],"that":[177,206],"not":[179],"relevant":[180],"(i.e.,":[181],"inhibition)":[182],"impacts":[183],"assessment":[185],"modality-assessors":[190],"higher":[192],"inhibition":[193],"more":[196],"than":[198],"those":[199],"lower":[201],"inhibition.":[202],"Our":[203],"results":[204],"indicate":[205],"can":[208],"reliably":[209],"leverage":[210],"means":[216],"effectively":[218],"collect":[219],"labels":[221],"crowdworkers.":[223]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2023-04-25T00:00:00"}