{"id":"https://openalex.org/W2058880306","doi":"https://doi.org/10.1145/2756406.2756942","title":"Improving Consistency of Crowdsourced Multimedia Similarity for Evaluation","display_name":"Improving Consistency of Crowdsourced Multimedia Similarity for Evaluation","publication_year":2015,"publication_date":"2015-06-12","ids":{"openalex":"https://openalex.org/W2058880306","doi":"https://doi.org/10.1145/2756406.2756942","mag":"2058880306"},"language":"en","primary_location":{"id":"doi:10.1145/2756406.2756942","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2756406.2756942","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM/IEEE-CS Joint Conference on Digital Libraries","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001038669","display_name":"Peter Organisciak","orcid":"https://orcid.org/0000-0002-9058-2280"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Organisciak","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Champaign, IL, USA","University of Illinois at Urbana-Champaign, Champaign, IL, USA;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Champaign, IL, USA","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Champaign, IL, USA;","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016957740","display_name":"J. Stephen Downie","orcid":"https://orcid.org/0000-0001-9784-5090"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"J. Stephen Downie","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Champaign, IL, USA","University of Illinois at Urbana-Champaign, Champaign, IL, USA;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Champaign, IL, USA","institution_ids":["https://openalex.org/I157725225"]},{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Champaign, IL, USA;","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8699,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.74386106,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"115","last_page":"118"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.854337215423584},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.789190411567688},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7825813293457031},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.758625864982605},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5833194851875305},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5806397795677185},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5122281908988953},{"id":"https://openalex.org/keywords/normative","display_name":"Normative","score":0.4552648365497589},{"id":"https://openalex.org/keywords/digital-library","display_name":"Digital library","score":0.42192164063453674},{"id":"https://openalex.org/keywords/digital-audio","display_name":"Digital audio","score":0.414205402135849},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3296498656272888},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32200735807418823},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2791920304298401},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21134048700332642},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.09120988845825195},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.08036014437675476}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.854337215423584},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.789190411567688},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7825813293457031},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.758625864982605},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5833194851875305},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5806397795677185},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5122281908988953},{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.4552648365497589},{"id":"https://openalex.org/C513874922","wikidata":"https://www.wikidata.org/wiki/Q212805","display_name":"Digital library","level":3,"score":0.42192164063453674},{"id":"https://openalex.org/C87687168","wikidata":"https://www.wikidata.org/wiki/Q173114","display_name":"Digital audio","level":4,"score":0.414205402135849},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3296498656272888},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32200735807418823},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2791920304298401},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21134048700332642},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.09120988845825195},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.08036014437675476},{"id":"https://openalex.org/C64922751","wikidata":"https://www.wikidata.org/wiki/Q4650799","display_name":"Audio signal","level":3,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C164913051","wikidata":"https://www.wikidata.org/wiki/Q482","display_name":"Poetry","level":2,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C13895895","wikidata":"https://www.wikidata.org/wiki/Q3270773","display_name":"Speech coding","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2756406.2756942","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2756406.2756942","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM/IEEE-CS Joint Conference on Digital Libraries","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1946435327","display_name":null,"funder_award_id":"31100663","funder_id":"https://openalex.org/F4320306146","funder_display_name":"Andrew W. Mellon Foundation"}],"funders":[{"id":"https://openalex.org/F4320306146","display_name":"Andrew W. Mellon Foundation","ror":"https://ror.org/04jsh2530"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W210231100","https://openalex.org/W1819763199","https://openalex.org/W1981164277","https://openalex.org/W1990420106","https://openalex.org/W2042659457","https://openalex.org/W2049455633","https://openalex.org/W2059975159","https://openalex.org/W2071256426","https://openalex.org/W2102791842","https://openalex.org/W2110071186","https://openalex.org/W2136449711","https://openalex.org/W2141538344","https://openalex.org/W2141714363","https://openalex.org/W2155486295","https://openalex.org/W2187193317","https://openalex.org/W6601599523","https://openalex.org/W6634134659","https://openalex.org/W6675362077"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"Building":[0],"evaluation":[1,28,51,111],"datasets":[2],"for":[3,108,113],"information":[4],"retrieval":[5],"is":[6],"a":[7,58,65],"time-consuming":[8],"and":[9,102,116],"exhausting":[10],"activity.":[11],"To":[12],"evaluate":[13],"research":[14],"over":[15],"novel":[16],"corpora,":[17],"researchers":[18],"are":[19,80],"increasingly":[20],"turning":[21],"to":[22,24,98],"crowdsourcing":[23],"efficiently":[25],"distribute":[26],"the":[27,41,84,99],"dataset":[29],"creation":[30],"among":[31],"many":[32],"workers.":[33],"However,":[34],"there":[35],"has":[36],"been":[37],"little":[38],"investigation":[39],"into":[40],"effect":[42],"of":[43,90],"instrument":[44],"design":[45,101],"on":[46],"data":[47],"quality":[48],"in":[49,64],"crowdsourced":[50],"datasets.":[52],"We":[53,86],"pursue":[54],"this":[55,91],"question":[56],"through":[57],"case":[59],"study,":[60],"music":[61,66,114],"similarity":[62,115],"judgments":[63,112],"digital":[67],"library":[68],"evaluation,":[69],"where":[70],"we":[71],"find":[72,87],"that":[73,88],"even":[74],"with":[75,106],"trusted":[76],"graders":[77],"song":[78],"pairs":[79],"not":[81],"consistently":[82],"rated":[83],"same.":[85],"much":[89],"low":[92],"intra-coder":[93],"consistency":[94],"can":[95],"be":[96],"attributed":[97],"task":[100],"judge":[103],"effects,":[104],"concluding":[105],"recommendations":[107],"achieving":[109],"reliable":[110],"other":[117],"normative":[118],"judgment":[119],"tasks.":[120]},"counts_by_year":[{"year":2020,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}