{"id":"https://openalex.org/W2569307494","doi":"https://doi.org/10.1145/3002172","title":"On Crowdsourcing Relevance Magnitudes for Information Retrieval Evaluation","display_name":"On Crowdsourcing Relevance Magnitudes for Information Retrieval Evaluation","publication_year":2017,"publication_date":"2017-01-04","ids":{"openalex":"https://openalex.org/W2569307494","doi":"https://doi.org/10.1145/3002172","mag":"2569307494"},"language":"en","primary_location":{"id":"doi:10.1145/3002172","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3002172","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020871436","display_name":"Eddy Maddalena","orcid":"https://orcid.org/0000-0002-5423-8669"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Eddy Maddalena","raw_affiliation_strings":["University of Udine, Italy"],"affiliations":[{"raw_affiliation_string":"University of Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005575421","display_name":"Stefano Mizzaro","orcid":"https://orcid.org/0000-0002-2852-168X"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Stefano Mizzaro","raw_affiliation_strings":["University of Udine, Italy"],"affiliations":[{"raw_affiliation_string":"University of Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014748363","display_name":"Andrew Turpin","orcid":"https://orcid.org/0000-0003-2559-8769"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Andrew Turpin","raw_affiliation_strings":["University of Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"University of Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5020871436"],"corresponding_institution_ids":["https://openalex.org/I129043915"],"apc_list":null,"apc_paid":null,"fwci":21.5503,"has_fulltext":false,"cited_by_count":130,"citation_normalized_percentile":{"value":0.99364096,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":"35","issue":"3","first_page":"1","last_page":"32"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9524999856948853,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7778524160385132},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7526909708976746},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7180504202842712},{"id":"https://openalex.org/keywords/magnitude","display_name":"Magnitude (astronomy)","score":0.6322619915008545},{"id":"https://openalex.org/keywords/estimation","display_name":"Estimation","score":0.5548104047775269},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.5136427283287048},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4285946786403656},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.3442055583000183},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.33597609400749207},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.2662097215652466},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.21177434921264648}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7778524160385132},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7526909708976746},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7180504202842712},{"id":"https://openalex.org/C126691448","wikidata":"https://www.wikidata.org/wiki/Q2028919","display_name":"Magnitude (astronomy)","level":2,"score":0.6322619915008545},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.5548104047775269},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.5136427283287048},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4285946786403656},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3442055583000183},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33597609400749207},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.2662097215652466},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21177434921264648},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3002172","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3002172","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:air.uniud.it:11390/1108797","is_oa":false,"landing_page_url":"http://www.acm.org/tois","pdf_url":null,"source":{"id":"https://openalex.org/S4306401163","display_name":"Institutional Research Information System (University of Udine)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I129043915","host_organization_name":"University of Udine","host_organization_lineage":["https://openalex.org/I129043915"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"internazionale"},{"id":"pmh:oai:alma.61RMIT_INST:11247223340001341","is_oa":false,"landing_page_url":"http://purl.org/au-research/grants/arc/DP130104007","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4584936334","display_name":null,"funder_award_id":"DP130104007","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W16426102","https://openalex.org/W60779023","https://openalex.org/W136732505","https://openalex.org/W645009165","https://openalex.org/W1483624827","https://openalex.org/W1546195864","https://openalex.org/W1938221098","https://openalex.org/W1962580118","https://openalex.org/W1966547126","https://openalex.org/W1981164277","https://openalex.org/W1982149803","https://openalex.org/W1991158387","https://openalex.org/W1996422672","https://openalex.org/W1996970545","https://openalex.org/W2001816027","https://openalex.org/W2004068087","https://openalex.org/W2009543464","https://openalex.org/W2009948657","https://openalex.org/W2013575152","https://openalex.org/W2025614130","https://openalex.org/W2026519996","https://openalex.org/W2026969101","https://openalex.org/W2029852711","https://openalex.org/W2037140704","https://openalex.org/W2039017419","https://openalex.org/W2053528567","https://openalex.org/W2056947815","https://openalex.org/W2061504941","https://openalex.org/W2062236692","https://openalex.org/W2069870183","https://openalex.org/W2074925565","https://openalex.org/W2088901214","https://openalex.org/W2089350162","https://openalex.org/W2094607109","https://openalex.org/W2110202502","https://openalex.org/W2113640060","https://openalex.org/W2143331230","https://openalex.org/W2143400823","https://openalex.org/W2210040668","https://openalex.org/W2221430606","https://openalex.org/W2294614122","https://openalex.org/W2313569764","https://openalex.org/W2319061465","https://openalex.org/W2406847633","https://openalex.org/W4237116629","https://openalex.org/W4255321213"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W1497983610"],"abstract_inverted_index":{"Magnitude":[0],"estimation":[1,30,56,65,99,130,143,153],"is":[2,159],"a":[3,43,195,204],"psychophysical":[4],"scaling":[5],"technique":[6],"for":[7,31,37,100,187],"the":[8,26,33,95,137,151,174],"measurement":[9],"of":[10,28,35,77,97,115,119,150,166,173,176,198,206],"sensation,":[11],"where":[12],"observers":[13],"assign":[14],"numbers":[15],"to":[16,20,163,202,210],"stimuli":[17],"in":[18,75,135,161,179],"response":[19],"their":[21],"perceived":[22],"intensity.":[23],"We":[24,93],"investigate":[25],"use":[27],"magnitude":[29,55,64,98,129,142,152],"judging":[32],"relevance":[34,91,131,199],"documents":[36],"information":[38],"retrieval":[39],"evaluation,":[40,102,189],"carrying":[41],"out":[42],"large-scale":[44],"user":[45],"study":[46],"across":[47],"18":[48],"TREC":[49,120],"topics":[50],"and":[51,80,109,128,144],"collecting":[52],"over":[53],"50,000":[54],"judgments":[57,66,87,146],"using":[58,71,141],"crowdsourcing.":[59],"Our":[60],"analysis":[61],"shows":[62,132,155],"that":[63,156,191],"can":[67],"be":[68],"reliably":[69],"collected":[70],"crowdsourcing,":[72],"are":[73,208],"competitive":[74],"terms":[76],"assessor":[78],"cost,":[79],"are,":[81],"on":[82,125],"average,":[83],"rank-aligned":[84],"with":[85],"ordinal":[86,145],"made":[88],"by":[89],"expert":[90],"assessors.":[92],"explore":[94],"application":[96],"IR":[101,188],"calibrating":[103],"two":[104],"gain-based":[105],"effectiveness":[106,122],"metrics,":[107],"nDCG":[108],"ERR,":[110],"directly":[111],"from":[112],"user-reported":[113],"perceptions":[114,165,172],"relevance.":[116,181],"A":[117],"comparison":[118],"system":[121],"rankings":[123],"based":[124],"binary,":[126],"ordinal,":[127],"substantial":[133],"variation;":[134],"particular,":[136],"top":[138],"systems":[139],"ranked":[140],"differ":[147],"substantially.":[148],"Analysis":[149],"scores":[154],"this":[157],"effect":[158],"due":[160],"part":[162],"varying":[164],"relevance:":[167],"different":[168,171],"users":[169,207],"have":[170,184],"impact":[175],"relative":[177],"differences":[178],"document":[180],"These":[182],"results":[183],"direct":[185],"implications":[186],"suggesting":[190],"current":[192],"assumptions":[193],"about":[194],"single":[196],"view":[197],"being":[200],"sufficient":[201],"represent":[203],"population":[205],"unlikely":[209],"hold.":[211]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":23},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":36},{"year":2020,"cited_by_count":28},{"year":2019,"cited_by_count":13},{"year":2018,"cited_by_count":10},{"year":2017,"cited_by_count":4}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2017-01-13T00:00:00"}