{"id":"https://openalex.org/W2986224017","doi":"https://doi.org/10.1145/3357384.3357988","title":"On Transforming Relevance Scales","display_name":"On Transforming Relevance Scales","publication_year":2019,"publication_date":"2019-11-03","ids":{"openalex":"https://openalex.org/W2986224017","doi":"https://doi.org/10.1145/3357384.3357988","mag":"2986224017"},"language":"en","primary_location":{"id":"doi:10.1145/3357384.3357988","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3357384.3357988","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101540918","display_name":"Lei Han","orcid":"https://orcid.org/0000-0002-7777-3592"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Lei Han","raw_affiliation_strings":["University of Queensland, Brisbane, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050287516","display_name":"Kevin Roitero","orcid":"https://orcid.org/0000-0002-9191-3280"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Kevin Roitero","raw_affiliation_strings":["University of Udine, Udine, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020871436","display_name":"Eddy Maddalena","orcid":"https://orcid.org/0000-0002-5423-8669"},"institutions":[{"id":"https://openalex.org/I43439940","display_name":"University of Southampton","ror":"https://ror.org/01ryk1543","country_code":"GB","type":"education","lineage":["https://openalex.org/I43439940"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Eddy Maddalena","raw_affiliation_strings":["University of Southampton, Southampton, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southampton, Southampton, United Kingdom","institution_ids":["https://openalex.org/I43439940"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005575421","display_name":"Stefano Mizzaro","orcid":"https://orcid.org/0000-0002-2852-168X"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Stefano Mizzaro","raw_affiliation_strings":["University of Udine, Udine, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["University of Queensland, Brisbane, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"39","last_page":"48"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.9238705635070801},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.914121150970459},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.7178072929382324},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.698742151260376},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6501951217651367},{"id":"https://openalex.org/keywords/transformation","display_name":"Transformation (genetics)","score":0.5827069282531738},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.49746230244636536},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.41760244965553284},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.08317616581916809}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.9238705635070801},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.914121150970459},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.7178072929382324},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.698742151260376},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6501951217651367},{"id":"https://openalex.org/C204241405","wikidata":"https://www.wikidata.org/wiki/Q461499","display_name":"Transformation (genetics)","level":3,"score":0.5827069282531738},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.49746230244636536},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.41760244965553284},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.08317616581916809},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3357384.3357988","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3357384.3357988","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:air.uniud.it:11390/1171142","is_oa":false,"landing_page_url":"http://hdl.handle.net/11390/1171142","pdf_url":null,"source":{"id":"https://openalex.org/S4306401163","display_name":"Institutional Research Information System (University of Udine)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I129043915","host_organization_name":"University of Udine","host_organization_lineage":["https://openalex.org/I129043915"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.75,"id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G6061249128","display_name":null,"funder_award_id":"732328","funder_id":"https://openalex.org/F4320338335","funder_display_name":"H2020 European Research Council"}],"funders":[{"id":"https://openalex.org/F4320338335","display_name":"H2020 European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W565003248","https://openalex.org/W1494734944","https://openalex.org/W1934428649","https://openalex.org/W1968927634","https://openalex.org/W2012070465","https://openalex.org/W2027247989","https://openalex.org/W2053528567","https://openalex.org/W2071446250","https://openalex.org/W2075893676","https://openalex.org/W2088901214","https://openalex.org/W2104822358","https://openalex.org/W2141649520","https://openalex.org/W2252197266","https://openalex.org/W2569307494","https://openalex.org/W2760642188","https://openalex.org/W2798782484","https://openalex.org/W2836296581","https://openalex.org/W3151947762","https://openalex.org/W6691644449"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W1497983610"],"abstract_inverted_index":{"Information":[0],"Retrieval":[1],"(IR)":[2],"researchers":[3],"have":[4,19,56],"often":[5,38],"used":[6,100,114],"existing":[7],"IR":[8,63,117,183],"evaluation":[9,184],"collections":[10],"and":[11,45,50,159,186],"transformed":[12],"the":[13,60,66,81,88,94,97,112,126,141,176],"relevance":[14,72,83,98,152,157,179],"scale":[15,35,99,129,180],"in":[16,131],"which":[17],"judgments":[18,29,73,103,144,153,195],"been":[20],"collected,":[21],"e.g.,":[22],"to":[23,43,48,70,90,101,115,139,147,174,187,192],"use":[24,67,150],"metrics":[25,113],"that":[26,54],"assume":[27],"binary":[28],"like":[30],"Mean":[31],"Average":[32],"Precision.":[33],"Such":[34,106],"transformations":[36,130,181],"are":[37],"arbitrary":[39],"(e.g.,":[40],"0,1":[41],"mapped":[42,47],"0":[44],"2,3":[46],"1)":[49],"it":[51],"is":[52,93,173],"assumed":[53],"they":[55],"no":[57],"impact":[58,177],"on":[59,111,155,182,190],"results":[61],"of":[62,68,87,128,143,169,178],"evaluation.":[64],"Moreover,":[65],"crowdsourcing":[69,82],"collect":[71,102],"has":[74,108],"become":[75],"a":[76,132,197],"standard":[77],"methodology.":[78],"When":[79],"designing":[80],"judgment":[84],"task,":[85],"one":[86],"decision":[89,107],"be":[91],"made":[92],"how":[95,191],"granular":[96],"should":[104],"be.":[105],"then":[109],"repercussions":[110],"measure":[116],"system":[118],"effectiveness.":[119],"In":[120],"this":[121],"paper":[122],"we":[123],"look":[124],"at":[125],"effect":[127],"systematic":[133],"way.":[134],"We":[135,149],"perform":[136],"extensive":[137],"experiments":[138],"study":[140],"transformation":[142],"from":[145],"fine-grained":[146],"coarse-grained.":[148],"different":[151,156,198],"expressed":[154,161],"scales":[158],"either":[160],"by":[162,167],"expert":[163],"annotators":[164],"or":[165],"collected":[166],"means":[168],"crowdsourcing.":[170],"The":[171],"objective":[172],"understand":[175],"outcomes":[185],"draw":[188],"conclusions":[189],"best":[193],"transform":[194],"into":[196],"scale,":[199],"when":[200],"necessary.":[201]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":9}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2019-11-22T00:00:00"}