{"id":"https://openalex.org/W2998679789","doi":"https://doi.org/10.1145/3336191.3371857","title":"Crowd Worker Strategies in Relevance Judgment Tasks","display_name":"Crowd Worker Strategies in Relevance Judgment Tasks","publication_year":2020,"publication_date":"2020-01-20","ids":{"openalex":"https://openalex.org/W2998679789","doi":"https://doi.org/10.1145/3336191.3371857","mag":"2998679789"},"language":"en","primary_location":{"id":"doi:10.1145/3336191.3371857","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3336191.3371857","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101540918","display_name":"Lei Han","orcid":"https://orcid.org/0000-0002-7777-3592"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Lei Han","raw_affiliation_strings":["University of Queensland, Brisbane, Australia"],"affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020871436","display_name":"Eddy Maddalena","orcid":"https://orcid.org/0000-0002-5423-8669"},"institutions":[{"id":"https://openalex.org/I43439940","display_name":"University of Southampton","ror":"https://ror.org/01ryk1543","country_code":"GB","type":"education","lineage":["https://openalex.org/I43439940"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Eddy Maddalena","raw_affiliation_strings":["University of Southampton, Southampton, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Southampton, Southampton, United Kingdom","institution_ids":["https://openalex.org/I43439940"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021006031","display_name":"Alessandro Checco","orcid":"https://orcid.org/0000-0002-0981-3409"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alessandro Checco","raw_affiliation_strings":["University of Sheffield, Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023061106","display_name":"Cristina Sarasua","orcid":"https://orcid.org/0000-0002-2076-9584"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Cristina Sarasua","raw_affiliation_strings":["University of Zurich, Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"University of Zurich, Zurich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038081564","display_name":"Ujwal Gadiraju","orcid":"https://orcid.org/0000-0002-6189-6539"},"institutions":[{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ujwal Gadiraju","raw_affiliation_strings":["L3S Research Center, Hannover, Germany"],"affiliations":[{"raw_affiliation_string":"L3S Research Center, Hannover, Germany","institution_ids":["https://openalex.org/I4210136150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050287516","display_name":"Kevin Roitero","orcid":"https://orcid.org/0000-0002-9191-3280"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Kevin Roitero","raw_affiliation_strings":["University of Udine, Udine, Italy"],"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["University of Queensland, Brisbane, Australia"],"affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5101540918"],"corresponding_institution_ids":["https://openalex.org/I165143802"],"apc_list":null,"apc_paid":null,"fwci":6.1406,"has_fulltext":false,"cited_by_count":56,"citation_normalized_percentile":{"value":0.9623535,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"241","last_page":"249"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9412999749183655,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9108999967575073,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9807641506195068},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8260589838027954},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7162885665893555},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.71213698387146},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6873459815979004},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5696275234222412},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5390291810035706},{"id":"https://openalex.org/keywords/quality-assurance","display_name":"Quality assurance","score":0.5257292985916138},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4055574834346771},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2492929995059967},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2357284128665924},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11471906304359436}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9807641506195068},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8260589838027954},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7162885665893555},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.71213698387146},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6873459815979004},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5696275234222412},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5390291810035706},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.5257292985916138},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4055574834346771},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2492929995059967},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2357284128665924},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11471906304359436},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C2778618615","wikidata":"https://www.wikidata.org/wiki/Q4008393","display_name":"External quality assessment","level":2,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3336191.3371857","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3336191.3371857","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},{"id":"pmh:oai:air.uniud.it:11390/1212332","is_oa":false,"landing_page_url":"http://hdl.handle.net/11390/1212332","pdf_url":null,"source":{"id":"https://openalex.org/S4306401163","display_name":"Institutional Research Information System (University of Udine)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I129043915","host_organization_name":"University of Udine","host_organization_lineage":["https://openalex.org/I129043915"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:eprints.soton.ac.uk:438319","is_oa":false,"landing_page_url":"https://eprints.soton.ac.uk/438319/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401019","display_name":"ePrints Soton (University of Southampton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I43439940","host_organization_name":"University of Southampton","host_organization_lineage":["https://openalex.org/I43439940"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference or Workshop Item"},{"id":"pmh:oai:iris.uniroma1.it:11573/1696316","is_oa":false,"landing_page_url":"https://hdl.handle.net/11573/1696316","pdf_url":null,"source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":42,"referenced_works":["https://openalex.org/W38029959","https://openalex.org/W119481552","https://openalex.org/W159136832","https://openalex.org/W1497983610","https://openalex.org/W1546195864","https://openalex.org/W1972978214","https://openalex.org/W1979290264","https://openalex.org/W2036431079","https://openalex.org/W2039017419","https://openalex.org/W2053528567","https://openalex.org/W2069870183","https://openalex.org/W2088901214","https://openalex.org/W2094567668","https://openalex.org/W2114269021","https://openalex.org/W2124994029","https://openalex.org/W2132232498","https://openalex.org/W2135504089","https://openalex.org/W2136606893","https://openalex.org/W2137366811","https://openalex.org/W2146436037","https://openalex.org/W2147603330","https://openalex.org/W2153222072","https://openalex.org/W2167913131","https://openalex.org/W2182992846","https://openalex.org/W2338705265","https://openalex.org/W2558888286","https://openalex.org/W2563948597","https://openalex.org/W2593021276","https://openalex.org/W2756079620","https://openalex.org/W2780163489","https://openalex.org/W2783946743","https://openalex.org/W2802354297","https://openalex.org/W2803462456","https://openalex.org/W2810116858","https://openalex.org/W2811107472","https://openalex.org/W2811371717","https://openalex.org/W2903257353","https://openalex.org/W2907147533","https://openalex.org/W2941486823","https://openalex.org/W3121257585","https://openalex.org/W4210764005","https://openalex.org/W7067714283"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W1497983610","https://openalex.org/W1980158193"],"abstract_inverted_index":{"Crowdsourcing":[0],"is":[1],"a":[2,34],"popular":[3],"technique":[4],"to":[5,17,59],"collect":[6],"large":[7],"amounts":[8],"of":[9,52,64],"human-generated":[10],"labels,":[11],"such":[12],"as":[13],"relevance":[14],"judgments":[15],"used":[16],"create":[18],"information":[19],"retrieval":[20],"(IR)":[21],"evaluation":[22],"collections.":[23],"Previous":[24],"research":[25],"has":[26],"shown":[27],"how":[28],"collecting":[29],"high":[30],"quality":[31,41,63],"labels":[32],"from":[33],"crowdsourcing":[35],"platform":[36],"can":[37],"be":[38],"challenging.":[39],"Existing":[40],"assurance":[42],"techniques":[43],"focus":[44],"on":[45,49],"answer":[46],"aggregation":[47],"or":[48],"the":[50,62,65],"use":[51],"gold":[53],"questions":[54],"where":[55],"ground-truth":[56],"data":[57],"allows":[58],"check":[60],"for":[61],"responses.":[66]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":26},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":10},{"year":2020,"cited_by_count":6}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}