{"id":"https://openalex.org/W2891077499","doi":"https://doi.org/10.3233/sw-200415","title":"Empirical methodology for crowdsourcing ground truth","display_name":"Empirical methodology for crowdsourcing ground truth","publication_year":2020,"publication_date":"2020-12-09","ids":{"openalex":"https://openalex.org/W2891077499","doi":"https://doi.org/10.3233/sw-200415","mag":"2891077499"},"language":"en","primary_location":{"id":"doi:10.3233/sw-200415","is_oa":false,"landing_page_url":"https://doi.org/10.3233/sw-200415","pdf_url":null,"source":{"id":"https://openalex.org/S4210177235","display_name":"Semantic Web","issn_l":"1570-0844","issn":["1570-0844","2210-4968"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Semantic Web","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021638892","display_name":"Anca Dumitrache","orcid":"https://orcid.org/0000-0003-1537-1763"},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Anca Dumitrache","raw_affiliation_strings":["FD Mediagroep, Amsterdam, Netherlands","Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"FD Mediagroep, Amsterdam, Netherlands","institution_ids":[]},{"raw_affiliation_string":"Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056796427","display_name":"Oana Inel","orcid":"https://orcid.org/0000-0003-4691-6586"},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]},{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Oana Inel","raw_affiliation_strings":["TU Delft, Delft, Netherlands","Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TU Delft, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]},{"raw_affiliation_string":"Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046133348","display_name":"Benjamin Timmermans","orcid":"https://orcid.org/0000-0002-5130-4309"},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Benjamin Timmermans","raw_affiliation_strings":["IBM Center for Advanced Studies Benelux, Johan Huizingalaan 765, Amsterdam, Netherlands. E-mails:\u00a0b.timmermans@nl.ibm.com,\u00a0rhjsips@gmail.com","Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Center for Advanced Studies Benelux, Johan Huizingalaan 765, Amsterdam, Netherlands. E-mails:\u00a0b.timmermans@nl.ibm.com,\u00a0rhjsips@gmail.com","institution_ids":[]},{"raw_affiliation_string":"Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100759804","display_name":"Carlos Ort\u00edz","orcid":"https://orcid.org/0000-0001-7983-7791"},"institutions":[{"id":"https://openalex.org/I4210095242","display_name":"Netherlands eScience Center","ror":"https://ror.org/00rbjv475","country_code":"NL","type":"funder","lineage":["https://openalex.org/I2800991832","https://openalex.org/I4210090210","https://openalex.org/I4210095242"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Carlos Ortiz","raw_affiliation_strings":["Netherlands eScience Center, Amsterdam, Netherlands. E-mail:\u00a0c.martinez@esciencecenter.nl"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Netherlands eScience Center, Amsterdam, Netherlands. E-mail:\u00a0c.martinez@esciencecenter.nl","institution_ids":["https://openalex.org/I4210095242"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040841362","display_name":"Robert-Jan Sips","orcid":null},"institutions":[{"id":"https://openalex.org/I2905213637","display_name":"TomTom (Netherlands)","ror":"https://ror.org/04as6y835","country_code":"NL","type":"company","lineage":["https://openalex.org/I2905213637"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Robert-Jan Sips","raw_affiliation_strings":["IBM Center for Advanced Studies Benelux, Johan Huizingalaan 765, Amsterdam, Netherlands. E-mails:\u00a0b.timmermans@nl.ibm.com,\u00a0rhjsips@gmail.com","myTomorrows, Amsterdam, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Center for Advanced Studies Benelux, Johan Huizingalaan 765, Amsterdam, Netherlands. E-mails:\u00a0b.timmermans@nl.ibm.com,\u00a0rhjsips@gmail.com","institution_ids":[]},{"raw_affiliation_string":"myTomorrows, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I2905213637"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016381010","display_name":"Lora Aroyo","orcid":"https://orcid.org/0000-0001-9402-1133"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL","US"],"is_corresponding":false,"raw_author_name":"Lora Aroyo","raw_affiliation_strings":["Google, New York, USA","Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, New York, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5106085231","display_name":"Chris Welty","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL","US"],"is_corresponding":false,"raw_author_name":"Chris Welty","raw_affiliation_strings":["Google, New York, USA","Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, New York, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Vrije Universiteit Amsterdam, De Boelelaan 1081, Amsterdam, Netherlands. E-mails:\u00a0anca.dmtrch@gmail.com,\u00a0oana.inel@gmail.com,\u00a0l.m.aroyo@gmail.com,\u00a0cawelty@gmail.com","institution_ids":["https://openalex.org/I865915315"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5021638892"],"corresponding_institution_ids":["https://openalex.org/I865915315"],"apc_list":null,"apc_paid":null,"fwci":1.2749,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.83927589,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"12","issue":"3","first_page":"403","last_page":"421"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9739000201225281,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.876915454864502},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8089450597763062},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.7403399348258972},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.6067255735397339},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.5786656141281128},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5576629638671875},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5288145542144775},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5123389959335327},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.49448633193969727},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.4692753851413727},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.44725847244262695},{"id":"https://openalex.org/keywords/data-quality","display_name":"Data quality","score":0.4224385619163513},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4200807213783264},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3622826337814331},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.360112726688385},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.356696754693985},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.15046238899230957},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.11086073517799377}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.876915454864502},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8089450597763062},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.7403399348258972},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.6067255735397339},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.5786656141281128},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5576629638671875},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5288145542144775},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5123389959335327},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.49448633193969727},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.4692753851413727},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.44725847244262695},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.4224385619163513},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4200807213783264},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3622826337814331},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.360112726688385},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.356696754693985},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.15046238899230957},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.11086073517799377},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"doi:10.3233/sw-200415","is_oa":false,"landing_page_url":"https://doi.org/10.3233/sw-200415","pdf_url":null,"source":{"id":"https://openalex.org/S4210177235","display_name":"Semantic Web","issn_l":"1570-0844","issn":["1570-0844","2210-4968"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Semantic Web","raw_type":"journal-article"},{"id":"pmh:oai:www.zora.uzh.ch:264735","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401281","display_name":"Zurich Open Repository and Archive (University of Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I202697423","host_organization_name":"University of Zurich","host_organization_lineage":["https://openalex.org/I202697423"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Dumitrache, Anca; Inel, Oana; Timmermans, Benjamin; Martinez-Ortiz, Carlos; Sips, Robert-Jan; Aroyo, Lora; Welty, Chris  (2021). Empirical methodology for crowdsourcing ground truth.  Semantic Web, 12(3):403-421.","raw_type":"info:eu-repo/semantics/acceptedVersion"},{"id":"pmh:oai:arXiv.org:1809.08888","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1809.08888","pdf_url":"https://arxiv.org/pdf/1809.08888","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2891077499","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1809.08888.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1809.08888","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1809.08888","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"},{"id":"doi:10.5167/uzh-264735","is_oa":true,"landing_page_url":"https://doi.org/10.5167/uzh-264735","pdf_url":null,"source":{"id":"https://openalex.org/S7407051291","display_name":"Universit\u00e4t Z\u00fcrich, ZORA","issn_l":null,"issn":[],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:www.zora.uzh.ch:264735","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401281","display_name":"Zurich Open Repository and Archive (University of Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I202697423","host_organization_name":"University of Zurich","host_organization_lineage":["https://openalex.org/I202697423"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Dumitrache, Anca; Inel, Oana; Timmermans, Benjamin; Martinez-Ortiz, Carlos; Sips, Robert-Jan; Aroyo, Lora; Welty, Chris  (2021). Empirical methodology for crowdsourcing ground truth.  Semantic Web, 12(3):403-421.","raw_type":"info:eu-repo/semantics/acceptedVersion"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":42,"referenced_works":["https://openalex.org/W62279109","https://openalex.org/W118259468","https://openalex.org/W1494186835","https://openalex.org/W1565732937","https://openalex.org/W1573900212","https://openalex.org/W1577358956","https://openalex.org/W1837605127","https://openalex.org/W1916072780","https://openalex.org/W1962580118","https://openalex.org/W1970381522","https://openalex.org/W2003497265","https://openalex.org/W2036356030","https://openalex.org/W2052217781","https://openalex.org/W2082802837","https://openalex.org/W2084413241","https://openalex.org/W2088058724","https://openalex.org/W2093044076","https://openalex.org/W2098865355","https://openalex.org/W2101349222","https://openalex.org/W2103330371","https://openalex.org/W2105578879","https://openalex.org/W2107598941","https://openalex.org/W2109429447","https://openalex.org/W2151401338","https://openalex.org/W2158428778","https://openalex.org/W2159583324","https://openalex.org/W2252039660","https://openalex.org/W2295145241","https://openalex.org/W2296107862","https://openalex.org/W2339482357","https://openalex.org/W2467897964","https://openalex.org/W2529095243","https://openalex.org/W2532694107","https://openalex.org/W2558888286","https://openalex.org/W2574781439","https://openalex.org/W2580299352","https://openalex.org/W2607311634","https://openalex.org/W2904031304","https://openalex.org/W4230282492","https://openalex.org/W4312235993","https://openalex.org/W6682780409","https://openalex.org/W6829516683"],"related_works":["https://openalex.org/W2886159104","https://openalex.org/W2208202786","https://openalex.org/W3162331882","https://openalex.org/W2402617095","https://openalex.org/W3041913476","https://openalex.org/W2116257150","https://openalex.org/W2216085305","https://openalex.org/W2056997037","https://openalex.org/W3206621920","https://openalex.org/W2614403205","https://openalex.org/W2158880898","https://openalex.org/W2290009368","https://openalex.org/W3213501996","https://openalex.org/W1761109129","https://openalex.org/W2913303681","https://openalex.org/W2185896664","https://openalex.org/W2475282416","https://openalex.org/W3175973996","https://openalex.org/W3200829055","https://openalex.org/W3013359827"],"abstract_inverted_index":{"The":[0],"process":[1],"of":[2,17,41,45,56,77,79,92,100,106,118,146,159,182,193,200,205],"gathering":[3,91],"ground":[4,93,137],"truth":[5,94],"data":[6,42,95,148],"through":[7],"human":[8],"annotation":[9,109],"is":[10,67,115,130],"a":[11,54,75,97,104,134,157,202],"major":[12],"bottleneck":[13],"in":[14,31,59,69,96,190],"the":[15,23,32,36,70,80,116,144,147,191,197],"use":[16,50,101,117],"information":[18,81],"extraction":[19],"methods":[20],"for":[21,89,132],"populating":[22],"Semantic":[24],"Web.":[25],"Crowdsourcing-based":[26],"approaches":[27],"are":[28],"gaining":[29],"popularity":[30],"attempt":[33],"to":[34,39,112,186],"solve":[35],"issues":[37],"related":[38],"volume":[40],"and":[43,108,172,188],"lack":[44],"annotators.":[46,206],"Typically":[47],"these":[48],"practices":[49],"inter-annotator":[51,123],"agreement":[52],"as":[53,63,72,74],"measure":[55],"quality.":[57],"However,":[58],"many":[60],"domains,":[61],"such":[62],"event":[64],"detection,":[65],"there":[66],"ambiguity":[68],"data,":[71],"well":[73],"multitude":[76],"perspectives":[78],"examples.":[82],"We":[83,125,139,175],"present":[84],"an":[85,179],"empirically":[86],"derived":[87],"methodology":[88],"efficiently":[90],"diverse":[98,160],"set":[99,158],"cases":[102],"covering":[103],"variety":[105],"domains":[107],"tasks.":[110],"Central":[111],"our":[113],"approach":[114],"CrowdTruth":[119,151],"metrics":[120,152],"that":[121,127,178],"capture":[122],"disagreement.":[124],"show":[126,177],"measuring":[128],"disagreement":[129],"essential":[131],"acquiring":[133],"high":[135],"quality":[136,145,192],"truth.":[138],"achieve":[140],"this":[141],"by":[142],"comparing":[143],"aggregated":[149],"with":[150,153],"majority":[154],"vote,":[155],"over":[156],"crowdsourcing":[161],"tasks:":[162],"Medical":[163],"Relation":[164],"Extraction,":[165],"Twitter":[166],"Event":[167,170],"Identification,":[168],"News":[169],"Extraction":[171],"Sound":[173],"Interpretation.":[174],"also":[176],"increased":[180],"number":[181,204],"crowd":[183],"workers":[184],"leads":[185],"growth":[187],"stabilization":[189],"annotations,":[194],"going":[195],"against":[196],"usual":[198],"practice":[199],"employing":[201],"small":[203]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}