{"id":"https://openalex.org/W1937987873","doi":"https://doi.org/10.1080/17538947.2015.1039609","title":"Assessing quality of volunteer crowdsourcing contributions: lessons from the Cropland Capture game","display_name":"Assessing quality of volunteer crowdsourcing contributions: lessons from the Cropland Capture game","publication_year":2015,"publication_date":"2015-06-02","ids":{"openalex":"https://openalex.org/W1937987873","doi":"https://doi.org/10.1080/17538947.2015.1039609","mag":"1937987873"},"language":"en","primary_location":{"id":"doi:10.1080/17538947.2015.1039609","is_oa":false,"landing_page_url":"https://doi.org/10.1080/17538947.2015.1039609","pdf_url":null,"source":{"id":"https://openalex.org/S199162493","display_name":"International Journal of Digital Earth","issn_l":"1753-8947","issn":["1753-8947","1753-8955"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Digital Earth","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006273022","display_name":"Carl Salk","orcid":"https://orcid.org/0000-0003-2833-8292"},"institutions":[{"id":"https://openalex.org/I298625061","display_name":"Swedish University of Agricultural Sciences","ror":"https://ror.org/02yy8x990","country_code":"SE","type":"education","lineage":["https://openalex.org/I298625061"]},{"id":"https://openalex.org/I1317774081","display_name":"International Institute for Applied Systems Analysis","ror":"https://ror.org/02wfhk785","country_code":"AT","type":"funder","lineage":["https://openalex.org/I1317774081"]}],"countries":["AT","SE"],"is_corresponding":true,"raw_author_name":"Carl F. Salk","raw_affiliation_strings":["Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria; Southern Swedish Forest Research Center, Swedish University of Agricultural Sciences, Alnarp, Sweden","Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria","Southern Swedish Forest Research Center, Swedish University of Agricultural Sciences, Alnarp, Sweden"],"affiliations":[{"raw_affiliation_string":"Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria; Southern Swedish Forest Research Center, Swedish University of Agricultural Sciences, Alnarp, Sweden","institution_ids":["https://openalex.org/I1317774081"]},{"raw_affiliation_string":"Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria","institution_ids":["https://openalex.org/I1317774081"]},{"raw_affiliation_string":"Southern Swedish Forest Research Center, Swedish University of Agricultural Sciences, Alnarp, Sweden","institution_ids":["https://openalex.org/I298625061"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034657026","display_name":"Tobias Sturn","orcid":null},"institutions":[{"id":"https://openalex.org/I1317774081","display_name":"International Institute for Applied Systems Analysis","ror":"https://ror.org/02wfhk785","country_code":"AT","type":"funder","lineage":["https://openalex.org/I1317774081"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Tobias Sturn","raw_affiliation_strings":["Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria"],"affiliations":[{"raw_affiliation_string":"Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria","institution_ids":["https://openalex.org/I1317774081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089565966","display_name":"Linda See","orcid":"https://orcid.org/0000-0002-2665-7065"},"institutions":[{"id":"https://openalex.org/I1317774081","display_name":"International Institute for Applied Systems Analysis","ror":"https://ror.org/02wfhk785","country_code":"AT","type":"funder","lineage":["https://openalex.org/I1317774081"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Linda See","raw_affiliation_strings":["Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria"],"affiliations":[{"raw_affiliation_string":"Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria","institution_ids":["https://openalex.org/I1317774081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040252235","display_name":"Steffen Fritz","orcid":"https://orcid.org/0000-0003-0420-8549"},"institutions":[{"id":"https://openalex.org/I1317774081","display_name":"International Institute for Applied Systems Analysis","ror":"https://ror.org/02wfhk785","country_code":"AT","type":"funder","lineage":["https://openalex.org/I1317774081"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Steffen Fritz","raw_affiliation_strings":["Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria"],"affiliations":[{"raw_affiliation_string":"Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria","institution_ids":["https://openalex.org/I1317774081"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065288270","display_name":"Christoph Perger","orcid":"https://orcid.org/0000-0002-7206-4148"},"institutions":[{"id":"https://openalex.org/I1317774081","display_name":"International Institute for Applied Systems Analysis","ror":"https://ror.org/02wfhk785","country_code":"AT","type":"funder","lineage":["https://openalex.org/I1317774081"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Christoph Perger","raw_affiliation_strings":["Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria"],"affiliations":[{"raw_affiliation_string":"Ecosystems Services and Management Program, International Institute for Applied Systems Analysis, Laxenburg, Austria","institution_ids":["https://openalex.org/I1317774081"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5006273022"],"corresponding_institution_ids":["https://openalex.org/I1317774081","https://openalex.org/I298625061"],"apc_list":{"value":2390,"currency":"USD","value_usd":2390},"apc_paid":null,"fwci":43.6381,"has_fulltext":false,"cited_by_count":66,"citation_normalized_percentile":{"value":0.9970943,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"9","issue":"4","first_page":"410","last_page":"426"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10757","display_name":"Geographic Information Systems Studies","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/3305","display_name":"Geography, Planning and Development"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10757","display_name":"Geographic Information Systems Studies","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/3305","display_name":"Geography, Planning and Development"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10895","display_name":"Species Distribution and Climate Change","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/2302","display_name":"Ecological Modeling"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11819","display_name":"Data-Driven Disease Surveillance","score":0.9757000207901001,"subfield":{"id":"https://openalex.org/subfields/2713","display_name":"Epidemiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9074870944023132},{"id":"https://openalex.org/keywords/volunteered-geographic-information","display_name":"Volunteered geographic information","score":0.8705145120620728},{"id":"https://openalex.org/keywords/citizen-science","display_name":"Citizen science","score":0.7267993688583374},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6335342526435852},{"id":"https://openalex.org/keywords/volunteer","display_name":"Volunteer","score":0.6319459676742554},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5480889081954956},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.46031755208969116},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.43522605299949646},{"id":"https://openalex.org/keywords/data-quality","display_name":"Data quality","score":0.41352155804634094},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3222447633743286},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.26995325088500977},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.21755659580230713},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.16312441229820251},{"id":"https://openalex.org/keywords/operations-management","display_name":"Operations management","score":0.13693547248840332},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.10584685206413269}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9074870944023132},{"id":"https://openalex.org/C57380593","wikidata":"https://www.wikidata.org/wiki/Q933625","display_name":"Volunteered geographic information","level":2,"score":0.8705145120620728},{"id":"https://openalex.org/C197352329","wikidata":"https://www.wikidata.org/wiki/Q1093434","display_name":"Citizen science","level":2,"score":0.7267993688583374},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6335342526435852},{"id":"https://openalex.org/C2776768635","wikidata":"https://www.wikidata.org/wiki/Q3427301","display_name":"Volunteer","level":2,"score":0.6319459676742554},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5480889081954956},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.46031755208969116},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.43522605299949646},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.41352155804634094},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3222447633743286},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26995325088500977},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.21755659580230713},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.16312441229820251},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.13693547248840332},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.10584685206413269},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1080/17538947.2015.1039609","is_oa":false,"landing_page_url":"https://doi.org/10.1080/17538947.2015.1039609","pdf_url":null,"source":{"id":"https://openalex.org/S199162493","display_name":"International Journal of Digital Earth","issn_l":"1753-8947","issn":["1753-8947","1753-8955"],"is_oa":false,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Digital Earth","raw_type":"journal-article"},{"id":"pmh:oai:pub.epsilon.slu.se:14273","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306402317","display_name":"Epsilon Open Archive (Sveriges lantbruksuniversitet biblioteket (Swedish University of Agricultural Sciences))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I298625061","host_organization_name":"Swedish University of Agricultural Sciences","host_organization_lineage":["https://openalex.org/I298625061"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Research article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2225102082","display_name":null,"funder_award_id":"CrowdLand (617754),SIGMA (603719)","funder_id":"https://openalex.org/F4320334678","funder_display_name":"European Research Council"}],"funders":[{"id":"https://openalex.org/F4320327324","display_name":"International Institute for Applied Systems Analysis","ror":"https://ror.org/02wfhk785"},{"id":"https://openalex.org/F4320334678","display_name":"European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W71896585","https://openalex.org/W606297877","https://openalex.org/W1559837540","https://openalex.org/W1581541037","https://openalex.org/W1805123227","https://openalex.org/W1894564341","https://openalex.org/W1971109292","https://openalex.org/W1980385468","https://openalex.org/W1992145496","https://openalex.org/W1993918804","https://openalex.org/W1998918746","https://openalex.org/W2001728294","https://openalex.org/W2004492980","https://openalex.org/W2018463030","https://openalex.org/W2021350657","https://openalex.org/W2022369034","https://openalex.org/W2022664803","https://openalex.org/W2023865077","https://openalex.org/W2040166608","https://openalex.org/W2042692910","https://openalex.org/W2047768285","https://openalex.org/W2049080106","https://openalex.org/W2053418852","https://openalex.org/W2058738785","https://openalex.org/W2063935068","https://openalex.org/W2075598345","https://openalex.org/W2093393699","https://openalex.org/W2105268242","https://openalex.org/W2105663391","https://openalex.org/W2117810730","https://openalex.org/W2118096631","https://openalex.org/W2118711140","https://openalex.org/W2123227534","https://openalex.org/W2151915598","https://openalex.org/W2167239677","https://openalex.org/W2226472208","https://openalex.org/W2549412929","https://openalex.org/W4285719527","https://openalex.org/W4391048313","https://openalex.org/W6885636619"],"related_works":["https://openalex.org/W2733029865","https://openalex.org/W3032998312","https://openalex.org/W2555391182","https://openalex.org/W2288471556","https://openalex.org/W2779614303","https://openalex.org/W194327752","https://openalex.org/W2315156699","https://openalex.org/W2566100235","https://openalex.org/W2301056841","https://openalex.org/W3110990129"],"abstract_inverted_index":{"Volunteered":[0],"geographic":[1],"information":[2,9],"(VGI)":[3],"is":[4],"the":[5,22,34,54,57,70,75,103,148,161,168],"assembly":[6],"of":[7,24,36,56,72,77,105,156,170,174],"spatial":[8],"based":[10],"on":[11,94,109],"public":[12],"input.":[13],"While":[14],"VGI":[15,183],"has":[16,27],"proliferated":[17],"in":[18,61],"recent":[19],"years,":[20],"assessing":[21],"quality":[23,45,138],"volunteer-contributed":[25],"data":[26,52],"proven":[28],"challenging,":[29],"leading":[30],"some":[31],"to":[32],"question":[33],"efficiency":[35],"such":[37],"programs.":[38],"In":[39],"this":[40],"paper,":[41],"we":[42],"compare":[43],"several":[44,63],"metrics":[46,132],"for":[47,69,181],"individual":[48],"volunteers\u2019":[49],"contributions.":[50],"The":[51],"were":[53,133,151],"product":[55],"\u2018Cropland":[58],"Capture\u2019":[59],"game,":[60],"which":[62],"thousand":[64],"volunteers":[65],"assessed":[66],"165,000":[67],"images":[68,96],"presence":[71],"cropland":[73],"over":[74],"course":[76],"6":[78],"months.":[79],"We":[80,100],"compared":[81],"agreement":[82,118,126],"between":[83],"volunteer":[84,92],"ratings":[85],"and":[86,97,107,122],"an":[87],"image's":[88],"majority":[89,120],"classification":[90],"with":[91,119,127,145],"self-agreement":[93,112],"repeated":[95],"expert":[98,128],"evaluations.":[99],"also":[101],"examined":[102],"impact":[104],"experience":[106],"learning":[108],"performance.":[110],"Volunteer":[111,137],"was":[113],"nearly":[114],"always":[115],"higher":[116],"than":[117,125],"classifications,":[121],"much":[123],"greater":[124],"validations":[129],"although":[130],"these":[131],"all":[134],"positively":[135],"correlated.":[136],"showed":[139],"a":[140,154,171,178],"broad":[141],"trend":[142],"toward":[143],"improvement":[144],"experience,":[146],"but":[147],"highest":[149],"accuracies":[150],"achieved":[152],"by":[153],"handful":[155],"moderately":[157],"active":[158,163],"contributors,":[159],"not":[160],"most":[162],"volunteers.":[164],"Our":[165],"results":[166],"emphasize":[167],"importance":[169],"universal":[172],"set":[173],"expert-validated":[175],"tasks":[176],"as":[177],"gold":[179],"standard":[180],"evaluating":[182],"quality.":[184]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":11},{"year":2017,"cited_by_count":9},{"year":2016,"cited_by_count":10},{"year":2015,"cited_by_count":2}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}