{"id":"https://openalex.org/W2346492130","doi":"https://doi.org/10.1109/pacificvis.2016.7465249","title":"A crowdsourcing system for integrated and reproducible evaluation in scientific visualization","display_name":"A crowdsourcing system for integrated and reproducible evaluation in scientific visualization","publication_year":2016,"publication_date":"2016-04-01","ids":{"openalex":"https://openalex.org/W2346492130","doi":"https://doi.org/10.1109/pacificvis.2016.7465249","mag":"2346492130"},"language":"en","primary_location":{"id":"doi:10.1109/pacificvis.2016.7465249","is_oa":false,"landing_page_url":"https://doi.org/10.1109/pacificvis.2016.7465249","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2016 IEEE Pacific Visualization Symposium (PacificVis)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-128702","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057685533","display_name":"Rickard Englund","orcid":null},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":true,"raw_author_name":"Rickard Englund","raw_affiliation_strings":["Linkopings universitet, Linkoping, SE"],"affiliations":[{"raw_affiliation_string":"Linkopings universitet, Linkoping, SE","institution_ids":["https://openalex.org/I102134673"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019831046","display_name":"Sathish Kottravel","orcid":"https://orcid.org/0000-0001-6511-4314"},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Sathish Kottravel","raw_affiliation_strings":["Interactive Visualization Group, Linkoping University, Sweden"],"affiliations":[{"raw_affiliation_string":"Interactive Visualization Group, Linkoping University, Sweden","institution_ids":["https://openalex.org/I102134673"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061059879","display_name":"Timo Ropinski","orcid":"https://orcid.org/0000-0002-7857-5512"},"institutions":[{"id":"https://openalex.org/I196349391","display_name":"Universit\u00e4t Ulm","ror":"https://ror.org/032000t02","country_code":"DE","type":"education","lineage":["https://openalex.org/I196349391"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Timo Ropinski","raw_affiliation_strings":["Visual Computing Group, Ulm University, Germany"],"affiliations":[{"raw_affiliation_string":"Visual Computing Group, Ulm University, Germany","institution_ids":["https://openalex.org/I196349391"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5057685533"],"corresponding_institution_ids":["https://openalex.org/I102134673"],"apc_list":null,"apc_paid":null,"fwci":1.67,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.89032843,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"40","last_page":"47"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8679201602935791},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8186756372451782},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.7869670987129211},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6208981871604919},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.48955783247947693},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4598245918750763},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4175783097743988},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.416273295879364},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.41211697459220886},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.19716399908065796},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1936594545841217}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8679201602935791},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8186756372451782},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.7869670987129211},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6208981871604919},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.48955783247947693},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4598245918750763},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4175783097743988},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.416273295879364},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.41211697459220886},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.19716399908065796},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1936594545841217},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/pacificvis.2016.7465249","is_oa":false,"landing_page_url":"https://doi.org/10.1109/pacificvis.2016.7465249","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2016 IEEE Pacific Visualization Symposium (PacificVis)","raw_type":"proceedings-article"},{"id":"pmh:oai:DiVA.org:liu-128702","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-128702","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference paper"}],"best_oa_location":{"id":"pmh:oai:DiVA.org:liu-128702","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:liu:diva-128702","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference paper"},"sustainable_development_goals":[{"score":0.4399999976158142,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W52809394","https://openalex.org/W1562630052","https://openalex.org/W1565761017","https://openalex.org/W1970282908","https://openalex.org/W1970569592","https://openalex.org/W1992743299","https://openalex.org/W2014703509","https://openalex.org/W2017521531","https://openalex.org/W2022625505","https://openalex.org/W2040726711","https://openalex.org/W2041633421","https://openalex.org/W2042361756","https://openalex.org/W2043523210","https://openalex.org/W2052511481","https://openalex.org/W2054901814","https://openalex.org/W2058199791","https://openalex.org/W2058203255","https://openalex.org/W2059745921","https://openalex.org/W2061863227","https://openalex.org/W2062196125","https://openalex.org/W2069228960","https://openalex.org/W2070664725","https://openalex.org/W2086301945","https://openalex.org/W2100396541","https://openalex.org/W2107031757","https://openalex.org/W2117470435","https://openalex.org/W2120831453","https://openalex.org/W2128555886","https://openalex.org/W2135415614","https://openalex.org/W2140469925","https://openalex.org/W2142491263","https://openalex.org/W2142493242","https://openalex.org/W2145459236","https://openalex.org/W2148479118","https://openalex.org/W2151022533","https://openalex.org/W2151401338","https://openalex.org/W2154946200","https://openalex.org/W2157887077","https://openalex.org/W2168440500","https://openalex.org/W2170989440","https://openalex.org/W2231327593","https://openalex.org/W2293451457","https://openalex.org/W2923742357","https://openalex.org/W4300870773","https://openalex.org/W6642852546","https://openalex.org/W6661527498","https://openalex.org/W6676028428","https://openalex.org/W6680141850","https://openalex.org/W6689594162","https://openalex.org/W6760590573","https://openalex.org/W6944797908"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"User":[0],"evaluations":[1,18,47,114,139,157,180,197,230],"have":[2,271],"gained":[3],"increasing":[4],"importance":[5],"in":[6,14,37,70,241,275],"visualization":[7,29,144,243,277],"research":[8],"over":[9],"the":[10,20,25,43,56,67,82,88,107,129,131,134,142,148,193,201,204,208,226,248,252,256,261,276],"past":[11],"years,":[12],"as":[13,190],"many":[15],"cases":[16],"these":[17],"are":[19,55,237],"only":[21,168],"way":[22],"to":[23,39,61,101,177,187,192],"support":[24],"claims":[26],"made":[27,254],"by":[28,263],"researchers.":[30],"Unfortunately,":[31],"recent":[32],"literature":[33],"reviews":[34],"show":[35],"that":[36,81],"comparison":[38],"algorithmic":[40],"performance":[41],"evaluations,":[42,267],"number":[44],"of":[45,59,84,92,136,203,221,228,269],"user":[46,138,165,196,266],"is":[48,115,210],"still":[49,116],"very":[50,117],"low.":[51],"Reasons":[52],"for":[53,109],"this":[54,96,120],"required":[57],"amount":[58],"time":[60,108,194],"conduct":[62,153],"such":[63,113],"studies":[64],"together":[65],"with":[66,167,231],"difficulties":[68],"involved":[69],"participant":[71,90],"recruitment":[72,91],"and":[73,87,111,133,154,225,251,259],"result":[74],"reporting.":[75],"While":[76],"it":[77,174],"could":[78],"be":[79],"shown":[80],"quality":[83],"evaluation":[85,223],"results":[86],"simplified":[89],"crowdsourcing":[93],"platforms":[94],"makes":[95],"technology":[97],"a":[98,124,159,169,213],"viable":[99],"alternative":[100],"lab":[102],"experiments":[103],"when":[104],"evaluating":[105],"visualizations,":[106],"conducting":[110,264],"reporting":[112,135],"high.":[118],"In":[119],"paper,":[121],"we":[122],"propose":[123],"software":[125],"system,":[126,150],"which":[127,184,216,236,270],"integrates":[128],"conduction,":[130],"analysis":[132],"crowdsourced":[137],"directly":[140],"into":[141],"scientific":[143,242],"development":[145],"process.":[146,206],"With":[147],"proposed":[149],"researchers":[151],"can":[152],"analyze":[155],"quantitative":[156],"on":[158],"large":[160],"scale":[161],"through":[162],"an":[163,218],"evaluation-centric":[164],"interface":[166],"few":[170],"mouse":[171],"clicks.":[172],"Thus,":[173],"becomes":[175],"possible":[176],"perform":[178],"iterative":[179],"during":[181,255],"algorithm":[182],"design,":[183],"potentially":[185],"leads":[186],"better":[188],"results,":[189],"compared":[191],"consuming":[195],"traditionally":[198],"conducted":[199],"at":[200],"end":[202],"design":[205,250,257],"Furthermore,":[207],"system":[209,262],"built":[211],"around":[212],"centralized":[214],"database,":[215],"supports":[217],"easy":[219],"reuse":[220],"old":[222,229],"designs":[224],"reproduction":[227],"new":[232],"or":[233],"additional":[234],"stimuli,":[235],"both":[238],"driving":[239],"challenges":[240],"research.":[244],"We":[245],"will":[246],"describe":[247],"system's":[249],"considerations":[253],"process,":[258],"demonstrate":[260],"three":[265],"all":[268],"been":[272],"published":[273],"before":[274],"literature.":[278]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}