{"id":"https://openalex.org/W2914556798","doi":"https://doi.org/10.1109/beliv.2018.8634201","title":"Lowering the Barrier for Successful Replication and Evaluation","display_name":"Lowering the Barrier for Successful Replication and Evaluation","publication_year":2018,"publication_date":"2018-10-01","ids":{"openalex":"https://openalex.org/W2914556798","doi":"https://doi.org/10.1109/beliv.2018.8634201","mag":"2914556798"},"language":"en","primary_location":{"id":"doi:10.1109/beliv.2018.8634201","is_oa":false,"landing_page_url":"https://doi.org/10.1109/beliv.2018.8634201","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE Evaluation and Beyond - Methodological Approaches for Visualization (BELIV)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://publica.fraunhofer.de/documents/N-515285.html","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018698322","display_name":"Hendrik L\u00fccke\u2010Tieke","orcid":"https://orcid.org/0000-0002-0934-6820"},"institutions":[{"id":"https://openalex.org/I3019423309","display_name":"Fraunhofer Institute for Computer Graphics Research","ror":"https://ror.org/04gmsar03","country_code":"DE","type":"facility","lineage":["https://openalex.org/I3019423309","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Hendrik Lucke-Tieke","raw_affiliation_strings":["Fraunhofer IGD"],"affiliations":[{"raw_affiliation_string":"Fraunhofer IGD","institution_ids":["https://openalex.org/I3019423309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020611302","display_name":"Marcel Beuth","orcid":null},"institutions":[{"id":"https://openalex.org/I3019423309","display_name":"Fraunhofer Institute for Computer Graphics Research","ror":"https://ror.org/04gmsar03","country_code":"DE","type":"facility","lineage":["https://openalex.org/I3019423309","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Marcel Beuth","raw_affiliation_strings":["Fraunhofer IGD"],"affiliations":[{"raw_affiliation_string":"Fraunhofer IGD","institution_ids":["https://openalex.org/I3019423309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066725732","display_name":"Philipp Schader","orcid":null},"institutions":[{"id":"https://openalex.org/I3019423309","display_name":"Fraunhofer Institute for Computer Graphics Research","ror":"https://ror.org/04gmsar03","country_code":"DE","type":"facility","lineage":["https://openalex.org/I3019423309","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Philipp Schader","raw_affiliation_strings":["Fraunhofer IGD"],"affiliations":[{"raw_affiliation_string":"Fraunhofer IGD","institution_ids":["https://openalex.org/I3019423309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025616122","display_name":"Thorsten May","orcid":"https://orcid.org/0000-0001-8027-2687"},"institutions":[{"id":"https://openalex.org/I3019423309","display_name":"Fraunhofer Institute for Computer Graphics Research","ror":"https://ror.org/04gmsar03","country_code":"DE","type":"facility","lineage":["https://openalex.org/I3019423309","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Thorsten May","raw_affiliation_strings":["Fraunhofer IGD"],"affiliations":[{"raw_affiliation_string":"Fraunhofer IGD","institution_ids":["https://openalex.org/I3019423309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078086722","display_name":"J\u00fcrgen Bernard","orcid":"https://orcid.org/0000-0001-8741-9709"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jurgen Bernard","raw_affiliation_strings":["TU Darmstadt"],"affiliations":[{"raw_affiliation_string":"TU Darmstadt","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032433963","display_name":"J\u00f6rn Kohlhammer","orcid":null},"institutions":[{"id":"https://openalex.org/I3019423309","display_name":"Fraunhofer Institute for Computer Graphics Research","ror":"https://ror.org/04gmsar03","country_code":"DE","type":"facility","lineage":["https://openalex.org/I3019423309","https://openalex.org/I4923324"]},{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jorn Kohlhammer","raw_affiliation_strings":["Fraunhofer IGD, TU Darmstadt"],"affiliations":[{"raw_affiliation_string":"Fraunhofer IGD, TU Darmstadt","institution_ids":["https://openalex.org/I3019423309","https://openalex.org/I31512782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5018698322"],"corresponding_institution_ids":["https://openalex.org/I3019423309"],"apc_list":null,"apc_paid":null,"fwci":0.6267,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.75231033,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"60","last_page":"68"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10637","display_name":"Advanced Clustering Algorithms Research","score":0.9754999876022339,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.8968288898468018},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8383196592330933},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.7466444373130798},{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.6951988339424133},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.6238807439804077},{"id":"https://openalex.org/keywords/data-visualization","display_name":"Data visualization","score":0.48698505759239197},{"id":"https://openalex.org/keywords/web-service","display_name":"Web service","score":0.47569215297698975},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.4523613452911377},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4389094412326813},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4038664698600769},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3362874388694763},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.21165618300437927},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.18065723776817322}],"concepts":[{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.8968288898468018},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8383196592330933},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.7466444373130798},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.6951988339424133},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.6238807439804077},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.48698505759239197},{"id":"https://openalex.org/C35578498","wikidata":"https://www.wikidata.org/wiki/Q193424","display_name":"Web service","level":2,"score":0.47569215297698975},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.4523613452911377},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4389094412326813},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4038664698600769},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3362874388694763},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.21165618300437927},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.18065723776817322},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/beliv.2018.8634201","is_oa":false,"landing_page_url":"https://doi.org/10.1109/beliv.2018.8634201","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2018 IEEE Evaluation and Beyond - Methodological Approaches for Visualization (BELIV)","raw_type":"proceedings-article"},{"id":"pmh:oai:fraunhofer.de:N-515285","is_oa":true,"landing_page_url":"http://publica.fraunhofer.de/documents/N-515285.html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400801","display_name":"Publikationsdatenbank der Fraunhofer-Gesellschaft (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Fraunhofer IGD","raw_type":"Conference Paper"},{"id":"pmh:oai:publica.fraunhofer.de:publica/401850","is_oa":false,"landing_page_url":"https://publica.fraunhofer.de/handle/publica/401850","pdf_url":null,"source":{"id":"https://openalex.org/S4306400318","display_name":"Fraunhofer-Publica (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"},{"id":"pmh:oai:tubiblio.ulb.tu-darmstadt.de:116175","is_oa":false,"landing_page_url":"http://tubiblio.ulb.tu-darmstadt.de/116175/","pdf_url":null,"source":{"id":"https://openalex.org/S4377196390","display_name":"TUbilio (Technical University of Darmstadt)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I31512782","host_organization_name":"Technische Universit\u00e4t Darmstadt","host_organization_lineage":["https://openalex.org/I31512782"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Konferenzver\u00f6ffentlichung"}],"best_oa_location":{"id":"pmh:oai:fraunhofer.de:N-515285","is_oa":true,"landing_page_url":"http://publica.fraunhofer.de/documents/N-515285.html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400801","display_name":"Publikationsdatenbank der Fraunhofer-Gesellschaft (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Fraunhofer IGD","raw_type":"Conference Paper"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W19399978","https://openalex.org/W1536091370","https://openalex.org/W1992743299","https://openalex.org/W2003339152","https://openalex.org/W2022625505","https://openalex.org/W2031331868","https://openalex.org/W2042361756","https://openalex.org/W2058203255","https://openalex.org/W2066079813","https://openalex.org/W2093464490","https://openalex.org/W2105467964","https://openalex.org/W2115977463","https://openalex.org/W2117470435","https://openalex.org/W2128555886","https://openalex.org/W2132862423","https://openalex.org/W2142491263","https://openalex.org/W2142493242","https://openalex.org/W2143964357","https://openalex.org/W2150076299","https://openalex.org/W2154946200","https://openalex.org/W2168440500","https://openalex.org/W2169082944","https://openalex.org/W2293451457","https://openalex.org/W2346492130","https://openalex.org/W2512030163","https://openalex.org/W2529094854","https://openalex.org/W2750514987","https://openalex.org/W2810117028","https://openalex.org/W2889072372","https://openalex.org/W2923742357"],"related_works":["https://openalex.org/W2053915529","https://openalex.org/W2111659373","https://openalex.org/W3013364101","https://openalex.org/W2378141881","https://openalex.org/W4230748207","https://openalex.org/W2025865172","https://openalex.org/W4388523091","https://openalex.org/W4220767635","https://openalex.org/W4221167770","https://openalex.org/W4253026353"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,21,31,92,110],"a":[2,11,108],"visualization":[3,33,39,58,67],"technique":[4],"is":[5],"complex":[6],"and":[7,19,44,51,64],"time-consuming.":[8],"We":[9,29,95],"present":[10],"system":[12,71],"that":[13],"aims":[14],"at":[15],"easing":[16],"design,":[17],"creation":[18],"execution":[20],"controlled":[22],"experiments":[23,54,112],"for":[24],"visualizations":[25],"in":[26,60],"the":[27,38,56,73,117],"web.":[28],"include":[30],"parameterizable":[32],"generation":[34,87],"services,":[35],"thus":[36],"separating":[37],"implementation":[40],"from":[41,75],"study":[42],"design":[43,50],"execution.":[45],"This":[46],"enables":[47],"experimenters":[48],"to":[49,78,99],"run":[52],"multiple":[53],"on":[55,89],"same":[57],"service":[59],"parallel,":[61],"replicate":[62],"experiments,":[63],"compare":[65],"different":[66],"services":[68],"quickly.":[69],"The":[70],"supports":[72],"range":[74],"simple":[76],"questionnaires":[77],"visualization-specific":[79],"interaction":[80],"techniques":[81],"as":[82,84],"well":[83],"automated":[85],"task":[86],"based":[88],"dynamic":[90],"sampling":[91],"parameter":[93],"spaces.":[94],"feature":[96],"two":[97],"examples":[98],"demonstrate":[100],"our":[101],"service-based":[102],"approach.":[103],"One":[104],"example":[105,119],"demonstrates":[106],"how":[107],"suite":[109],"successive":[111],"can":[113],"be":[114],"conducted,":[115],"while":[116],"other":[118],"includes":[120],"an":[121],"extended":[122],"replication":[123],"study.":[124]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":1}],"updated_date":"2026-02-26T08:16:20.718346","created_date":"2025-10-10T00:00:00"}