{"id":"https://openalex.org/W2883256923","doi":"https://doi.org/10.1145/3239570","title":"Evaluation-as-a-Service for the Computational Sciences","display_name":"Evaluation-as-a-Service for the Computational Sciences","publication_year":2018,"publication_date":"2018-10-29","ids":{"openalex":"https://openalex.org/W2883256923","doi":"https://doi.org/10.1145/3239570","mag":"2883256923"},"language":"en","primary_location":{"id":"doi:10.1145/3239570","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3239570","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080524660","display_name":"Frank Hopfgartner","orcid":"https://orcid.org/0000-0003-0380-6088"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Frank Hopfgartner","raw_affiliation_strings":["University of Sheffield, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0003-0380-6088","affiliations":[{"raw_affiliation_string":"University of Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020665735","display_name":"Allan Hanbury","orcid":"https://orcid.org/0000-0002-7149-5843"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]},{"id":"https://openalex.org/I4210112164","display_name":"Complexity Science Hub","ror":"https://ror.org/023dz9m50","country_code":"AT","type":"nonprofit","lineage":["https://openalex.org/I4210112164"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Allan Hanbury","raw_affiliation_strings":["TU Wien, Complexity Science Hub Vienna, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TU Wien, Complexity Science Hub Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I145847075","https://openalex.org/I4210112164"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061961859","display_name":"Henning M\u00fcller","orcid":"https://orcid.org/0000-0001-6800-9878"},"institutions":[{"id":"https://openalex.org/I173439891","display_name":"HES-SO University of Applied Sciences and Arts Western Switzerland","ror":"https://ror.org/01xkakk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I173439891"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Henning M\u00fcller","raw_affiliation_strings":["University of Applied Sciences Western Switzerland (HES-SO), Sierre, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Applied Sciences Western Switzerland (HES-SO), Sierre, Switzerland","institution_ids":["https://openalex.org/I173439891"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072406581","display_name":"Ivan Eggel","orcid":null},"institutions":[{"id":"https://openalex.org/I173439891","display_name":"HES-SO University of Applied Sciences and Arts Western Switzerland","ror":"https://ror.org/01xkakk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I173439891"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Ivan Eggel","raw_affiliation_strings":["University of Applied Sciences Western Switzerland (HES-SO), Sierre, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Applied Sciences Western Switzerland (HES-SO), Sierre, Switzerland","institution_ids":["https://openalex.org/I173439891"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059926999","display_name":"Krisztian Balog","orcid":"https://orcid.org/0000-0003-2762-721X"},"institutions":[{"id":"https://openalex.org/I92008406","display_name":"University of Stavanger","ror":"https://ror.org/02qte9q33","country_code":"NO","type":"education","lineage":["https://openalex.org/I92008406"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Krisztian Balog","raw_affiliation_strings":["University of Stavanger, Stavanger, Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Stavanger, Stavanger, Norway","institution_ids":["https://openalex.org/I92008406"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020270103","display_name":"Torben Brodt","orcid":"https://orcid.org/0009-0008-0650-7490"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Torben Brodt","raw_affiliation_strings":["plista GmbH, Berlin Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"plista GmbH, Berlin Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084091807","display_name":"Gordon V. Cormack","orcid":"https://orcid.org/0000-0002-5890-0293"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Gordon V. Cormack","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039535916","display_name":"Jayashree Kalpathy\u2013Cramer","orcid":"https://orcid.org/0000-0001-8906-9618"},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]},{"id":"https://openalex.org/I4210087915","display_name":"Massachusetts General Hospital","ror":"https://ror.org/002pd6e78","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210087915","https://openalex.org/I48633490"]},{"id":"https://openalex.org/I4210127055","display_name":"Athinoula A. Martinos Center for Biomedical Imaging","ror":"https://ror.org/032q5ym94","country_code":"US","type":"facility","lineage":["https://openalex.org/I136199984","https://openalex.org/I4210087915","https://openalex.org/I4210127055","https://openalex.org/I48633490","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jayashree Kalpathy-Cramer","raw_affiliation_strings":["Athinoula A. Martinos Center for Biomedical Imaging at Massachusetts General Hospital and Harvard Medical School, Charlestown, MA USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Athinoula A. Martinos Center for Biomedical Imaging at Massachusetts General Hospital and Harvard Medical School, Charlestown, MA USA","institution_ids":["https://openalex.org/I136199984","https://openalex.org/I4210087915","https://openalex.org/I4210127055"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045875579","display_name":"Noriko Kando","orcid":"https://orcid.org/0000-0002-2133-0215"},"institutions":[{"id":"https://openalex.org/I184597095","display_name":"National Institute of Informatics","ror":"https://ror.org/04ksd4g47","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I184597095","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Noriko Kando","raw_affiliation_strings":["National Institute of Informatics, Tokyo, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Informatics, Tokyo, Japan","institution_ids":["https://openalex.org/I184597095"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015183832","display_name":"Makoto P. Kato","orcid":"https://orcid.org/0000-0002-9351-0901"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Makoto P. Kato","raw_affiliation_strings":["Kyoto University, Yoshida Honmachi, Sakyo, Kyoto, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kyoto University, Yoshida Honmachi, Sakyo, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058877446","display_name":"Anastasia Krithara","orcid":"https://orcid.org/0000-0003-0491-4507"},"institutions":[{"id":"https://openalex.org/I203474044","display_name":"National Centre of Scientific Research \"Demokritos\"","ror":"https://ror.org/038jp4m40","country_code":"GR","type":"facility","lineage":["https://openalex.org/I203474044"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Anastasia Krithara","raw_affiliation_strings":["National Center for Scientific Research \u201cDemokritos\u201d, Paraskevi, Athens, Greece","National Center for Scientific Research \"Demokritos\", Paraskevi, Athens, Greece"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Center for Scientific Research \u201cDemokritos\u201d, Paraskevi, Athens, Greece","institution_ids":["https://openalex.org/I203474044"]},{"raw_affiliation_string":"National Center for Scientific Research \"Demokritos\", Paraskevi, Athens, Greece","institution_ids":["https://openalex.org/I203474044"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087022274","display_name":"Tim Gollub","orcid":"https://orcid.org/0000-0003-1737-6517"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Tim Gollub","raw_affiliation_strings":["Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083712311","display_name":"Martin Potthast","orcid":"https://orcid.org/0000-0003-2451-0665"},"institutions":[{"id":"https://openalex.org/I926574661","display_name":"Leipzig University","ror":"https://ror.org/03s7gtk40","country_code":"DE","type":"education","lineage":["https://openalex.org/I926574661"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Martin Potthast","raw_affiliation_strings":["Leipzig University, Leipzig, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Leipzig University, Leipzig, Germany","institution_ids":["https://openalex.org/I926574661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069349084","display_name":"Evelyne Viegas","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evelyne Viegas","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007133493","display_name":"Simon Mercer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Simon Mercer","raw_affiliation_strings":["Independent Consultant"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent Consultant","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":9,"institutions_distinct_count":15,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.1729,"has_fulltext":false,"cited_by_count":45,"citation_normalized_percentile":{"value":0.93219679,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"10","issue":"4","first_page":"1","last_page":"32"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8119193315505981},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.7181113958358765},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6525217294692993},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5972950458526611},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.5957844853401184},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5153869390487671},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.48658692836761475},{"id":"https://openalex.org/keywords/confidentiality","display_name":"Confidentiality","score":0.4816130995750427},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.45390215516090393},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.41493451595306396},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.27592527866363525},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.20143389701843262},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.14346462488174438}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8119193315505981},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.7181113958358765},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6525217294692993},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5972950458526611},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.5957844853401184},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5153869390487671},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.48658692836761475},{"id":"https://openalex.org/C71745522","wikidata":"https://www.wikidata.org/wiki/Q2476929","display_name":"Confidentiality","level":2,"score":0.4816130995750427},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.45390215516090393},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.41493451595306396},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.27592527866363525},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.20143389701843262},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.14346462488174438},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3239570","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3239570","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"},{"id":"pmh:oai:eprints.whiterose.ac.uk:133293","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400854","display_name":"White Rose Research Online (University of Leeds, The University of Sheffield, University of York)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2800616092","host_organization_name":"White Rose University Consortium","host_organization_lineage":["https://openalex.org/I2800616092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.6600000262260437}],"awards":[{"id":"https://openalex.org/G7529661274","display_name":"Research on Information Elicitation Technologies for Estimating Latent Information Needs","funder_award_id":"26700009","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"}],"funders":[{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":79,"referenced_works":["https://openalex.org/W39360085","https://openalex.org/W56959776","https://openalex.org/W88603068","https://openalex.org/W114599062","https://openalex.org/W135190683","https://openalex.org/W196002257","https://openalex.org/W789651694","https://openalex.org/W806995027","https://openalex.org/W1482274359","https://openalex.org/W1494137514","https://openalex.org/W1507150160","https://openalex.org/W1524281572","https://openalex.org/W1534477342","https://openalex.org/W1546847154","https://openalex.org/W1553544680","https://openalex.org/W1586649869","https://openalex.org/W1635892993","https://openalex.org/W1757621696","https://openalex.org/W1981208470","https://openalex.org/W1981745715","https://openalex.org/W1990244040","https://openalex.org/W1995054497","https://openalex.org/W2006996936","https://openalex.org/W2009545460","https://openalex.org/W2015770094","https://openalex.org/W2017419154","https://openalex.org/W2023616424","https://openalex.org/W2036417821","https://openalex.org/W2036448357","https://openalex.org/W2037140704","https://openalex.org/W2038061949","https://openalex.org/W2038885675","https://openalex.org/W2049297495","https://openalex.org/W2064818677","https://openalex.org/W2066682493","https://openalex.org/W2073893432","https://openalex.org/W2075754392","https://openalex.org/W2089711563","https://openalex.org/W2124972954","https://openalex.org/W2132862423","https://openalex.org/W2145684485","https://openalex.org/W2149085946","https://openalex.org/W2169113736","https://openalex.org/W2177816635","https://openalex.org/W2178147882","https://openalex.org/W2187139527","https://openalex.org/W2250201115","https://openalex.org/W2250384498","https://openalex.org/W2271209846","https://openalex.org/W2294992630","https://openalex.org/W2339562433","https://openalex.org/W2396476612","https://openalex.org/W2397447617","https://openalex.org/W2415809604","https://openalex.org/W2463109969","https://openalex.org/W2464171116","https://openalex.org/W2468776465","https://openalex.org/W2485755916","https://openalex.org/W2492484388","https://openalex.org/W2516976960","https://openalex.org/W2517508341","https://openalex.org/W2561828810","https://openalex.org/W2566134098","https://openalex.org/W2572848331","https://openalex.org/W2621144345","https://openalex.org/W2625835497","https://openalex.org/W2734973975","https://openalex.org/W2740840489","https://openalex.org/W2746418139","https://openalex.org/W2783670593","https://openalex.org/W2789228244","https://openalex.org/W2794432940","https://openalex.org/W2897807698","https://openalex.org/W2967583913","https://openalex.org/W3026272994","https://openalex.org/W3125148015","https://openalex.org/W4232709175","https://openalex.org/W4255767193","https://openalex.org/W4402526844"],"related_works":["https://openalex.org/W4387497383","https://openalex.org/W3183948672","https://openalex.org/W3173606202","https://openalex.org/W3110381201","https://openalex.org/W2948807893","https://openalex.org/W2935909890","https://openalex.org/W2778153218","https://openalex.org/W2758277628","https://openalex.org/W1531601525","https://openalex.org/W3032998312"],"abstract_inverted_index":{"Evaluation":[0],"in":[1,80,92,100,124,244,264],"empirical":[2],"computer":[3],"science":[4],"is":[5,132,143],"essential":[6],"to":[7,27,115,118,151,180,189,203,221,255,261,289],"show":[8],"progress":[9],"and":[10,42,84,110,153,166,191,196,223,228,232,241,259],"assess":[11],"technologies":[12],"developed.":[13],"Several":[14],"research":[15,213,280,326,340],"domains":[16],"such":[17,322],"as":[18,78,89,313,315],"information":[19],"retrieval":[20],"have":[21,61,154],"long":[22],"relied":[23],"on":[24,120,134,139,157],"systematic":[25],"evaluation":[26],"measure":[28],"progress:":[29],"here,":[30],"the":[31,81,98,125,144,158,163,193,198,205,230,242,249,278,302,336],"Cranfield":[32],"paradigm":[33,68,145],"of":[34,127,146,185,200,208,246,277,293,304,338],"creating":[35],"shared":[36],"test":[37],"collections,":[38],"defining":[39],"search":[40],"tasks,":[41],"collecting":[43],"ground":[44],"truth":[45],"for":[46,248,268,325],"these":[47,201],"tasks":[48],"has":[49,95],"persisted":[50],"up":[51],"until":[52],"now.":[53],"In":[54],"recent":[55],"years,":[56],"however,":[57,308],"several":[58,218],"new":[59],"challenges":[60,109,319],"emerged":[62],"that":[63],"do":[64],"not":[65,147,287,299],"fit":[66],"this":[67,186],"very":[69],"well:":[70],"extremely":[71],"large":[72],"data":[73,76,87,149,159,164,283,311],"sets,":[74],"confidential":[75],"sets":[77,88,150,284,312],"found":[79],"medical":[82],"domain,":[83],"rapidly":[85],"changing":[86],"often":[90,286,298],"encountered":[91],"industry.":[93],"Crowdsourcing":[94],"also":[96,229,328],"changed":[97],"way":[99],"which":[101,269],"industry":[102,262],"approaches":[103,195,202,220],"problem-solving":[104],"with":[105],"companies":[106],"now":[107],"organizing":[108],"handing":[111],"out":[112],"monetary":[113],"awards":[114],"incentivize":[116],"people":[117],"work":[119,156],"their":[121,225],"challenges,":[122],"particularly":[123,210],"field":[126],"machine":[128],"learning.":[129],"This":[130],"article":[131,187,216],"based":[133],"discussions":[135],"at":[136],"a":[137,323],"workshop":[138],"Evaluation-as-a-Service":[140],"(EaaS).":[141],"EaaS":[142,222,238,273],"providing":[148],"participants":[152],"them":[155],"locally,":[160],"but":[161,321],"keeping":[162],"central":[165],"allowing":[167],"access":[168],"via":[169],"Application":[170],"Programming":[171],"Interfaces":[172],"(API),":[173],"Virtual":[174],"Machines":[175],"(VM),":[176],"or":[177],"other":[178],"possibilities":[179],"ship":[181],"executables.":[182,317],"The":[183,215,234],"objectives":[184],"are":[188,239,285,296],"summarize":[190],"compare":[192],"current":[194,279],"consolidate":[197],"experiences":[199],"outline":[204],"next":[206],"steps":[207],"EaaS,":[209,307],"toward":[211],"sustainable":[212],"infrastructures.":[214],"summarizes":[217],"existing":[219],"analyzes":[224],"usage":[226],"scenarios":[227],"advantages":[231],"disadvantages.":[233],"many":[235,275,290],"factors":[236],"influencing":[237],"summarized,":[240],"environment":[243],"terms":[245],"motivations":[247],"various":[250],"stakeholders,":[251],"from":[252],"funding":[253],"agencies":[254],"challenge":[256],"organizers,":[257],"researchers":[258],"participants,":[260],"interested":[263],"supplying":[265],"real-world":[266],"problems":[267,276],"they":[270],"require":[271],"solutions.":[272],"solves":[274],"environment,":[281],"where":[282],"accessible":[288],"researchers.":[291],"Executables":[292],"published":[294],"tools":[295],"equally":[297],"available":[300,316],"making":[301],"reproducibility":[303],"results":[305],"impossible.":[306],"creates":[309],"reusable/citable":[310],"well":[314],"Many":[318],"remain,":[320],"framework":[324],"can":[327],"foster":[329],"more":[330],"collaboration":[331],"between":[332],"researchers,":[333],"potentially":[334],"increasing":[335],"speed":[337],"obtaining":[339],"results.":[341]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":8}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2018-08-03T00:00:00"}