{"id":"https://openalex.org/W789651694","doi":"https://doi.org/10.1145/2795403.2795416","title":"Report on the Evaluation-as-a-Service (EaaS) Expert Workshop","display_name":"Report on the Evaluation-as-a-Service (EaaS) Expert Workshop","publication_year":2015,"publication_date":"2015-06-23","ids":{"openalex":"https://openalex.org/W789651694","doi":"https://doi.org/10.1145/2795403.2795416","mag":"789651694"},"language":"en","primary_location":{"id":"doi:10.1145/2795403.2795416","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2795403.2795416","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080524660","display_name":"Frank Hopfgartner","orcid":"https://orcid.org/0000-0003-0380-6088"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Frank Hopfgartner","raw_affiliation_strings":["","University of Glasgow, Glasgow, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020665735","display_name":"Allan Hanbury","orcid":"https://orcid.org/0000-0002-7149-5843"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Allan Hanbury","raw_affiliation_strings":["","TU Wien, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"TU Wien, Vienna, Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061961859","display_name":"Henning M\u00fcller","orcid":"https://orcid.org/0000-0001-6800-9878"},"institutions":[{"id":"https://openalex.org/I173439891","display_name":"HES-SO University of Applied Sciences and Arts Western Switzerland","ror":"https://ror.org/01xkakk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I173439891"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Henning M\u00fcller","raw_affiliation_strings":["","HES-SO University of Applied Sciences and Arts Western Switzerland, Del\u00e9mont, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"HES-SO University of Applied Sciences and Arts Western Switzerland, Del\u00e9mont, Switzerland","institution_ids":["https://openalex.org/I173439891"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045875579","display_name":"Noriko Kando","orcid":"https://orcid.org/0000-0002-2133-0215"},"institutions":[{"id":"https://openalex.org/I184597095","display_name":"National Institute of Informatics","ror":"https://ror.org/04ksd4g47","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I184597095","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Noriko Kando","raw_affiliation_strings":["","National Institute of Informatics, Tokyo, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"National Institute of Informatics, Tokyo, Japan","institution_ids":["https://openalex.org/I184597095"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007133493","display_name":"Simon Mercer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Simon Mercer","raw_affiliation_strings":[""],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039535916","display_name":"Jayashree Kalpathy\u2013Cramer","orcid":"https://orcid.org/0000-0001-8906-9618"},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jayashree Kalpathy-Cramer","raw_affiliation_strings":["","Harvard University, Cambridge, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"Harvard University, Cambridge, United States","institution_ids":["https://openalex.org/I2801851002"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083712311","display_name":"Martin Potthast","orcid":"https://orcid.org/0000-0003-2451-0665"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Martin Potthast","raw_affiliation_strings":["","Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087022274","display_name":"Tim Gollub","orcid":"https://orcid.org/0000-0003-1737-6517"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Tim Gollub","raw_affiliation_strings":["","Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058877446","display_name":"Anastasia Krithara","orcid":"https://orcid.org/0000-0003-0491-4507"},"institutions":[{"id":"https://openalex.org/I33976269","display_name":"Xerox (France)","ror":"https://ror.org/033q0mv79","country_code":"FR","type":"company","lineage":["https://openalex.org/I33976269","https://openalex.org/I4210132870"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Anastasia Krithara","raw_affiliation_strings":["","Xerox (France), Saint-Denis, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"Xerox (France), Saint-Denis, France","institution_ids":["https://openalex.org/I33976269"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["","University of Maryland, College Park, College Park, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"University of Maryland, College Park, College Park, United States","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059926999","display_name":"Krisztian Balog","orcid":"https://orcid.org/0000-0003-2762-721X"},"institutions":[{"id":"https://openalex.org/I92008406","display_name":"University of Stavanger","ror":"https://ror.org/02qte9q33","country_code":"NO","type":"education","lineage":["https://openalex.org/I92008406"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Krisztian Balog","raw_affiliation_strings":["","University of Stavanger, Stavanger, Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"University of Stavanger, Stavanger, Norway","institution_ids":["https://openalex.org/I92008406"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072406581","display_name":"Ivan Eggel","orcid":null},"institutions":[{"id":"https://openalex.org/I173439891","display_name":"HES-SO University of Applied Sciences and Arts Western Switzerland","ror":"https://ror.org/01xkakk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I173439891"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Ivan Eggel","raw_affiliation_strings":["","HES-SO University of Applied Sciences and Arts Western Switzerland, Del\u00e9mont, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"","institution_ids":[]},{"raw_affiliation_string":"HES-SO University of Applied Sciences and Arts Western Switzerland, Del\u00e9mont, Switzerland","institution_ids":["https://openalex.org/I173439891"]}]}],"institutions":[],"countries_distinct_count":8,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":14.6275,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.98630137,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"49","issue":"1","first_page":"57","last_page":"65"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9873999953269958,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9685999751091003,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.872854471206665},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.6781244277954102},{"id":"https://openalex.org/keywords/application-programming-interface","display_name":"Application programming interface","score":0.6163297891616821},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.5659604072570801},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.5481187105178833},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.4645618498325348},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.41721999645233154},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4093252420425415},{"id":"https://openalex.org/keywords/engineering-management","display_name":"Engineering management","score":0.3815539479255676},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.32179898023605347},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.14630424976348877}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.872854471206665},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.6781244277954102},{"id":"https://openalex.org/C99613125","wikidata":"https://www.wikidata.org/wiki/Q165194","display_name":"Application programming interface","level":2,"score":0.6163297891616821},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.5659604072570801},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.5481187105178833},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.4645618498325348},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.41721999645233154},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4093252420425415},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.3815539479255676},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.32179898023605347},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.14630424976348877},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2795403.2795416","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2795403.2795416","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},{"id":"pmh:oai:eprints.gla.ac.uk:105922","is_oa":false,"landing_page_url":"https://eprints.gla.ac.uk/view/journal_volume/SIGIR_Forum.html>,","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":false,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Articles"},{"id":"pmh:oai:arodes.hes-so.ch:1022","is_oa":false,"landing_page_url":"http://arodes.hes-so.ch/record/1022","pdf_url":null,"source":{"id":"https://openalex.org/S4306402432","display_name":"ArODES (HES-SO (https://www.hes-so.ch/))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210088449","host_organization_name":"HES-SO Gen\u00e8ve","host_organization_lineage":["https://openalex.org/I4210088449"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://arodes.hes-so.ch/record/1022","raw_type":"Text"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6299999952316284,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W39360085","https://openalex.org/W88603068","https://openalex.org/W196002257","https://openalex.org/W2023616424","https://openalex.org/W2036448357","https://openalex.org/W2066682493","https://openalex.org/W2089711563","https://openalex.org/W2145684485","https://openalex.org/W2187139527","https://openalex.org/W2296024507","https://openalex.org/W2296751301","https://openalex.org/W2339562433","https://openalex.org/W2396476612","https://openalex.org/W2397447617","https://openalex.org/W2399079233"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2383532021","https://openalex.org/W2366230866","https://openalex.org/W2357906747","https://openalex.org/W2372895414","https://openalex.org/W4298211017","https://openalex.org/W2348049494","https://openalex.org/W4200184607","https://openalex.org/W1981617837","https://openalex.org/W1992987618"],"abstract_inverted_index":{"In":[0],"this":[1],"report,":[2],"we":[3],"summarize":[4],"the":[5,8,15,27],"outcome":[6],"of":[7,26,50],"\"Evaluation-as-a-Service\"":[9],"workshop":[10],"that":[11,34,47],"was":[12,29],"held":[13],"on":[14],"5th":[16],"and":[17,44],"6th":[18],"March":[19],"2015":[20],"in":[21],"Sierre,":[22],"Switzerland.":[23],"The":[24],"objective":[25],"meeting":[28],"to":[30],"bring":[31],"together":[32],"initiatives":[33],"use":[35],"cloud":[36],"infrastructures,":[37],"virtual":[38],"machines,":[39],"APIs":[40],"(Application":[41],"Programming":[42],"Interface)":[43],"related":[45],"projects":[46],"provide":[48],"evaluation":[49],"information":[51],"retrieval":[52],"or":[53],"machine":[54],"learning":[55],"tools":[56],"as":[57],"a":[58],"service.":[59]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":3}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2016-06-24T00:00:00"}