{"id":"https://openalex.org/W3004654365","doi":"https://doi.org/10.1007/s13222-020-00335-x","title":"Evaluation Infrastructures for Academic Shared Tasks","display_name":"Evaluation Infrastructures for Academic Shared Tasks","publication_year":2020,"publication_date":"2020-02-07","ids":{"openalex":"https://openalex.org/W3004654365","doi":"https://doi.org/10.1007/s13222-020-00335-x","mag":"3004654365"},"language":"en","primary_location":{"id":"doi:10.1007/s13222-020-00335-x","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13222-020-00335-x","pdf_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00335-x.pdf","source":{"id":"https://openalex.org/S73012565","display_name":"Datenbank-Spektrum","issn_l":"1610-1995","issn":["1610-1995","1618-2162"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Datenbank-Spektrum","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00335-x.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001363347","display_name":"Johann Schaible","orcid":"https://orcid.org/0000-0002-5441-7640"},"institutions":[{"id":"https://openalex.org/I4210101898","display_name":"GESIS - Leibniz-Institute for the Social Sciences","ror":"https://ror.org/018afyw53","country_code":"DE","type":"facility","lineage":["https://openalex.org/I315704651","https://openalex.org/I4210101898"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Johann Schaible","raw_affiliation_strings":["GESIS \u2013 Leibniz Institute for the Social Sciences, Cologne, Germany"],"raw_orcid":"https://orcid.org/0000-0002-5441-7640","affiliations":[{"raw_affiliation_string":"GESIS \u2013 Leibniz Institute for the Social Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I4210101898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067984355","display_name":"Timo Breuer","orcid":"https://orcid.org/0000-0002-1765-2449"},"institutions":[{"id":"https://openalex.org/I102520234","display_name":"TH K\u00f6ln - University of Applied Sciences","ror":"https://ror.org/014nnvj65","country_code":"DE","type":"education","lineage":["https://openalex.org/I102520234"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Timo Breuer","raw_affiliation_strings":["TH K\u00f6ln \u2013 University of Applied Sciences, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TH K\u00f6ln \u2013 University of Applied Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I102520234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048388461","display_name":"Narges Tavakolpoursaleh","orcid":"https://orcid.org/0000-0001-9324-3252"},"institutions":[{"id":"https://openalex.org/I4210101898","display_name":"GESIS - Leibniz-Institute for the Social Sciences","ror":"https://ror.org/018afyw53","country_code":"DE","type":"facility","lineage":["https://openalex.org/I315704651","https://openalex.org/I4210101898"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Narges Tavakolpoursaleh","raw_affiliation_strings":["GESIS \u2013 Leibniz Institute for the Social Sciences, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"GESIS \u2013 Leibniz Institute for the Social Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I4210101898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015732254","display_name":"B. G. M\u00fcller","orcid":"https://orcid.org/0000-0003-3062-8192"},"institutions":[{"id":"https://openalex.org/I4210112663","display_name":"ZB MED - Information Centre for Life Sciences","ror":"https://ror.org/0259fwx54","country_code":"DE","type":"archive","lineage":["https://openalex.org/I4210112663"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Bernd M\u00fcller","raw_affiliation_strings":["ZB MED \u2013 Information Centre for Life Sciences, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ZB MED \u2013 Information Centre for Life Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I4210112663"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002751141","display_name":"Benjamin Wolff","orcid":"https://orcid.org/0000-0001-9345-8958"},"institutions":[{"id":"https://openalex.org/I4210112663","display_name":"ZB MED - Information Centre for Life Sciences","ror":"https://ror.org/0259fwx54","country_code":"DE","type":"archive","lineage":["https://openalex.org/I4210112663"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Benjamin Wolff","raw_affiliation_strings":["ZB MED \u2013 Information Centre for Life Sciences, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ZB MED \u2013 Information Centre for Life Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I4210112663"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087564658","display_name":"Philipp Schaer","orcid":"https://orcid.org/0000-0002-8817-4632"},"institutions":[{"id":"https://openalex.org/I102520234","display_name":"TH K\u00f6ln - University of Applied Sciences","ror":"https://ror.org/014nnvj65","country_code":"DE","type":"education","lineage":["https://openalex.org/I102520234"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Philipp Schaer","raw_affiliation_strings":["TH K\u00f6ln \u2013 University of Applied Sciences, Cologne, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TH K\u00f6ln \u2013 University of Applied Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I102520234"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5001363347"],"corresponding_institution_ids":["https://openalex.org/I4210101898"],"apc_list":{"value":2380,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2380,"currency":"EUR","value_usd":2890},"fwci":1.6702,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.87845717,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"20","issue":"1","first_page":"29","last_page":"36"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8087143898010254},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.6168889403343201},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5581468343734741},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.5074164271354675},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5002210140228271},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4610869288444519},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36154064536094666}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8087143898010254},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.6168889403343201},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5581468343734741},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.5074164271354675},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5002210140228271},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4610869288444519},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36154064536094666},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1007/s13222-020-00335-x","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13222-020-00335-x","pdf_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00335-x.pdf","source":{"id":"https://openalex.org/S73012565","display_name":"Datenbank-Spektrum","issn_l":"1610-1995","issn":["1610-1995","1618-2162"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Datenbank-Spektrum","raw_type":"journal-article"},{"id":"pmh:oai:epb.bibl.th-koeln.de:2254","is_oa":true,"landing_page_url":"https://nbn-resolving.org/urn:nbn:de:hbz:832-epub4-22548","pdf_url":null,"source":{"id":"https://openalex.org/S4306401363","display_name":"ePublications (Technical University of Cologne)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"article"},{"id":"pmh:oai:gesis.izsoz.de:document/86702","is_oa":true,"landing_page_url":"https://www.ssoar.info/ssoar/handle/document/86702","pdf_url":null,"source":{"id":"https://openalex.org/S4306401996","display_name":"Social Science Open Access Repository (GESIS \u2013 Leibniz Institute for the Social Sciences)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210101898","host_organization_name":"GESIS - Leibniz-Institute for the Social Sciences","host_organization_lineage":["https://openalex.org/I4210101898"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Datenbank-Spektrum : Zeitschrift f\u00fcr Datenbanktechnologien und Information Retrieval","raw_type":"journal article"}],"best_oa_location":{"id":"doi:10.1007/s13222-020-00335-x","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13222-020-00335-x","pdf_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00335-x.pdf","source":{"id":"https://openalex.org/S73012565","display_name":"Datenbank-Spektrum","issn_l":"1610-1995","issn":["1610-1995","1618-2162"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Datenbank-Spektrum","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.6299999952316284}],"awards":[{"id":"https://openalex.org/G3150620016","display_name":null,"funder_award_id":"407518790","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3004654365.pdf","grobid_xml":"https://content.openalex.org/works/W3004654365.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W298464722","https://openalex.org/W789651694","https://openalex.org/W1792839667","https://openalex.org/W1981208470","https://openalex.org/W1985476803","https://openalex.org/W1991725352","https://openalex.org/W2023616424","https://openalex.org/W2036448357","https://openalex.org/W2052739481","https://openalex.org/W2100574107","https://openalex.org/W2145684485","https://openalex.org/W2294303215","https://openalex.org/W2464171116","https://openalex.org/W2468776465","https://openalex.org/W2552478800","https://openalex.org/W2561675875","https://openalex.org/W2619621170","https://openalex.org/W2870281694","https://openalex.org/W2883163430","https://openalex.org/W2894680593","https://openalex.org/W2899715207","https://openalex.org/W2966427072","https://openalex.org/W2967620349","https://openalex.org/W2968343274","https://openalex.org/W2968887325","https://openalex.org/W2969044820","https://openalex.org/W2977457369","https://openalex.org/W3015952299","https://openalex.org/W3105114834","https://openalex.org/W3130434286","https://openalex.org/W4234071252"],"related_works":["https://openalex.org/W4390273403","https://openalex.org/W4386781444","https://openalex.org/W3092950680","https://openalex.org/W4246980185","https://openalex.org/W2150182025","https://openalex.org/W3197542405","https://openalex.org/W2418190244","https://openalex.org/W4238861846","https://openalex.org/W3125580266","https://openalex.org/W44246808"],"abstract_inverted_index":{"Abstract":[0],"Academic":[1],"search":[2,68,94,119,162],"systems":[3,22,69],"aid":[4],"users":[5],"in":[6,92,158],"finding":[7],"information":[8],"covering":[9],"specific":[10,93],"topics":[11],"of":[12,31,42,56,113,135,140],"scientific":[13],"interest":[14],"and":[15,95,111,120,130,138,142,163],"have":[16,49],"evolved":[17],"from":[18],"early":[19],"catalog-based":[20],"library":[21],"to":[23,50,61,65,87],"modern":[24],"web-scale":[25],"systems.":[26,165],"However,":[27],"evaluating":[28],"the":[29,32,57,109,124,156],"performance":[30],"underlying":[33],"retrieval":[34,47,73,90],"approaches":[35,91],"remains":[36],"a":[37],"challenge.":[38],"An":[39],"increasing":[40],"amount":[41],"requirements":[43],"for":[44,78,128,161],"producing":[45],"accurate":[46],"results":[48],"be":[51],"considered,":[52],"e.g.,":[53],"close":[54],"integration":[55],"system\u2019s":[58],"users.":[59],"Due":[60],"these":[62],"requirements,":[63],"small":[64],"mid-size":[66],"academic":[67],"cannot":[70],"evaluate":[71],"their":[72,100],"system":[74],"in-house.":[75],"Evaluation":[76],"infrastructures":[77,117],"shared":[79,136,159],"tasks":[80,122,160],"alleviate":[81],"this":[82,104],"situation.":[83],"They":[84],"allow":[85],"researchers":[86],"experiment":[88],"with":[89],"recommendation":[96,121],"scenarios":[97],"without":[98],"building":[99],"own":[101],"infrastructure.":[102],"In":[103,144],"paper,":[105],"we":[106,146],"elaborate":[107],"on":[108,118],"benefits":[110],"shortcomings":[112,157],"four":[114],"state-of-the-art":[115],"evaluation":[116,149],"concerning":[123],"following":[125],"requirements:":[126],"support":[127],"online":[129],"offline":[131],"evaluations,":[132],"domain":[133],"specificity":[134],"tasks,":[137],"reproducibility":[139],"experiments":[141],"results.":[143],"addition,":[145],"introduce":[147],"an":[148],"infrastructure":[150],"concept":[151],"design":[152],"aiming":[153],"at":[154],"reducing":[155],"recommender":[164]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2020-02-14T00:00:00"}