{"id":"https://openalex.org/W2739800617","doi":"https://doi.org/10.1145/3130332.3130344","title":"Practical Issues in Information Access System Evaluation","display_name":"Practical Issues in Information Access System Evaluation","publication_year":2017,"publication_date":"2017-08-02","ids":{"openalex":"https://openalex.org/W2739800617","doi":"https://doi.org/10.1145/3130332.3130344","mag":"2739800617"},"language":"en","primary_location":{"id":"doi:10.1145/3130332.3130344","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3130332.3130344","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pure.uva.nl/ws/files/25497208/p67_kanoulas.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055639036","display_name":"Evangelos Kanoulas","orcid":"https://orcid.org/0000-0002-8312-0694"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Evangelos Kanoulas","raw_affiliation_strings":["University of Amsterdam"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001872735","display_name":"Jussi Karlgren","orcid":"https://orcid.org/0000-0003-4042-4919"},"institutions":[{"id":"https://openalex.org/I4210122946","display_name":"Gavagai (Sweden)","ror":"https://ror.org/021zf9412","country_code":"SE","type":"company","lineage":["https://openalex.org/I4210122946"]},{"id":"https://openalex.org/I86987016","display_name":"KTH Royal Institute of Technology","ror":"https://ror.org/026vcq606","country_code":"SE","type":"education","lineage":["https://openalex.org/I86987016"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Jussi Karlgren","raw_affiliation_strings":["KTH Royal Institute of Technology and Gavagai, Stockholm"],"affiliations":[{"raw_affiliation_string":"KTH Royal Institute of Technology and Gavagai, Stockholm","institution_ids":["https://openalex.org/I86987016","https://openalex.org/I4210122946"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5055639036"],"corresponding_institution_ids":["https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":1.264,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.86896545,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"51","issue":"1","first_page":"67","last_page":"72"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10953","display_name":"E-Government and Public Services","score":0.9463000297546387,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10953","display_name":"E-Government and Public Services","score":0.9463000297546387,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9334999918937683,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12028","display_name":"Knowledge Management and Sharing","score":0.9176999926567078,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8573165535926819},{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.7426658272743225},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.43320223689079285},{"id":"https://openalex.org/keywords/library-science","display_name":"Library science","score":0.34106236696243286}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8573165535926819},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.7426658272743225},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.43320223689079285},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.34106236696243286}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3130332.3130344","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3130332.3130344","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/1b095894-10f3-4bbf-ac9e-26c6c743bac9","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/practical-issues-in-information-access-system-evaluation(1b095894-10f3-4bbf-ac9e-26c6c743bac9).html","pdf_url":"https://pure.uva.nl/ws/files/25497208/p67_kanoulas.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Kanoulas, E & Karlgren, J 2017, 'Practical Issues in Information Access System Evaluation : BCS-IRSG Search Solutions Workshop Report', SIGIR Forum, vol. 51, no. 1, pp. 67-72. https://doi.org/10.1145/3130332.3130344","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:dare.uva.nl:publications/1b095894-10f3-4bbf-ac9e-26c6c743bac9","is_oa":true,"landing_page_url":null,"pdf_url":"https://dare.uva.nl/personal/pure/en/publications/practical-issues-in-information-access-system-evaluation(1b095894-10f3-4bbf-ac9e-26c6c743bac9).html","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""},{"id":"pmh:uvapub:oai:dare.uva.nl:publications/1b095894-10f3-4bbf-ac9e-26c6c743bac9","is_oa":true,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/practical-issues-in-information-access-system-evaluation(1b095894-10f3-4bbf-ac9e-26c6c743bac9).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SIGIR Forum, 51(1), 67 - 72. Association for Computing Machinery (ACM)","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/1b095894-10f3-4bbf-ac9e-26c6c743bac9","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/practical-issues-in-information-access-system-evaluation(1b095894-10f3-4bbf-ac9e-26c6c743bac9).html","pdf_url":"https://pure.uva.nl/ws/files/25497208/p67_kanoulas.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Kanoulas, E & Karlgren, J 2017, 'Practical Issues in Information Access System Evaluation : BCS-IRSG Search Solutions Workshop Report', SIGIR Forum, vol. 51, no. 1, pp. 67-72. https://doi.org/10.1145/3130332.3130344","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5799999833106995,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320320301","display_name":"European Science Foundation","ror":"https://ror.org/04esata81"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2739800617.pdf","grobid_xml":"https://content.openalex.org/works/W2739800617.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W2130043461","https://openalex.org/W2530322880","https://openalex.org/W1596801655"],"abstract_inverted_index":{"Initial":[0],"motivationsThe":[1],"last":[2],"six":[3],"decades":[4],"information":[5,40,59,249],"retrieval":[6],"scholars":[7],"and":[8,22,32,44,70,72,102,124,185,192,199,209,213,228,239,245,257],"practitioners":[9],"have":[10,76],"put":[11],"an":[12,17,121,177],"enormous":[13],"effort":[14],"to":[15,29,35,56,133,140,163,215,220,259],"develop":[16,134],"arsenal":[18],"of":[19,39,46,80,108,148,166,190,202],"evaluation":[20,110,136,165,198],"techniques":[21],"frameworks":[23],"that":[24,93],"vary":[25],"from":[26,235,267],"collection-based":[27],"approaches":[28],"user":[30],"studies":[31],"in-situ":[33],"methods":[34,48],"assess":[36],"the":[37,53,106,130,142,146,164,188,200,206,210,263],"effectiveness":[38],"access":[41,60,250],"algorithms,":[42],"technologies,":[43],"systems.Most":[45],"these":[47,260],"take":[49],"adhoc":[50],"search":[51,83],"as":[52,86,115,270],"use":[54,91],"case":[55],"address.Simultaneously,":[57],"practical":[58,157,255],"services":[61,69,75],"are":[62,98,112,218],"now":[63],"embedded":[64],"in":[65,120,145,153,248],"many":[66,73,79],"other":[67],"commercial":[68,227],"products,":[71],"new":[74],"been":[77],"introduced,":[78],"which":[81,195],"incorporate":[82],"or":[84],"classification":[85],"central":[87],"components":[88,97],"but":[89,138],"with":[90,205,217],"cases":[92],"range":[94],"beyond":[95],"search.These":[96],"seldom":[99,113],"evaluated":[100],"rigorously,":[101],"if":[103],"they":[104],"are,":[105],"results":[107],"those":[109],"exercises":[111],"perceived":[114],"relevant":[116],"for":[117,225,251,274],"system":[118],"design":[119],"industrial":[122,243,264],"setting.Small":[123],"medium":[125],"size":[126],"enterprises":[127,204],"often":[128,174],"lack":[129],"resources":[131,208],"needed":[132],"proper":[135],"infrastructures,":[137],"also":[139],"follow":[141],"research":[143,247],"development":[144],"field":[147],"evaluation.Similarly,":[149],"academics":[150],"lag":[151],"behind":[152],"(a)":[154],"understanding":[155],"real":[156,167],"issues":[158,256],"raised":[159],"when":[160,176],"it":[161],"comes":[162],"systems":[168,196],"-e.g.":[169],"even":[170],"depth-k":[171],"pooling":[172],"is":[173],"infeasible":[175],"SME":[178],"has":[179],"a":[180,252,271],"single":[181],"ranking":[182],"algorithm":[183],"developed,":[184],"(b)":[186],"sensing":[187],"breadth":[189],"applications":[191],"tasks":[193],"on":[194,254],"require":[197],"challenges":[201],"them.Large":[203],"necessary":[207],"data":[211],"sets":[212],"flows":[214],"work":[216],"hesitant":[219],"make":[221],"their":[222],"tests":[223],"public,":[224],"both":[226],"legal":[229],"reasons.This":[230],"workshop":[231],"brought":[232],"together":[233],"representatives":[234],"technology":[236],"companies,":[237],"large":[238],"small,":[240],"media":[241],"houses,":[242],"consultants":[244],"academic":[246],"discussion":[253],"solutions":[258],"issues.It":[261],"took":[262],"day":[265],"panel":[266],"SIGIR":[268],"2016":[269],"partial":[272],"inspiration":[273],"further":[275],"discussions.":[276]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-03-18T14:38:29.013473","created_date":"2025-10-10T00:00:00"}