{"id":"https://openalex.org/W2122343331","doi":"https://doi.org/10.1002/asi.23600","title":"A framework for evaluating automatic indexing or classification in the context of retrieval","display_name":"A framework for evaluating automatic indexing or classification in the context of retrieval","publication_year":2015,"publication_date":"2015-10-22","ids":{"openalex":"https://openalex.org/W2122343331","doi":"https://doi.org/10.1002/asi.23600","mag":"2122343331"},"language":"en","primary_location":{"id":"doi:10.1002/asi.23600","is_oa":false,"landing_page_url":"https://doi.org/10.1002/asi.23600","pdf_url":null,"source":{"id":"https://openalex.org/S4210197613","display_name":"Journal of the Association for Information Science and Technology","issn_l":"2330-1635","issn":["2330-1635","2330-1643"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the Association for Information Science and Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://openaccess.city.ac.uk/id/eprint/12537/1/EvaluationFramework_v19_notable.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072826753","display_name":"Koraljka Golub","orcid":"https://orcid.org/0000-0003-4169-4777"},"institutions":[{"id":"https://openalex.org/I223464139","display_name":"Linnaeus University","ror":"https://ror.org/00j9qag85","country_code":"SE","type":"education","lineage":["https://openalex.org/I223464139"]}],"countries":["SE"],"is_corresponding":true,"raw_author_name":"Koraljka Golub","raw_affiliation_strings":["Department of Library and Information Science School of Cultural Sciences Faculty of Arts and Humanities Linnaeus University  V\u00e4xj\u00f6 351 95 Sweden"],"affiliations":[{"raw_affiliation_string":"Department of Library and Information Science School of Cultural Sciences Faculty of Arts and Humanities Linnaeus University  V\u00e4xj\u00f6 351 95 Sweden","institution_ids":["https://openalex.org/I223464139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036787563","display_name":"Dagobert Soergel","orcid":"https://orcid.org/0000-0001-5806-2384"},"institutions":[{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dagobert Soergel","raw_affiliation_strings":["Department of Library and Information Studies University of Buffalo  Room 534 Baldy Buffalo NY 14260\u20101020","Department of Library and Information Studies University of Buffalo Room 534 Baldy Buffalo NY 14260\u20101020"],"affiliations":[{"raw_affiliation_string":"Department of Library and Information Studies University of Buffalo  Room 534 Baldy Buffalo NY 14260\u20101020","institution_ids":["https://openalex.org/I63190737"]},{"raw_affiliation_string":"Department of Library and Information Studies University of Buffalo Room 534 Baldy Buffalo NY 14260\u20101020","institution_ids":["https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024942135","display_name":"George Buchanan","orcid":"https://orcid.org/0000-0001-9044-6644"},"institutions":[{"id":"https://openalex.org/I125768316","display_name":"Northampton Community College","ror":"https://ror.org/05j27jv73","country_code":"US","type":"education","lineage":["https://openalex.org/I125768316"]},{"id":"https://openalex.org/I180825142","display_name":"City, University of London","ror":"https://ror.org/04489at23","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I180825142"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"George Buchanan","raw_affiliation_strings":["Department of Computer Science City University  Northampton Square London EC1V 0HB United Kingdom","Department of Computer Science City University Northampton Square London EC1V 0HB United Kingdom"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science City University  Northampton Square London EC1V 0HB United Kingdom","institution_ids":["https://openalex.org/I125768316"]},{"raw_affiliation_string":"Department of Computer Science City University Northampton Square London EC1V 0HB United Kingdom","institution_ids":["https://openalex.org/I180825142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013952812","display_name":"Douglas Tudhope","orcid":"https://orcid.org/0000-0002-5222-0430"},"institutions":[{"id":"https://openalex.org/I128993996","display_name":"University of South Wales","ror":"https://ror.org/02mzn7s88","country_code":"GB","type":"education","lineage":["https://openalex.org/I128993996"]},{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU","GB"],"is_corresponding":false,"raw_author_name":"Douglas Tudhope","raw_affiliation_strings":["Hypermedia Research Group University of South Wales  Pontypridd Wales CF37 1DL United Kingdom","Hypermedia Research Group, University of South Wales, Pontypridd, Wales, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"Hypermedia Research Group University of South Wales  Pontypridd Wales CF37 1DL United Kingdom","institution_ids":["https://openalex.org/I128993996"]},{"raw_affiliation_string":"Hypermedia Research Group, University of South Wales, Pontypridd, Wales, United Kingdom#TAB#","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048174522","display_name":"Marianne Lykke","orcid":"https://orcid.org/0000-0003-3635-3943"},"institutions":[{"id":"https://openalex.org/I891191580","display_name":"Aalborg University","ror":"https://ror.org/04m5j1k67","country_code":"DK","type":"education","lineage":["https://openalex.org/I891191580"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Marianne Lykke","raw_affiliation_strings":["E\u2010Learning Lab University of Aalborg  Rensburggade 14 Aalborg 9000 Denmark","E\u2010Learning Lab University of Aalborg Rensburggade 14 Aalborg 9000 Denmark"],"affiliations":[{"raw_affiliation_string":"E\u2010Learning Lab University of Aalborg  Rensburggade 14 Aalborg 9000 Denmark","institution_ids":["https://openalex.org/I891191580"]},{"raw_affiliation_string":"E\u2010Learning Lab University of Aalborg Rensburggade 14 Aalborg 9000 Denmark","institution_ids":["https://openalex.org/I891191580"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089526504","display_name":"Debra A Hiom","orcid":"https://orcid.org/0000-0001-6106-4006"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]},{"id":"https://openalex.org/I3132500319","display_name":"Bristol and Bath Science Park","ror":"https://ror.org/0105p2r38","country_code":"GB","type":"education","lineage":["https://openalex.org/I3132500319"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Debra Hiom","raw_affiliation_strings":["IT Services R&D University of Bristol  8\u201010 Berkeley Square Bristol BS8 1HH United Kingdom","IT Services R&D University of Bristol 8\u201010 Berkeley Square Bristol BS8 1HH United Kingdom"],"affiliations":[{"raw_affiliation_string":"IT Services R&D University of Bristol  8\u201010 Berkeley Square Bristol BS8 1HH United Kingdom","institution_ids":["https://openalex.org/I36234482","https://openalex.org/I3132500319"]},{"raw_affiliation_string":"IT Services R&D University of Bristol 8\u201010 Berkeley Square Bristol BS8 1HH United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5072826753"],"corresponding_institution_ids":["https://openalex.org/I223464139"],"apc_list":{"value":3000,"currency":"USD","value_usd":3000},"apc_paid":null,"fwci":3.883,"has_fulltext":false,"cited_by_count":44,"citation_normalized_percentile":{"value":0.94132073,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"67","issue":"1","first_page":"3","last_page":"16"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/search-engine-indexing","display_name":"Search engine indexing","score":0.8448895215988159},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7465479373931885},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6995114684104919},{"id":"https://openalex.org/keywords/automatic-indexing","display_name":"Automatic indexing","score":0.6462271809577942},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5811644792556763},{"id":"https://openalex.org/keywords/document-retrieval","display_name":"Document retrieval","score":0.4208308458328247},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.11174333095550537}],"concepts":[{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.8448895215988159},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7465479373931885},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6995114684104919},{"id":"https://openalex.org/C2778330532","wikidata":"https://www.wikidata.org/wiki/Q4826577","display_name":"Automatic indexing","level":3,"score":0.6462271809577942},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5811644792556763},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.4208308458328247},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.11174333095550537},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":7,"locations":[{"id":"doi:10.1002/asi.23600","is_oa":false,"landing_page_url":"https://doi.org/10.1002/asi.23600","pdf_url":null,"source":{"id":"https://openalex.org/S4210197613","display_name":"Journal of the Association for Information Science and Technology","issn_l":"2330-1635","issn":["2330-1635","2330-1643"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the Association for Information Science and Technology","raw_type":"journal-article"},{"id":"pmh:oai:research-information.bris.ac.uk:openaire_cris_publications/5b26439e-505d-499a-8894-3ee8663d9d50","is_oa":false,"landing_page_url":"https://research-information.bris.ac.uk/en/publications/5b26439e-505d-499a-8894-3ee8663d9d50","pdf_url":null,"source":{"id":"https://openalex.org/S4306400895","display_name":"Bristol Research (University of Bristol)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I36234482","host_organization_name":"University of Bristol","host_organization_lineage":["https://openalex.org/I36234482"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Golub, K, Soergel, D, Buchanan, G, Tudhope, D, Lykke, M & Hiom, D 2015, 'A framework for evaluating automatic indexing or classification in the context of retrieval', Journal of the Association for Information Science and Technology. https://doi.org/10.1002/asi.23600","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:openaccess.city.ac.uk:12537","is_oa":true,"landing_page_url":"https://openaccess.city.ac.uk/view/creators_id/george=2Ebuchanan=2E1.html>","pdf_url":"https://openaccess.city.ac.uk/id/eprint/12537/1/EvaluationFramework_v19_notable.pdf","source":{"id":"https://openalex.org/S4306401940","display_name":"City Research Online (City University London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I180825142","host_organization_name":"City, University of London","host_organization_lineage":["https://openalex.org/I180825142"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"},{"id":"pmh:oai:DiVA.org:lnu-45521","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:lnu:diva-45521","pdf_url":null,"source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article in journal"},{"id":"pmh:oai:RePEc:bla:jinfst:v:67:y:2016:i:1:p:3-16","is_oa":false,"landing_page_url":"http://hdl.handle.net/10.1002/asi.23600","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:pure.atira.dk:publications/3dd01b9d-a89a-44ba-a1ba-44fa67ab912c","is_oa":false,"landing_page_url":"https://vbn.aau.dk/da/publications/3dd01b9d-a89a-44ba-a1ba-44fa67ab912c","pdf_url":null,"source":{"id":"https://openalex.org/S4306401731","display_name":"VBN Forskningsportal (Aalborg Universitet)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I891191580","host_organization_name":"Aalborg University","host_organization_lineage":["https://openalex.org/I891191580"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Golub, K, Soergel, D, Buchanan, G, Tudhope, D, Lykke, M & Hiom, D 2016, 'A framework for evaluating automatic indexing or classification in the context of retrieval', American Society for Information Science and Technology. Journal, vol. 67, no. 1, pp. 3-16. https://doi.org/10.1002/asi.23600","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:research-information.bris.ac.uk:publications/5b26439e-505d-499a-8894-3ee8663d9d50","is_oa":false,"landing_page_url":"https://hdl.handle.net/1983/5b26439e-505d-499a-8894-3ee8663d9d50","pdf_url":null,"source":{"id":"https://openalex.org/S4306400895","display_name":"Bristol Research (University of Bristol)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I36234482","host_organization_name":"University of Bristol","host_organization_lineage":["https://openalex.org/I36234482"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Golub, K, Soergel, D, Buchanan, G, Tudhope, D, Lykke, M & Hiom, D 2015, 'A framework for evaluating automatic indexing or classification in the context of retrieval', Journal of the Association for Information Science and Technology. https://doi.org/10.1002/asi.23600","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"pmh:oai:openaccess.city.ac.uk:12537","is_oa":true,"landing_page_url":"https://openaccess.city.ac.uk/view/creators_id/george=2Ebuchanan=2E1.html>","pdf_url":"https://openaccess.city.ac.uk/id/eprint/12537/1/EvaluationFramework_v19_notable.pdf","source":{"id":"https://openalex.org/S4306401940","display_name":"City Research Online (City University London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I180825142","host_organization_name":"City, University of London","host_organization_lineage":["https://openalex.org/I180825142"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"},"sustainable_development_goals":[{"score":0.5699999928474426,"display_name":"Responsible consumption and production","id":"https://metadata.un.org/sdg/12"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2122343331.pdf"},"referenced_works_count":84,"referenced_works":["https://openalex.org/W8513375","https://openalex.org/W129635239","https://openalex.org/W206402474","https://openalex.org/W650616204","https://openalex.org/W852797041","https://openalex.org/W1484522762","https://openalex.org/W1485344421","https://openalex.org/W1493535761","https://openalex.org/W1495821370","https://openalex.org/W1580741461","https://openalex.org/W1597521809","https://openalex.org/W1839790462","https://openalex.org/W1938221098","https://openalex.org/W1966774441","https://openalex.org/W1968129103","https://openalex.org/W1968801340","https://openalex.org/W1976398589","https://openalex.org/W1980047209","https://openalex.org/W1982149803","https://openalex.org/W1988294080","https://openalex.org/W1989525913","https://openalex.org/W1991546151","https://openalex.org/W1991885828","https://openalex.org/W1994960885","https://openalex.org/W2000569830","https://openalex.org/W2008883775","https://openalex.org/W2011471399","https://openalex.org/W2018264495","https://openalex.org/W2018311971","https://openalex.org/W2025672980","https://openalex.org/W2026969101","https://openalex.org/W2033943395","https://openalex.org/W2035820422","https://openalex.org/W2040249527","https://openalex.org/W2040927952","https://openalex.org/W2042050931","https://openalex.org/W2045828803","https://openalex.org/W2048987619","https://openalex.org/W2053528567","https://openalex.org/W2060959387","https://openalex.org/W2065522562","https://openalex.org/W2073722401","https://openalex.org/W2075310148","https://openalex.org/W2087511931","https://openalex.org/W2096276775","https://openalex.org/W2102500012","https://openalex.org/W2102698119","https://openalex.org/W2113076637","https://openalex.org/W2114373757","https://openalex.org/W2117966604","https://openalex.org/W2118020653","https://openalex.org/W2119020571","https://openalex.org/W2123107811","https://openalex.org/W2130229804","https://openalex.org/W2135985057","https://openalex.org/W2141649520","https://openalex.org/W2144701020","https://openalex.org/W2145790509","https://openalex.org/W2147297395","https://openalex.org/W2150102617","https://openalex.org/W2156296249","https://openalex.org/W2156304974","https://openalex.org/W2156754333","https://openalex.org/W2160171761","https://openalex.org/W2162379288","https://openalex.org/W2167945625","https://openalex.org/W2168066506","https://openalex.org/W2168808398","https://openalex.org/W2255895239","https://openalex.org/W2289729049","https://openalex.org/W2407710372","https://openalex.org/W2524432660","https://openalex.org/W2529448089","https://openalex.org/W2616553306","https://openalex.org/W2998276394","https://openalex.org/W3021088434","https://openalex.org/W3200788958","https://openalex.org/W4233760227","https://openalex.org/W4236645963","https://openalex.org/W4244706380","https://openalex.org/W4245915609","https://openalex.org/W4254168416","https://openalex.org/W4255321213","https://openalex.org/W4289257076"],"related_works":["https://openalex.org/W204619012","https://openalex.org/W2398073981","https://openalex.org/W1558342070","https://openalex.org/W2389273994","https://openalex.org/W2188505374","https://openalex.org/W2075310148","https://openalex.org/W2071964087","https://openalex.org/W2294586269","https://openalex.org/W2085542554","https://openalex.org/W4254725359"],"abstract_inverted_index":{"Tools":[0],"for":[1,56],"automatic":[2],"subject":[3,39],"assignment":[4],"help":[5],"deal":[6],"with":[7,81,142],"scale":[8],"and":[9,13,20,23,30,73,78,90,108,110,160],"sustainability":[10],"in":[11,47,64,153],"creating":[12],"enriching":[14],"metadata,":[15],"establishing":[16],"more":[17,98],"connections":[18],"across":[19],"between":[21],"resources":[22],"enhancing":[24],"consistency.":[25],"Although":[26],"some":[27],"software":[28],"vendors":[29],"experimental":[31],"researchers":[32],"claim":[33],"the":[34,68,94,125,147,154],"tools":[35],"can":[36],"replace":[37],"manual":[38],"indexing,":[40],"hard":[41],"scientific":[42],"evidence":[43],"of":[44,70,88,124,149,156],"their":[45],"performance":[46],"operating":[48],"information":[49],"environments":[50],"is":[51,58,61,118],"scarce.":[52],"A":[53],"major":[54],"reason":[55],"this":[57],"that":[59],"research":[60],"usually":[62],"conducted":[63],"laboratory":[65],"conditions,":[66],"excluding":[67],"complexities":[69],"real\u2010life":[71],"systems":[72],"situations.":[74],"The":[75,116],"article":[76],"reviews":[77],"discusses":[79],"issues":[80],"existing":[82],"evaluation":[83,114,128],"approaches":[84],"such":[85],"as":[86],"problems":[87],"aboutness":[89],"relevance":[91],"assessments,":[92],"implying":[93],"need":[95],"to":[96],"use":[97],"than":[99],"a":[100,112,121,143],"single":[101],"\u201cgold":[102],"standard\u201d":[103],"method":[104],"when":[105],"evaluating":[106,130,146,161],"indexing":[107,131,151,158,162],"retrieval,":[109],"proposes":[111],"comprehensive":[113],"framework.":[115],"framework":[117],"informed":[119],"by":[120,136],"systematic":[122],"review":[123],"literature":[126],"on":[127],"approaches:":[129],"quality":[132,148,163],"directly":[133,152],"through":[134,140,165],"assessment":[135],"an":[137,157],"evaluator":[138],"or":[139],"comparison":[141],"gold":[144],"standard,":[145],"computer\u2010assisted":[150],"context":[155],"workflow,":[159],"indirectly":[164],"analyzing":[166],"retrieval":[167],"performance.":[168]},"counts_by_year":[{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}