{"id":"https://openalex.org/W2767908683","doi":"https://doi.org/10.1145/3132847.3132989","title":"A Neural Candidate-Selector Architecture for Automatic Structured Clinical Text Annotation","display_name":"A Neural Candidate-Selector Architecture for Automatic Structured Clinical Text Annotation","publication_year":2017,"publication_date":"2017-11-06","ids":{"openalex":"https://openalex.org/W2767908683","doi":"https://doi.org/10.1145/3132847.3132989","mag":"2767908683","pmid":"https://pubmed.ncbi.nlm.nih.gov/29308293"},"language":"en","primary_location":{"id":"doi:10.1145/3132847.3132989","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3132847.3132989","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 ACM on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://kclpure.kcl.ac.uk/portal/en/publications/3967c796-7a66-41b6-9be2-3a60dd7a4c51","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103279363","display_name":"Gaurav Singh","orcid":"https://orcid.org/0000-0001-8514-0739"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Gaurav Singh","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036860076","display_name":"Iain Marshall","orcid":"https://orcid.org/0000-0003-2594-2654"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Iain J. Marshall","raw_affiliation_strings":["King's College London, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"King's College London, London, United Kingdom","institution_ids":["https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090183643","display_name":"James Thomas","orcid":"https://orcid.org/0000-0003-4805-4190"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"James Thomas","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047980012","display_name":"John Shawe\u2010Taylor","orcid":"https://orcid.org/0000-0002-2030-0073"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"John Shawe-Taylor","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036790226","display_name":"Byron Wallace","orcid":"https://orcid.org/0000-0003-2409-7735"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Byron C. Wallace","raw_affiliation_strings":["Northeastern University, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University, Boston, MA, USA","institution_ids":["https://openalex.org/I12912129"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"2017","issue":null,"first_page":"1519","last_page":"1528"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7906262874603271},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.6644794940948486},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5833382606506348},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.539534330368042},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.530283510684967},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5299635529518127},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5092884302139282},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.48164549469947815},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.46438300609588623},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4589132070541382},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3579045534133911},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3226577043533325}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7906262874603271},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.6644794940948486},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5833382606506348},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.539534330368042},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.530283510684967},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5299635529518127},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5092884302139282},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.48164549469947815},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.46438300609588623},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4589132070541382},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3579045534133911},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3226577043533325},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"doi:10.1145/3132847.3132989","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3132847.3132989","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 ACM on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},{"id":"pmid:29308293","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/29308293","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ... ACM International Conference on Information & Knowledge Management. ACM International Conference on Information and Knowledge Management","raw_type":null},{"id":"pmh:oai:kclpure.kcl.ac.uk:openaire/3967c796-7a66-41b6-9be2-3a60dd7a4c51","is_oa":true,"landing_page_url":"https://kclpure.kcl.ac.uk/portal/en/publications/3967c796-7a66-41b6-9be2-3a60dd7a4c51","pdf_url":null,"source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Singh, G, Marshall, I J, Thomas, J, Shawe-Taylor, J & Wallace, B C 2017, A neural candidate-selector architecture for automatic structured clinical text annotation. in CIKM 2017 - Proceedings of the 2017 ACM Conference on Information and Knowledge Management. vol. Part F131841, pp. 1519-1528, 26th ACM International Conference on Information and Knowledge Management, CIKM 2017, Singapore, Singapore, 6/11/2017. https://doi.org/10.1145/3132847.3132989","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:10055844","is_oa":false,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10055844/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"In: (Proceedings) ACM Conference on Information and Knowledge Management (CIKM). (pp. pp. 1519-1528). ACM (2017)","raw_type":"Proceedings paper"},{"id":"pmh:oai:europepmc.org:4666205","is_oa":false,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/5752318","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},{"id":"pmh:oai:kclpure.kcl.ac.uk:publications/3967c796-7a66-41b6-9be2-3a60dd7a4c51","is_oa":true,"landing_page_url":"http://www.scopus.com/inward/record.url?scp=85037357460&partnerID=8YFLogxK","pdf_url":null,"source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Singh, G, Marshall, I J, Thomas, J, Shawe-Taylor, J & Wallace, B C 2017, A neural candidate-selector architecture for automatic structured clinical text annotation. in CIKM 2017 - Proceedings of the 2017 ACM Conference on Information and Knowledge Management. vol. Part F131841, pp. 1519-1528, 26th ACM International Conference on Information and Knowledge Management, CIKM 2017, Singapore, Singapore, 6/11/2017. https://doi.org/10.1145/3132847.3132989","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"pmh:oai:kclpure.kcl.ac.uk:openaire/3967c796-7a66-41b6-9be2-3a60dd7a4c51","is_oa":true,"landing_page_url":"https://kclpure.kcl.ac.uk/portal/en/publications/3967c796-7a66-41b6-9be2-3a60dd7a4c51","pdf_url":null,"source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Singh, G, Marshall, I J, Thomas, J, Shawe-Taylor, J & Wallace, B C 2017, A neural candidate-selector architecture for automatic structured clinical text annotation. in CIKM 2017 - Proceedings of the 2017 ACM Conference on Information and Knowledge Management. vol. Part F131841, pp. 1519-1528, 26th ACM International Conference on Information and Knowledge Management, CIKM 2017, Singapore, Singapore, 6/11/2017. https://doi.org/10.1145/3132847.3132989","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.6100000143051147,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G2127598164","display_name":"RobotReviewer: development and evaluation of a machine learning tool to speed up evidence synthesis in cardiovascular diseases","funder_award_id":"MR/N015185/1","funder_id":"https://openalex.org/F4320334626","funder_display_name":"Medical Research Council"},{"id":"https://openalex.org/G2166638282","display_name":null,"funder_award_id":"MR/J005037/1","funder_id":"https://openalex.org/F4320334626","funder_display_name":"Medical Research Council"},{"id":"https://openalex.org/G7591678011","display_name":null,"funder_award_id":"R01LM012086","funder_id":"https://openalex.org/F4320337372","funder_display_name":"U.S. National Library of Medicine"},{"id":"https://openalex.org/G8729937839","display_name":null,"funder_award_id":"R01 LM012086","funder_id":"https://openalex.org/F4320337372","funder_display_name":"U.S. National Library of Medicine"},{"id":"https://openalex.org/G8824100907","display_name":null,"funder_award_id":"MR/N015185/1","funder_id":"https://openalex.org/F4320334626","funder_display_name":"Medical Research Council"}],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320334626","display_name":"Medical Research Council","ror":"https://ror.org/03x94j517"},{"id":"https://openalex.org/F4320337372","display_name":"U.S. National Library of Medicine","ror":"https://ror.org/0060t0j89"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W183078661","https://openalex.org/W1546425806","https://openalex.org/W1574574070","https://openalex.org/W1832693441","https://openalex.org/W2020959315","https://openalex.org/W2042954874","https://openalex.org/W2053301680","https://openalex.org/W2067284187","https://openalex.org/W2099369363","https://openalex.org/W2114393161","https://openalex.org/W2116485318","https://openalex.org/W2118712128","https://openalex.org/W2122402213","https://openalex.org/W2153579005","https://openalex.org/W2158899491","https://openalex.org/W2166912588","https://openalex.org/W2184378182","https://openalex.org/W2271587360","https://openalex.org/W2429914308","https://openalex.org/W2484269232","https://openalex.org/W2557060223","https://openalex.org/W2913340405","https://openalex.org/W2963082289","https://openalex.org/W2963083845","https://openalex.org/W2963271116","https://openalex.org/W2964006392","https://openalex.org/W2985962305","https://openalex.org/W2997617958"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W4389116644","https://openalex.org/W2153315159","https://openalex.org/W3103844505","https://openalex.org/W259157601","https://openalex.org/W4205463238","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2761785940"],"abstract_inverted_index":{"We":[0,132,142,200],"consider":[1],"the":[2,40,43,46,50,54,70,76,98,149,165,197,202,224],"task":[3],"of":[4,31,39,101,153,205,223],"automatically":[5],"annotating":[6],"free":[7],"texts":[8],"describing":[9,34],"clinical":[10],"trials":[11],"with":[12],"concepts":[13,33,102,117,155],"from":[14],"a":[15,25,62,134,144,174,220],"controlled,":[16],"structured":[17],"medical":[18],"vocabulary.":[19],"Specifically,":[20],"we":[21,218],"aim":[22],"to":[23,27,92,168,208,231],"build":[24],"model":[26,137,150,190],"infer":[28],"distinct":[29],"sets":[30],"(ontological)":[32],"complementary":[35],"clinically":[36],"salient":[37],"aspects":[38],"underlying":[41],"trials:":[42],"populations":[44,110],"enrolled,":[45],"interventions":[47],"administered":[48],"and":[49,94,159,211],"outcomes":[51],"measured,":[52],"i.e.,":[53],"PICO":[55,105,157],"elements.":[56],"This":[57,171],"important":[58],"practical":[59],"problem":[60],"poses":[61],"few":[63],"key":[64],"challenges.":[65,141],"One":[66],"issue":[67],"is":[68,73,90],"that":[69,138,213],"output":[71],"space":[72],"vast,":[74],"because":[75],"vocabulary":[77],"comprises":[78],"many":[79],"unique":[80],"concepts.":[81],"Compounding":[82],"this":[83,88],"problem,":[84],"annotated":[85],"data":[86],"in":[87,147],"domain":[89,229],"expensive":[91],"collect":[93],"hence":[95],"sparse.":[96],"Furthermore,":[97],"outputs":[99],"(sets":[100],"for":[103,156],"each":[104],"element)":[106],"are":[107],"correlated:":[108],"specific":[109],"(e.g.,":[111],"diabetics)":[112],"will":[113],"render":[114],"certain":[115],"intervention":[116],"likely":[118],"(insulin":[119],"therapy)":[120],"while":[121],"effectively":[122],"precluding":[123],"others":[124],"(radiation":[125],"therapy).":[126],"Such":[127],"correlations":[128],"should":[129],"be":[130,169,180],"exploited.":[131],"propose":[133],"novel":[135],"neural":[136,189],"addresses":[139],"these":[140],"introduce":[143],"Candidate-Selector":[145],"architecture":[146],"which":[148,178],"considers":[151],"setes":[152],"candidate":[154,194],"elements,":[158],"assesses":[160],"their":[161,233],"plausibility":[162],"conditioned":[163],"on":[164,173,184],"input":[166,198],"text":[167],"annotated.":[170],"relies":[172,183],"'candidate":[175],"set'":[176],"generator,":[177],"may":[179],"learned":[181],"or":[182],"heuristics.":[185],"A":[186],"conditional":[187],"discriminative":[188],"then":[191],"jointly":[192],"selects":[193],"concepts,":[195],"given":[196],"text.":[199],"compare":[201],"predictive":[203],"performance":[204],"our":[206],"approach":[207],"strong":[209],"baselines,":[210],"show":[212],"it":[214],"outperforms":[215],"them.":[216],"Finally,":[217],"perform":[219],"qualitative":[221],"evaluation":[222],"generated":[225],"annotations":[226],"by":[227],"asking":[228],"experts":[230],"assess":[232],"quality.":[234]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}