{"id":"https://openalex.org/W7118191069","doi":"https://doi.org/10.3390/info17010035","title":"KGEval: Evaluating Scientific Knowledge Graphs with Large Language Models","display_name":"KGEval: Evaluating Scientific Knowledge Graphs with Large Language Models","publication_year":2026,"publication_date":"2026-01-03","ids":{"openalex":"https://openalex.org/W7118191069","doi":"https://doi.org/10.3390/info17010035"},"language":"en","primary_location":{"id":"doi:10.3390/info17010035","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info17010035","pdf_url":"https://www.mdpi.com/2078-2489/17/1/35/pdf?version=1767429984","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2078-2489/17/1/35/pdf?version=1767429984","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086250814","display_name":"Vladyslav Nechakhin","orcid":"https://orcid.org/0000-0003-0146-1207"},"institutions":[{"id":"https://openalex.org/I2802635041","display_name":"Technische Informationsbibliothek (TIB)","ror":"https://ror.org/04aj4c181","country_code":"DE","type":"archive","lineage":["https://openalex.org/I2802635041","https://openalex.org/I315704651"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Vladyslav Nechakhin","raw_affiliation_strings":["Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany"],"raw_orcid":"https://orcid.org/0000-0003-0146-1207","affiliations":[{"raw_affiliation_string":"Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany","institution_ids":["https://openalex.org/I2802635041"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jennifer D\u2019Souza","orcid":"https://orcid.org/0000-0002-6616-9509"},"institutions":[{"id":"https://openalex.org/I2802635041","display_name":"Technische Informationsbibliothek (TIB)","ror":"https://ror.org/04aj4c181","country_code":"DE","type":"archive","lineage":["https://openalex.org/I2802635041","https://openalex.org/I315704651"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jennifer D\u2019Souza","raw_affiliation_strings":["Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany"],"raw_orcid":"https://orcid.org/0000-0002-6616-9509","affiliations":[{"raw_affiliation_string":"Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany","institution_ids":["https://openalex.org/I2802635041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053947568","display_name":"Steffen Eger","orcid":"https://orcid.org/0000-0003-4663-8336"},"institutions":[{"id":"https://openalex.org/I3130238102","display_name":"Lutheran University of Applied Sciences Nuremberg","ror":"https://ror.org/01tbgce33","country_code":"DE","type":"education","lineage":["https://openalex.org/I3130238102"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Steffen Eger","raw_affiliation_strings":["Natural Language Learning & Generation (NLLG), University of Technology Nuremberg (UTN), 90461 Nuremberg, Germany"],"raw_orcid":"https://orcid.org/0000-0003-4663-8336","affiliations":[{"raw_affiliation_string":"Natural Language Learning & Generation (NLLG), University of Technology Nuremberg (UTN), 90461 Nuremberg, Germany","institution_ids":["https://openalex.org/I3130238102"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5121973802","display_name":"S\u00f6ren Auer","orcid":null},"institutions":[{"id":"https://openalex.org/I2802635041","display_name":"Technische Informationsbibliothek (TIB)","ror":"https://ror.org/04aj4c181","country_code":"DE","type":"archive","lineage":["https://openalex.org/I2802635041","https://openalex.org/I315704651"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"S\u00f6ren Auer","raw_affiliation_strings":["Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany"],"raw_orcid":"https://orcid.org/0000-0002-0698-2864","affiliations":[{"raw_affiliation_string":"Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany","institution_ids":["https://openalex.org/I2802635041"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5086250814"],"corresponding_institution_ids":["https://openalex.org/I2802635041"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02875821,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"17","issue":"1","first_page":"35","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.7261000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.7261000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.12890000641345978,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.010700000450015068,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.7398999929428101},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.423799991607666},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4172999858856201},{"id":"https://openalex.org/keywords/knowledge-transfer","display_name":"Knowledge transfer","score":0.3971000015735626},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.3905999958515167},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.3804999887943268},{"id":"https://openalex.org/keywords/knowledge-graph","display_name":"Knowledge graph","score":0.3763999938964844},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.37560001015663147}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7570000290870667},{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.7398999929428101},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48170000314712524},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.43070000410079956},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.423799991607666},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4172999858856201},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4050000011920929},{"id":"https://openalex.org/C2776960227","wikidata":"https://www.wikidata.org/wiki/Q2586354","display_name":"Knowledge transfer","level":2,"score":0.3971000015735626},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.3905999958515167},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C2987255567","wikidata":"https://www.wikidata.org/wiki/Q33002955","display_name":"Knowledge graph","level":2,"score":0.3763999938964844},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.37560001015663147},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3637999892234802},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36010000109672546},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.350600004196167},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.323199987411499},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.321399986743927},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.29510000348091125},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.2863999903202057},{"id":"https://openalex.org/C2779561248","wikidata":"https://www.wikidata.org/wiki/Q743861","display_name":"Tacit knowledge","level":2,"score":0.2718000113964081},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.27129998803138733},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C84685590","wikidata":"https://www.wikidata.org/wiki/Q1540472","display_name":"Knowledge engineering","level":2,"score":0.2648000121116638},{"id":"https://openalex.org/C120567893","wikidata":"https://www.wikidata.org/wiki/Q1582085","display_name":"Knowledge extraction","level":2,"score":0.2554999887943268}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3390/info17010035","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info17010035","pdf_url":"https://www.mdpi.com/2078-2489/17/1/35/pdf?version=1767429984","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},{"id":"pmh:oai:ub-madoc.bib.uni-mannheim.de:71795","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4377196315","display_name":"MADOC (University of Mannheim)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I177802217","host_organization_name":"University of Mannheim","host_organization_lineage":["https://openalex.org/I177802217"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"NonPeerReviewed"},{"id":"pmh:oai:doaj.org/article:931fc3496ed1467b9cc5988543e17d6f","is_oa":true,"landing_page_url":"https://doaj.org/article/931fc3496ed1467b9cc5988543e17d6f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information, Vol 17, Iss 1, p 35 (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/info17010035","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info17010035","pdf_url":"https://www.mdpi.com/2078-2489/17/1/35/pdf?version=1767429984","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8007442355155945}],"awards":[{"id":"https://openalex.org/G268059635","display_name":null,"funder_award_id":"460234259","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G2761004381","display_name":"NFDI4DS  NFDI f\u00fcr Datenwissenschaften und K\u00fcnstliche Intelligenz","funder_award_id":"460234259","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G359036363","display_name":null,"funder_award_id":"01lS22070","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G3775756354","display_name":null,"funder_award_id":"819536","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G7328517233","display_name":null,"funder_award_id":"01lS22070","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G7594458915","display_name":null,"funder_award_id":"819536","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7118191069.pdf","grobid_xml":"https://content.openalex.org/works/W7118191069.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2302501749","https://openalex.org/W2970785793","https://openalex.org/W3099942180","https://openalex.org/W3112005222","https://openalex.org/W4384211370","https://openalex.org/W4385571232","https://openalex.org/W4389519239","https://openalex.org/W4389519254","https://openalex.org/W4394834299","https://openalex.org/W4396697170","https://openalex.org/W4401043863"],"related_works":[],"abstract_inverted_index":{"This":[0,107],"paper":[1],"explores":[2],"the":[3,29,81,119,134,142,148,188,191],"novel":[4],"application":[5,169],"of":[6,38,83,121,144,193],"large":[7],"language":[8,21],"models":[9],"(LLMs)":[10],"as":[11,35],"evaluators":[12,89],"for":[13],"structured":[14,122,197],"scientific":[15],"summaries\u2014a":[16],"task":[17],"where":[18],"traditional":[19],"natural":[20],"evaluation":[22,120,153],"metrics":[23],"may":[24],"not":[25],"readily":[26],"apply.":[27],"Leveraging":[28],"Open":[30],"Research":[31],"Knowledge":[32],"Graph":[33],"(ORKG)":[34],"a":[36,43,157],"repository":[37],"human-curated":[39],"properties,":[40,85],"we":[41,86],"augment":[42],"gold-standard":[44],"dataset":[45],"by":[46],"generating":[47],"corresponding":[48],"properties":[49,127],"using":[50],"three":[51,57],"distinct":[52,175],"LLMs\u2014Llama,":[53],"Mistral,":[54,91],"and":[55,69,71,92,105,168,182,190],"Qwen\u2014under":[56],"contextual":[58,145,180],"settings:":[59],"context-lean":[60],"(research":[61,65,73],"problem":[62,66,74],"only),":[63],"context-rich":[64],"with":[67,75],"title":[68],"abstract),":[70],"context-dense":[72],"multiple":[76],"similar":[77],"papers).":[78],"To":[79],"assess":[80],"quality":[82],"these":[84],"employ":[87],"LLM":[88],"(Deepseek,":[90],"Qwen)":[93],"to":[94,118,129,164],"rate":[95],"them":[96],"on":[97],"criteria,":[98],"including":[99],"similarity,":[100],"relevance,":[101],"factuality,":[102],"informativeness,":[103],"coherence,":[104],"specificity.":[106],"study":[108],"addresses":[109],"key":[110],"research":[111],"questions:":[112],"How":[113,124,140],"do":[114,125],"LLM-as-a-judge":[115],"rubrics":[116],"transfer":[117],"summaries?":[123],"LLM-generated":[126],"compare":[128],"human-annotated":[130],"ones?":[131],"What":[132],"are":[133],"performance":[135],"differences":[136],"among":[137],"various":[138],"LLMs?":[139],"does":[141],"amount":[143],"input":[146],"affect":[147],"generation":[149],"quality?":[150],"The":[151],"resulting":[152],"framework,":[154],"KGEval,":[155],"offers":[156],"customizable":[158],"approach":[159],"that":[160],"can":[161],"be":[162],"extended":[163],"diverse":[165],"knowledge":[166],"graphs":[167],"domains.":[170],"Our":[171],"experimental":[172],"findings":[173],"reveal":[174],"patterns":[176],"in":[177],"evaluator":[178],"biases,":[179],"sensitivity,":[181],"inter-model":[183],"performance,":[184],"thereby":[185],"highlighting":[186],"both":[187],"promise":[189],"challenges":[192],"integrating":[194],"LLMs":[195],"into":[196],"science":[198],"evaluation.":[199]},"counts_by_year":[],"updated_date":"2026-06-22T08:00:12.763002","created_date":"2026-01-05T00:00:00"}