{"id":"https://openalex.org/W4399358184","doi":"https://doi.org/10.3390/info15060328","title":"Evaluating Large Language Models for Structured Science Summarization in the Open Research Knowledge Graph","display_name":"Evaluating Large Language Models for Structured Science Summarization in the Open Research Knowledge Graph","publication_year":2024,"publication_date":"2024-06-05","ids":{"openalex":"https://openalex.org/W4399358184","doi":"https://doi.org/10.3390/info15060328"},"language":"en","primary_location":{"id":"doi:10.3390/info15060328","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info15060328","pdf_url":"https://www.mdpi.com/2078-2489/15/6/328/pdf?version=1717571427","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2078-2489/15/6/328/pdf?version=1717571427","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086250814","display_name":"Vladyslav Nechakhin","orcid":"https://orcid.org/0000-0003-0146-1207"},"institutions":[{"id":"https://openalex.org/I114112103","display_name":"Leibniz University Hannover","ror":"https://ror.org/0304hq317","country_code":"DE","type":"education","lineage":["https://openalex.org/I114112103"]},{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Vladyslav Nechakhin","raw_affiliation_strings":["L3S Research Center, Leibniz University Hannover, 30167 Hannover, Germany"],"affiliations":[{"raw_affiliation_string":"L3S Research Center, Leibniz University Hannover, 30167 Hannover, Germany","institution_ids":["https://openalex.org/I4210136150","https://openalex.org/I114112103"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017940779","display_name":"Jennifer D\u2019Souza","orcid":"https://orcid.org/0000-0002-6616-9509"},"institutions":[{"id":"https://openalex.org/I2802635041","display_name":"Technische Informationsbibliothek (TIB)","ror":"https://ror.org/04aj4c181","country_code":"DE","type":"archive","lineage":["https://openalex.org/I2802635041","https://openalex.org/I315704651"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Jennifer D\u2019Souza","raw_affiliation_strings":["Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany"],"affiliations":[{"raw_affiliation_string":"Leibniz Information Centre for Science and Technology, 30167 Hannover, Germany","institution_ids":["https://openalex.org/I2802635041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053947568","display_name":"Steffen Eger","orcid":"https://orcid.org/0000-0003-4663-8336"},"institutions":[{"id":"https://openalex.org/I177802217","display_name":"University of Mannheim","ror":"https://ror.org/031bsb921","country_code":"DE","type":"education","lineage":["https://openalex.org/I177802217"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Steffen Eger","raw_affiliation_strings":["School of Business Informatics and Mathematics, University of Mannheim, 68161 Mannheim, Germany"],"affiliations":[{"raw_affiliation_string":"School of Business Informatics and Mathematics, University of Mannheim, 68161 Mannheim, Germany","institution_ids":["https://openalex.org/I177802217"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5017940779","https://openalex.org/A5086250814"],"corresponding_institution_ids":["https://openalex.org/I114112103","https://openalex.org/I2802635041","https://openalex.org/I4210136150"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":1.366,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.83380775,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"15","issue":"6","first_page":"328","last_page":"328"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.7882544994354248},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6948494911193848},{"id":"https://openalex.org/keywords/handwriting","display_name":"Handwriting","score":0.504203200340271},{"id":"https://openalex.org/keywords/structuring","display_name":"Structuring","score":0.4823250472545624},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.47236281633377075},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4516271650791168},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.44309064745903015},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.42391538619995117},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3168575167655945}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.7882544994354248},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6948494911193848},{"id":"https://openalex.org/C2779386606","wikidata":"https://www.wikidata.org/wiki/Q2393642","display_name":"Handwriting","level":2,"score":0.504203200340271},{"id":"https://openalex.org/C2775945657","wikidata":"https://www.wikidata.org/wiki/Q381442","display_name":"Structuring","level":2,"score":0.4823250472545624},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.47236281633377075},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4516271650791168},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.44309064745903015},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42391538619995117},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3168575167655945},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3390/info15060328","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info15060328","pdf_url":"https://www.mdpi.com/2078-2489/15/6/328/pdf?version=1717571427","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},{"id":"pmh:oai:ub-madoc.bib.uni-mannheim.de:67321","is_oa":true,"landing_page_url":null,"pdf_url":"https://madoc.bib.uni-mannheim.de/67321/1/information-15-00328.pdf","source":{"id":"https://openalex.org/S4377196315","display_name":"MADOC (University of Mannheim)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I177802217","host_organization_name":"University of Mannheim","host_organization_lineage":["https://openalex.org/I177802217"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Zeitschriftenartikel"},{"id":"pmh:oai:doaj.org/article:0bf9593365c2422b822bae6554ece678","is_oa":true,"landing_page_url":"https://doaj.org/article/0bf9593365c2422b822bae6554ece678","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information, Vol 15, Iss 6, p 328 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/info15060328","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info15060328","pdf_url":"https://www.mdpi.com/2078-2489/15/6/328/pdf?version=1717571427","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G268059635","display_name":null,"funder_award_id":"460234259","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G2761004381","display_name":null,"funder_award_id":"460234259","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G2952219261","display_name":null,"funder_award_id":"819536","funder_id":"https://openalex.org/F4320334678","funder_display_name":"European Research Council"},{"id":"https://openalex.org/G359036363","display_name":null,"funder_award_id":"01lS22070","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G3775756354","display_name":null,"funder_award_id":"819536","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G4412456161","display_name":null,"funder_award_id":"460234259","funder_id":"https://openalex.org/F4320334678","funder_display_name":"European Research Council"},{"id":"https://openalex.org/G7328517233","display_name":null,"funder_award_id":"01lS22070","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G7594458915","display_name":null,"funder_award_id":"819536","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G8139095068","display_name":null,"funder_award_id":"01lS22070","funder_id":"https://openalex.org/F4320334678","funder_display_name":"European Research Council"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320334678","display_name":"European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4399358184.pdf"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W8870360","https://openalex.org/W1988680150","https://openalex.org/W2131191620","https://openalex.org/W2905257126","https://openalex.org/W2970771982","https://openalex.org/W3035324702","https://openalex.org/W3035726715","https://openalex.org/W3112005222","https://openalex.org/W4221143046","https://openalex.org/W4292779060","https://openalex.org/W4303449713","https://openalex.org/W4385567522","https://openalex.org/W4391521451","https://openalex.org/W4391828195","https://openalex.org/W4391992904","https://openalex.org/W4392669714","https://openalex.org/W6600367688","https://openalex.org/W6778883912","https://openalex.org/W6809646742"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W2091301346","https://openalex.org/W3148229873","https://openalex.org/W4389760904","https://openalex.org/W2150160875","https://openalex.org/W4242223894","https://openalex.org/W4306886878","https://openalex.org/W2973759123","https://openalex.org/W1517524280"],"abstract_inverted_index":{"Structured":[0],"science":[1,14,146],"summaries":[2],"or":[3,8],"research":[4,35],"contributions":[5,37],"using":[6,54],"properties":[7,32,98],"dimensions":[9],"beyond":[10],"traditional":[11],"keywords":[12],"enhance":[13],"findability.":[15],"Current":[16],"methods,":[17],"such":[18],"as":[19,152],"those":[20,100],"used":[21],"by":[22,102],"the":[23,70,94,103],"Open":[24],"Research":[25],"Knowledge":[26],"Graph":[27],"(ORKG),":[28],"involve":[29],"manually":[30,96],"curating":[31],"to":[33,59,68,163],"describe":[34],"papers\u2019":[36],"in":[38],"a":[39,89,144],"structured":[40],"manner,":[41],"but":[42,158],"this":[43,81],"is":[44,66,161],"labor-intensive":[45],"and":[46,78,99,118,131,170],"inconsistent":[47],"among":[48],"human":[49,173],"domain-expert":[50],"curators.":[51],"We":[52,107],"propose":[53],"Large":[55],"Language":[56],"Models":[57],"(LLMs)":[58],"automatically":[60],"suggest":[61],"these":[62],"properties.":[63],"However,":[64],"it":[65],"essential":[67],"assess":[69],"readiness":[71],"of":[72,172],"LLMs":[73,149],"like":[74],"GPT-3.5,":[75],"Llama":[76],"2,":[77],"Mistral":[79],"for":[80,155],"task":[82],"before":[83],"their":[84,165],"application.":[85],"Our":[86],"study":[87],"performs":[88],"comprehensive":[90],"comparative":[91],"analysis":[92],"between":[93],"ORKG\u2019s":[95],"curated":[97],"generated":[101],"aforementioned":[104],"state-of-the-art":[105],"LLMs.":[106],"evaluate":[108],"LLM":[109,138],"performance":[110],"from":[111,120],"four":[112],"unique":[113],"perspectives:":[114],"semantic":[115],"alignment":[116,166],"with":[117,137,167],"deviation":[119],"ORKG":[121],"properties,":[122],"fine-grained":[123],"property":[124],"mapping":[125],"accuracy,":[126],"SciNCL":[127],"embedding-based":[128],"cosine":[129],"similarity,":[130],"expert":[132],"surveys":[133],"comparing":[134],"manual":[135],"annotations":[136],"outputs.":[139],"These":[140],"evaluations":[141],"occur":[142],"within":[143],"multidisciplinary":[145],"setting.":[147],"Overall,":[148],"show":[150],"potential":[151],"recommendation":[153],"systems":[154],"structuring":[156],"science,":[157],"further":[159],"fine-tuning":[160],"recommended":[162],"improve":[164],"scientific":[168],"tasks":[169],"mimicry":[171],"expertise.":[174]},"counts_by_year":[{"year":2025,"cited_by_count":4}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}