{"id":"https://openalex.org/W4402191657","doi":"https://doi.org/10.1007/s10270-024-01206-9","title":"Towards Standardized benchmarks of LLMs in software modeling tasks: a conceptual framework","display_name":"Towards Standardized benchmarks of LLMs in software modeling tasks: a conceptual framework","publication_year":2024,"publication_date":"2024-09-03","ids":{"openalex":"https://openalex.org/W4402191657","doi":"https://doi.org/10.1007/s10270-024-01206-9"},"language":"en","primary_location":{"id":"doi:10.1007/s10270-024-01206-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10270-024-01206-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10270-024-01206-9.pdf","source":{"id":"https://openalex.org/S64245694","display_name":"Software & Systems Modeling","issn_l":"1619-1366","issn":["1619-1366","1619-1374"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software and Systems Modeling","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10270-024-01206-9.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091731762","display_name":"Javier C\u00e1mara","orcid":"https://orcid.org/0000-0001-6717-4775"},"institutions":[{"id":"https://openalex.org/I82767444","display_name":"Universidad de M\u00e1laga","ror":"https://ror.org/036b2ww28","country_code":"ES","type":"education","lineage":["https://openalex.org/I82767444"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Javier C\u00e1mara","raw_affiliation_strings":["ITIS Software, Universidad de M\u00e1laga, Blvd. Louis Pasteur, M\u00e1laga, 29071, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITIS Software, Universidad de M\u00e1laga, Blvd. Louis Pasteur, M\u00e1laga, 29071, Spain","institution_ids":["https://openalex.org/I82767444","https://openalex.org/I97750245"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037353994","display_name":"Loli Burgue\u00f1o","orcid":"https://orcid.org/0000-0002-7779-8810"},"institutions":[{"id":"https://openalex.org/I82767444","display_name":"Universidad de M\u00e1laga","ror":"https://ror.org/036b2ww28","country_code":"ES","type":"education","lineage":["https://openalex.org/I82767444"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Lola Burgue\u00f1o","raw_affiliation_strings":["ITIS Software, Universidad de M\u00e1laga, Blvd. Louis Pasteur, M\u00e1laga, 29071, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITIS Software, Universidad de M\u00e1laga, Blvd. Louis Pasteur, M\u00e1laga, 29071, Spain","institution_ids":["https://openalex.org/I82767444","https://openalex.org/I97750245"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055078557","display_name":"Javier Troya","orcid":"https://orcid.org/0000-0002-1314-9694"},"institutions":[{"id":"https://openalex.org/I82767444","display_name":"Universidad de M\u00e1laga","ror":"https://ror.org/036b2ww28","country_code":"ES","type":"education","lineage":["https://openalex.org/I82767444"]},{"id":"https://openalex.org/I97750245","display_name":"Software (Spain)","ror":"https://ror.org/02ethns06","country_code":"ES","type":"company","lineage":["https://openalex.org/I4210087817","https://openalex.org/I97750245"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Javier Troya","raw_affiliation_strings":["ITIS Software, Universidad de M\u00e1laga, Blvd. Louis Pasteur, M\u00e1laga, 29071, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ITIS Software, Universidad de M\u00e1laga, Blvd. Louis Pasteur, M\u00e1laga, 29071, Spain","institution_ids":["https://openalex.org/I82767444","https://openalex.org/I97750245"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5091731762"],"corresponding_institution_ids":["https://openalex.org/I82767444","https://openalex.org/I97750245"],"apc_list":{"value":2890,"currency":"USD","value_usd":2890},"apc_paid":{"value":2890,"currency":"USD","value_usd":2890},"fwci":6.4015,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.9724366,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"23","issue":"6","first_page":"1309","last_page":"1318"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.9865999817848206,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9825000166893005,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5502146482467651},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.45189476013183594},{"id":"https://openalex.org/keywords/conceptual-framework","display_name":"Conceptual framework","score":0.42924734950065613},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.1691167652606964},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11891362071037292},{"id":"https://openalex.org/keywords/social-science","display_name":"Social science","score":0.0937509834766388}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5502146482467651},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.45189476013183594},{"id":"https://openalex.org/C14224292","wikidata":"https://www.wikidata.org/wiki/Q13600188","display_name":"Conceptual framework","level":2,"score":0.42924734950065613},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.1691167652606964},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11891362071037292},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0937509834766388}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s10270-024-01206-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10270-024-01206-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10270-024-01206-9.pdf","source":{"id":"https://openalex.org/S64245694","display_name":"Software & Systems Modeling","issn_l":"1619-1366","issn":["1619-1366","1619-1374"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software and Systems Modeling","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s10270-024-01206-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10270-024-01206-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10270-024-01206-9.pdf","source":{"id":"https://openalex.org/S64245694","display_name":"Software & Systems Modeling","issn_l":"1619-1366","issn":["1619-1366","1619-1374"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software and Systems Modeling","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2170680412","display_name":null,"funder_award_id":"TED2021-130523B-I00","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G301372621","display_name":null,"funder_award_id":"PID2021\u2010125527NB\u2010I00","funder_id":"https://openalex.org/F4320322930","funder_display_name":"Ministerio de Ciencia e Innovaci\u00f3n"},{"id":"https://openalex.org/G5210454185","display_name":null,"funder_award_id":"TED2021-130523B-I00","funder_id":"https://openalex.org/F4320322930","funder_display_name":"Ministerio de Ciencia e Innovaci\u00f3n"},{"id":"https://openalex.org/G7434292347","display_name":null,"funder_award_id":"PID2021-125527NB-I00","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"}],"funders":[{"id":"https://openalex.org/F4320311265","display_name":"Universidad de M\u00e1laga","ror":"https://ror.org/036b2ww28"},{"id":"https://openalex.org/F4320322930","display_name":"Ministerio de Ciencia e Innovaci\u00f3n","ror":"https://ror.org/034900433"},{"id":"https://openalex.org/F4320335598","display_name":"Agencia Estatal de Investigaci\u00f3n","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402191657.pdf","grobid_xml":"https://content.openalex.org/works/W4402191657.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W2990695366","https://openalex.org/W3090459478","https://openalex.org/W4281763794","https://openalex.org/W4367059011","https://openalex.org/W4377291442","https://openalex.org/W4390638022","https://openalex.org/W4392414327","https://openalex.org/W4402665833","https://openalex.org/W6600438464","https://openalex.org/W6600466347"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Abstract":[0],"The":[1],"integration":[2],"of":[3,27,72,81],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"in":[8,24,42,59,83],"software":[9,60,84],"modeling":[10],"tasks":[11],"presents":[12],"both":[13],"opportunities":[14],"and":[15,47,78],"challenges.":[16],"This":[17,63],"Expert":[18],"Voice":[19],"addresses":[20],"a":[21,52,96],"significant":[22],"gap":[23],"the":[25,32,39,68,73],"evaluation":[26,80],"these":[28],"models,":[29],"advocating":[30],"for":[31,34,70],"need":[33],"standardized":[35],"benchmarking":[36,74],"frameworks.":[37],"Recognizing":[38],"potential":[40],"variability":[41],"prompt":[43],"strategies,":[44],"LLM":[45],"outputs,":[46],"solution":[48],"space,":[49],"we":[50],"propose":[51],"conceptual":[53,87],"framework":[54,64,88],"to":[55,66],"assess":[56],"their":[57],"quality":[58],"model":[61],"generation.":[62],"aims":[65],"pave":[67],"way":[69],"standardization":[71],"process,":[75],"ensuring":[76],"consistent":[77],"objective":[79],"LLMs":[82],"modeling.":[85],"Our":[86],"is":[89],"illustrated":[90],"using":[91],"UML":[92],"class":[93],"diagrams":[94],"as":[95],"running":[97],"example.":[98]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":9}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2024-09-04T00:00:00"}