{"id":"https://openalex.org/W7124426555","doi":"https://doi.org/10.1109/access.2026.3654948","title":"CodeGen-3D: A Benchmark for Evaluating LLMs in Zero-Shot and Iterative 3D Modeling in Blender","display_name":"CodeGen-3D: A Benchmark for Evaluating LLMs in Zero-Shot and Iterative 3D Modeling in Blender","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7124426555","doi":"https://doi.org/10.1109/access.2026.3654948"},"language":"en","primary_location":{"id":"doi:10.1109/access.2026.3654948","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3654948","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2026.3654948","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101969395","display_name":"Hao Ji","orcid":"https://orcid.org/0000-0002-5481-5788"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hao Ji","raw_affiliation_strings":["Center for Advanced Research Computing, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Advanced Research Computing, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123214137","display_name":"Kotha Aditya","orcid":null},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kotha Aditya","raw_affiliation_strings":["Department of Computer Science, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123208913","display_name":"Sebastian Escalante","orcid":null},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sebastian Escalante","raw_affiliation_strings":["Department of Computer Science, University of Southern California, Los Angeles, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-3607-2682","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123238976","display_name":"Yunjian Qiu","orcid":null},"institutions":[{"id":"https://openalex.org/I51504820","display_name":"San Jose State University","ror":"https://ror.org/04qyvz380","country_code":"US","type":"education","lineage":["https://openalex.org/I51504820"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yunjian Qiu","raw_affiliation_strings":["Department of Mechanical Engineering, San José State University, San Jose, CA, USA"],"raw_orcid":"https://orcid.org/0009-0003-4312-4046","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, San José State University, San Jose, CA, USA","institution_ids":["https://openalex.org/I51504820"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.07239976,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":null,"first_page":"18181","last_page":"18192"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10719","display_name":"3D Shape Modeling and Analysis","score":0.288100004196167,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10719","display_name":"3D Shape Modeling and Analysis","score":0.288100004196167,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.10890000313520432,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.08889999985694885,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.7986000180244446},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6255000233650208},{"id":"https://openalex.org/keywords/iterative-refinement","display_name":"Iterative refinement","score":0.5346999764442444},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.47839999198913574},{"id":"https://openalex.org/keywords/rendering","display_name":"Rendering (computer graphics)","score":0.4699999988079071},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4129999876022339},{"id":"https://openalex.org/keywords/iterative-method","display_name":"Iterative method","score":0.4075999855995178},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.40230000019073486},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.3919000029563904}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8346999883651733},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.7986000180244446},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6255000233650208},{"id":"https://openalex.org/C2779982483","wikidata":"https://www.wikidata.org/wiki/Q6094420","display_name":"Iterative refinement","level":2,"score":0.5346999764442444},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.47839999198913574},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.4699999988079071},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4129999876022339},{"id":"https://openalex.org/C159694833","wikidata":"https://www.wikidata.org/wiki/Q2321565","display_name":"Iterative method","level":2,"score":0.4075999855995178},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.40230000019073486},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3919000029563904},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.3831000030040741},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.37959998846054077},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.3628000020980835},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.3467000126838684},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3278000056743622},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.32179999351501465},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30550000071525574},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.30489999055862427},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.30230000615119934},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.30169999599456787},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2937999963760376},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.27630001306533813},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.273499995470047},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.26910001039505005},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.26260000467300415},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2590000033378601},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.25850000977516174},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.2556000053882599},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2026.3654948","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3654948","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:scholarworks.sjsu.edu:faculty_rsca-7835","is_oa":true,"landing_page_url":"https://scholarworks.sjsu.edu/faculty_rsca/6835","pdf_url":null,"source":{"id":"https://openalex.org/S4377196389","display_name":"San Jos\u00e9 State University ScholarWorks (San Jose State University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I51504820","host_organization_name":"San Jose State University","host_organization_lineage":["https://openalex.org/I51504820"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Faculty Research, Scholarly, and Creative Activity","raw_type":"text"},{"id":"pmh:oai:doaj.org/article:015d7912d7564532bf674838745cac0f","is_oa":false,"landing_page_url":"https://doaj.org/article/015d7912d7564532bf674838745cac0f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 14, Pp 18181-18192 (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2026.3654948","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3654948","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.4420376121997833}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W2117088012","https://openalex.org/W3162843412","https://openalex.org/W4385318467","https://openalex.org/W4386065887","https://openalex.org/W4386075660","https://openalex.org/W4387394725","https://openalex.org/W4390873331","https://openalex.org/W4403770356","https://openalex.org/W4411067687","https://openalex.org/W4413144965","https://openalex.org/W4413156313","https://openalex.org/W4413556737","https://openalex.org/W4415798936"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"that":[4,105,157,211],"generate":[5],"executable":[6],"code":[7,50,73,232,252],"are":[8,162,193],"opening":[9],"a":[10,30,63,92,101,126,146,236],"new":[11],"path":[12],"to":[13],"text-to-3D":[14,213],"by":[15],"converting":[16],"natural":[17],"language":[18],"prompts":[19,81],"into":[20],"scripts":[21],"for":[22,34,68,121,133,142,171,185,206,244],"modeling":[23,71],"software":[24],"like":[25,49],"Blender.":[26],"However,":[27],"there":[28],"is":[29],"lack":[31],"of":[32,78,250],"evaluation":[33,103],"such":[35],"methods":[36],"currently.":[37],"Existing":[38],"benchmarks":[39],"focus":[40],"on":[41,230],"direct":[42],"mesh":[43],"synthesis":[44],"and":[45,53,66,90,96,100,111,119,124,136,149,159,174,200,240,254],"overlook":[46],"critical":[47],"factors":[48],"execution":[51,95],"reliability":[52],"human-aligned":[54],"quality.":[55],"To":[56],"address":[57],"this":[58],"gap,":[59],"we":[60],"introduce":[61],"CodeGen-3D,":[62],"comprehensive":[64],"benchmark":[65,76,239],"protocol":[67,104],"assessing":[69],"3D":[70,255],"via":[72],"generation.":[74,233],"The":[75],"consists":[77],"100":[79],"diverse":[80],"derived":[82],"from":[83],"Objaverse":[84],"with":[85,220],"rich":[86],"captions":[87],"(Cap3D,":[88],"DiffuRank,":[89],"Llama-based),":[91],"standardized":[93],"Blender":[94],"six-view":[97],"rendering":[98],"pipeline,":[99],"three-part":[102],"assesses:":[106],"(i)":[107],"error":[108],"rate":[109,168,184],"(compilation":[110],"runtime":[112],"failures),":[113],"(ii)":[114],"multi-view":[115],"CLIP":[116,191],"alignment":[117,199],"(average":[118],"max)":[120],"semantic":[122],"similarity,":[123],"(iii)":[125],"VLM-as-a-Judge":[127],"pairwise":[128],"preference":[129,179],"task":[130],"(using":[131],"GPT-4o)":[132],"perceptual":[134],"quality":[135],"prompt":[137],"adherence.":[138],"We":[139],"report":[140],"results":[141],"eight":[143],"general-purpose":[144],"LLMs,":[145,251],"specialized":[147,158],"BlenderLLM,":[148],"an":[150],"iterative":[151,160,187],"self-correcting":[152],"agent.":[153],"Our":[154],"analysis":[155],"shows":[156],"systems":[161,214],"substantially":[163],"more":[164],"reliable":[165],"(3\u20134%":[166],"failure":[167],"vs.":[169],"21\u201392%":[170],"general":[172,218],"LLMs)":[173],"obtain":[175],"the":[176,186,204,248],"highest":[177],"GPT-4o":[178],"rates":[180],"(e.g.,":[181],"33.3%":[182],"win":[183],"agent),":[188],"even":[189],"when":[190],"scores":[192],"similar.":[194],"This":[195],"gap":[196],"between":[197],"CLIP-based":[198],"judge-based":[201],"preferences":[202],"underscores":[203],"need":[205],"feedback-driven,":[207],"multi-signal":[208],"evaluation,":[209],"suggesting":[210],"practical":[212],"should":[215],"combine":[216],"strong":[217,242],"LLMs":[219],"domain":[221],"specialization":[222],"or":[223],"inexpensive":[224],"self-refinement":[225],"loops,":[226],"rather":[227],"than":[228],"relying":[229],"single-pass":[231],"CodeGen-3D":[234],"establishes":[235],"practical,":[237],"reproducible":[238],"offers":[241],"baselines":[243],"future":[245],"research":[246],"at":[247],"intersection":[249],"generation,":[253],"modeling.":[256]},"counts_by_year":[],"updated_date":"2026-02-09T05:59:30.833894","created_date":"2026-01-17T00:00:00"}