{"id":"https://openalex.org/W4405033162","doi":"https://doi.org/10.48550/arxiv.2412.00535","title":"FullStack Bench: Evaluating LLMs as Full Stack Coders","display_name":"FullStack Bench: Evaluating LLMs as Full Stack Coders","publication_year":2024,"publication_date":"2024-11-30","ids":{"openalex":"https://openalex.org/W4405033162","doi":"https://doi.org/10.48550/arxiv.2412.00535"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2412.00535","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.00535","pdf_url":"https://arxiv.org/pdf/2412.00535","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2412.00535","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Bytedance-Seed-Foundation-Code-Team","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Bytedance-Seed-Foundation-Code-Team","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":":","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":":","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Cheng, Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Yao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Jianfeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jianfeng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Jie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100379178","display_name":"Li Chen","orcid":"https://orcid.org/0000-0001-7922-0599"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Li","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Liyu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Liyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Wentao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Wentao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Zhengyu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zhengyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Geng, Shijie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Geng, Shijie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048313538","display_name":"Aoyan Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Aoyan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028016065","display_name":"Zhaoxiang Zhang","orcid":"https://orcid.org/0000-0003-2648-3875"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Bo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Bowen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Bowen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Linyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Linyi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Boyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Boyi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Jiaheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiaheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Kaibo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Kaibo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Qi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035078709","display_name":"Shukai Liu","orcid":"https://orcid.org/0000-0001-7369-2958"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shukai","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100765398","display_name":"Siyao Liu","orcid":"https://orcid.org/0009-0009-7131-2187"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Siyao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Tianyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Tianyi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Tingkai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Tingkai","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Yongfei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yongfei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030798993","display_name":"Rui Long","orcid":"https://orcid.org/0000-0003-4911-1716"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Long, Rui","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Mai, Jing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mai, Jing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ning, Guanghan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ning, Guanghan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103096847","display_name":"Z. Y. Peng","orcid":"https://orcid.org/0009-0001-7026-5421"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Z. Y.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114858716","display_name":"Kai Shen","orcid":"https://orcid.org/0000-0002-3564-0281"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Kai","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Su, Jiahao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Jiahao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Su, Jing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Jing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Sun, Tao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Tao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Sun, Yifan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Yifan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Tao, Yunzhe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tao, Yunzhe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Guoyin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Guoyin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Siwei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Siwei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Xuwu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xuwu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Yite","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yite","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Zihan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zihan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xia, Jinxiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Jinxiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100945219","display_name":"Xiang Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiang, Liang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xiao, Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Xia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xiao, Yongsheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Yongsheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xi, Chenguang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xi, Chenguang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xin, Shulin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xin, Shulin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xu, Jingjing","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Jingjing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xu, Shikun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Shikun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yang, Hongxia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Hongxia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035696034","display_name":"Jack Yang","orcid":"https://orcid.org/0000-0002-2329-5900"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Jack","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yang, Yingxiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yingxiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yuan, Jianbo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Jianbo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhang, Jun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhang, Yufeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yufeng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhang, Yuyu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuyu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zheng, Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Shen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072810180","display_name":"Zhu He","orcid":"https://orcid.org/0000-0003-3675-9441"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, He","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Zhu, Ming","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Ming","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":56,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.8895000219345093,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.8895000219345093,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.8569999933242798,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stack","display_name":"Stack (abstract data type)","score":0.7598752379417419},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.397308886051178},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.36032634973526},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.06997036933898926}],"concepts":[{"id":"https://openalex.org/C9395851","wikidata":"https://www.wikidata.org/wiki/Q177929","display_name":"Stack (abstract data type)","level":2,"score":0.7598752379417419},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.397308886051178},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.36032634973526},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.06997036933898926}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2412.00535","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.00535","pdf_url":"https://arxiv.org/pdf/2412.00535","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2412.00535","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2412.00535","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2412.00535","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.00535","pdf_url":"https://arxiv.org/pdf/2412.00535","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4405033162.pdf","grobid_xml":"https://content.openalex.org/works/W4405033162.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2380576232","https://openalex.org/W2937054111","https://openalex.org/W2066223521","https://openalex.org/W2013178899","https://openalex.org/W373327546","https://openalex.org/W2321534397","https://openalex.org/W2058958858"],"abstract_inverted_index":{"As":[0],"the":[1,120,135],"capabilities":[2],"of":[3,54,122,139],"code":[4,16,40,106],"large":[5],"language":[6],"models":[7],"(LLMs)":[8],"continue":[9],"to":[10,69,91,118],"expand,":[11],"their":[12],"applications":[13],"across":[14],"diverse":[15],"intelligence":[17],"domains":[18,56],"are":[19],"rapidly":[20],"increasing.":[21],"However,":[22],"most":[23],"existing":[24],"datasets":[25],"only":[26],"evaluate":[27,119],"limited":[28],"application":[29,55],"domains.":[30],"To":[31],"address":[32],"this":[33],"gap,":[34],"we":[35,77,101],"have":[36],"developed":[37],"a":[38,51],"comprehensive":[39],"evaluation":[41],"dataset":[42],"FullStack":[43,75,124,132,141],"Bench":[44,125,133,142],"focusing":[45],"on":[46,130],"full-stack":[47],"programming,":[48,59],"which":[49],"encompasses":[50],"wide":[52],"range":[53],"(e.g.,":[57],"basic":[58],"data":[60],"analysis,":[61],"software":[62],"engineering,":[63],"mathematics,":[64],"and":[65,81,116,137,143],"machine":[66],"learning).":[67],"Besides,":[68],"assess":[70],"multilingual":[71],"programming":[72,89,114],"capabilities,":[73],"in":[74],"Bench,":[76],"design":[78],"real-world":[79,93],"instructions":[80],"corresponding":[82],"unit":[83],"test":[84],"cases":[85],"from":[86],"16":[87],"widely-used":[88],"languages":[90,115],"reflect":[92],"usage":[94],"scenarios":[95],"rather":[96],"than":[97],"simple":[98],"translations.":[99],"Moreover,":[100],"also":[102],"release":[103],"an":[104],"effective":[105],"sandbox":[107],"execution":[108],"tool":[109],"(i.e.,":[110],"SandboxFusion)":[111],"supporting":[112],"various":[113],"packages":[117],"performance":[121],"our":[123,131,140],"efficiently.":[126],"Comprehensive":[127],"experimental":[128],"results":[129],"demonstrate":[134],"necessity":[136],"effectiveness":[138],"SandboxFusion.":[144]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2024-12-05T00:00:00"}