{"id":"https://openalex.org/W7204866911","doi":"https://doi.org/10.48550/arxiv.2608.28641","title":"Terminal-Bench-LILT: Multilingual Agentic Coding Benchmark Grounded in Language, Region, and Culture","display_name":"Terminal-Bench-LILT: Multilingual Agentic Coding Benchmark Grounded in Language, Region, and Culture","publication_year":2026,"publication_date":"2026-08-11","ids":{"openalex":"https://openalex.org/W7204866911","doi":"https://doi.org/10.48550/arxiv.2608.28641"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.28641","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.28641","pdf_url":"https://arxiv.org/pdf/2608.28641","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.28641","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134861455","display_name":"Yunsu Kim","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Kim, Yunsu","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109639336","display_name":"K Uhlig","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Uhlig, Kaden","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043326962","display_name":"Ashwin Purohit","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Purohit, Ashwin","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056307183","display_name":"Milind Agarwal","orcid":"https://orcid.org/0000-0002-6794-8707","observed_orcids":["https://orcid.org/0000-0002-6794-8707"]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Agarwal, Milind","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038893703","display_name":"Patrick Simianer","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Simianer, Patrick","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148895246","display_name":"Anil Arslan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Arslan, Anil","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056232919","display_name":"Kiarash Mokhtari","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Mokhtari, Kiarash","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028321737","display_name":"Thomas Zenkel","orcid":"https://orcid.org/0009-0002-2060-156X","observed_orcids":["https://orcid.org/0009-0002-2060-156X"]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Zenkel, Thomas","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072860790","display_name":"Johannes Mosig","orcid":"https://orcid.org/0000-0002-9189-4219","observed_orcids":["https://orcid.org/0000-0002-9189-4219"]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Mosig, Johannes","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085119673","display_name":"Gabriel Bretschner","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Bretschner, Gabriel","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110946701","display_name":"Shamik Bose","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Bose, Shamik","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067263769","display_name":"Joern Wuebker","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Wuebker, Joern","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072072153","display_name":"John DeNero","orcid":"https://orcid.org/0000-0001-9152-3891","observed_orcids":["https://orcid.org/0000-0001-9152-3891"]},"institutions":[{"id":"https://openalex.org/I4210123482","display_name":"Lega Italiana per la Lotta ai Tumori","ror":"https://ror.org/02r2t8x54","country_code":"IT","type":"nonprofit","lineage":["https://openalex.org/I4210123482"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"DeNero, John","raw_affiliation_strings":["LILT, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LILT, Inc","institution_ids":["https://openalex.org/I4210123482"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210123482"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9717000126838684,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9717000126838684,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.007000000216066837,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.004600000102072954,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-coding-agents","display_name":"coding agents","score":0.6800000071525574},{"id":"https://openalex.org/keywords/llm-benchmarking","display_name":"LLM benchmarking","score":0.6169999837875366},{"id":"https://openalex.org/keywords/internationalization","display_name":"internationalization","score":0.8220000267028809},{"id":"https://openalex.org/keywords/text-normalization","display_name":"text normalization","score":0.6230000257492065}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6891999840736389},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.6539999842643738},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.6103000044822693},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5378000140190125},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4537000060081482},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.38119998574256897},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3741999864578247},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.37380000948905945},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3400000035762787},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2833999991416931},{"id":"https://openalex.org/C2778883600","wikidata":"https://www.wikidata.org/wiki/Q2390977","display_name":"Language proficiency","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26100000739097595},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.259799987077713}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.28641","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.28641","pdf_url":"https://arxiv.org/pdf/2608.28641","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.28641","doi":"https://doi.org/10.48550/arxiv.2608.28641","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.28641","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.28641","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.28641","pdf_url":"https://arxiv.org/pdf/2608.28641","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204866911.pdf","grobid_xml":"https://content.openalex.org/works/W7204866911.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W3177813494","https://openalex.org/W4307479317","https://openalex.org/W4310428868","https://openalex.org/W4366204357","https://openalex.org/W4385562549","https://openalex.org/W4387561453","https://openalex.org/W4392822190","https://openalex.org/W4401042855","https://openalex.org/W4402671169","https://openalex.org/W4410313682","https://openalex.org/W4411119259","https://openalex.org/W4415241223","https://openalex.org/W4415797205","https://openalex.org/W4416445988","https://openalex.org/W6891998059","https://openalex.org/W7118067677","https://openalex.org/W7125193990","https://openalex.org/W7131786218","https://openalex.org/W7166318274"],"related_works":["https://openalex.org/W7131786218","https://openalex.org/W6929495831","https://openalex.org/W4412944815","https://openalex.org/W7214002252","https://openalex.org/W4417081359","https://openalex.org/W4388747891","https://openalex.org/W4323570365","https://openalex.org/W7165376407","https://openalex.org/W7158446072","https://openalex.org/W4416034047"],"abstract_inverted_index":{"Most":[0],"evaluations":[1],"for":[2],"coding":[3,25,112,118],"agents":[4],"are":[5,66,129],"conducted":[6],"exclusively":[7],"in":[8,27],"English,":[9],"which":[10],"does":[11,108],"not":[12,109],"reflect":[13],"real-world":[14],"multilingual":[15,117],"deployment.":[16],"We":[17],"present":[18],"Terminal-Bench-LILT,":[19],"a":[20,74,121],"suite":[21],"of":[22,80],"300":[23],"authentic":[24],"tasks":[26,65,97,128],"ten":[28],"languages:":[29],"Arabic,":[30],"Czech,":[31],"German,":[32],"Spanish,":[33],"Hindi,":[34],"Japanese,":[35],"Korean,":[36],"Serbian,":[37],"Turkish,":[38],"and":[39,61,71,107,123],"Chinese.":[40],"Each":[41],"task":[42],"targets":[43],"issues":[44],"specific":[45],"to":[46],"non-English":[47],"software":[48],"development":[49],"that":[50,85,116],"have":[51],"no":[52],"direct":[53],"English":[54],"equivalent,":[55],"e.g.,":[56],"internationalization,":[57],"encoding,":[58],"text":[59],"normalization,":[60],"cultural":[62],"conventions.":[63],"All":[64],"authored":[67],"by":[68,99,105],"native-speaker":[69],"programmers":[70],"validated":[72],"through":[73],"multi-stage":[75],"quality":[76],"control":[77],"pipeline.":[78],"Evaluation":[79],"six":[81],"frontier":[82],"models":[83],"reveals":[84],"even":[86],"the":[87],"strongest":[88],"model":[89],"reaches":[90],"only":[91],"63.1\\%":[92],"pass":[93],"rate,":[94],"with":[95],"many":[96],"unsolved":[98],"any":[100],"model.":[101],"Performance":[102],"varies":[103],"substantially":[104],"language":[106],"track":[110],"general":[111],"benchmark":[113],"rankings,":[114],"highlighting":[115],"competence":[119],"is":[120],"distinct":[122],"underexplored":[124],"capability":[125],"axis.":[126],"Sample":[127],"available":[130],"at":[131],"https://github.com/lilt/terminal-bench-lilt":[132]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-02T00:00:00"}