{"id":"https://openalex.org/W4415035879","doi":"https://doi.org/10.18653/v1/2026.findings-acl.560","title":"Large Language Models for IT Automation Tasks: Are We There Yet?","display_name":"Large Language Models for IT Automation Tasks: Are We There Yet?","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W4415035879","doi":"https://doi.org/10.18653/v1/2026.findings-acl.560"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.560","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.560","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.18653/v1/2026.findings-acl.560","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102741432","display_name":"Md Mahadi Hassan","orcid":"https://orcid.org/0000-0001-6139-3133"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Md. Mahadi Hassan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102623315","display_name":"John Salvador","orcid":"https://orcid.org/0009-0009-8360-4734"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"John Salvador","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069063267","display_name":"Akond Rahman","orcid":"https://orcid.org/0000-0002-5056-757X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akond Ashfaque Ur Rahman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5114621407","display_name":"Santu Karmaker","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Santu Karmaker","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":159.3846,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.99871217,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"11534","last_page":"11573"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.9225999712944031,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.9225999712944031,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.842199981212616},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6258000135421753},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5188000202178955},{"id":"https://openalex.org/keywords/automated-reasoning","display_name":"Automated reasoning","score":0.5092999935150146},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.49410000443458557},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.483599990606308},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4481000006198883}],"concepts":[{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.842199981212616},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7386999726295471},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6258000135421753},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5756999850273132},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5188000202178955},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.5092999935150146},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.49410000443458557},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.483599990606308},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4481000006198883},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.392300009727478},{"id":"https://openalex.org/C64260653","wikidata":"https://www.wikidata.org/wiki/Q1194864","display_name":"Electronic design automation","level":2,"score":0.3151000142097473},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.31349998712539673},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.28700000047683716},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.27959999442100525},{"id":"https://openalex.org/C85135244","wikidata":"https://www.wikidata.org/wiki/Q1799487","display_name":"Laboratory automation","level":3,"score":0.27869999408721924},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.2757999897003174},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27570000290870667},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.27230000495910645},{"id":"https://openalex.org/C21457203","wikidata":"https://www.wikidata.org/wiki/Q4056293","display_name":"Process automation system","level":3,"score":0.2572000026702881},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.25440001487731934}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.560","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.560","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.20505","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.20505","pdf_url":"https://arxiv.org/pdf/2505.20505","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.20505","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.20505","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.560","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.560","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLMs":[0,119],"show":[1],"promise":[2],"in":[3,88,130,149,169,191],"code":[4],"generation,":[5],"yet":[6],"their":[7],"effectiveness":[8],"for":[9,14,65],"IT":[10,37,72,185],"automation":[11,38,73,83,186],"tasks,":[12],"particularly":[13],"tools":[15],"like":[16],"Ansible,":[17],"remains":[18],"understudied.":[19],"Existing":[20],"benchmarks":[21],"rely":[22],"primarily":[23],"on":[24],"synthetic":[25],"tasks":[26,55],"that":[27,183],"fail":[28],"to":[29,71,79,173],"capture":[30],"the":[31,117],"needs":[32],"of":[33,52,97,125],"practitioners":[34],"who":[35],"use":[36],"tools,":[39],"such":[40],"as":[41],"Ansible.":[42],"We":[43,91],"present":[44],"ITAB":[45,75],"(IT":[46],"Automation":[47],"Task":[48],"Benchmark),":[49],"a":[50,68,102],"benchmark":[51],"126":[53],"diverse":[54],"(e.g.,":[56],"configuring":[57],"servers,":[58],"managing":[59],"files)":[60],"where":[61],"each":[62],"task":[63],"accounts":[64],"state":[66,131,175,192],"reconciliation:":[67],"property":[69],"unique":[70],"tools.":[74],"evaluates":[76],"LLMs'":[77,171],"ability":[78,172],"generate":[80],"functional":[81],"Ansible":[82],"scripts":[84],"via":[85],"dynamic":[86],"execution":[87,114,151,196],"controlled":[89],"environments.":[90],"evaluate":[92],"14":[93],"open-source":[94,170],"LLMs,":[95],"none":[96],"which":[98],"accomplish":[99],"pass@10":[100],"at":[101],"rate":[103],"beyond":[104],"12%.":[105],"To":[106],"explain":[107],"these":[108],"low":[109],"scores,":[110],"we":[111],"analyze":[112],"1,411":[113],"failures":[115,129],"across":[116],"evaluated":[118],"and":[120,143,147,157,160,177,194],"identify":[121],"two":[122],"main":[123],"categories":[124],"prevalent":[126],"semantic":[127],"errors:":[128],"reconciliation":[132],"related":[133],"reasoning":[134,193],"(44.87%":[135],"combined":[136,154],"from":[137,155],"variable":[138],"(11.43%),":[139],"host":[140],"(11.84%),":[141],"path(11.63%),":[142],"template":[144],"(9.97%)":[145],"issues)":[146],"deficiencies":[148],"module-specific":[150],"knowledge":[152],"(24.37%":[153],"Attribute":[156],"parameter":[158],"(14.44%)":[159],"module":[161,180],"(9.93%)":[162],"errors).":[163],"Our":[164],"findings":[165],"reveal":[166],"key":[167],"limitations":[168],"track":[174],"changes":[176],"apply":[178],"specialized":[179],"knowledge,":[181],"indicating":[182],"reliable":[184],"will":[187],"require":[188],"major":[189],"advances":[190],"domain-specific":[195],"understanding.":[197]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}