{"id":"https://openalex.org/W7166836249","doi":"https://doi.org/10.1145/3805773.3806005","title":"Syntax Is Easy, Semantics Is Hard: Evaluating LLMs for LTL Translation","display_name":"Syntax Is Easy, Semantics Is Hard: Evaluating LLMs for LTL Translation","publication_year":2026,"publication_date":"2026-06-30","ids":{"openalex":"https://openalex.org/W7166836249","doi":"https://doi.org/10.1145/3805773.3806005"},"language":null,"primary_location":{"id":"doi:10.1145/3805773.3806005","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805773.3806005","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Secure Development Conference","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805773.3806005","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079454058","display_name":"Priscilla Kyei Danso","orcid":"https://orcid.org/0000-0002-1262-844X"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Priscilla Kyei Danso","raw_affiliation_strings":["Stony Brook University, Department of Computer Science, Stony Brook, USA"],"raw_orcid":"https://orcid.org/0000-0002-1262-844X","affiliations":[{"raw_affiliation_string":"Stony Brook University, Department of Computer Science, Stony Brook, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028103833","display_name":"Md. Saqib Hasan","orcid":"https://orcid.org/0000-0001-7588-8591"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammad Saqib Hasan","raw_affiliation_strings":["Stony Brook University, Department of Computer Science, Stony Brook, USA"],"raw_orcid":"https://orcid.org/0000-0001-7588-8591","affiliations":[{"raw_affiliation_string":"Stony Brook University, Department of Computer Science, Stony Brook, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101768349","display_name":"Niranjan Balasubramanian","orcid":"https://orcid.org/0000-0003-4187-9368"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Niranjan Balasubramanian","raw_affiliation_strings":["Stony Brook University, Department of Computer Science, Stony Brook, USA"],"raw_orcid":"https://orcid.org/0000-0003-4187-9368","affiliations":[{"raw_affiliation_string":"Stony Brook University, Department of Computer Science, Stony Brook, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070136662","display_name":"Omar Chowdhury","orcid":"https://orcid.org/0000-0002-1356-6279"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Omar Chowdhury","raw_affiliation_strings":["Stony Brook University, Department of computer Science, Stony Brook, USA"],"raw_orcid":"https://orcid.org/0000-0002-1356-6279","affiliations":[{"raw_affiliation_string":"Stony Brook University, Department of computer Science, Stony Brook, USA","institution_ids":["https://openalex.org/I59553526"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I59553526"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"152","last_page":"168"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.09319999814033508,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.09319999814033508,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09290000051259995,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.07079999893903732,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/premise","display_name":"Premise","score":0.517300009727478},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.5037000179290771},{"id":"https://openalex.org/keywords/linear-temporal-logic","display_name":"Linear temporal logic","score":0.4551999866962433},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.4235000014305115},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.40849998593330383},{"id":"https://openalex.org/keywords/temporal-logic","display_name":"Temporal logic","score":0.37700000405311584},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.3671000003814697}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7113999724388123},{"id":"https://openalex.org/C2778023277","wikidata":"https://www.wikidata.org/wiki/Q321703","display_name":"Premise","level":2,"score":0.517300009727478},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.5037000179290771},{"id":"https://openalex.org/C4777664","wikidata":"https://www.wikidata.org/wiki/Q1536492","display_name":"Linear temporal logic","level":2,"score":0.4551999866962433},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.4235000014305115},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.40849998593330383},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3856000006198883},{"id":"https://openalex.org/C25016198","wikidata":"https://www.wikidata.org/wiki/Q781833","display_name":"Temporal logic","level":2,"score":0.37700000405311584},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3671000003814697},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3573000133037567},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3458000123500824},{"id":"https://openalex.org/C73301696","wikidata":"https://www.wikidata.org/wiki/Q5469984","display_name":"Formalism (music)","level":3,"score":0.31529998779296875},{"id":"https://openalex.org/C154690210","wikidata":"https://www.wikidata.org/wiki/Q1668499","display_name":"Rewriting","level":2,"score":0.31209999322891235},{"id":"https://openalex.org/C192562157","wikidata":"https://www.wikidata.org/wiki/Q316694","display_name":"Rhetorical question","level":2,"score":0.3003000020980835},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.29350000619888306},{"id":"https://openalex.org/C27508121","wikidata":"https://www.wikidata.org/wiki/Q210841","display_name":"Modal logic","level":3,"score":0.2797999978065491},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2718999981880188},{"id":"https://openalex.org/C40305131","wikidata":"https://www.wikidata.org/wiki/Q2616305","display_name":"Obfuscation","level":2,"score":0.2624000012874603},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.25519999861717224}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805773.3806005","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805773.3806005","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Secure Development Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805773.3806005","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805773.3806005","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Secure Development Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.5392686128616333}],"awards":[{"id":"https://openalex.org/G6181661201","display_name":null,"funder_award_id":"HR00112520301","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1507758008","https://openalex.org/W1547304883","https://openalex.org/W1608490958","https://openalex.org/W1996832037","https://openalex.org/W2005176430","https://openalex.org/W2023083005","https://openalex.org/W2023808162","https://openalex.org/W2027769154","https://openalex.org/W2169354977","https://openalex.org/W2233387800","https://openalex.org/W2573798417","https://openalex.org/W2750938873","https://openalex.org/W2792636363","https://openalex.org/W2912640545","https://openalex.org/W2953940064","https://openalex.org/W2987215725","https://openalex.org/W3003444615","https://openalex.org/W3198685994","https://openalex.org/W3203444134","https://openalex.org/W3213858345","https://openalex.org/W4237885699","https://openalex.org/W4246913434","https://openalex.org/W4281961561","https://openalex.org/W4284708843","https://openalex.org/W4313563695","https://openalex.org/W4382396170","https://openalex.org/W4382463985","https://openalex.org/W4384521943","https://openalex.org/W4388726324","https://openalex.org/W4392517661","https://openalex.org/W4400583090","https://openalex.org/W4401043240","https://openalex.org/W4402364429","https://openalex.org/W4402897118","https://openalex.org/W4402957942","https://openalex.org/W4403877748","https://openalex.org/W4406614566","https://openalex.org/W4408566181","https://openalex.org/W4411721775"],"related_works":[],"abstract_inverted_index":{"Propositional":[0],"Linear":[1],"Temporal":[2],"Logic":[3],"(LTL)":[4],"is":[5],"a":[6,151,165],"popular":[7],"formalism":[8],"for":[9,18,57,175],"specifying":[10],"desirable":[11],"requirements":[12,26],"and":[13,15,21,27,41,60,102,111,145,171],"security":[14,40],"privacy":[16,42],"policies":[17,28],"software,":[19],"networks,":[20],"systems.":[22],"Yet":[23],"expressing":[24],"such":[25,70],"in":[29,120,163],"LTL":[30,46,78,97,132],"remains":[31],"challenging":[32],"because":[33],"of":[34,55,131],"its":[35],"intricate":[36],"semantics.":[37],"Since":[38],"many":[39,58],"analysis":[43],"tools":[44,71],"require":[45],"formulas":[47],"as":[48,150],"input,":[49],"this":[50,169],"difficulty":[51],"places":[52],"them":[53],"out":[54],"reach":[56],"developers":[59],"analysts.":[61],"Large":[62],"Language":[63],"Models":[64],"(LLMs)":[65],"could":[66],"broaden":[67],"access":[68],"to":[69],"by":[72,85],"translating":[73],"natural":[74],"language":[75],"fragments":[76],"into":[77,96],"formulas.":[79,98],"This":[80],"paper":[81],"evaluates":[82],"that":[83],"premise":[84],"assessing":[86],"how":[87],"effectively":[88],"several":[89],"representative":[90],"LLMs":[91,125],"translate":[92],"assertive":[93],"English":[94],"sentences":[95],"Using":[99],"both":[100],"human-generated":[101],"synthetic":[103],"ground-truth":[104],"data,":[105],"we":[106],"evaluate":[107],"effectiveness":[108],"along":[109],"syntactic":[110,129],"semantic":[112,135],"dimensions.":[113],"The":[114],"results":[115],"reveal":[116],"three":[117],"findings:":[118],"(1)":[119],"line":[121],"with":[122,173],"prior":[123],"findings,":[124],"perform":[126],"better":[127],"on":[128,134,168],"aspects":[130],"than":[133],"ones;":[136],"(2)":[137],"they":[138],"generally":[139],"benefit":[140],"from":[141],"more":[142],"detailed":[143],"prompts;":[144],"(3)":[146],"reformulating":[147],"the":[148],"task":[149,170],"Python":[152],"code-completion":[153],"problem":[154],"substantially":[155],"improves":[156],"overall":[157],"performance.":[158],"We":[159],"also":[160],"discuss":[161],"challenges":[162],"conducting":[164],"fair":[166],"evaluation":[167],"conclude":[172],"recommendations":[174],"future":[176],"work.":[177]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-07-02T00:00:00"}