{"id":"https://openalex.org/W4414359713","doi":"https://doi.org/10.24963/ijcai.2025/858","title":"BinMetric: A Comprehensive Binary Code Analysis Benchmark for Large Language Models","display_name":"BinMetric: A Comprehensive Binary Code Analysis Benchmark for Large Language Models","publication_year":2025,"publication_date":"2025-09-01","ids":{"openalex":"https://openalex.org/W4414359713","doi":"https://doi.org/10.24963/ijcai.2025/858"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2025/858","is_oa":false,"landing_page_url":"https://doi.org/10.24963/ijcai.2025/858","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Fourth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038253295","display_name":"Xiuwei Shang","orcid":"https://orcid.org/0009-0009-6660-9947"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiuwei Shang","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100634295","display_name":"Guoqiang Chen","orcid":"https://orcid.org/0000-0002-6094-7883"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guoqiang Chen","raw_affiliation_strings":["QI-ANXIN Technology Research Institute, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"QI-ANXIN Technology Research Institute, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085001934","display_name":"Shaoyin Cheng","orcid":"https://orcid.org/0000-0002-3992-9509"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I143868143","display_name":"Anhui University","ror":"https://ror.org/05th6yx34","country_code":"CN","type":"education","lineage":["https://openalex.org/I143868143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaoyin Cheng","raw_affiliation_strings":["Anhui Province Key Laboratory of Digital Security, Hefei, China","University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Anhui Province Key Laboratory of Digital Security, Hefei, China","institution_ids":["https://openalex.org/I143868143"]},{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113311055","display_name":"Benlong Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Benlong Wu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103452489","display_name":"Li Hu","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Hu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101338646","display_name":"Gangyang Li","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Gangyang Li","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100340726","display_name":"Weiming Zhang","orcid":"https://orcid.org/0000-0002-3782-2855"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I143868143","display_name":"Anhui University","ror":"https://ror.org/05th6yx34","country_code":"CN","type":"education","lineage":["https://openalex.org/I143868143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weiming Zhang","raw_affiliation_strings":["Anhui Province Key Laboratory of Digital Security, Hefei, China","University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Anhui Province Key Laboratory of Digital Security, Hefei, China","institution_ids":["https://openalex.org/I143868143"]},{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064573190","display_name":"Nenghai Yu","orcid":"https://orcid.org/0000-0003-4417-9316"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]},{"id":"https://openalex.org/I143868143","display_name":"Anhui University","ror":"https://ror.org/05th6yx34","country_code":"CN","type":"education","lineage":["https://openalex.org/I143868143"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nenghai Yu","raw_affiliation_strings":["Anhui Province Key Laboratory of Digital Security, Hefei, China","University of Science and Technology of China, Hefei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Anhui Province Key Laboratory of Digital Security, Hefei, China","institution_ids":["https://openalex.org/I143868143"]},{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5038253295"],"corresponding_institution_ids":["https://openalex.org/I126520041"],"apc_list":null,"apc_paid":null,"fwci":7.6229,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.97063475,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"7715","last_page":"7723"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9057000279426575,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9057000279426575,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8557999730110168},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.5891000032424927},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.49799999594688416},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.44510000944137573},{"id":"https://openalex.org/keywords/binary-data","display_name":"Binary data","score":0.4397999942302704},{"id":"https://openalex.org/keywords/binary-code","display_name":"Binary code","score":0.42340001463890076},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.3847000002861023}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8557999730110168},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7111999988555908},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.5891000032424927},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.49799999594688416},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.44510000944137573},{"id":"https://openalex.org/C2779190172","wikidata":"https://www.wikidata.org/wiki/Q4913888","display_name":"Binary data","level":3,"score":0.4397999942302704},{"id":"https://openalex.org/C63435697","wikidata":"https://www.wikidata.org/wiki/Q864135","display_name":"Binary code","level":3,"score":0.42340001463890076},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.42250001430511475},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.40299999713897705},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.3847000002861023},{"id":"https://openalex.org/C207850805","wikidata":"https://www.wikidata.org/wiki/Q269608","display_name":"Reverse engineering","level":2,"score":0.3734999895095825},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3555000126361847},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35530000925064087},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.33899998664855957},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32679998874664307},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.2824999988079071},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.27320000529289246},{"id":"https://openalex.org/C97686452","wikidata":"https://www.wikidata.org/wiki/Q7604153","display_name":"Static analysis","level":2,"score":0.27230000495910645},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26339998841285706},{"id":"https://openalex.org/C95986675","wikidata":"https://www.wikidata.org/wiki/Q185168","display_name":"Quantitative analysis (chemistry)","level":2,"score":0.2533000111579895}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2025/858","is_oa":false,"landing_page_url":"https://doi.org/10.24963/ijcai.2025/858","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Fourth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Binary":[0],"analysis":[1,67,83,145],"is":[2,32],"crucial":[3],"for":[4,26,160],"software":[5,164],"security,":[6],"offering":[7],"insights":[8,159],"into":[9],"compiled":[10],"programs":[11],"without":[12],"source":[13],"code.":[14],"As":[15],"large":[16],"language":[17,22],"models":[18],"(LLMs)":[19],"excel":[20],"in":[21,45,124,142,163],"tasks,":[23,84],"their":[24,41,107],"potential":[25],"complex":[27],"decoding":[28],"binary":[29,66,82,129,144],"data":[30],"structures":[31],"growing.":[33],"However,":[34],"the":[35,125],"lack":[36],"of":[37,127,147],"standardized":[38],"benchmarks":[39],"hinders":[40],"evaluation":[42],"and":[43,109,131,154],"progress":[44],"this":[46,50,100],"domain.":[47],"To":[48],"bridge":[49],"gap,":[51],"we":[52],"introduce":[53],"BinMetric,":[54],"a":[55,138,150],"first":[56],"comprehensive":[57],"benchmark":[58,101,152],"designed":[59],"specifically":[60],"to":[61],"evaluate":[62],"LLMs":[63,116,162],"performance":[64],"on":[65,99],"tasks.":[68],"BinMetric":[69,136],"comprises":[70],"1,000":[71],"questions":[72],"derived":[73],"from":[74],"20":[75],"real-world":[76],"open-source":[77],"projects":[78],"across":[79],"6":[80],"practical":[81],"including":[85],"decompilation,":[86],"code":[87],"summarization,":[88],"etc.,":[89],"which":[90],"reflect":[91],"actual":[92],"reverse":[93],"engineering":[94],"scenarios.":[95],"Our":[96],"empirical":[97],"study":[98,156],"investigates":[102],"various":[103],"state-of-the-art":[104],"LLMs,":[105,148],"revealing":[106],"strengths":[108],"limitations.":[110],"The":[111],"findings":[112],"indicate":[113],"that":[114],"while":[115],"show":[117],"strong":[118],"potential,":[119],"challenges":[120],"still":[121],"exist,":[122],"particularly":[123],"areas":[126],"precise":[128],"lifting":[130],"assembly":[132],"synthesis.":[133],"In":[134],"summary,":[135],"makes":[137],"significant":[139],"step":[140],"forward":[141],"measuring":[143],"capabilities":[146],"establishing":[149],"new":[151],"leaderboard,":[153],"our":[155],"offers":[157],"valuable":[158],"advancing":[161],"security.":[165]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}