{"id":"https://openalex.org/W7162576742","doi":"https://doi.org/10.48550/arxiv.2605.27210","title":"Qiskit QuantumKatas: Adapting Microsoft's Quantum Computing exercises for LLM evaluation","display_name":"Qiskit QuantumKatas: Adapting Microsoft's Quantum Computing exercises for LLM evaluation","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162576742","doi":"https://doi.org/10.48550/arxiv.2605.27210"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.27210","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.27210","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085459979","display_name":"Juan Cruz-Benito","orcid":"https://orcid.org/0000-0003-2045-8329"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cruz-Benito, Juan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5055771565","display_name":"Ismael Faro","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Faro, Ismael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10682","display_name":"Quantum Computing Algorithms and Architecture","score":0.9337000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10682","display_name":"Quantum Computing Algorithms and Architecture","score":0.9337000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10622","display_name":"Quantum Mechanics and Applications","score":0.006000000052154064,"subfield":{"id":"https://openalex.org/subfields/3107","display_name":"Atomic and Molecular Physics, and Optics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7692000269889832},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.63919997215271},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.555400013923645},{"id":"https://openalex.org/keywords/quantum-computer","display_name":"Quantum computer","score":0.5101000070571899},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.4982999861240387},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.42649999260902405},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.4002000093460083},{"id":"https://openalex.org/keywords/quantum","display_name":"Quantum","score":0.366100013256073}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7692000269889832},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6866000294685364},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.63919997215271},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.555400013923645},{"id":"https://openalex.org/C58053490","wikidata":"https://www.wikidata.org/wiki/Q176555","display_name":"Quantum computer","level":3,"score":0.5101000070571899},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4982999861240387},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.42649999260902405},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.4002000093460083},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.383899986743927},{"id":"https://openalex.org/C84114770","wikidata":"https://www.wikidata.org/wiki/Q46344","display_name":"Quantum","level":2,"score":0.366100013256073},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35420000553131104},{"id":"https://openalex.org/C58849907","wikidata":"https://www.wikidata.org/wiki/Q2118982","display_name":"Quantum gate","level":4,"score":0.3359000086784363},{"id":"https://openalex.org/C137019171","wikidata":"https://www.wikidata.org/wiki/Q2623817","display_name":"Quantum algorithm","level":3,"score":0.33180001378059387},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.32580000162124634},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.320499986410141},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.31929999589920044},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.305400013923645},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.30070000886917114},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C95466800","wikidata":"https://www.wikidata.org/wiki/Q1027298","display_name":"Quantum key distribution","level":3,"score":0.2728999853134155},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27070000767707825},{"id":"https://openalex.org/C203087015","wikidata":"https://www.wikidata.org/wiki/Q378201","display_name":"Qubit","level":3,"score":0.25679999589920044},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.25529998540878296}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.27210","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.27210","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27210","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.643165111541748,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,108,223],"adapt":[1],"Microsoft's":[2],"QuantumKatas":[3],"--":[4,10,116,123,190],"a":[5,60,90,117,150,186],"well-established":[6],"quantum":[7,18,55,239],"computing":[8,19],"curriculum":[9],"from":[11,86,145],"Q#":[12],"to":[13,124,147,232],"Qiskit,":[14],"the":[15,77,100,126,134,193,211,225],"most":[16],"widely-adopted":[17],"framework,":[20,228],"and":[21,54,66,94,105,149,157,181,229],"package":[22],"it":[23,191,214],"with":[24,140,174],"an":[25],"evaluation":[26,103,227],"framework":[27,101],"for":[28,196,210],"systematic":[29],"LLM":[30,236],"assessment.":[31],"The":[32],"resulting":[33],"benchmark":[34,135],"comprises":[35],"350":[36],"tasks":[37,85],"across":[38,112],"26":[39],"categories,":[40],"spanning":[41],"fundamental":[42],"gates":[43],"through":[44],"advanced":[45],"algorithms":[46,167],"(Grover's,":[47],"Simon's,":[48],"Deutsch-Jozsa),":[49],"error":[50],"correction,":[51],"key":[52,130],"distribution,":[53],"games.":[56],"Each":[57],"task":[58],"includes":[59],"natural":[61],"language":[62],"prompt,":[63],"canonical":[64],"solution,":[65],"deterministic":[67],"test":[68],"verification":[69],"via":[70],"classical":[71],"circuit":[72],"simulation.":[73],"By":[74],"building":[75],"on":[76,235],"QuantumKatas'":[78],"proven":[79],"pedagogical":[80],"design":[81],"rather":[82],"than":[83],"creating":[84],"scratch,":[87],"we":[88],"inherit":[89],"principled":[91],"difficulty":[92],"progression":[93],"comprehensive":[95],"concept":[96],"coverage":[97],"while":[98],"contributing":[99],"adaptation,":[102],"infrastructure,":[104],"empirical":[106],"analysis.":[107],"evaluate":[109],"16":[110],"LLMs":[111],"7":[113],"prompting":[114,184],"configurations":[115],"total":[118],"of":[119,200],"39,200":[120],"model":[121,138],"runs":[122],"demonstrate":[125],"benchmark's":[127],"utility.":[128],"Three":[129],"findings":[131],"emerge:":[132],"(1)":[133],"effectively":[136],"differentiates":[137],"capabilities,":[139],"best-configuration":[141],"pass":[142],"rates":[143],"ranging":[144],"32.3%":[146],"83.1%":[148],"26.1":[151],"pp":[152],"average":[153],"gap":[154],"between":[155],"frontier":[156],"open-source":[158],"models;":[159],"(2)":[160],"models":[161,198],"perform":[162],"well":[163],"at":[164],"implementing":[165],"known":[166],"(SimonsAlgorithm":[168],"82.1%,":[169],"BasicGates":[170],"81.6%)":[171],"but":[172,207],"struggle":[173],"problem":[175],"encoding":[176],"(SolveSATWithGrover":[177],"34.4%,":[178],"DistinguishUnitaries":[179],"40.0%);":[180],"(3)":[182],"chain-of-thought":[183],"shows":[185],"modestly":[187],"bimodal":[188],"effect":[189],"is":[192],"best":[194],"strategy":[195],"three":[197],"(two":[199],"them":[201],"explicitly":[202],"reasoning-tuned":[203],"per":[204],"vendor":[205],"documentation)":[206],"degrades":[208],"performance":[209],"rest,":[212],"leaving":[213],"mid-pack":[215],"in":[216,238],"aggregate":[217],"(56.3%":[218],"mean)":[219],"behind":[220],"few-shot-5":[221],"(57.8%).":[222],"release":[224],"benchmark,":[226],"baseline":[230],"results":[231],"support":[233],"research":[234],"capabilities":[237],"computing.":[240]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-28T00:00:00"}