{"id":"https://openalex.org/W7214708994","doi":"https://doi.org/10.48550/arxiv.2609.35581","title":"QC-Stark: A Multi-Task Benchmark Revealing Capability Dissociations in LLMs Evaluated on Quantum Computing Tasks","display_name":"QC-Stark: A Multi-Task Benchmark Revealing Capability Dissociations in LLMs Evaluated on Quantum Computing Tasks","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214708994","doi":"https://doi.org/10.48550/arxiv.2609.35581"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.35581","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35581","pdf_url":"https://arxiv.org/pdf/2609.35581","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.35581","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058904450","display_name":"Pranav Gupta","orcid":"https://orcid.org/0000-0003-2640-1390","observed_orcids":["https://orcid.org/0000-0003-2640-1390"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Gupta, Pranav","raw_affiliation_strings":["Cisco , 2901 3rd Ave , Suite 600 , Seattle , WA 98121 , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cisco , 2901 3rd Ave , Suite 600 , Seattle , WA 98121 , USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5058904450"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10682","display_name":"Quantum Computing Algorithms and Architecture","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10682","display_name":"Quantum Computing Algorithms and Architecture","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8500000238418579},{"id":"https://openalex.org/keywords/quantum-computing","display_name":"quantum computing","score":0.8109999895095825},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.5989999771118164},{"id":"https://openalex.org/keywords/multi-task-evaluation","display_name":"multi-task evaluation","score":0.8109999895095825},{"id":"https://openalex.org/keywords/quantum-circuit-construction","display_name":"quantum circuit construction","score":0.8799999952316284},{"id":"https://openalex.org/keywords/quantum-compilation","display_name":"quantum circuit compilation","score":0.8769999742507935},{"id":"https://openalex.org/keywords/quantum-error-correction","display_name":"quantum error correction","score":0.9610000252723694},{"id":"https://openalex.org/keywords/quantum-simulation","display_name":"quantum simulation","score":0.8930000066757202},{"id":"https://openalex.org/keywords/item-response-theory","display_name":"item response theory","score":0.9150000214576721},{"id":"https://openalex.org/keywords/prompt-sensitivity","display_name":"prompt sensitivity","score":0.8460000157356262}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6983000040054321},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6908000111579895},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6747999787330627},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4867999851703644},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4781999886035919},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.4697999954223633},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45080000162124634},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.4302000105381012},{"id":"https://openalex.org/C58053490","wikidata":"https://www.wikidata.org/wiki/Q176555","display_name":"Quantum computer","level":3,"score":0.39959999918937683},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38920000195503235},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.38749998807907104},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3555000126361847},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.34310001134872437},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.3212999999523163},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.31310001015663147},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.31189998984336853},{"id":"https://openalex.org/C84114770","wikidata":"https://www.wikidata.org/wiki/Q46344","display_name":"Quantum","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.2818000018596649},{"id":"https://openalex.org/C2778061373","wikidata":"https://www.wikidata.org/wiki/Q1315146","display_name":"Predictive coding","level":3,"score":0.2782000005245209}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.35581","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35581","pdf_url":"https://arxiv.org/pdf/2609.35581","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.35581","doi":"https://doi.org/10.48550/arxiv.2609.35581","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.35581","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.35581","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35581","pdf_url":"https://arxiv.org/pdf/2609.35581","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214708994.pdf","grobid_xml":"https://content.openalex.org/works/W7214708994.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W4320003957","https://openalex.org/W4404987838","https://openalex.org/W4406261974","https://openalex.org/W4414974057","https://openalex.org/W4415798388","https://openalex.org/W6949549214","https://openalex.org/W7128618764","https://openalex.org/W7133219754","https://openalex.org/W7162576742","https://openalex.org/W7196972484"],"related_works":["https://openalex.org/W7223417885","https://openalex.org/W7162576742","https://openalex.org/W4416231581","https://openalex.org/W7128618764","https://openalex.org/W7125908344","https://openalex.org/W7221537065","https://openalex.org/W4406261974","https://openalex.org/W7163559521","https://openalex.org/W7154157285","https://openalex.org/W4415933609"],"abstract_inverted_index":{"We":[0,104],"introduce":[1],"QC-Stark,":[2],"a":[3],"benchmark":[4],"for":[5,61],"evaluating":[6],"large":[7],"language":[8],"models":[9,30],"(LLMs)":[10],"on":[11,112],"11":[12,32,66],"quantum":[13],"computing":[14],"(QC)":[15],"tasks,":[16],"spanning":[17],"circuit":[18],"construction,":[19],"debugging,":[20],"compilation,":[21],"error":[22],"correction,":[23],"and":[24,55,82,108],"simulation.":[25],"Across":[26],"2,750":[27],"evaluations":[28],"(10":[29],"$\\times$":[31],"tasks":[33,67,93],"x":[34,38],"5":[35,39],"difficulty":[36],"levels":[37],"seeds),":[40],"we":[41],"find":[42],"that":[43],"overall":[44,54],"rankings":[45,57],"mask":[46],"substantial":[47],"per-task":[48,56],"variation.":[49],"The":[50],"Spearman":[51],"correlation":[52],"between":[53],"is":[58],"statistically":[59],"insignificant":[60],"4":[62],"out":[63],"of":[64],"the":[65,106],"included":[68],"in":[69],"this":[70],"benchmark.":[71],"A":[72],"2-parameter":[73],"Item":[74],"Response":[75],"Theory":[76],"(IRT)":[77],"model":[78],"validates":[79],"measurement":[80],"quality,":[81],"prompt":[83,90],"sensitivity":[84],"analysis":[85],"confirms":[86],"ranking":[87],"robustness":[88],"across":[89],"conditions.":[91],"All":[92],"are":[94],"auto-verifiable":[95],"via":[96],"execution,":[97],"thus":[98],"not":[99],"requiring":[100],"any":[101],"manual":[102],"evaluation.":[103],"make":[105],"code":[107],"data":[109],"publicly":[110],"available":[111],"Huggingface.":[113]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}