{"id":"https://openalex.org/W4416249868","doi":"https://doi.org/10.1109/ijcnn64981.2025.11227545","title":"TechSupportEval: An Automated Evaluation Framework for Technical Support Question Answering","display_name":"TechSupportEval: An Automated Evaluation Framework for Technical Support Question Answering","publication_year":2025,"publication_date":"2025-06-30","ids":{"openalex":"https://openalex.org/W4416249868","doi":"https://doi.org/10.1109/ijcnn64981.2025.11227545"},"language":null,"primary_location":{"id":"doi:10.1109/ijcnn64981.2025.11227545","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn64981.2025.11227545","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100774168","display_name":"Bohan Chen","orcid":"https://orcid.org/0000-0002-4159-076X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Bohan Chen","raw_affiliation_strings":["Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027731066","display_name":"Yongqian Sun","orcid":"https://orcid.org/0000-0003-0266-7899"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongqian Sun","raw_affiliation_strings":["Nankai University"],"affiliations":[{"raw_affiliation_string":"Nankai University","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001504472","display_name":"Yuhe Liu","orcid":"https://orcid.org/0000-0002-7470-1905"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuhe Liu","raw_affiliation_strings":["Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101779169","display_name":"Longlong Xu","orcid":"https://orcid.org/0009-0008-7364-1307"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Longlong Xu","raw_affiliation_strings":["Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101650951","display_name":"Zhihao Xie","orcid":"https://orcid.org/0009-0005-3522-4703"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhe Xie","raw_affiliation_strings":["Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073273454","display_name":"Changhua Pei","orcid":"https://orcid.org/0000-0001-9288-4787"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"funder","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210108629","display_name":"Computer Network Information Center","ror":"https://ror.org/01s0wyf50","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210108629"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changhua Pei","raw_affiliation_strings":["Chinese Academy of Sciences,Computer Network Information Center"],"affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences,Computer Network Information Center","institution_ids":["https://openalex.org/I4210108629","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055987265","display_name":"Jing Han","orcid":"https://orcid.org/0000-0003-3251-6549"},"institutions":[{"id":"https://openalex.org/I75746372","display_name":"ZTE (United States)","ror":"https://ror.org/0518yg160","country_code":"US","type":"company","lineage":["https://openalex.org/I4210098582","https://openalex.org/I75746372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jing Han","raw_affiliation_strings":["ZTE Corporation"],"affiliations":[{"raw_affiliation_string":"ZTE Corporation","institution_ids":["https://openalex.org/I75746372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061017872","display_name":"Ni Fan","orcid":"https://orcid.org/0000-0003-3007-3174"},"institutions":[{"id":"https://openalex.org/I75746372","display_name":"ZTE (United States)","ror":"https://ror.org/0518yg160","country_code":"US","type":"company","lineage":["https://openalex.org/I4210098582","https://openalex.org/I75746372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fan Ni","raw_affiliation_strings":["ZTE Corporation"],"affiliations":[{"raw_affiliation_string":"ZTE Corporation","institution_ids":["https://openalex.org/I75746372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111319570","display_name":"Xuhui Cai","orcid":"https://orcid.org/0009-0005-8228-9817"},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuhui Cai","raw_affiliation_strings":["China Mobile Communications Group Co., Ltd"],"affiliations":[{"raw_affiliation_string":"China Mobile Communications Group Co., Ltd","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114973808","display_name":"Ce Yang","orcid":"https://orcid.org/0000-0002-0446-641X"},"institutions":[{"id":"https://openalex.org/I180662265","display_name":"China Mobile (China)","ror":"https://ror.org/05gftfe97","country_code":"CN","type":"company","lineage":["https://openalex.org/I180662265"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ce Yang","raw_affiliation_strings":["China Mobile Communications Group Co., Ltd"],"affiliations":[{"raw_affiliation_string":"China Mobile Communications Group Co., Ltd","institution_ids":["https://openalex.org/I180662265"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046419834","display_name":"Dan Pei","orcid":"https://orcid.org/0000-0002-5113-838X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dan Pei","raw_affiliation_strings":["Tsinghua University"],"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5100774168"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18207814,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5580000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5580000281333923,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10209999978542328,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.08169999718666077,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.7117999792098999},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6370999813079834},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5299000144004822},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.40880000591278076},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3594000041484833},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.35269999504089355}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8264999985694885},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.7117999792098999},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6370999813079834},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5299000144004822},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.40880000591278076},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3903000056743622},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3594000041484833},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.35269999504089355},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.33329999446868896},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.319599986076355},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.30230000615119934},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29660001397132874},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2662999927997589},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.25999999046325684},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.25429999828338623}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn64981.2025.11227545","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn64981.2025.11227545","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2887103446","https://openalex.org/W2889557241","https://openalex.org/W3034478991","https://openalex.org/W3101786835","https://openalex.org/W3200174022","https://openalex.org/W3217387898","https://openalex.org/W4382202658","https://openalex.org/W4389519254","https://openalex.org/W4389519321","https://openalex.org/W4389519598","https://openalex.org/W4396823873","https://openalex.org/W4400267554","https://openalex.org/W4401042808","https://openalex.org/W4403391008","https://openalex.org/W4403560390","https://openalex.org/W4404534210","https://openalex.org/W4404783114","https://openalex.org/W4406892656","https://openalex.org/W4411630045"],"related_works":[],"abstract_inverted_index":{"Technical":[0],"support":[1,50,89,149],"question-answering":[2],"(QA)":[3],"systems":[4,22,64],"assist":[5],"users":[6],"in":[7,113,140],"diagnosing":[8],"and":[9,31,71,75,107,123,131,147,187],"resolving":[10],"technical":[11,49,88],"issues,":[12],"but":[13],"ensuring":[14,53],"their":[15],"reliability":[16],"remains":[17],"a":[18,104,154],"challenge.":[19],"Existing":[20],"QA":[21,51,63,171],"may":[23],"generate":[24,55],"inaccurate":[25],"responses":[26,165],"due":[27],"to":[28,37,66,111,117,135],"LLM":[29,110,134],"hallucinations":[30],"retrieval":[32],"errors,":[33],"which":[34,99,126],"can":[35],"lead":[36],"misleading":[38],"guidance.":[39,57],"A":[40],"reliable":[41],"evaluation":[42,60,85],"framework":[43,86,92],"is":[44],"essential":[45],"for":[46,62,87],"systematically":[47],"improving":[48],"systems,":[52],"they":[54],"accurate":[56],"However,":[58],"existing":[59],"methods":[61],"struggle":[65],"precisely":[67],"match":[68],"key":[69,115,120],"terms,":[70],"verify":[72],"step":[73,145],"order":[74,146],"completeness.To":[76,148],"address":[77],"these":[78],"challenges,":[79],"we":[80,152],"propose":[81,153],"TechSupportEval,":[82],"an":[83,109,133,175],"automated":[84],"QA.":[90],"Our":[91],"introduces":[93],"two":[94],"novel":[95],"techniques:":[96],"(1)":[97],"ClozeFact,":[98],"formulates":[100],"fact":[101],"verification":[102],"as":[103],"cloze":[105],"test":[106],"uses":[108,132],"fill":[112],"missing":[114],"terms":[116],"ensure":[118],"precise":[119],"term":[121],"matching,":[122],"(2)":[124],"StepRestore,":[125],"shuffles":[127],"ground":[128],"truth":[129],"steps":[130],"reconstruct":[136],"the":[137,141,159,180],"actionable":[138],"instructions":[139],"correct":[142],"order,":[143],"verifying":[144],"comprehensive":[150],"evaluation,":[151],"benchmark":[155],"dataset":[156,188],"built":[157],"upon":[158],"publicly":[160],"available":[161,190],"TechQA":[162],"dataset,":[163],"containing":[164],"generated":[166],"by":[167,183],"different":[168],"levels":[169],"of":[170,177],"systems.":[172],"TechSupportEval":[173],"achieves":[174],"AUC":[176],"0.91,":[178],"outperforming":[179],"state-of-the-art":[181],"method":[182],"7.6%.":[184],"The":[185],"code":[186],"are":[189],"at":[191],"https://github.com/NetManAIOps/TechSupportEval.":[192]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-11-14T00:00:00"}