{"id":"https://openalex.org/W7169511535","doi":"https://doi.org/10.1145/3803437.3805214","title":"Eagle: Leveraging Operations Documents for Comprehensive Benchmark Question Generation","display_name":"Eagle: Leveraging Operations Documents for Comprehensive Benchmark Question Generation","publication_year":2026,"publication_date":"2026-07-05","ids":{"openalex":"https://openalex.org/W7169511535","doi":"https://doi.org/10.1145/3803437.3805214"},"language":null,"primary_location":{"id":"doi:10.1145/3803437.3805214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805214","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3803437.3805214","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001504472","display_name":"Yuhe Liu","orcid":"https://orcid.org/0000-0002-7470-1905"},"institutions":[{"id":"https://openalex.org/I4210156423","display_name":"National Engineering Research Center for Information Technology in Agriculture","ror":"https://ror.org/04c3j3t84","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210156423"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuhe Liu","raw_affiliation_strings":["Beijing National Research Center for Information Science and Technology, Beijing, Beijing, China","Tsinghua University, Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-0324-7749","affiliations":[{"raw_affiliation_string":"Beijing National Research Center for Information Science and Technology, Beijing, Beijing, China","institution_ids":["https://openalex.org/I4210156423"]},{"raw_affiliation_string":"Tsinghua University, Beijing, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073273454","display_name":"Changhua Pei","orcid":"https://orcid.org/0000-0001-9288-4787"},"institutions":[{"id":"https://openalex.org/I4210108629","display_name":"Computer Network Information Center","ror":"https://ror.org/01s0wyf50","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210108629"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changhua Pei","raw_affiliation_strings":["Computer Network Information Center, Chinese Academy of Sciences, Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9288-4787","affiliations":[{"raw_affiliation_string":"Computer Network Information Center, Chinese Academy of Sciences, Beijing, Beijing, China","institution_ids":["https://openalex.org/I4210108629"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141073501","display_name":"Hang Wang","orcid":"https://orcid.org/0009-0004-7469-6511"},"institutions":[{"id":"https://openalex.org/I4210108629","display_name":"Computer Network Information Center","ror":"https://ror.org/01s0wyf50","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210108629"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hang Wang","raw_affiliation_strings":["Computer Network Information Center, Chinese Academy of Sciences, Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0004-7469-6511","affiliations":[{"raw_affiliation_string":"Computer Network Information Center, Chinese Academy of Sciences, Beijing, Beijing, China","institution_ids":["https://openalex.org/I4210108629"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101779169","display_name":"Longlong Xu","orcid":"https://orcid.org/0009-0008-7364-1307"},"institutions":[{"id":"https://openalex.org/I4210156423","display_name":"National Engineering Research Center for Information Technology in Agriculture","ror":"https://ror.org/04c3j3t84","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210156423"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Longlong Xu","raw_affiliation_strings":["Beijing National Research Center for Information Science and Technology, Beijing, Beijing, China","Tsinghua University, Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-7364-1307","affiliations":[{"raw_affiliation_string":"Beijing National Research Center for Information Science and Technology, Beijing, Beijing, China","institution_ids":["https://openalex.org/I4210156423"]},{"raw_affiliation_string":"Tsinghua University, Beijing, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141112770","display_name":"Xiaogang Dong","orcid":"https://orcid.org/0009-0002-1558-2843"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaogang Dong","raw_affiliation_strings":["Huawei, Xian'an, China"],"raw_orcid":"https://orcid.org/0009-0002-1558-2843","affiliations":[{"raw_affiliation_string":"Huawei, Xian'an, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141087124","display_name":"Zhen Feng","orcid":"https://orcid.org/0009-0000-3345-199X"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhen Feng","raw_affiliation_strings":["Huawei, Xi'an, China"],"raw_orcid":"https://orcid.org/0009-0000-3345-199X","affiliations":[{"raw_affiliation_string":"Huawei, Xi'an, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141051060","display_name":"Li Zheng","orcid":"https://orcid.org/0009-0007-0562-4786"},"institutions":[{"id":"https://openalex.org/I4210130112","display_name":"China Academy of Information and Communications Technology","ror":"https://ror.org/038dte259","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210130112","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Zheng","raw_affiliation_strings":["China Academy of Information and Communications Technology, Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-0562-4786","affiliations":[{"raw_affiliation_string":"China Academy of Information and Communications Technology, Beijing, Beijing, China","institution_ids":["https://openalex.org/I4210130112"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141070109","display_name":"Kehang Ji","orcid":"https://orcid.org/0009-0005-2400-7340"},"institutions":[{"id":"https://openalex.org/I4210130112","display_name":"China Academy of Information and Communications Technology","ror":"https://ror.org/038dte259","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210130112","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kehang Ji","raw_affiliation_strings":["China Academy of Information and Communications Technology, Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0005-2400-7340","affiliations":[{"raw_affiliation_string":"China Academy of Information and Communications Technology, Beijing, Beijing, China","institution_ids":["https://openalex.org/I4210130112"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046419834","display_name":"Dan Pei","orcid":"https://orcid.org/0000-0002-5113-838X"},"institutions":[{"id":"https://openalex.org/I4210156423","display_name":"National Engineering Research Center for Information Technology in Agriculture","ror":"https://ror.org/04c3j3t84","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210156423"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dan Pei","raw_affiliation_strings":["Beijing National Research Center for Information Science and Technology, Beijing, Beijing, China","Tsinghua University, Beijing, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5113-838X","affiliations":[{"raw_affiliation_string":"Beijing National Research Center for Information Science and Technology, Beijing, Beijing, China","institution_ids":["https://openalex.org/I4210156423"]},{"raw_affiliation_string":"Tsinghua University, Beijing, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"404","last_page":"414"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.47189998626708984},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.46540001034736633},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.45100000500679016},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.41440001130104065},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.323199987411499},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.30730000138282776}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7032999992370605},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.47189998626708984},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.46540001034736633},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.45100000500679016},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4253999888896942},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.41440001130104065},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3488999903202057},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33390000462532043},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.323199987411499},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.30730000138282776},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.2928999960422516},{"id":"https://openalex.org/C157170001","wikidata":"https://www.wikidata.org/wiki/Q4781507","display_name":"Applications of artificial intelligence","level":2,"score":0.2736000120639801},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2703999876976013},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.25130000710487366},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3803437.3805214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805214","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3803437.3805214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805214","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1983247737","display_name":null,"funder_award_id":"2024YFB4505903","funder_id":"https://openalex.org/F4320336026","funder_display_name":"National Key Research and Development Program of China Stem Cell and Translational Research"}],"funders":[{"id":"https://openalex.org/F4320336026","display_name":"National Key Research and Development Program of China Stem Cell and Translational Research","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W2757978590","https://openalex.org/W3155743842","https://openalex.org/W3155952370","https://openalex.org/W3187045993","https://openalex.org/W4221146870","https://openalex.org/W4221148741","https://openalex.org/W4385572634","https://openalex.org/W4386721326","https://openalex.org/W4394769528","https://openalex.org/W4400484838","https://openalex.org/W4405270453"],"related_works":[],"abstract_inverted_index":{"The":[0],"rapid":[1],"growth":[2],"in":[3],"scale":[4],"and":[5,17,49,84],"complexity":[6],"of":[7,72,78],"modern":[8],"software":[9],"systems":[10],"has":[11],"intensified":[12],"the":[13,70],"need":[14],"for":[15,24,88],"intelligent":[16],"reliable":[18],"IT":[19,25],"operations.":[20],"While":[21],"Artificial":[22],"Intelligence":[23],"Operations":[26,53],"(AIOps)":[27],"addresses":[28],"some":[29],"challenges,":[30],"existing":[31],"solutions":[32],"predominantly":[33],"rely":[34],"on":[35],"isolated,":[36],"task-specific":[37],"models":[38],"that":[39],"struggle":[40],"with":[41],"interpreting":[42],"multimodal":[43],"data,":[44],"incur":[45],"high":[46],"maintenance":[47],"costs,":[48],"lack":[50],"sufficient":[51],"transparency.":[52],"Large":[54],"Language":[55],"Models":[56],"(OpsLLMs)":[57],"offer":[58],"unified,":[59],"knowledge-rich":[60],"reasoning":[61],"capabilities,":[62],"yet":[63],"their":[64],"evaluation":[65,74],"faces":[66],"significant":[67],"barriers,":[68],"including":[69],"absence":[71],"Ops-centric":[73],"taxonomies,":[75],"limited":[76],"availability":[77],"public":[79],"datasets,":[80],"simplistic":[81],"question-generation":[82],"methods,":[83],"inadequate":[85],"quality":[86],"standards":[87],"comprehensive":[89],"operations":[90],"tasks.":[91]},"counts_by_year":[],"updated_date":"2026-07-18T05:58:53.570196","created_date":"2026-07-18T00:00:00"}