{"id":"https://openalex.org/W4308642998","doi":"https://doi.org/10.1145/3540250.3558941","title":"Improving ML-based information retrieval software with user-driven functional testing and defect class analysis","display_name":"Improving ML-based information retrieval software with user-driven functional testing and defect class analysis","publication_year":2022,"publication_date":"2022-11-07","ids":{"openalex":"https://openalex.org/W4308642998","doi":"https://doi.org/10.1145/3540250.3558941"},"language":"en","primary_location":{"id":"doi:10.1145/3540250.3558941","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3540250.3558941","pdf_url":null,"source":{"id":"https://openalex.org/S4363608883","display_name":"Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100698979","display_name":"Junjie Zhu","orcid":"https://orcid.org/0000-0001-7177-8694"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junjie Zhu","raw_affiliation_strings":["Apple, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054182168","display_name":"Teng Long","orcid":"https://orcid.org/0000-0002-7465-3843"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Teng Long","raw_affiliation_strings":["Apple, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100392089","display_name":"Wei Wang","orcid":"https://orcid.org/0000-0002-8180-2886"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei Wang","raw_affiliation_strings":["Apple, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, USA","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103489081","display_name":"Atif M. Memon","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Atif Memon","raw_affiliation_strings":["Apple, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, USA","institution_ids":["https://openalex.org/I4210153776"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210153776"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1291","last_page":"1301"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7860607504844666},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6368098258972168},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.541711688041687},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5121054649353027},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.480033278465271},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4465102553367615},{"id":"https://openalex.org/keywords/software-bug","display_name":"Software bug","score":0.4243428111076355},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4131249785423279},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38608261942863464},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3467981219291687},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.13017424941062927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7860607504844666},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6368098258972168},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.541711688041687},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5121054649353027},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.480033278465271},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4465102553367615},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.4243428111076355},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4131249785423279},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38608261942863464},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3467981219291687},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13017424941062927},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3540250.3558941","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3540250.3558941","pdf_url":null,"source":{"id":"https://openalex.org/S4363608883","display_name":"Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W1736726159","https://openalex.org/W1996574389","https://openalex.org/W1996898994","https://openalex.org/W2020278455","https://openalex.org/W2026784708","https://openalex.org/W2027691696","https://openalex.org/W2065219958","https://openalex.org/W2087198174","https://openalex.org/W2110441383","https://openalex.org/W2110933113","https://openalex.org/W2126752493","https://openalex.org/W2150608210","https://openalex.org/W2173213060","https://openalex.org/W2293505944","https://openalex.org/W2464171116","https://openalex.org/W2531638282","https://openalex.org/W2787894218","https://openalex.org/W2883656129","https://openalex.org/W2922234936","https://openalex.org/W2945883466","https://openalex.org/W2963856968","https://openalex.org/W2973084513","https://openalex.org/W2998064020","https://openalex.org/W3003212147","https://openalex.org/W3007157104","https://openalex.org/W3034708216","https://openalex.org/W3039522238","https://openalex.org/W3043238989","https://openalex.org/W3048821638","https://openalex.org/W3100925971","https://openalex.org/W3110368543","https://openalex.org/W3158950919","https://openalex.org/W3162654950","https://openalex.org/W3163698227","https://openalex.org/W3194588521","https://openalex.org/W4205947740","https://openalex.org/W4232633635","https://openalex.org/W4287254792","https://openalex.org/W4288092818","https://openalex.org/W4394654521"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W2384888906","https://openalex.org/W4224009465","https://openalex.org/W4362564695","https://openalex.org/W4285233257","https://openalex.org/W4310720697","https://openalex.org/W2088791420","https://openalex.org/W2179540415","https://openalex.org/W4206403383"],"abstract_inverted_index":{"Machine":[0],"Learning":[1],"(ML)":[2],"has":[3],"become":[4],"the":[5,28,46,87,91,95,106,111,124,155,203,223],"cornerstone":[6],"of":[7,31,90,97,105,138],"information":[8,53],"retrieval":[9],"(IR)":[10],"software,":[11,47,92],"as":[12],"it":[13],"can":[14,35],"drive":[15],"better":[16],"user":[17],"experience":[18],"by":[19],"leveraging":[20],"information-rich":[21],"data":[22],"and":[23,149,189,215],"complex":[24],"models.":[25],"However,":[26],"evaluating":[27],"emergent":[29],"behavior":[30,89],"ML-based":[32,112],"IR":[33,156],"software":[34,40,113,157],"be":[36],"challenging":[37,129],"with":[38,198],"traditional":[39],"testing":[41],"approaches:":[42],"when":[43],"developers":[44],"modify":[45],"they":[48,59],"cannot":[49],"often":[50],"extract":[51],"useful":[52],"from":[54,158],"individual":[55],"test":[56,140],"instances;":[57],"rather,":[58],"seek":[60],"to":[61,85,144,153,237],"holistically":[62],"verify":[63],"whether\u2014and":[64],"where\u2014their":[65],"modifications":[66],"caused":[67],"significant":[68,231],"regressions":[69,188],"or":[70,121],"improvements":[71],"at":[72,222],"scale.":[73],"In":[74],"this":[75],"paper,":[76],"we":[77,178,217,234],"introduce":[78],"not":[79],"only":[80],"such":[81],"a":[82,98,103,131,166,173],"holistic":[83,183],"approach":[84,164],"evaluate":[86],"system-level":[88],"but":[93],"also":[94,218],"concept":[96],"defect":[99,147,206,220],"class,":[100],"which":[101,110,123,228,233],"represents":[102],"partition":[104],"input":[107],"space":[108],"on":[109,122,165],"does":[114],"measurably":[115],"worse":[116],"for":[117,130,227],"an":[118,159],"existing":[119],"feature":[120],"ML":[125,176,213],"task":[126],"is":[127],"more":[128],"new":[132,151],"feature.":[133],"We":[134],"leverage":[135],"large":[136],"volumes":[137],"functional":[139],"cases,":[141],"automatically":[142,204],"obtained,":[143],"derive":[145],"these":[146],"classes,":[148],"propose":[150],"ways":[152],"improve":[154],"end-user\u2019s":[160],"perspective.":[161],"Applying":[162],"our":[163,182],"real":[167],"production":[168],"Search-AutoComplete":[169],"system":[170],"that":[171,180],"contains":[172],"query":[174],"interpretation":[175],"component,":[177],"demonstrate":[179],"(1)":[181],"metrics":[184],"successfully":[185],"identified":[186],"two":[187],"one":[190],"improvement,":[191],"where":[192],"all":[193],"3":[194],"were":[195,230],"independently":[196],"verified":[197],"retrospective":[199],"A/B":[200],"experiments,":[201],"(2)":[202],"obtained":[205],"classes":[207,221],"provided":[208],"actionable":[209],"insights":[210],"during":[211],"early-stage":[212],"development,":[214],"(3)":[216],"detected":[219],"finer":[224],"sub-component":[225],"level":[226],"there":[229],"regressions,":[232],"blocked":[235],"prior":[236],"different":[238],"releases.":[239]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}