{"id":"https://openalex.org/W3199199750","doi":"https://doi.org/10.1145/3459637.3482352","title":"Identifying Untrustworthy Samples","display_name":"Identifying Untrustworthy Samples","publication_year":2021,"publication_date":"2021-10-26","ids":{"openalex":"https://openalex.org/W3199199750","doi":"https://doi.org/10.1145/3459637.3482352","mag":"3199199750"},"language":"en","primary_location":{"id":"doi:10.1145/3459637.3482352","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3459637.3482352","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM International Conference on Information & Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/abs/2109.06471","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027658278","display_name":"Lei Shen","orcid":"https://orcid.org/0000-0003-0307-696X"},"institutions":[{"id":"https://openalex.org/I1321014770","display_name":"Association for Computing Machinery","ror":"https://ror.org/03wsadn68","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1321014770"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Lei Shen","raw_affiliation_strings":["Institute of Computing Technology, Chinese Academy of Sciences & University of Chinese Academy of Sciences, Beijing, China","Association for Computing Machinery"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology, Chinese Academy of Sciences & University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I4210165038"]},{"raw_affiliation_string":"Association for Computing Machinery","institution_ids":["https://openalex.org/I1321014770"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043429023","display_name":"Haolan Zhan","orcid":"https://orcid.org/0000-0001-9628-5992"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haolan Zhan","raw_affiliation_strings":["University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100624454","display_name":"Xin Shen","orcid":"https://orcid.org/0009-0006-2699-3292"},"institutions":[{"id":"https://openalex.org/I118347636","display_name":"Australian National University","ror":"https://ror.org/019wvm592","country_code":"AU","type":"education","lineage":["https://openalex.org/I118347636"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Xin Shen","raw_affiliation_strings":["Australian National University, Australia, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Australian National University, Australia, Australia","institution_ids":["https://openalex.org/I118347636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102966812","display_name":"Hongshen Chen","orcid":"https://orcid.org/0000-0002-9164-2898"},"institutions":[{"id":"https://openalex.org/I4210103986","display_name":"Jingdong (China)","ror":"https://ror.org/01dkjkq64","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210103986"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongshen Chen","raw_affiliation_strings":["JD.com, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JD.com, Beijing, China","institution_ids":["https://openalex.org/I4210103986"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100750786","display_name":"Xiaofang Zhao","orcid":"https://orcid.org/0000-0002-1898-1359"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaofang Zhao","raw_affiliation_strings":["Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China","Institute of Computing Technology; Chinese Academy of Sciences; Beijing; China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]},{"raw_affiliation_string":"Institute of Computing Technology; Chinese Academy of Sciences; Beijing; China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016892586","display_name":"Xiaodan Zhu","orcid":"https://orcid.org/0000-0003-3856-3696"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Xiaodan Zhu","raw_affiliation_strings":["Queen's University, Kingston, Canada","Queen\u2019s University,,,Kingston,,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Queen's University, Kingston, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Queen\u2019s University,,,Kingston,,Canada","institution_ids":["https://openalex.org/I204722609"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1598","last_page":"1608"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7849929332733154},{"id":"https://openalex.org/keywords/filter","display_name":"Filter (signal processing)","score":0.6025485992431641},{"id":"https://openalex.org/keywords/sort","display_name":"sort","score":0.5872236490249634},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5808541178703308},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5244001746177673},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5194644331932068},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.47484949231147766},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4705265760421753},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.46424660086631775},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.44884005188941956},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4299566149711609},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4294852316379547},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3535923361778259},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.18792951107025146}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7849929332733154},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.6025485992431641},{"id":"https://openalex.org/C88548561","wikidata":"https://www.wikidata.org/wiki/Q347599","display_name":"sort","level":2,"score":0.5872236490249634},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5808541178703308},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5244001746177673},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5194644331932068},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.47484949231147766},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4705265760421753},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.46424660086631775},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44884005188941956},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4299566149711609},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4294852316379547},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3535923361778259},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.18792951107025146},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3459637.3482352","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3459637.3482352","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM International Conference on Information & Knowledge Management","raw_type":"proceedings-article"},{"id":"mag:3199199750","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2109.06471","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3199199750","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2109.06471","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8100000023841858,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W1840435438","https://openalex.org/W2099201756","https://openalex.org/W2101105183","https://openalex.org/W2133012565","https://openalex.org/W2151814822","https://openalex.org/W2157331557","https://openalex.org/W2192203593","https://openalex.org/W2250539671","https://openalex.org/W2296712013","https://openalex.org/W2419539795","https://openalex.org/W2590513900","https://openalex.org/W2727581617","https://openalex.org/W2752172973","https://openalex.org/W2799037524","https://openalex.org/W2803437449","https://openalex.org/W2807873315","https://openalex.org/W2808293489","https://openalex.org/W2890969459","https://openalex.org/W2941500661","https://openalex.org/W2949918260","https://openalex.org/W2950142196","https://openalex.org/W2951114218","https://openalex.org/W2962717182","https://openalex.org/W2962786758","https://openalex.org/W2962796276","https://openalex.org/W2962801572","https://openalex.org/W2962879692","https://openalex.org/W2962989446","https://openalex.org/W2963206148","https://openalex.org/W2963330684","https://openalex.org/W2963403868","https://openalex.org/W2963544536","https://openalex.org/W2963790827","https://openalex.org/W2963903950","https://openalex.org/W2963962154","https://openalex.org/W2964125718","https://openalex.org/W2964134121","https://openalex.org/W2964308564","https://openalex.org/W2964352131","https://openalex.org/W2970988759","https://openalex.org/W2972437240","https://openalex.org/W2983160116","https://openalex.org/W2990563493","https://openalex.org/W2997657234","https://openalex.org/W2998083599","https://openalex.org/W3013310839","https://openalex.org/W3031414376","https://openalex.org/W3034242983","https://openalex.org/W3034424015","https://openalex.org/W3034808773","https://openalex.org/W3034950505","https://openalex.org/W3035239386","https://openalex.org/W3035656139","https://openalex.org/W3091913192","https://openalex.org/W3103621832","https://openalex.org/W3160293015","https://openalex.org/W3172448816","https://openalex.org/W3199193983"],"related_works":["https://openalex.org/W5879097","https://openalex.org/W4771408","https://openalex.org/W12783365","https://openalex.org/W4272546","https://openalex.org/W2990258","https://openalex.org/W11219696","https://openalex.org/W10719664","https://openalex.org/W2804300","https://openalex.org/W2777878","https://openalex.org/W12125165"],"abstract_inverted_index":{"Being":[0],"able":[1],"to":[2,22,114,149,200,212],"reply":[3],"with":[4,40,48,93,132,189],"a":[5,61,79,94,162,182],"related,":[6],"fluent,":[7],"and":[8,51,77,141,171,192,234],"informative":[9],"response":[10],"is":[11,198,210],"an":[12,116],"indispensable":[13],"requirement":[14],"for":[15,83,119],"building":[16],"high-quality":[17],"conversational":[18,45],"agents.":[19],"In":[20,68],"order":[21],"generate":[23],"better":[24,242],"responses,":[25],"some":[26],"approaches":[27],"have":[28],"been":[29],"proposed,":[30],"such":[31],"as":[32],"feeding":[33],"extra":[34],"information":[35],"by":[36],"collecting":[37],"large-scale":[38,158],"datasets":[39,224,240],"human":[41],"annotations,":[42],"designing":[43],"neural":[44],"models":[46],"(NCMs)":[47],"complex":[49],"architecture":[50],"loss":[52],"functions,":[53],"or":[54,66],"filtering":[55,81],"out":[56,143],"untrustworthy":[57,88,232],"samples":[58,89,131],"based":[59],"on":[60,123,157,185,222,237],"dialogue":[62,101,120],"attribute,":[63],"e.g.,":[64],"Relatedness":[65],"Genericness.":[67],"this":[69],"paper,":[70],"we":[71,128,160],"follow":[72],"the":[73,124,133,146,151,202,214,238],"third":[74],"research":[75],"branch":[76],"present":[78],"data":[80,92],"method":[82,174,178,228],"open-domain":[84],"dialogues,":[85],"which":[86],"identifies":[87],"from":[90],"training":[91,130,163,173,177],"quality":[95,134],"measure":[96],"that":[97,112,165,226],"linearly":[98],"combines":[99],"seven":[100],"attributes.":[102],"The":[103,176],"attribute":[104],"weights":[105],"are":[106],"obtained":[107],"via":[108],"Bayesian":[109],"Optimization":[110],"(BayesOpt)":[111],"aims":[113],"optimize":[115],"objective":[117],"function":[118],"generation":[121],"iteratively":[122],"validation":[125],"set.":[126],"Then":[127],"score":[129],"measure,":[135],"sort":[136],"them":[137],"in":[138,155],"descending":[139],"order,":[140],"filter":[142],"those":[144],"at":[145],"bottom.":[147],"Furthermore,":[148],"accelerate":[150],"\"filter-train-evaluate''":[152],"iterations":[153],"involved":[154],"BayesOpt":[156],"datasets,":[159],"propose":[161],"framework":[164],"integrates":[166],"maximum":[167],"likelihood":[168],"estimation":[169],"(MLE)":[170],"negative":[172],"(NEG).":[175],"updates":[179],"parameters":[180],"of":[181,204,216],"trained":[183,236],"NCMs":[184,235],"two":[186,223],"small":[187],"sets":[188],"newly":[190,205,217],"maintained":[191,206],"removed":[193,218],"samples,":[194,207,233],"respectively.":[195],"Specifically,":[196],"MLE":[197],"applied":[199],"maximize":[201],"log-likelihood":[203,215],"while":[208],"NEG":[209],"used":[211],"minimize":[213],"ones.":[219],"Experimental":[220],"results":[221],"show":[225],"our":[227],"can":[229],"effectively":[230],"identify":[231],"filtered":[239],"achieve":[241],"performance.":[243]},"counts_by_year":[],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}