{"id":"https://openalex.org/W2916049517","doi":"https://doi.org/10.48550/arxiv.1902.09711","title":"Detecting Data Errors with Statistical Constraints","display_name":"Detecting Data Errors with Statistical Constraints","publication_year":2019,"publication_date":"2019-02-26","ids":{"openalex":"https://openalex.org/W2916049517","doi":"https://doi.org/10.48550/arxiv.1902.09711","mag":"2916049517"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:1902.09711","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.09711","pdf_url":"https://arxiv.org/pdf/1902.09711","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1902.09711","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113817523","display_name":"Jing Nathan Yan","orcid":"https://orcid.org/0000-0002-3886-8687","observed_orcids":["https://orcid.org/0000-0002-3886-8687"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yan, Jing Nathan","raw_affiliation_strings":["UNIVERSITY OF HONG KONG"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNIVERSITY OF HONG KONG","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026297632","display_name":"Oliver Schulte","orcid":"https://orcid.org/0000-0002-2805-4313","observed_orcids":["https://orcid.org/0000-0002-2805-4313"]},"institutions":[{"id":"https://openalex.org/I18014758","display_name":"Simon Fraser University","ror":"https://ror.org/0213rcc28","country_code":"CA","type":"education","lineage":["https://openalex.org/I18014758"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Schulte, Oliver","raw_affiliation_strings":["[Simon Fraser University]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"[Simon Fraser University]","institution_ids":["https://openalex.org/I18014758"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101850961","display_name":"Jiannan Wang","orcid":"https://orcid.org/0000-0002-8229-3622","observed_orcids":["https://orcid.org/0000-0002-8229-3622","https://orcid.org/0009-0002-8978-312X"]},"institutions":[{"id":"https://openalex.org/I18014758","display_name":"Simon Fraser University","ror":"https://ror.org/0213rcc28","country_code":"CA","type":"education","lineage":["https://openalex.org/I18014758"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Wang, Jiannan","raw_affiliation_strings":["[Simon Fraser University]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"[Simon Fraser University]","institution_ids":["https://openalex.org/I18014758"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005847882","display_name":"Reynold C. K. Cheng","orcid":"https://orcid.org/0000-0002-9480-9809","observed_orcids":["https://orcid.org/0000-0002-9480-9809"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Cheng, Reynold","raw_affiliation_strings":["UNIVERSITY OF HONG KONG"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UNIVERSITY OF HONG KONG","institution_ids":["https://openalex.org/I889458895"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.006099999882280827,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.00139999995008111,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-quality","display_name":"data quality","score":0.6959999799728394},{"id":"https://openalex.org/keywords/data-error-detection","display_name":"data error detection","score":0.824999988079071},{"id":"https://openalex.org/keywords/statistical-constraints","display_name":"statistical constraints","score":0.9610000252723694},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"domain knowledge","score":0.6150000095367432},{"id":"https://openalex.org/keywords/integrity-constraints","display_name":"integrity constraints","score":0.46700000762939453}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7863609790802002},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.6043083667755127},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.5254287719726562},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5127688050270081},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5043796300888062},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.49508824944496155},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4872647523880005},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4831206500530243},{"id":"https://openalex.org/C35651441","wikidata":"https://www.wikidata.org/wiki/Q625303","display_name":"Independence (probability theory)","level":2,"score":0.47044864296913147},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4599650502204895},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4572155475616455},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.45165833830833435},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.42961210012435913},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.41084355115890503},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.4107978343963623},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33967870473861694},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.33312028646469116},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3180096745491028},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14089688658714294},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12939724326133728},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.11198502779006958},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:1902.09711","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.09711","pdf_url":"https://arxiv.org/pdf/1902.09711","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.1902.09711","doi":"https://doi.org/10.48550/arxiv.1902.09711","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1902.09711","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1902.09711","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1902.09711","pdf_url":"https://arxiv.org/pdf/1902.09711","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.7699999809265137,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, justice, and strong institutions"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W30853191","https://openalex.org/W132374903","https://openalex.org/W1492518272","https://openalex.org/W1496357020","https://openalex.org/W1502954553","https://openalex.org/W1524326598","https://openalex.org/W1562811233","https://openalex.org/W1571341547","https://openalex.org/W1579798859","https://openalex.org/W1595443289","https://openalex.org/W1860991815","https://openalex.org/W1967167578","https://openalex.org/W1983170052","https://openalex.org/W1992479406","https://openalex.org/W2000809552","https://openalex.org/W2009563364","https://openalex.org/W2011039300","https://openalex.org/W2024834471","https://openalex.org/W2025440555","https://openalex.org/W2029469881","https://openalex.org/W2030326671","https://openalex.org/W2031648200","https://openalex.org/W2038254774","https://openalex.org/W2044469685","https://openalex.org/W2047182010","https://openalex.org/W2047745978","https://openalex.org/W2048957655","https://openalex.org/W2053503869","https://openalex.org/W2087615914","https://openalex.org/W2089206172","https://openalex.org/W2102489964","https://openalex.org/W2106895292","https://openalex.org/W2110810394","https://openalex.org/W2119803607","https://openalex.org/W2133990480","https://openalex.org/W2137362862","https://openalex.org/W2138949658","https://openalex.org/W2142472956","https://openalex.org/W2143891888","https://openalex.org/W2144620969","https://openalex.org/W2153531471","https://openalex.org/W2162774438","https://openalex.org/W2164187405","https://openalex.org/W2167333415","https://openalex.org/W2170712852","https://openalex.org/W2238711864","https://openalex.org/W2250162459","https://openalex.org/W2295468252","https://openalex.org/W2298871042","https://openalex.org/W2394811972","https://openalex.org/W2402007731","https://openalex.org/W2437617937","https://openalex.org/W2437990191","https://openalex.org/W2544486974","https://openalex.org/W2617213571","https://openalex.org/W2797602122","https://openalex.org/W2798323405","https://openalex.org/W2798546256","https://openalex.org/W2798682670","https://openalex.org/W2888944692","https://openalex.org/W2949244208","https://openalex.org/W3106124465"],"related_works":["https://openalex.org/W2067317451","https://openalex.org/W2154771632","https://openalex.org/W4211085505","https://openalex.org/W3122478268","https://openalex.org/W2084758217","https://openalex.org/W408804804","https://openalex.org/W4231021675","https://openalex.org/W3086365953","https://openalex.org/W4226072953","https://openalex.org/W2392606101"],"abstract_inverted_index":{"A":[0],"powerful":[1],"approach":[2,47],"to":[3,8,27,87,143,179],"detecting":[4],"erroneous":[5],"data":[6,13,174],"is":[7,126],"check":[9],"which":[10],"potentially":[11],"dirty":[12],"records":[14,138],"are":[15,82],"incompatible":[16],"with":[17,103],"a":[18,50,63,76,89,131,153],"user's":[19],"domain":[20,29,93],"knowledge.":[21],"Previous":[22],"approaches":[23],"allow":[24],"the":[25,32,45,85,136,141,144],"user":[26,86],"specify":[28,88],"knowledge":[30],"in":[31],"form":[33],"of":[34,53,92,146,155],"logical":[35],"constraints":[36,55,81],"(e.g.,":[37],"functional":[38],"dependency":[39],"and":[40,66,149,172,182],"denial":[41],"constraints).":[42],"We":[43,107,162],"extend":[44],"constraint-based":[46],"by":[48],"introducing":[49],"novel":[51],"class":[52],"statistical":[54,105],"(SCs).":[56],"An":[57],"SC":[58,125],"treats":[59],"each":[60,167],"column":[61],"as":[62],"random":[64,78],"variable,":[65],"enforces":[67],"an":[68,110,124,147],"independence":[69],"or":[70,128,160],"dependence":[71],"relationship":[72],"between":[73],"two":[74],"(or":[75],"few)":[77],"variables.":[79],"Statistical":[80],"expressive,":[83],"allowing":[84],"wide":[90],"range":[91],"knowledge,":[94],"beyond":[95],"traditional":[96],"integrity":[97],"constraints.":[98],"Furthermore,":[99],"they":[100],"work":[101],"harmoniously":[102],"downstream":[104],"modeling.":[106],"develop":[108],"CODED,":[109],"SC-Oriented":[111],"Data":[112],"Error":[113],"Detection":[114],"system":[115],"that":[116,139,185],"supports":[117],"three":[118],"key":[119],"tasks:":[120],"(1)":[121],"Checking":[122,151],"whether":[123,152],"violated":[127],"not":[129],"on":[130,170],"given":[132],"dataset,":[133],"(2)":[134],"Identify":[135],"top-k":[137],"contribute":[140],"most":[142],"violation":[145],"SC,":[148],"(3)":[150],"set":[154],"input":[156],"SCs":[157,177],"have":[158],"conflicts":[159],"not.":[161],"present":[163],"effective":[164],"solutions":[165],"for":[166],"task.":[168],"Experiments":[169],"synthetic":[171],"real-world":[173],"illustrate":[175],"how":[176],"apply":[178],"error":[180],"detection,":[181],"provide":[183],"evidence":[184],"CODED":[186],"performs":[187],"better":[188],"than":[189],"state-of-the-art":[190],"approaches.":[191]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-10-01T08:54:57.152042","created_date":"2025-10-10T00:00:00"}