{"id":"https://openalex.org/W2899507349","doi":"https://doi.org/10.1145/3242180","title":"To Clean or Not to Clean","display_name":"To Clean or Not to Clean","publication_year":2018,"publication_date":"2018-10-29","ids":{"openalex":"https://openalex.org/W2899507349","doi":"https://doi.org/10.1145/3242180","mag":"2899507349"},"language":"en","primary_location":{"id":"doi:10.1145/3242180","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3242180","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000936680","display_name":"Dwaipayan Roy","orcid":"https://orcid.org/0000-0002-5962-5983"},"institutions":[{"id":"https://openalex.org/I6498739","display_name":"Indian Statistical Institute","ror":"https://ror.org/00q2w1j53","country_code":"IN","type":"education","lineage":["https://openalex.org/I6498739"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Dwaipayan Roy","raw_affiliation_strings":["Indian Statistical Institute, Kolkata, India"],"affiliations":[{"raw_affiliation_string":"Indian Statistical Institute, Kolkata, India","institution_ids":["https://openalex.org/I6498739"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052357764","display_name":"Mandar Mitra","orcid":"https://orcid.org/0009-0007-8026-3220"},"institutions":[{"id":"https://openalex.org/I6498739","display_name":"Indian Statistical Institute","ror":"https://ror.org/00q2w1j53","country_code":"IN","type":"education","lineage":["https://openalex.org/I6498739"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Mandar Mitra","raw_affiliation_strings":["Indian Statistical Institute, Kolkata, India"],"affiliations":[{"raw_affiliation_string":"Indian Statistical Institute, Kolkata, India","institution_ids":["https://openalex.org/I6498739"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082339849","display_name":"Debasis Ganguly","orcid":"https://orcid.org/0000-0003-0050-7138"},"institutions":[{"id":"https://openalex.org/I4210145784","display_name":"IBM Research - Ireland","ror":"https://ror.org/04jnxr720","country_code":"IE","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210145784"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Debasis Ganguly","raw_affiliation_strings":["IBM Research, Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"IBM Research, Dublin, Ireland","institution_ids":["https://openalex.org/I4210145784"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5000936680"],"corresponding_institution_ids":["https://openalex.org/I6498739"],"apc_list":null,"apc_paid":null,"fwci":2.3561,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.9166395,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"10","issue":"4","first_page":"1","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8708054423332214},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7548006176948547},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.6965780854225159},{"id":"https://openalex.org/keywords/markup-language","display_name":"Markup language","score":0.6545118093490601},{"id":"https://openalex.org/keywords/search-engine-indexing","display_name":"Search engine indexing","score":0.5619708299636841},{"id":"https://openalex.org/keywords/preprocessor","display_name":"Preprocessor","score":0.541715681552887},{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.5341958403587341},{"id":"https://openalex.org/keywords/query-expansion","display_name":"Query expansion","score":0.5016450881958008},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4952651560306549},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4884113669395447},{"id":"https://openalex.org/keywords/xml","display_name":"XML","score":0.3017053008079529},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.269159734249115},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.24431565403938293},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.17377111315727234},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.09449169039726257}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8708054423332214},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7548006176948547},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.6965780854225159},{"id":"https://openalex.org/C45874996","wikidata":"https://www.wikidata.org/wiki/Q37045","display_name":"Markup language","level":3,"score":0.6545118093490601},{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.5619708299636841},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.541715681552887},{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.5341958403587341},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.5016450881958008},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4952651560306549},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4884113669395447},{"id":"https://openalex.org/C8797682","wikidata":"https://www.wikidata.org/wiki/Q2115","display_name":"XML","level":2,"score":0.3017053008079529},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.269159734249115},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.24431565403938293},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.17377111315727234},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.09449169039726257},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3242180","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3242180","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5099999904632568,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":81,"referenced_works":["https://openalex.org/W60779023","https://openalex.org/W94637519","https://openalex.org/W95791645","https://openalex.org/W100562005","https://openalex.org/W179016587","https://openalex.org/W299870200","https://openalex.org/W1509991416","https://openalex.org/W1539587635","https://openalex.org/W1566513354","https://openalex.org/W1576064612","https://openalex.org/W1606874436","https://openalex.org/W1685426458","https://openalex.org/W1776265914","https://openalex.org/W1966093341","https://openalex.org/W1972594981","https://openalex.org/W1973289172","https://openalex.org/W1988711048","https://openalex.org/W1990380106","https://openalex.org/W2000569744","https://openalex.org/W2006996936","https://openalex.org/W2012575882","https://openalex.org/W2014415866","https://openalex.org/W2016802218","https://openalex.org/W2019577381","https://openalex.org/W2020929650","https://openalex.org/W2025486348","https://openalex.org/W2040631151","https://openalex.org/W2045966063","https://openalex.org/W2047662094","https://openalex.org/W2053878916","https://openalex.org/W2057695077","https://openalex.org/W2066055909","https://openalex.org/W2067506377","https://openalex.org/W2073562093","https://openalex.org/W2084929085","https://openalex.org/W2100738695","https://openalex.org/W2105157020","https://openalex.org/W2113365997","https://openalex.org/W2117473841","https://openalex.org/W2118131693","https://openalex.org/W2136542423","https://openalex.org/W2151450030","https://openalex.org/W2155482025","https://openalex.org/W2158051716","https://openalex.org/W2168859760","https://openalex.org/W2169213601","https://openalex.org/W2169270715","https://openalex.org/W2287148974","https://openalex.org/W2294145134","https://openalex.org/W2307814545","https://openalex.org/W2338477879","https://openalex.org/W2404616702","https://openalex.org/W2468776465","https://openalex.org/W2510769428","https://openalex.org/W2517555848","https://openalex.org/W2532134734","https://openalex.org/W2533392838","https://openalex.org/W2536015822","https://openalex.org/W2537986768","https://openalex.org/W2539107637","https://openalex.org/W2570788028","https://openalex.org/W2583976214","https://openalex.org/W2588670714","https://openalex.org/W2610935556","https://openalex.org/W2612012962","https://openalex.org/W2613589950","https://openalex.org/W2742396803","https://openalex.org/W2751100699","https://openalex.org/W2758725343","https://openalex.org/W2759558800","https://openalex.org/W2915170155","https://openalex.org/W3013019162","https://openalex.org/W3212575067","https://openalex.org/W4210911708","https://openalex.org/W4234200495","https://openalex.org/W4236789328","https://openalex.org/W4240913316","https://openalex.org/W4246858749","https://openalex.org/W4252076394","https://openalex.org/W4285719527","https://openalex.org/W4298361434"],"related_works":["https://openalex.org/W4385386698","https://openalex.org/W4235354490","https://openalex.org/W2381082207","https://openalex.org/W2025266532","https://openalex.org/W2136793911","https://openalex.org/W2143258726","https://openalex.org/W1971521465","https://openalex.org/W2377737135","https://openalex.org/W2100702185","https://openalex.org/W2355331657"],"abstract_inverted_index":{"Web":[0],"document":[1,159],"collections":[2,19,148],"such":[3],"as":[4,140],"WT10G,":[5],"GOV2,":[6],"and":[7,33,93],"ClueWeb":[8],"are":[9,161],"widely":[10],"used":[11,124],"for":[12,206],"text":[13],"retrieval":[14,118],"experiments.":[15],"Documents":[16],"in":[17,27,74,132,145],"these":[18,40],"contain":[20],"a":[21,171],"fair":[22],"amount":[23,142],"of":[24,30,143,158,166,168,174,181],"non-content-related":[25],"markup":[26,51,202],"the":[28,75,88,134,141,146,156,164,179],"form":[29],"tags,":[31],"hyperlinks,":[32],"so":[34],"on.":[35],"Published":[36],"articles":[37],"that":[38,69,155,195],"use":[39],"corpora":[41],"generally":[42,103,198],"do":[43],"not":[44],"provide":[45],"specific":[46],"details":[47,157],"about":[48],"how":[49],"this":[50,58,151,189],"information":[52],"is":[53,99,123,197],"handled":[54],"during":[55],"indexing.":[56],"However,":[57],"question":[59],"turns":[60],"out":[61],"to":[62,101,200],"be":[63,102],"important:":[64],"Through":[65],"experiments,":[66,175],"we":[67,153,176,193],"find":[68,194],"including":[70],"or":[71,108],"excluding":[72],"metadata":[73,97,144],"index":[76],"can":[77,115],"produce":[78],"significantly":[79,116],"different":[80],"results":[81],"with":[82,111,125],"standard":[83],"IR":[84],"models.":[85],"More":[86],"importantly,":[87],"effect":[89,180],"varies":[90],"across":[91],"models":[92],"collections.":[94],"For":[95],"example,":[96],"filtering":[98],"found":[100],"beneficial":[104],"when":[105],"using":[106,186,204],"BM25,":[107],"language":[109,121],"modeling":[110,122],"Dirichlet":[112],"smoothing,":[113],"but":[114],"reduce":[117],"effectiveness":[119],"if":[120],"Jelinek-Mercer":[126],"smoothing.":[127],"We":[128],"also":[129,177],"observe":[130],"that,":[131],"general,":[133],"performance":[135],"differences":[136],"become":[137],"more":[138],"noticeable":[139],"test":[147],"increase.":[149],"Given":[150],"variability,":[152],"believe":[154],"preprocessing":[160,182],"significant":[162],"from":[163],"point":[165],"view":[167],"reproducibility.":[169],"In":[170,188],"second":[172],"set":[173],"study":[178],"on":[183],"query":[184,207],"expansion":[185],"RM3.":[187],"case,":[190],"once":[191],"again,":[192],"it":[196],"better":[199],"remove":[201],"before":[203],"documents":[205],"expansion.":[208]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}