{"id":"https://openalex.org/W2068329940","doi":"https://doi.org/10.1177/0165551515577912","title":"Detecting near-duplicate text documents with a hybrid approach","display_name":"Detecting near-duplicate text documents with a hybrid approach","publication_year":2015,"publication_date":"2015-03-24","ids":{"openalex":"https://openalex.org/W2068329940","doi":"https://doi.org/10.1177/0165551515577912","mag":"2068329940"},"language":"en","primary_location":{"id":"doi:10.1177/0165551515577912","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0165551515577912","pdf_url":null,"source":{"id":"https://openalex.org/S68913162","display_name":"Journal of Information Science","issn_l":"0165-5515","issn":["0165-5515","1741-6485"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Information Science","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061796733","display_name":"Cihan Varol","orcid":"https://orcid.org/0000-0002-4940-6808"},"institutions":[{"id":"https://openalex.org/I191429286","display_name":"Sam Houston State University","ror":"https://ror.org/00yh3cz06","country_code":"US","type":"education","lineage":["https://openalex.org/I191429286"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Cihan Varol","raw_affiliation_strings":["Sam Houston State University, Texas, USA"],"affiliations":[{"raw_affiliation_string":"Sam Houston State University, Texas, USA","institution_ids":["https://openalex.org/I191429286"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085622459","display_name":"Sairam Hari","orcid":null},"institutions":[{"id":"https://openalex.org/I191429286","display_name":"Sam Houston State University","ror":"https://ror.org/00yh3cz06","country_code":"US","type":"education","lineage":["https://openalex.org/I191429286"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sairam Hari","raw_affiliation_strings":["Sam Houston State University, Texas, USA"],"affiliations":[{"raw_affiliation_string":"Sam Houston State University, Texas, USA","institution_ids":["https://openalex.org/I191429286"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5061796733"],"corresponding_institution_ids":["https://openalex.org/I191429286"],"apc_list":null,"apc_paid":null,"fwci":0.8012,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.7788503,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"41","issue":"4","first_page":"405","last_page":"414"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9776999950408936,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8065136671066284},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.5622201561927795},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4892451763153076},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4582582712173462},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.4224355220794678},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.38762038946151733},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3651088774204254},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3578817546367645},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.33560535311698914},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08491069078445435}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8065136671066284},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.5622201561927795},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4892451763153076},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4582582712173462},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.4224355220794678},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.38762038946151733},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3651088774204254},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3578817546367645},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.33560535311698914},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08491069078445435},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1177/0165551515577912","is_oa":false,"landing_page_url":"https://doi.org/10.1177/0165551515577912","pdf_url":null,"source":{"id":"https://openalex.org/S68913162","display_name":"Journal of Information Science","issn_l":"0165-5515","issn":["0165-5515","1741-6485"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Information Science","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W1484851474","https://openalex.org/W1522405883","https://openalex.org/W1536700701","https://openalex.org/W1647671624","https://openalex.org/W1941707302","https://openalex.org/W1966389434","https://openalex.org/W1976526581","https://openalex.org/W1979016021","https://openalex.org/W1980073965","https://openalex.org/W2007842132","https://openalex.org/W2010595692","https://openalex.org/W2012833704","https://openalex.org/W2018978908","https://openalex.org/W2023358833","https://openalex.org/W2024770506","https://openalex.org/W2057515359","https://openalex.org/W2065259291","https://openalex.org/W2067432306","https://openalex.org/W2081666882","https://openalex.org/W2085922539","https://openalex.org/W2109803107","https://openalex.org/W2110166424","https://openalex.org/W2121135610","https://openalex.org/W2140655242","https://openalex.org/W2145349611","https://openalex.org/W2148885851","https://openalex.org/W2150862929","https://openalex.org/W2342204193","https://openalex.org/W4229912654","https://openalex.org/W4249371737","https://openalex.org/W4255111644"],"related_works":["https://openalex.org/W4390608645","https://openalex.org/W4247566972","https://openalex.org/W4394895745","https://openalex.org/W2960264696","https://openalex.org/W3090563135","https://openalex.org/W2497432351","https://openalex.org/W4206777497","https://openalex.org/W2910064364","https://openalex.org/W4200136508","https://openalex.org/W2296205523"],"abstract_inverted_index":{"Near":[0],"duplicate":[1],"data":[2],"not":[3],"only":[4],"increase":[5,16,89],"the":[6,40,80,84,90,96,117,125,130],"cost":[7],"of":[8,53,56,63,83,95,111],"information":[9,25],"processing":[10],"in":[11,36,58,67],"big":[12],"data,":[13,39],"but":[14],"also":[15],"decision":[17],"time.":[18],"Therefore,":[19,87],"detecting":[20],"and":[21,92,108,139],"eliminating":[22],"nearly":[23],"identical":[24],"is":[26,49,74],"vital":[27],"to":[28,88],"enhance":[29],"overall":[30,81],"business":[31],"decisions.":[32],"To":[33],"identify":[34],"near-duplicates":[35],"large-scale":[37],"text":[38,123],"shingling":[41,97,131],"algorithm":[42,48,85],"has":[43],"been":[44],"widely":[45],"used.":[46],"This":[47],"based":[50],"on":[51,137],"occurrences":[52],"contiguous":[54],"subsequences":[55],"tokens":[57],"two":[59],"or":[60],"more":[61],"sets":[62],"information,":[64],"such":[65],"as":[66],"documents.":[68],"In":[69,120],"other":[70],"words,":[71],"if":[72],"there":[73],"a":[75,101,121],"slight":[76],"variation":[77],"among":[78],"documents,":[79],"performance":[82,134],"decreases.":[86],"efficiency":[91],"accuracy":[93,133],"performances":[94],"algorithm,":[98],"we":[99],"propose":[100],"hybrid":[102,127],"approach":[103,128],"that":[104],"embeds":[105],"Jaro":[106],"distance":[107],"statistical":[109],"results":[110],"word":[112],"usage":[113],"frequency":[114],"for":[115],"fixing":[116],"ill-defined":[118],"data.":[119],"real":[122],"dataset,":[124],"proposed":[126],"improved":[129],"algorithm\u2019s":[132],"by":[135],"27%":[136],"average":[138],"achieved":[140],"above":[141],"90%":[142],"common":[143],"shingles.":[144]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2016,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}