{"id":"https://openalex.org/W3008479101","doi":"https://doi.org/10.1109/bigdata47090.2019.9005594","title":"Web Scraping: State-of-the-Art and Areas of Application","display_name":"Web Scraping: State-of-the-Art and Areas of Application","publication_year":2019,"publication_date":"2019-12-01","ids":{"openalex":"https://openalex.org/W3008479101","doi":"https://doi.org/10.1109/bigdata47090.2019.9005594","mag":"3008479101"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata47090.2019.9005594","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9005594","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035578875","display_name":"Rabiyatou Diouf","orcid":null},"institutions":[{"id":"https://openalex.org/I79793008","display_name":"Universit\u00e9 de Thi\u00e8s","ror":"https://ror.org/05nfkgg69","country_code":"SN","type":"education","lineage":["https://openalex.org/I79793008"]}],"countries":["SN"],"is_corresponding":true,"raw_author_name":"Rabiyatou Diouf","raw_affiliation_strings":["Universit\u00e9 de Thies, Thies, SENEGAL","UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Thies, Thies, SENEGAL","institution_ids":["https://openalex.org/I79793008"]},{"raw_affiliation_string":"UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)","institution_ids":["https://openalex.org/I79793008"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005258285","display_name":"Edouard Ngor Sarr","orcid":null},"institutions":[{"id":"https://openalex.org/I79793008","display_name":"Universit\u00e9 de Thi\u00e8s","ror":"https://ror.org/05nfkgg69","country_code":"SN","type":"education","lineage":["https://openalex.org/I79793008"]},{"id":"https://openalex.org/I4210151855","display_name":"Sonatel (Senegal)","ror":"https://ror.org/05j79r106","country_code":"SN","type":"company","lineage":["https://openalex.org/I4210151855"]}],"countries":["SN"],"is_corresponding":false,"raw_author_name":"Edouard Ngor Sarr","raw_affiliation_strings":["UCAO-Saint Michel, Dakar, SENEGAL","UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)"],"affiliations":[{"raw_affiliation_string":"UCAO-Saint Michel, Dakar, SENEGAL","institution_ids":["https://openalex.org/I4210151855"]},{"raw_affiliation_string":"UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)","institution_ids":["https://openalex.org/I79793008"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011347992","display_name":"Ousmane Sall","orcid":null},"institutions":[{"id":"https://openalex.org/I79793008","display_name":"Universit\u00e9 de Thi\u00e8s","ror":"https://ror.org/05nfkgg69","country_code":"SN","type":"education","lineage":["https://openalex.org/I79793008"]}],"countries":["SN"],"is_corresponding":false,"raw_author_name":"Ousmane Sall","raw_affiliation_strings":["Universit\u00e9 de Thies, Thies, SENEGAL","UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Thies, Thies, SENEGAL","institution_ids":["https://openalex.org/I79793008"]},{"raw_affiliation_string":"UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)","institution_ids":["https://openalex.org/I79793008"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008419222","display_name":"Babiga Birregah","orcid":"https://orcid.org/0000-0002-1274-3034"},"institutions":[{"id":"https://openalex.org/I140494188","display_name":"Universit\u00e9 de Technologie de Troyes","ror":"https://ror.org/01qhqcj41","country_code":"FR","type":"education","lineage":["https://openalex.org/I140494188"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Babiga Birregah","raw_affiliation_strings":["Universit\u00e9 de Technologie de Troyes, Troyes, France","LM2S - Laboratoire Mod\u00e9lisation et S\u00fbret\u00e9 des Syst\u00e8mes (UTT - 12, rue Marie Curie - BP 2060 - 10010 TROYES CEDEX - France)"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Technologie de Troyes, Troyes, France","institution_ids":["https://openalex.org/I140494188"]},{"raw_affiliation_string":"LM2S - Laboratoire Mod\u00e9lisation et S\u00fbret\u00e9 des Syst\u00e8mes (UTT - 12, rue Marie Curie - BP 2060 - 10010 TROYES CEDEX - France)","institution_ids":["https://openalex.org/I140494188"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084598524","display_name":"Mamadou Bousso","orcid":null},"institutions":[{"id":"https://openalex.org/I79793008","display_name":"Universit\u00e9 de Thi\u00e8s","ror":"https://ror.org/05nfkgg69","country_code":"SN","type":"education","lineage":["https://openalex.org/I79793008"]}],"countries":["SN"],"is_corresponding":false,"raw_author_name":"Mamadou Bousso","raw_affiliation_strings":["Universit\u00e9 de Thies, Thies, SENEGAL","UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Thies, Thies, SENEGAL","institution_ids":["https://openalex.org/I79793008"]},{"raw_affiliation_string":"UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)","institution_ids":["https://openalex.org/I79793008"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013307886","display_name":"Seny Ndiaye Mbaye","orcid":null},"institutions":[{"id":"https://openalex.org/I79793008","display_name":"Universit\u00e9 de Thi\u00e8s","ror":"https://ror.org/05nfkgg69","country_code":"SN","type":"education","lineage":["https://openalex.org/I79793008"]}],"countries":["SN"],"is_corresponding":false,"raw_author_name":"Seny Ndiaye Mbaye","raw_affiliation_strings":["Universit\u00e9 de Thies, Thies, SENEGAL","UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Thies, Thies, SENEGAL","institution_ids":["https://openalex.org/I79793008"]},{"raw_affiliation_string":"UIDT - Universit\u00e9 Iba Der Thiam [Thi\u00e8s] (Thi\u00e8s - Senegal)","institution_ids":["https://openalex.org/I79793008"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5035578875"],"corresponding_institution_ids":["https://openalex.org/I79793008"],"apc_list":null,"apc_paid":null,"fwci":13.727,"has_fulltext":false,"cited_by_count":117,"citation_normalized_percentile":{"value":0.98787243,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"6040","last_page":"6042"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9828000068664551,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8141535520553589},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6911892294883728},{"id":"https://openalex.org/keywords/web-application","display_name":"Web application","score":0.6457901000976562},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.6339150071144104},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5580280423164368},{"id":"https://openalex.org/keywords/web-page","display_name":"Web page","score":0.4804636836051941},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.45222359895706177},{"id":"https://openalex.org/keywords/data-web","display_name":"Data Web","score":0.43561452627182007},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.42269569635391235},{"id":"https://openalex.org/keywords/web-modeling","display_name":"Web modeling","score":0.4153093695640564},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.4103235602378845},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3687572479248047},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.10405194759368896},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09157684445381165}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8141535520553589},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6911892294883728},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.6457901000976562},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.6339150071144104},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5580280423164368},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.4804636836051941},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.45222359895706177},{"id":"https://openalex.org/C162005631","wikidata":"https://www.wikidata.org/wiki/Q54837","display_name":"Data Web","level":3,"score":0.43561452627182007},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.42269569635391235},{"id":"https://openalex.org/C130436687","wikidata":"https://www.wikidata.org/wiki/Q7978591","display_name":"Web modeling","level":3,"score":0.4153093695640564},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.4103235602378845},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3687572479248047},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.10405194759368896},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09157684445381165},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/bigdata47090.2019.9005594","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata47090.2019.9005594","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},{"id":"pmh:oai:HAL:hal-02492481v1","is_oa":false,"landing_page_url":"https://utt.hal.science/hal-02492481","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2019 IEEE International Conference on Big Data (Big Data), Dec 2019, Los Angeles, United States. pp.6040-6042, ⟨10.1109/BigData47090.2019.9005594⟩","raw_type":"Conference papers"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W1502424938","https://openalex.org/W1529013893","https://openalex.org/W1574901103","https://openalex.org/W1835244612","https://openalex.org/W1883869625","https://openalex.org/W2030144199","https://openalex.org/W2066590388","https://openalex.org/W2109904151","https://openalex.org/W2123354009","https://openalex.org/W2124591829","https://openalex.org/W2134585533","https://openalex.org/W2142809703","https://openalex.org/W2145632240","https://openalex.org/W2147859400","https://openalex.org/W2155106456","https://openalex.org/W2159094788","https://openalex.org/W2170146596","https://openalex.org/W2171960770","https://openalex.org/W2461816915","https://openalex.org/W2601476733","https://openalex.org/W2616421135","https://openalex.org/W2764104808","https://openalex.org/W2789893667","https://openalex.org/W2902778042","https://openalex.org/W2949480888","https://openalex.org/W3122137447","https://openalex.org/W4236322898","https://openalex.org/W4255475599"],"related_works":["https://openalex.org/W1495666889","https://openalex.org/W2963140294","https://openalex.org/W2100071482","https://openalex.org/W2729360854","https://openalex.org/W2780890200","https://openalex.org/W2165565716","https://openalex.org/W2121592681","https://openalex.org/W3088578348","https://openalex.org/W2137912379","https://openalex.org/W4249203355"],"abstract_inverted_index":{"Main":[0],"objective":[1],"of":[2,59,78],"Web":[3,36,68],"Scraping":[4,37,69],"is":[5,38,46,62],"to":[6,30,63],"extract":[7],"information":[8],"from":[9],"one":[10],"or":[11,24],"many":[12],"websites":[13],"and":[14,40,72],"process":[15],"it":[16,45],"into":[17],"simple":[18],"structures":[19],"such":[20],"as":[21],"spreadsheets,":[22],"database":[23],"CSV":[25],"file.":[26],"However,":[27],"in":[28],"addition":[29],"be":[31],"a":[32],"very":[33],"complicated":[34],"task,":[35],"resource":[39],"time":[41],"consuming,":[42],"mainly":[43],"when":[44],"carried":[47],"out":[48],"manually.":[49],"Previous":[50],"studies":[51],"have":[52],"developed":[53],"several":[54],"automated":[55],"solutions.":[56],"The":[57],"purpose":[58],"this":[60],"article":[61],"revisit":[64],"the":[65],"different":[66],"existing":[67],"approaches,":[70],"categories,":[71],"tools,":[73],"but":[74],"also":[75],"its":[76],"areas":[77],"application.":[79]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":25},{"year":2023,"cited_by_count":33},{"year":2022,"cited_by_count":21},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":5}],"updated_date":"2026-03-31T07:56:22.981413","created_date":"2025-10-10T00:00:00"}