{"id":"https://openalex.org/W4410464977","doi":"https://doi.org/10.14445/22312803/ijctt-v73i4p120","title":"Building Robust Data Pipelines: Best Practices for Error Handling, Monitoring, and Recovery","display_name":"Building Robust Data Pipelines: Best Practices for Error Handling, Monitoring, and Recovery","publication_year":2025,"publication_date":"2025-04-30","ids":{"openalex":"https://openalex.org/W4410464977","doi":"https://doi.org/10.14445/22312803/ijctt-v73i4p120"},"language":"en","primary_location":{"id":"doi:10.14445/22312803/ijctt-v73i4p120","doi":"https://doi.org/10.14445/22312803/ijctt-v73i4p120","is_oa":true,"landing_page_url":"https://doi.org/10.14445/22312803/ijctt-v73i4p120","pdf_url":"https://www.ijcttjournal.org/2025/Volume-73/Issue-4/IJCTT-V73I4P120.pdf","source":{"id":"https://openalex.org/S2764857185","display_name":"International Journal of Computer Trends and Technology","issn_l":"2231-2803","issn":["2231-2803","2349-0829"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-1"],"host_organization":"https://openalex.org/P4323288697","host_organization_name":"Seventh Sense Research Group","host_organization_lineage":["https://openalex.org/P4323288697"],"host_organization_lineage_names":["Seventh Sense Research Group"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Computer Trends and Technology","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://www.ijcttjournal.org/2025/Volume-73/Issue-4/IJCTT-V73I4P120.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5117582120","display_name":"Dharanidhar Vuppu","orcid":"https://orcid.org/0009-0009-1677-0986","observed_orcids":["https://orcid.org/0009-0009-1677-0986"]},"institutions":[{"id":"https://openalex.org/I4210149662","display_name":"SurveyMonkey (United States)","ror":"https://ror.org/04p0q7p15","country_code":"US","type":"company","lineage":["https://openalex.org/I4210149662"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dharanidhar Vuppu","raw_affiliation_strings":["Sr Data Engineer , SurveyMonkey , Texas , United States of America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sr Data Engineer , SurveyMonkey , Texas , United States of America","institution_ids":["https://openalex.org/I4210149662"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5117582121","display_name":"Mounica Achanta","orcid":"https://orcid.org/0009-0009-2711-1652","observed_orcids":["https://orcid.org/0009-0009-2711-1652"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mounica Achanta","raw_affiliation_strings":["Independent Research at IEE , Texas , United States of America"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent Research at IEE , Texas , United States of America","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"73","issue":"4","first_page":"140","last_page":"148"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9253000020980835,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9253000020980835,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.021400000900030136,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.014700000174343586,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-pipelines","display_name":"data pipelines","score":0.984000027179718},{"id":"https://openalex.org/keywords/error-handling","display_name":"error handling","score":0.8429999947547913},{"id":"https://openalex.org/keywords/pipeline-monitoring","display_name":"pipeline monitoring","score":0.8669999837875366},{"id":"https://openalex.org/keywords/automatic-recovery","display_name":"automatic recovery","score":0.7879999876022339},{"id":"https://openalex.org/keywords/self-healing-systems","display_name":"self-healing systems","score":0.7950000166893005},{"id":"https://openalex.org/keywords/alerting","display_name":"alerting","score":0.6679999828338623},{"id":"https://openalex.org/keywords/data-engineering","display_name":"data engineering","score":0.6520000100135803},{"id":"https://openalex.org/keywords/pipeline-failure","display_name":"pipeline failure","score":0.621999979019165}],"concepts":[{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.5763223767280579},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4598454236984253},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3654819130897522},{"id":"https://openalex.org/C77595967","wikidata":"https://www.wikidata.org/wiki/Q3151013","display_name":"Forensic engineering","level":1,"score":0.36480939388275146},{"id":"https://openalex.org/C39432304","wikidata":"https://www.wikidata.org/wiki/Q188847","display_name":"Environmental science","level":0,"score":0.332618772983551},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2961759567260742},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.07301610708236694}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.14445/22312803/ijctt-v73i4p120","doi":"https://doi.org/10.14445/22312803/ijctt-v73i4p120","is_oa":true,"landing_page_url":"https://doi.org/10.14445/22312803/ijctt-v73i4p120","pdf_url":"https://www.ijcttjournal.org/2025/Volume-73/Issue-4/IJCTT-V73I4P120.pdf","source":{"id":"https://openalex.org/S2764857185","display_name":"International Journal of Computer Trends and Technology","issn_l":"2231-2803","issn":["2231-2803","2349-0829"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-1"],"host_organization":"https://openalex.org/P4323288697","host_organization_name":"Seventh Sense Research Group","host_organization_lineage":["https://openalex.org/P4323288697"],"host_organization_lineage_names":["Seventh Sense Research Group"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Computer Trends and Technology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.14445/22312803/ijctt-v73i4p120","doi":"https://doi.org/10.14445/22312803/ijctt-v73i4p120","is_oa":true,"landing_page_url":"https://doi.org/10.14445/22312803/ijctt-v73i4p120","pdf_url":"https://www.ijcttjournal.org/2025/Volume-73/Issue-4/IJCTT-V73I4P120.pdf","source":{"id":"https://openalex.org/S2764857185","display_name":"International Journal of Computer Trends and Technology","issn_l":"2231-2803","issn":["2231-2803","2349-0829"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-1"],"host_organization":"https://openalex.org/P4323288697","host_organization_name":"Seventh Sense Research Group","host_organization_lineage":["https://openalex.org/P4323288697"],"host_organization_lineage_names":["Seventh Sense Research Group"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Computer Trends and Technology","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.428600013256073}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4410464977.pdf","grobid_xml":"https://content.openalex.org/works/W4410464977.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2936971138","https://openalex.org/W3198730098","https://openalex.org/W7203792485","https://openalex.org/W7202287232","https://openalex.org/W2902344783","https://openalex.org/W7214743239","https://openalex.org/W7204099337","https://openalex.org/W7217302663","https://openalex.org/W6912109962","https://openalex.org/W4414361841"],"abstract_inverted_index":{"In":[0],"today's":[1],"data-driven":[2],"world,":[3],"businesses":[4],"depend":[5],"heavily":[6],"on":[7],"solid":[8],"data":[9,22,34,50,115,166,186],"pipelines":[10,40,119,174],"to":[11,18,41,164],"support":[12],"everything":[13],"from":[14],"analytics":[15],"and":[16,28,45,86,97,123,136,142,147,170,182],"reporting":[17],"day-to-day":[19],"decision-making.":[20],"As":[21],"ecosystems":[23],"scale":[24],"in":[25,95,101,149],"volume,":[26],"velocity,":[27],"complexity,":[29],"the":[30,33,55,102,157,185],"role":[31],"of":[32,57,114,159],"engineer":[35],"has":[36],"evolved-from":[37],"simply":[38],"building":[39,148],"architecting":[42],"resilient,":[43],"observable,":[44],"recovery-aware":[46],"systems.":[47],"However,":[48],"as":[49],"platforms":[51],"grow":[52],"more":[53,83],"complex,":[54],"chances":[56],"something":[58],"going":[59],"wrong":[60],"also":[61,180],"increase.":[62],"Whether":[63],"it's":[64],"a":[65,68,76,93],"schema":[66],"change,":[67],"broken":[69],"upstream":[70],"dependency,":[71],"an":[72,108],"infrastructure":[73],"hiccup,":[74],"or":[75],"resource":[77],"crunch,":[78],"pipeline":[79],"failures":[80],"are":[81],"becoming":[82],"common":[84],"-":[85],"when":[87,193],"they":[88,90],"happen,":[89],"can":[91,120],"throw":[92],"wrench":[94],"operations":[96],"shake":[98],"people's":[99],"confidence":[100],"data.In":[103],"this":[104],"paper,":[105],"we":[106],"highlight":[107],"important":[109],"but":[110,179],"often":[111],"neglected":[112],"area":[113],"engineering:":[116],"making":[117],"sure":[118],"fail":[121],"gracefully":[122],"recover":[124],"without":[125],"manual":[126],"intervention.":[127],"We'll":[128],"dig":[129],"into":[130],"practical,":[131],"real-world":[132],"techniques":[133],"for":[134,172],"identifying":[135],"handling":[137],"errors,":[138],"setting":[139],"up":[140],"alerts":[141],"monitoring":[143],"that":[144,154,175],"actually":[145],"matters,":[146],"automatic":[150],"recovery":[151],"using":[152],"patterns":[153],"have":[155],"stood":[156],"test":[158],"time.":[160],"The":[161],"goal":[162],"is":[163],"give":[165],"engineers":[167],"practical":[168],"tools":[169],"approaches":[171],"creating":[173],"aren't":[176],"just":[177],"scalable":[178],"resilient":[181],"self":[183],"healing-so":[184],"systems":[187],"behind":[188],"them":[189],"stay":[190],"reliable,":[191],"even":[192],"things":[194],"go":[195],"wrong.":[196]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2025-05-18T00:00:00"}