{"id":"https://openalex.org/W2139803001","doi":"https://doi.org/10.1109/e-science.2009.52","title":"Building Reliable Data Pipelines for Managing Community Data Using Scientific Workflows","display_name":"Building Reliable Data Pipelines for Managing Community Data Using Scientific Workflows","publication_year":2009,"publication_date":"2009-12-01","ids":{"openalex":"https://openalex.org/W2139803001","doi":"https://doi.org/10.1109/e-science.2009.52","mag":"2139803001"},"language":"en","primary_location":{"id":"doi:10.1109/e-science.2009.52","doi":"https://doi.org/10.1109/e-science.2009.52","is_oa":false,"landing_page_url":"https://doi.org/10.1109/e-science.2009.52","pdf_url":null,"source":{"id":"https://openalex.org/S4306418677","display_name":"IEEE International Conference on eScience","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2009 Fifth IEEE International Conference on e-Science","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041794289","display_name":"Yogesh L. Simmhan","orcid":"https://orcid.org/0000-0003-4140-7774","observed_orcids":["https://orcid.org/0000-0003-4140-7774"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yogesh Simmhan","raw_affiliation_strings":["eScience Group, Microsoft Research, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"eScience Group, Microsoft Research, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074465144","display_name":"Catharine van Ingen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Catharine van Ingen","raw_affiliation_strings":["Microsoft Research, Redmond, WA, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, US","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019891335","display_name":"Alexander S. Szalay","orcid":"https://orcid.org/0000-0002-4108-3282","observed_orcids":["https://orcid.org/0000-0002-4108-3282"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Szalay","raw_affiliation_strings":["Department of Physics and Astronomy, Johns Hopkins University, Baltimore, MD, USA","Extreme Computing Group, Microsoft Research, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Physics and Astronomy, Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Extreme Computing Group, Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089941827","display_name":"Roger Barga","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Roger Barga","raw_affiliation_strings":["Department of Physics and Astronomy, Johns Hopkins University, Baltimore, MD, USA","Extreme Computing Group, Microsoft Research Limited, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Physics and Astronomy, Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Extreme Computing Group, Microsoft Research Limited, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020408940","display_name":"J. N. Heasley","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I117965899","display_name":"University of Hawai\u02bbi at M\u0101noa","ror":"https://ror.org/01wspgy28","country_code":"US","type":"education","lineage":["https://openalex.org/I117965899"]},{"id":"https://openalex.org/I1331384533","display_name":"University of Hawaii System","ror":"https://ror.org/03tzaeb71","country_code":"US","type":"education","lineage":["https://openalex.org/I1331384533"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jim Heasley","raw_affiliation_strings":["Institute for Astronomy, University of Hawaii, Honolulu, HI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Astronomy, University of Hawaii, Honolulu, HI, USA","institution_ids":["https://openalex.org/I117965899","https://openalex.org/I1331384533"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.3083,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.9213264,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"321","last_page":"328"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9711999893188477,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9711999893188477,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.018300000578165054,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10772","display_name":"Distributed systems and fault tolerance","score":0.0034000000450760126,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scientific-workflows","display_name":"scientific workflows","score":0.9620000123977661},{"id":"https://openalex.org/keywords/data-pipelines","display_name":"data pipelines","score":0.8330000042915344},{"id":"https://openalex.org/keywords/data-repositories","display_name":"data repositories","score":0.7229999899864197},{"id":"https://openalex.org/keywords/data-ingestion","display_name":"data ingestion","score":0.5569999814033508},{"id":"https://openalex.org/keywords/fault-tolerance","display_name":"fault tolerance","score":0.7080000042915344},{"id":"https://openalex.org/keywords/data-consistency","display_name":"data consistency","score":0.6610000133514404},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"distributed computing","score":0.6740000247955322},{"id":"https://openalex.org/keywords/pan-starrs","display_name":"Pan-STARRS","score":0.8629999756813049},{"id":"https://openalex.org/keywords/astronomical-surveys","display_name":"astronomical surveys","score":0.5509999990463257},{"id":"https://openalex.org/keywords/commodity-clusters","display_name":"commodity clusters","score":0.6179999709129333}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8698627948760986},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7781594395637512},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6118155717849731},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5430521368980408},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5038852095603943},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.47916021943092346},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4668586552143097},{"id":"https://openalex.org/C2776397876","wikidata":"https://www.wikidata.org/wiki/Q1450531","display_name":"Cyberinfrastructure","level":2,"score":0.411678284406662},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.39001065492630005},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1171737015247345},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/e-science.2009.52","doi":"https://doi.org/10.1109/e-science.2009.52","is_oa":false,"landing_page_url":"https://doi.org/10.1109/e-science.2009.52","pdf_url":null,"source":{"id":"https://openalex.org/S4306418677","display_name":"IEEE International Conference on eScience","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2009 Fifth IEEE International Conference on e-Science","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.5199999809265137,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W87446106","https://openalex.org/W122889433","https://openalex.org/W1483326864","https://openalex.org/W1501507312","https://openalex.org/W1516988610","https://openalex.org/W1517833827","https://openalex.org/W1528589539","https://openalex.org/W1588239089","https://openalex.org/W1603776115","https://openalex.org/W1673447925","https://openalex.org/W1858703999","https://openalex.org/W1968526517","https://openalex.org/W1972743486","https://openalex.org/W1991767575","https://openalex.org/W1994326726","https://openalex.org/W2001372508","https://openalex.org/W2026398165","https://openalex.org/W2055631879","https://openalex.org/W2061197966","https://openalex.org/W2064032871","https://openalex.org/W2066972244","https://openalex.org/W2073669509","https://openalex.org/W2079501787","https://openalex.org/W2088348363","https://openalex.org/W2091045579","https://openalex.org/W2107523227","https://openalex.org/W2107571548","https://openalex.org/W2109230338","https://openalex.org/W2110546525","https://openalex.org/W2113174917","https://openalex.org/W2113642306","https://openalex.org/W2116680417","https://openalex.org/W2119565742","https://openalex.org/W2120210035","https://openalex.org/W2132771461","https://openalex.org/W2140240997","https://openalex.org/W2147504831","https://openalex.org/W2150206209","https://openalex.org/W2154513453","https://openalex.org/W2161407849","https://openalex.org/W2162733761","https://openalex.org/W2163955799","https://openalex.org/W2164806428","https://openalex.org/W2168636496","https://openalex.org/W2985125481","https://openalex.org/W2999144257","https://openalex.org/W3010210491","https://openalex.org/W3034016781","https://openalex.org/W4239670432","https://openalex.org/W4248578633","https://openalex.org/W4254233144","https://openalex.org/W4285719527","https://openalex.org/W4300280897","https://openalex.org/W6628713582","https://openalex.org/W6666104759"],"related_works":["https://openalex.org/W2105500738","https://openalex.org/W7099618352","https://openalex.org/W2113642306","https://openalex.org/W1569231427","https://openalex.org/W2031226732","https://openalex.org/W3080759945","https://openalex.org/W2038696232","https://openalex.org/W2974696587","https://openalex.org/W4402673683","https://openalex.org/W2521379111"],"abstract_inverted_index":{"The":[0],"growing":[1],"amount":[2],"of":[3,57,98,137,145],"scientific":[4,100],"data":[5,23,35,64,77,111,130,146],"from":[6],"sensors":[7],"and":[8,37,55,76,117],"field":[9],"observations":[10],"is":[11,71],"posing":[12],"a":[13,42,114],"challenge":[14],"to":[15,32,41,50,80,122,148],"\u00bfdata":[16],"valets\u00bf":[17],"responsible":[18],"for":[19,109,132],"managing":[20],"them":[21],"in":[22,74,113],"repositories.":[24],"These":[25],"repositories":[26],"built":[27],"on":[28,90],"commodity":[29],"clusters":[30],"need":[31,79],"reliably":[33,89],"ingest":[34,65,112],"continuously":[36],"ensure":[38],"its":[39],"availability":[40],"wide":[43],"user":[44],"community.":[45],"Workflows":[46],"provide":[47],"several":[48,82],"benefits":[49,59],"modeling":[51],"data-intensive":[52],"science":[53],"applications":[54],"many":[56],"these":[58,127],"can":[60],"help":[61],"manage":[62],"the":[63,96,99,129,133,138],"pipelines":[66],"too.":[67],"But":[68],"using":[69,128],"workflows":[70,86],"not":[72],"panacea":[73],"itself":[75],"valets":[78],"consider":[81],"issues":[83],"when":[84],"designing":[85],"that":[87,142],"behave":[88],"fault":[91],"prone":[92],"hardware":[93],"while":[94],"retaining":[95],"consistency":[97],"data.":[101],"In":[102],"this":[103],"paper,":[104],"we":[105],"propose":[106],"workflow":[107,119],"designs":[108],"reliable":[110],"distributed":[115],"environment":[116],"identify":[118],"framework":[120],"features":[121],"support":[123,149],"resilience.":[124],"We":[125],"illustrate":[126],"pipeline":[131],"Pan-STARRS":[134],"repository,":[135],"one":[136],"largest":[139],"digital":[140],"surveys":[141],"accumulates":[143],"100TB":[144],"annually":[147],"300":[150],"astronomers.":[151]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2016-06-24T00:00:00"}