{"id":"https://openalex.org/W1670721435","doi":"https://doi.org/10.21427/d7gw28","title":"Building a Scalable Global Data Processing Pipeline for Large Astronomical Photometric Datasets","display_name":"Building a Scalable Global Data Processing Pipeline for Large Astronomical Photometric Datasets","publication_year":2015,"publication_date":"2015-01-01","ids":{"openalex":"https://openalex.org/W1670721435","doi":"https://doi.org/10.21427/d7gw28","mag":"1670721435"},"language":"en","primary_location":{"id":"pmh:oai:arrow.tudublin.ie:sciendoc-1160","is_oa":true,"landing_page_url":"https://arrow.tudublin.ie/sciendoc/158","pdf_url":"https://arrow.tudublin.ie/sciendoc/158","source":{"id":"https://openalex.org/S4377196307","display_name":"Arrow - TU Dublin (Technological University Dublin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210144925","host_organization_name":"Technological University Dublin","host_organization_lineage":["https://openalex.org/I4210144925"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Doctoral","raw_type":"phd"},"type":"dissertation","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arrow.tudublin.ie/sciendoc/158","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012770794","display_name":"Paul F. Doyle","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Doyle, Paul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5012770794"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/terabyte","display_name":"Terabyte","score":0.8352364301681519},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7576926350593567},{"id":"https://openalex.org/keywords/upload","display_name":"Upload","score":0.6473528742790222},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.62140291929245},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.6165474057197571},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.5161653161048889},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.48226577043533325},{"id":"https://openalex.org/keywords/data-processing","display_name":"Data processing","score":0.4482927620410919},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.42124488949775696},{"id":"https://openalex.org/keywords/real-time-computing","display_name":"Real-time computing","score":0.39326396584510803},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3374032974243164},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.2705749273300171},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.2113017439842224}],"concepts":[{"id":"https://openalex.org/C199683683","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Terabyte","level":2,"score":0.8352364301681519},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7576926350593567},{"id":"https://openalex.org/C71901391","wikidata":"https://www.wikidata.org/wiki/Q7126699","display_name":"Upload","level":2,"score":0.6473528742790222},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.62140291929245},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.6165474057197571},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.5161653161048889},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.48226577043533325},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.4482927620410919},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.42124488949775696},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.39326396584510803},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3374032974243164},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2705749273300171},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2113017439842224}],"mesh":[],"locations_count":6,"locations":[{"id":"pmh:oai:arrow.tudublin.ie:sciendoc-1160","is_oa":true,"landing_page_url":"https://arrow.tudublin.ie/sciendoc/158","pdf_url":"https://arrow.tudublin.ie/sciendoc/158","source":{"id":"https://openalex.org/S4377196307","display_name":"Arrow - TU Dublin (Technological University Dublin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210144925","host_organization_name":"Technological University Dublin","host_organization_lineage":["https://openalex.org/I4210144925"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Doctoral","raw_type":"phd"},{"id":"mag:1670721435","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1502.02821.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:arrow.dit.ie:sciendoc-1160","is_oa":true,"landing_page_url":"https://arrow.dit.ie/sciendoc/158","pdf_url":null,"source":{"id":"https://openalex.org/S4377196307","display_name":"Arrow - TU Dublin (Technological University Dublin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210144925","host_organization_name":"Technological University Dublin","host_organization_lineage":["https://openalex.org/I4210144925"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Doctoral","raw_type":"phd"},{"id":"pmh:oai:arrow.tudublin.ie:ascnetoth-1003","is_oa":false,"landing_page_url":"https://arrow.tudublin.ie/ascnetoth/6","pdf_url":null,"source":{"id":"https://openalex.org/S4377196307","display_name":"Arrow - TU Dublin (Technological University Dublin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210144925","host_organization_name":"Technological University Dublin","host_organization_lineage":["https://openalex.org/I4210144925"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Other","raw_type":"text"},{"id":"pmh:oai:cds.cern.ch:1988171","is_oa":false,"landing_page_url":"http://cds.cern.ch/record/1988171","pdf_url":null,"source":{"id":"https://openalex.org/S4306402195","display_name":"CERN Document Server (European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"doi:10.21427/d7gw28","is_oa":true,"landing_page_url":"https://doi.org/10.21427/d7gw28","pdf_url":null,"source":{"id":"https://openalex.org/S4377196307","display_name":"Arrow - TU Dublin (Technological University Dublin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210144925","host_organization_name":"Technological University Dublin","host_organization_lineage":["https://openalex.org/I4210144925"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arrow.tudublin.ie:sciendoc-1160","is_oa":true,"landing_page_url":"https://arrow.tudublin.ie/sciendoc/158","pdf_url":"https://arrow.tudublin.ie/sciendoc/158","source":{"id":"https://openalex.org/S4377196307","display_name":"Arrow - TU Dublin (Technological University Dublin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210144925","host_organization_name":"Technological University Dublin","host_organization_lineage":["https://openalex.org/I4210144925"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Doctoral","raw_type":"phd"},"sustainable_development_goals":[{"score":0.5699999928474426,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320314503","display_name":"Space Telescope Science Institute","ror":"https://ror.org/036f5mx38"},{"id":"https://openalex.org/F4320332178","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416"},{"id":"https://openalex.org/F4320332375","display_name":"Jet Propulsion Laboratory","ror":"https://ror.org/027k65916"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1670721435.pdf","grobid_xml":"https://content.openalex.org/works/W1670721435.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W32311115","https://openalex.org/W48315014","https://openalex.org/W1515554977","https://openalex.org/W1645146351","https://openalex.org/W2018652200","https://openalex.org/W2048295208","https://openalex.org/W2074181799","https://openalex.org/W2077113723","https://openalex.org/W2100964354","https://openalex.org/W2103363198","https://openalex.org/W2104171511","https://openalex.org/W2127769446","https://openalex.org/W2143853136","https://openalex.org/W2150058384","https://openalex.org/W2162509246","https://openalex.org/W2950062608","https://openalex.org/W2984345892","https://openalex.org/W2990075684","https://openalex.org/W2997084743","https://openalex.org/W3000648408","https://openalex.org/W3013164817","https://openalex.org/W3014099495","https://openalex.org/W3015261479","https://openalex.org/W3036150282","https://openalex.org/W3101547990","https://openalex.org/W3105451511"],"related_works":["https://openalex.org/W3170862004","https://openalex.org/W2473596205","https://openalex.org/W2913559868","https://openalex.org/W2597378585","https://openalex.org/W2968184796","https://openalex.org/W3132736081","https://openalex.org/W2981186140","https://openalex.org/W3115817058","https://openalex.org/W2960911172","https://openalex.org/W2564927364","https://openalex.org/W3214018980","https://openalex.org/W2466982395","https://openalex.org/W3017900719","https://openalex.org/W1500356253","https://openalex.org/W3152841505","https://openalex.org/W2897943079","https://openalex.org/W2967152538","https://openalex.org/W2180453900","https://openalex.org/W2794076253","https://openalex.org/W2566003192"],"abstract_inverted_index":{"Astronomical":[0,128],"photometry":[1],"is":[2,180,183,207,256],"the":[3,7,16,20,28,52,58,127,216,233,236,251,259,263],"science":[4,113],"of":[5,9,23,31,43,54,151,177,191,199,201,218,221,235,262,271],"measuring":[6,24],"flux":[8,25],"a":[10,41,122,132,139,149,162,168,197,210,268],"celestial":[11],"object.":[12,33],"Since":[13],"its":[14,49],"introduction,":[15],"CCD":[17,35,193],"has":[18,266],"been":[19],"principle":[21],"method":[22],"to":[26,48,85,104,110,161,187,214,223,231],"calculate":[27],"apparent":[29],"magnitude":[30],"an":[32],"Each":[34],"image":[36,64,194],"taken":[37],"must":[38],"go":[39],"through":[40],"process":[42,86],"cleaning":[44],"and":[45,82,102,137,159,189,227],"calibration":[46],"prior":[47],"use.":[50],"As":[51],"number":[53],"research":[55,116,179],"telescopes":[56],"increases":[57],"overall":[59],"computing":[60,247],"resources":[61,108,248],"required":[62],"for":[63],"processing":[65,69,192,269,281],"also":[66],"increases.":[67],"Existing":[68,88],"techniques":[70],"are":[71,98,244,283],"primarily":[72,109],"sequential":[73],"in":[74,100],"nature,":[75],"requiring":[76],"increasingly":[77],"powerful":[78],"servers,":[79,226],"faster":[80,83,170],"disks":[81],"networks":[84],"data.":[87,237],"High":[89],"Performance":[90],"Computing":[91],"solutions":[92],"involving":[93],"high":[94,111],"capacity":[95],"data":[96,147,157,172,195,222],"centres":[97],"complex":[99],"design":[101],"expensive":[103],"maintain,":[105],"while":[106],"providing":[107],"profile":[112],"projects.":[114],"This":[115,205],"describes":[117],"three":[118],"distributed":[119,141,224,239,249],"pipeline":[120,145,206],"architectures,":[121],"virtualised":[123],"cloud":[124,134,164],"based":[125,135],"IRAF,":[126],"Compute":[129],"Node":[130],"(ACN),":[131],"private":[133],"pipeline,":[136],"NIMBUS,":[138,181],"globally":[140],"system.":[142],"The":[143,174],"ACN":[144],"processed":[146],"at":[148,167,196],"rate":[150,169,198,270],"4":[152],"Terabytes":[153,202,273],"per":[154,203,274],"day":[155,275],"demonstrating":[156],"compression":[158,217],"upload":[160],"central":[163],"storage":[165],"service":[166],"than":[171],"generation.":[173],"primary":[175],"contribution":[176],"this":[178],"which":[182,265],"rapidly":[184],"scalable,":[185],"resilient":[186],"failure":[188],"capable":[190],"hundreds":[200],"day.":[204],"implemented":[208],"using":[209],"decentralised":[211],"web":[212,225,229,240],"queue":[213,241],"control":[215],"data,":[219],"uploading":[220],"creating":[228],"messages":[230],"identify":[232],"location":[234],"Using":[238],"messages,":[242],"images":[243],"downloaded":[245],"by":[246],"around":[250],"globe.":[252],"Rigorous":[253],"experimental":[254],"evidence":[255],"presented":[257],"verifying":[258],"horizontal":[260],"scalability":[261],"system":[264],"demonstrated":[267],"192":[272],"with":[276],"clear":[277],"indications":[278],"that":[279],"higher":[280],"rates":[282],"possible.":[284]},"counts_by_year":[],"updated_date":"2026-08-18T07:49:30.821534","created_date":"2025-10-10T00:00:00"}