{"id":"https://openalex.org/W3162922567","doi":"https://doi.org/10.1007/978-3-662-63519-3_3","title":"pygrametl: A Powerful Programming Framework for Easy Creation and Testing of ETL Flows","display_name":"pygrametl: A Powerful Programming Framework for Easy Creation and Testing of ETL Flows","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3162922567","doi":"https://doi.org/10.1007/978-3-662-63519-3_3","mag":"3162922567"},"language":"en","primary_location":{"id":"doi:10.1007/978-3-662-63519-3_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-662-63519-3_3","pdf_url":"https://link.springer.com/content/pdf/10.1007%2F978-3-662-63519-3_3.pdf","source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007%2F978-3-662-63519-3_3.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029735205","display_name":"S\u00f8ren Kejser Jensen","orcid":"https://orcid.org/0000-0003-1184-1000"},"institutions":[{"id":"https://openalex.org/I891191580","display_name":"Aalborg University","ror":"https://ror.org/04m5j1k67","country_code":"DK","type":"education","lineage":["https://openalex.org/I891191580"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"S\u00f8ren Kejser Jensen","raw_affiliation_strings":["Department of Computer Science, Aalborg University, Aalborg, Denmark"],"raw_orcid":"https://orcid.org/0000-0003-1184-1000","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Aalborg University, Aalborg, Denmark","institution_ids":["https://openalex.org/I891191580"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010748912","display_name":"Christian Thomsen","orcid":"https://orcid.org/0000-0002-0862-0509"},"institutions":[{"id":"https://openalex.org/I891191580","display_name":"Aalborg University","ror":"https://ror.org/04m5j1k67","country_code":"DK","type":"education","lineage":["https://openalex.org/I891191580"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Christian Thomsen","raw_affiliation_strings":["Department of Computer Science, Aalborg University, Aalborg, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-0862-0509","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Aalborg University, Aalborg, Denmark","institution_ids":["https://openalex.org/I891191580"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061064237","display_name":"Torben Bach Pedersen","orcid":"https://orcid.org/0000-0002-1615-777X"},"institutions":[{"id":"https://openalex.org/I891191580","display_name":"Aalborg University","ror":"https://ror.org/04m5j1k67","country_code":"DK","type":"education","lineage":["https://openalex.org/I891191580"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Torben Bach Pedersen","raw_affiliation_strings":["Department of Computer Science, Aalborg University, Aalborg, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-1615-777X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Aalborg University, Aalborg, Denmark","institution_ids":["https://openalex.org/I891191580"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046730446","display_name":"Ove Andersen","orcid":"https://orcid.org/0000-0001-7981-4565"},"institutions":[{"id":"https://openalex.org/I2801055619","display_name":"Danish Geodata Agency","ror":"https://ror.org/02av6zw72","country_code":"DK","type":"other","lineage":["https://openalex.org/I2801055619","https://openalex.org/I4210102581"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Ove Andersen","raw_affiliation_strings":["Danish Geodata Agency, N\u00f8rresundby, Denmark"],"raw_orcid":"https://orcid.org/0000-0001-7981-4565","affiliations":[{"raw_affiliation_string":"Danish Geodata Agency, N\u00f8rresundby, Denmark","institution_ids":["https://openalex.org/I2801055619"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":5000,"currency":"EUR","value_usd":5392},"apc_paid":{"value":5000,"currency":"EUR","value_usd":5392},"fwci":0.577,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.62117452,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"45","last_page":"84"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8644325137138367},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.5621659159660339},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4210224151611328}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8644325137138367},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5621659159660339},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4210224151611328}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/978-3-662-63519-3_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-662-63519-3_3","pdf_url":"https://link.springer.com/content/pdf/10.1007%2F978-3-662-63519-3_3.pdf","source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},{"id":"pmh:oai:pure.atira.dk:openaire/aa6fe477-97f3-4430-8255-b140b7452875","is_oa":true,"landing_page_url":"https://vbn.aau.dk/da/publications/aa6fe477-97f3-4430-8255-b140b7452875","pdf_url":null,"source":{"id":"https://openalex.org/S4306401731","display_name":"VBN Forskningsportal (Aalborg Universitet)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I891191580","host_organization_name":"Aalborg University","host_organization_lineage":["https://openalex.org/I891191580"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Jensen, S K, Thomsen, C, Pedersen, T B & Andersen, O 2021, pygrametl: A Powerful Programming Framework for Easy Creation and Testing of ETL Flows. in Transactions on Large-Scale Data- and Knowledge-Centered Systems XLVIII : Special Issue In Memory of Univ. Prof. Dr. Roland Wagner. vol. XLVIII, Springer, Lecture Notes in Computer Science, vol. 12670, pp. 45-84. https://doi.org/10.1007/978-3-662-63519-3_3","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1007/978-3-662-63519-3_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-662-63519-3_3","pdf_url":"https://link.springer.com/content/pdf/10.1007%2F978-3-662-63519-3_3.pdf","source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5983835024","display_name":"Generalized Operational FLEXibility for Integrating Renewables in the Distribution Grid","funder_award_id":"731232","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7813569064","display_name":null,"funder_award_id":"8022-00284B","funder_id":"https://openalex.org/F4320322928","funder_display_name":"Danmarks Frie Forskningsfond"}],"funders":[{"id":"https://openalex.org/F4320313796","display_name":"Innovationsfonden","ror":"https://ror.org/00daj4111"},{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320322928","display_name":"Danmarks Frie Forskningsfond","ror":"https://ror.org/02sptwz63"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3162922567.pdf","grobid_xml":"https://content.openalex.org/works/W3162922567.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W33602417","https://openalex.org/W1482268585","https://openalex.org/W1510220247","https://openalex.org/W1515821046","https://openalex.org/W1517055698","https://openalex.org/W1556842621","https://openalex.org/W1624831767","https://openalex.org/W1976175488","https://openalex.org/W1993892970","https://openalex.org/W2014309790","https://openalex.org/W2022678827","https://openalex.org/W2036992657","https://openalex.org/W2040753378","https://openalex.org/W2041388219","https://openalex.org/W2065697793","https://openalex.org/W2078783226","https://openalex.org/W2083755493","https://openalex.org/W2089438716","https://openalex.org/W2092499505","https://openalex.org/W2152236140","https://openalex.org/W2158275487","https://openalex.org/W2170634549","https://openalex.org/W2173213060","https://openalex.org/W2344432548","https://openalex.org/W2400147603","https://openalex.org/W2584003902","https://openalex.org/W2752854719","https://openalex.org/W2773910439","https://openalex.org/W2790247697","https://openalex.org/W2793918510","https://openalex.org/W2838566359","https://openalex.org/W2973379277","https://openalex.org/W2990038669","https://openalex.org/W3145100113","https://openalex.org/W4248072204","https://openalex.org/W6601124404","https://openalex.org/W6810243994","https://openalex.org/W6922735323"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Abstract":[0],"Extract-Transform-Load":[1],"(ETL)":[2],"flows":[3,77,131,214,258,282,309],"are":[4,194],"used":[5,116,254,270],"to":[6,46,74,126,190,205,219,255],"extract":[7],"data,":[8],"transform":[9],"it,":[10],"and":[11,36,59,72,102,122,184,235,286,304,326],"load":[12],"it":[13,217],"into":[14,247],"data":[15],"warehouses":[16],"(DWs).":[17],"The":[18],"dominating":[19],"ETL":[20,51,76,90,120,130,213,231,243,257,262,281,308],"tools":[21],"use":[22],"graphical":[23],"user":[24,125],"interfaces":[25],"(GUIs)":[26],"where":[27],"users":[28,278],"must":[29],"manually":[30,37],"place":[31],"steps/components":[32],"on":[33],"a":[34,143,157,162,170,198,248,266,317,321],"canvas":[35],"connect":[38],"them":[39],"using":[40,283,323],"lines.":[41],"This":[42],"provides":[43,265,300],"an":[44,108,295,339],"easy":[45,218],"understand":[47],"overview":[48,110],"of":[49,111,136,156,179,186,226,268,329,337],"the":[50,83,88,99,112,124,133,148,154,180,224,227,230,238,242,306,330,335],"flow":[52,232,244],"but":[53],"can":[54,251],"also":[55,252,264],"be":[56,253],"rather":[57],"tedious":[58],"require":[60],"much":[61],"trivial":[62],"work":[63],"for":[64,118,272,289],"simple":[65,284],"things.":[66],"We,":[67],"therefore,":[68],"challenge":[69],"this":[70,104],"approach":[71],"propose":[73],"develop":[75],"by":[78,142,197,215],"writing":[79],"code.":[80],"To":[81],"make":[82],"programming":[84],"easy,":[85],"we":[86,106,315,332],"proposed":[87],"Python-based":[89],"framework":[91],"in":[92,103,153,260,324],"2009.":[93],"We":[94,293],"have":[95,310],"extended":[96],"significantly":[97],"since":[98],"original":[100],"release,":[101],"paper,":[105],"present":[107,294,316],"up-to-date":[109],"framework.":[113,342],"offers":[114],"commonly":[115,269],"functionality":[117],"programmatic":[119],"development":[121,336],"enables":[123],"efficiently":[127],"create":[128],"effective":[129],"with":[132,291],"full":[134],"power":[135],"programming.":[137],"Each":[138],"dimension":[139,144,167],"is":[140,233],"represented":[141,196],"object":[145],"that":[146,276,297,299,305],"manages":[147],"underlying":[149],"table":[150,200],"or":[151,165],"tables":[152,193],"case":[155,318],"snowflaked":[158,166],"dimension.":[159],"Thus,":[160],"filling":[161],"slowly":[163],"changing":[164],"only":[168],"requires":[169],"single":[171],"method":[172],"call":[173],"per":[174],"row":[175],"as":[176,338],"performs":[177],"all":[178],"required":[181],"lookups,":[182],"insertions,":[183],"assignment":[185],"surrogate":[187],"keys.":[188],"Similarly":[189],"dimensions,":[191],"fact":[192,199],"each":[195],"object.":[201],"Our":[202],"latest":[203],"addition":[204],",":[206],"Drawn":[207],"Table":[208],"Testing":[209],"(DTT),":[210],"simplifies":[211],"testing":[212],"making":[216],"define":[220],"both":[221],"preconditions":[222],"(i.e.,":[223,237],"state":[225,240],"database":[228],"before":[229],"run)":[234,246],"postconditions":[236],"expected":[239],"after":[241],"has":[245],"test.":[249],"DTT":[250],"test":[256],"created":[259,307],"other":[261],"tools.":[263],"set":[267],"functions":[271],"transforming":[273],"rows,":[274],"classes":[275],"help":[277],"parallelize":[279],"their":[280],"abstractions,":[285],"editor":[287],"support":[288],"working":[290],"DTT.":[292],"evaluation":[296],"shows":[298],"high":[301],"programmer":[302],"productivity":[303],"good":[311],"run-time":[312],"performance.":[313],"Last,":[314],"study":[319],"from":[320],"company":[322],"production":[325],"consider":[327],"some":[328],"lessons":[331],"learned":[333],"during":[334],"open":[340],"source":[341]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}