{"id":"https://openalex.org/W2744523068","doi":"https://doi.org/10.5408/16-225.1","title":"Using Systems Thinking in the Design, Implementation, and Evaluation of Complex Educational Innovations, With Examples From the InTeGrate Project","display_name":"Using Systems Thinking in the Design, Implementation, and Evaluation of Complex Educational Innovations, With Examples From the InTeGrate Project","publication_year":2017,"publication_date":"2017-08-07","ids":{"openalex":"https://openalex.org/W2744523068","doi":"https://doi.org/10.5408/16-225.1","mag":"2744523068"},"language":"en","primary_location":{"id":"doi:10.5408/16-225.1","doi":"https://doi.org/10.5408/16-225.1","is_oa":false,"landing_page_url":"https://doi.org/10.5408/16-225.1","pdf_url":null,"source":{"id":"https://openalex.org/S2765076859","display_name":"Journal of Geoscience Education","issn_l":"1089-9995","issn":["1089-9995","2158-1428"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","jufo-1","norway-1","poland-70","russia-white-list-2"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Geoscience Education","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073472781","display_name":"Kim Anne Kastens","orcid":"https://orcid.org/0000-0001-8292-5098","observed_orcids":["https://orcid.org/0000-0001-8292-5098"]},"institutions":[{"id":"https://openalex.org/I152304114","display_name":"Lamont-Doherty Earth Observatory","ror":"https://ror.org/02e2tgs60","country_code":"US","type":"facility","lineage":["https://openalex.org/I152304114","https://openalex.org/I78577930"]},{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kim A. Kastens","raw_affiliation_strings":["Lamont-Doherty Earth Observatory of Columbia University, 61 Rt 9W, Palisades, New York 10964, USA, , 1-914-671-2341"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lamont-Doherty Earth Observatory of Columbia University, 61 Rt 9W, Palisades, New York 10964, USA, , 1-914-671-2341","institution_ids":["https://openalex.org/I152304114","https://openalex.org/I78577930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031304523","display_name":"Cathryn A. Manduca","orcid":"https://orcid.org/0000-0001-9125-3318","observed_orcids":["https://orcid.org/0000-0001-9125-3318"]},"institutions":[{"id":"https://openalex.org/I188497080","display_name":"Carleton College","ror":"https://ror.org/03jep7677","country_code":"US","type":"education","lineage":["https://openalex.org/I188497080"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cathryn A. Manduca","raw_affiliation_strings":["Science Education Resource Center, Carleton College, One North College Street, Northfield, Minnesota 55057, USA,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Science Education Resource Center, Carleton College, One North College Street, Northfield, Minnesota 55057, USA,","institution_ids":["https://openalex.org/I188497080"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.1081,"has_fulltext":false,"cited_by_count":32,"citation_normalized_percentile":{"value":0.93985763,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"65","issue":"3","first_page":"219","last_page":"230"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11810","display_name":"Complex Systems and Decision Making","score":0.7792999744415283,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11810","display_name":"Complex Systems and Decision Making","score":0.7792999744415283,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12469","display_name":"Chaos, Complexity, and Education","score":0.08079999685287476,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10072","display_name":"Science Education and Pedagogy","score":0.038600001484155655,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/systems-thinking","display_name":"systems thinking","score":1.0},{"id":"https://openalex.org/keywords/educational-innovation","display_name":"educational innovation","score":0.8600000143051147},{"id":"https://openalex.org/keywords/integrate","display_name":"InTeGrate","score":0.8080000281333923},{"id":"https://openalex.org/keywords/geoscience-education","display_name":"geoscience education","score":0.9539999961853027},{"id":"https://openalex.org/keywords/program-design","display_name":"program design","score":0.8569999933242798},{"id":"https://openalex.org/keywords/program-evaluation","display_name":"program evaluation","score":0.8180000185966492},{"id":"https://openalex.org/keywords/feedback-loops","display_name":"feedback loops","score":0.7820000052452087},{"id":"https://openalex.org/keywords/systems-mapping","display_name":"systems mapping","score":0.8970000147819519},{"id":"https://openalex.org/keywords/emergence","display_name":"emergence","score":0.800000011920929},{"id":"https://openalex.org/keywords/higher-education-transformation","display_name":"higher education transformation","score":0.5529999732971191},{"id":"https://openalex.org/keywords/program-leadership","display_name":"program leadership","score":0.38999998569488525},{"id":"https://openalex.org/keywords/complex-systems","display_name":"complex systems","score":0.3619999885559082}],"concepts":[{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6062983870506287},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.5975016951560974},{"id":"https://openalex.org/C26011011","wikidata":"https://www.wikidata.org/wiki/Q6030243","display_name":"Influencer marketing","level":4,"score":0.5472337007522583},{"id":"https://openalex.org/C27177047","wikidata":"https://www.wikidata.org/wiki/Q269699","display_name":"Systems thinking","level":2,"score":0.5175894498825073},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.5074760317802429},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49637871980667114},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.42946699261665344},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.41037410497665405},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3933611214160919},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.38452231884002686},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.23644116520881653},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.22676357626914978},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.16830116510391235},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.16764557361602783},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C192975520","wikidata":"https://www.wikidata.org/wiki/Q1143466","display_name":"Marketing management","level":2,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C54649085","wikidata":"https://www.wikidata.org/wiki/Q574424","display_name":"Relationship marketing","level":3,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5408/16-225.1","doi":"https://doi.org/10.5408/16-225.1","is_oa":false,"landing_page_url":"https://doi.org/10.5408/16-225.1","pdf_url":null,"source":{"id":"https://openalex.org/S2765076859","display_name":"Journal of Geoscience Education","issn_l":"1089-9995","issn":["1089-9995","2158-1428"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","jufo-1","norway-1","poland-70","russia-white-list-2"],"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547","https://openalex.org/P4310320449"],"host_organization_lineage_names":["Taylor & Francis","Informa"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Geoscience Education","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality education","id":"https://metadata.un.org/sdg/4","score":0.975600004196167}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W243776677","https://openalex.org/W286571147","https://openalex.org/W313365726","https://openalex.org/W606131240","https://openalex.org/W1268037625","https://openalex.org/W1515735338","https://openalex.org/W1544145404","https://openalex.org/W1761828760","https://openalex.org/W1946093131","https://openalex.org/W1968329320","https://openalex.org/W1969319546","https://openalex.org/W1995056010","https://openalex.org/W1996496792","https://openalex.org/W1998829946","https://openalex.org/W2010593734","https://openalex.org/W2011534468","https://openalex.org/W2022041506","https://openalex.org/W2043787017","https://openalex.org/W2043863312","https://openalex.org/W2044489495","https://openalex.org/W2054710998","https://openalex.org/W2067386609","https://openalex.org/W2093256570","https://openalex.org/W2094595478","https://openalex.org/W2095121463","https://openalex.org/W2101892546","https://openalex.org/W2113942717","https://openalex.org/W2116199508","https://openalex.org/W2135306643","https://openalex.org/W2140317779","https://openalex.org/W2149157563","https://openalex.org/W2164086087","https://openalex.org/W2169592465","https://openalex.org/W2297705604","https://openalex.org/W2423609147","https://openalex.org/W2466577542","https://openalex.org/W2480148616","https://openalex.org/W2517199338","https://openalex.org/W2545436971","https://openalex.org/W2588811889","https://openalex.org/W2796391277","https://openalex.org/W2994207713","https://openalex.org/W3080315074","https://openalex.org/W4238149961","https://openalex.org/W4245961435","https://openalex.org/W4247941329","https://openalex.org/W4252541056","https://openalex.org/W4301379491","https://openalex.org/W4307038068"],"related_works":["https://openalex.org/W2504051924","https://openalex.org/W2616037536","https://openalex.org/W2899687501","https://openalex.org/W2770563988","https://openalex.org/W2984726036","https://openalex.org/W7096780412","https://openalex.org/W2566273182","https://openalex.org/W2068778941","https://openalex.org/W2915989455","https://openalex.org/W1985993596"],"abstract_inverted_index":{"Many":[0],"geoscience":[1],"education":[2],"initiatives":[3],"now":[4],"involve":[5],"cross-departmental":[6],"or":[7,21,27],"multi-institutional":[8],"programs.":[9],"However,":[10],"the":[11,34,57,69,99,103,162,202,238],"geoscientists":[12,37],"who":[13],"lead":[14],"such":[15],"programs":[16],"typically":[17],"have":[18],"little":[19],"experience":[20,138],"training":[22],"in":[23,118,145,233],"program":[24],"design,":[25],"leadership,":[26],"evaluation.":[28],"In":[29,204],"this":[30],"commentary,":[31],"we":[32,72],"make":[33],"case":[35],"that":[36,53,130],"taking":[38],"on":[39,46,106],"these":[40,220],"ambitious":[41],"leadership":[42],"roles":[43],"can":[44],"draw":[45],"a":[47,91,124],"set":[48,125],"of":[49,61,63,85,110,126,187,226,235,241],"understandings":[50],"and":[51,59,78,89,112,153,174,181,190,215,229],"skills":[52],"they":[54],"already":[55],"have:":[56],"tools":[58],"habits":[60],"mind":[62],"systems":[64,207],"thinking.":[65],"Using":[66],"examples":[67],"from":[68,176],"InTeGrate":[70],"program,":[71],"suggest":[73],"ways":[74],"to":[75,116,168,184,195,209,237],"envision,":[76],"shape,":[77],"monitor":[79],"an":[80,107],"educational":[81],"intervention":[82],"by":[83],"thinking":[84],"oneself":[86],"as":[87,137,194],"building":[88],"improving":[90],"complex":[92],"system":[93,240],"with":[94,135],"constituent":[95],"subsystems.":[96],"Suggestions":[97],"for":[98,114,133,141],"design":[100],"phase":[101],"include":[102],"following:":[104],"Decide":[105],"essential":[108],"suite":[109],"subsystems":[111,129],"plan":[113],"them":[115],"interact":[117],"mutually":[119],"beneficial":[120,151],"ways.":[121],"Consider":[122],"using":[123],"semi-autonomous":[127],"parallel":[128],"will":[131],"allow":[132],"replication":[134],"adaptation":[136],"accrues.":[139],"Plan":[140],"nonlinear":[142],"causality":[143],"chains":[144],"which":[146],"one":[147],"activity":[148],"has":[149],"multiple":[150,159],"outputs,":[152],"desired":[154,172],"outcomes":[155,232],"are":[156],"supported":[157],"through":[158],"influencers.":[160],"At":[161],"implementation":[163],"phase,":[164],"leverage":[165],"feedback":[166],"loops":[167],"nudge":[169],"actors":[170],"toward":[171],"behaviors":[173],"away":[175],"problematic":[177],"choices.":[178],"Build":[179],"technical":[180],"social":[182],"mechanisms":[183],"regulate":[185],"flows":[186],"information":[188,199],"between":[189],"within":[191],"subsystems,":[192],"so":[193],"deliver":[196],"timely,":[197],"actionable":[198],"without":[200],"overloading":[201],"actors.":[203],"evaluation,":[205],"use":[206],"mapping":[208],"understand":[210],"where":[211],"critical":[212],"dependencies":[213],"occur":[214],"insert":[216],"evaluative":[217],"probes":[218],"at":[219],"locations.":[221],"Seek":[222],"out":[223],"early":[224],"indicators":[225],"emergent":[227],"phenomena":[228],"conceptualize":[230],"long-term":[231],"terms":[234],"modifications":[236],"larger":[239],"higher":[242],"education.":[243]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":4}],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2017-08-17T00:00:00"}