{"id":"https://openalex.org/W4383224321","doi":"https://doi.org/10.1007/s41233-023-00059-2","title":"Towards enhancing ecological validity in user studies: a systematic review of guidelines and implications for QoE research","display_name":"Towards enhancing ecological validity in user studies: a systematic review of guidelines and implications for QoE research","publication_year":2023,"publication_date":"2023-07-05","ids":{"openalex":"https://openalex.org/W4383224321","doi":"https://doi.org/10.1007/s41233-023-00059-2"},"language":"en","primary_location":{"id":"doi:10.1007/s41233-023-00059-2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41233-023-00059-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41233-023-00059-2.pdf","source":{"id":"https://openalex.org/S4210197207","display_name":"Quality and User Experience","issn_l":"2366-0139","issn":["2366-0139","2366-0147"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quality and User Experience","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s41233-023-00059-2.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020688894","display_name":"Sruti Subramanian","orcid":"https://orcid.org/0000-0001-6286-9789"},"institutions":[{"id":"https://openalex.org/I204778367","display_name":"Norwegian University of Science and Technology","ror":"https://ror.org/05xg72x27","country_code":"NO","type":"education","lineage":["https://openalex.org/I204778367"]}],"countries":["NO"],"is_corresponding":true,"raw_author_name":"Sruti Subramanian","raw_affiliation_strings":["Department of Information Security and Communication Technology, Norwegian University of Science and Technology, Trondheim, Norway"],"raw_orcid":"https://orcid.org/0000-0001-6286-9789","affiliations":[{"raw_affiliation_string":"Department of Information Security and Communication Technology, Norwegian University of Science and Technology, Trondheim, Norway","institution_ids":["https://openalex.org/I204778367"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041533076","display_name":"Katrien De Moor","orcid":"https://orcid.org/0000-0002-8752-3351"},"institutions":[{"id":"https://openalex.org/I204778367","display_name":"Norwegian University of Science and Technology","ror":"https://ror.org/05xg72x27","country_code":"NO","type":"education","lineage":["https://openalex.org/I204778367"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Katrien De Moor","raw_affiliation_strings":["Department of Information Security and Communication Technology, Norwegian University of Science and Technology, Trondheim, Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Security and Communication Technology, Norwegian University of Science and Technology, Trondheim, Norway","institution_ids":["https://openalex.org/I204778367"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051117813","display_name":"Markus Fiedler","orcid":"https://orcid.org/0000-0001-8929-4911"},"institutions":[{"id":"https://openalex.org/I52719799","display_name":"Blekinge Institute of Technology","ror":"https://ror.org/0093a8w51","country_code":"SE","type":"education","lineage":["https://openalex.org/I52719799"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Markus Fiedler","raw_affiliation_strings":["Department of Technology and Aesthetics, Blekinge Institute of Technology, Karlskrona, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Technology and Aesthetics, Blekinge Institute of Technology, Karlskrona, Sweden","institution_ids":["https://openalex.org/I52719799"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042726747","display_name":"Kamil Koniuch","orcid":"https://orcid.org/0000-0002-8243-7155"},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Kamil Koniuch","raw_affiliation_strings":["Department of Electronics and Telecommunications, AGH University of Science and Technology, Krakow, Poland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electronics and Telecommunications, AGH University of Science and Technology, Krakow, Poland","institution_ids":["https://openalex.org/I686019"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064855801","display_name":"Lucjan Janowski","orcid":"https://orcid.org/0000-0002-3151-2944"},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Lucjan Janowski","raw_affiliation_strings":["Department of Electronics and Telecommunications, AGH University of Science and Technology, Krakow, Poland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electronics and Telecommunications, AGH University of Science and Technology, Krakow, Poland","institution_ids":["https://openalex.org/I686019"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5020688894"],"corresponding_institution_ids":["https://openalex.org/I204778367"],"apc_list":{"value":2590,"currency":"EUR","value_usd":3390},"apc_paid":{"value":2590,"currency":"EUR","value_usd":3390},"fwci":1.9351,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.88277696,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"8","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10803","display_name":"Innovative Human-Technology Interaction","score":0.9460999965667725,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10648","display_name":"Virtual Reality Applications and Impacts","score":0.9049999713897705,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/internal-validity","display_name":"Internal validity","score":0.7204166650772095},{"id":"https://openalex.org/keywords/external-validity","display_name":"External validity","score":0.6928626298904419},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.628414511680603},{"id":"https://openalex.org/keywords/ecological-validity","display_name":"Ecological validity","score":0.5896203517913818},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.5309628248214722},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4676271080970764},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.45489802956581116},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.24981683492660522},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.1282036304473877},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.12549623847007751},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.1110287606716156},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1094372570514679},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.10053044557571411}],"concepts":[{"id":"https://openalex.org/C51082289","wikidata":"https://www.wikidata.org/wiki/Q4113800","display_name":"Internal validity","level":2,"score":0.7204166650772095},{"id":"https://openalex.org/C174106493","wikidata":"https://www.wikidata.org/wiki/Q1057880","display_name":"External validity","level":2,"score":0.6928626298904419},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.628414511680603},{"id":"https://openalex.org/C190210774","wikidata":"https://www.wikidata.org/wiki/Q15427384","display_name":"Ecological validity","level":3,"score":0.5896203517913818},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.5309628248214722},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4676271080970764},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.45489802956581116},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.24981683492660522},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.1282036304473877},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.12549623847007751},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.1110287606716156},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1094372570514679},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.10053044557571411},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s41233-023-00059-2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41233-023-00059-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41233-023-00059-2.pdf","source":{"id":"https://openalex.org/S4210197207","display_name":"Quality and User Experience","issn_l":"2366-0139","issn":["2366-0139","2366-0147"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quality and User Experience","raw_type":"journal-article"},{"id":"pmh:oai:ntnuopen.ntnu.no:11250/3113329","is_oa":true,"landing_page_url":"https://hdl.handle.net/11250/3113329","pdf_url":null,"source":{"id":"https://openalex.org/S4306401716","display_name":"Duo Research Archive (University of Oslo)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I184942183","host_organization_name":"University of Oslo","host_organization_lineage":["https://openalex.org/I184942183"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"8","raw_type":"info:eu-repo/semantics/other"}],"best_oa_location":{"id":"doi:10.1007/s41233-023-00059-2","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s41233-023-00059-2","pdf_url":"https://link.springer.com/content/pdf/10.1007/s41233-023-00059-2.pdf","source":{"id":"https://openalex.org/S4210197207","display_name":"Quality and User Experience","issn_l":"2366-0139","issn":["2366-0139","2366-0147"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quality and User Experience","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/15","score":0.5199999809265137,"display_name":"Life in Land"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320311040","display_name":"Norges Teknisk-Naturvitenskapelige Universitet","ror":"https://ror.org/05xg72x27"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4383224321.pdf"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W51844010","https://openalex.org/W82984000","https://openalex.org/W88836578","https://openalex.org/W140775045","https://openalex.org/W160153222","https://openalex.org/W1530622094","https://openalex.org/W1647397793","https://openalex.org/W2012238399","https://openalex.org/W2025988517","https://openalex.org/W2039683595","https://openalex.org/W2052679081","https://openalex.org/W2067949228","https://openalex.org/W2120911096","https://openalex.org/W2128378281","https://openalex.org/W2149170403","https://openalex.org/W2163031058","https://openalex.org/W2171366442","https://openalex.org/W2173536757","https://openalex.org/W2260402465","https://openalex.org/W2284531499","https://openalex.org/W2473804596","https://openalex.org/W2500671607","https://openalex.org/W2581348306","https://openalex.org/W2612843443","https://openalex.org/W2750168540","https://openalex.org/W2767122676","https://openalex.org/W2893614580","https://openalex.org/W2941693935","https://openalex.org/W2953655670","https://openalex.org/W2955051043","https://openalex.org/W2963912515","https://openalex.org/W2969120675","https://openalex.org/W3037266308","https://openalex.org/W3083348805","https://openalex.org/W3102446926","https://openalex.org/W3174579619","https://openalex.org/W4200592174","https://openalex.org/W4229003875","https://openalex.org/W4240257942","https://openalex.org/W4281715872","https://openalex.org/W4290098718","https://openalex.org/W4301494176"],"related_works":["https://openalex.org/W3014130253","https://openalex.org/W4321603748","https://openalex.org/W4256013822","https://openalex.org/W2890570810","https://openalex.org/W4284881548","https://openalex.org/W1979863452","https://openalex.org/W4247244042","https://openalex.org/W4256571315","https://openalex.org/W4383224321","https://openalex.org/W2792624737"],"abstract_inverted_index":{"Abstract":[0],"The":[1,106],"concept":[2],"of":[3,15,17,30,41,64,85,92,113,165,186,248,262,273],"conducting":[4,76,120],"ecologically":[5,52,77,121],"valid":[6,53,78,122],"user":[7,79,123,250],"studies":[8,31,94,274],"is":[9,275],"gaining":[10],"traction":[11],"in":[12,102,110,201,235],"the":[13,27,45,96,103,111,149,159,162,168,183,191,240,260],"field":[14],"Quality":[16],"Experience":[18],"(QoE).":[19],"However,":[20],"despite":[21],"previous":[22],"research":[23,65,197,206],"exploring":[24],"this":[25,56],"concept,":[26],"increasing":[28],"volume":[29],"has":[32],"made":[33],"it":[34],"challenging":[35],"to":[36,48,59,237,267],"obtain":[37],"a":[38,61,89,177,187,268],"comprehensive":[39],"overview":[40],"existing":[42],"guidelines":[43,115,126],"and":[44,70,99,144,155,204,218],"key":[46,241],"aspects":[47],"consider":[49],"when":[50,205],"designing":[51],"studies.":[54,80,124],"Therefor":[55],"paper":[57],"aims":[58],"provide":[60,117],"systematic":[62,107],"review":[63,108,160,193],"articles":[66],"published":[67],"between":[68],"2011":[69],"2021":[71],"that":[72,116,195,256],"offer":[73],"insight":[74],"into":[75],"From":[81],"an":[82],"initial":[83],"count":[84,91],"782":[86],"retrieved":[87],"studies,":[88],"final":[90,104],"12":[93],"met":[95],"predefined":[97],"criteria":[98],"were":[100],"included":[101],"review.":[105],"resulted":[109],"extraction":[112],"55":[114],"guidance":[118],"towards":[119],"These":[125],"have":[127],"been":[128],"grouped":[129],"within":[130],"8":[131],"categories":[132],"(":[133],"Environment,":[134],"Technology,":[135],"Content,":[136],"Participant":[137],"Recruitment,":[138],"User":[139],"Behavior,":[140],"Study":[141],"Design,":[142],"Task":[143],"data":[145],"collection":[146],")":[147],"overarching":[148],"three":[150],"main":[151],"dimensions":[152],"(Setting,":[153],"Users":[154],"Research":[156],"Methodology).":[157],"Furthermore,":[158],"discusses:":[161],"flip":[163],"side":[164],"ecological":[166,184,211,223,263],"validity,":[167],"implications":[169],"for":[170,181,245],"QoE":[171,244],"research,":[172],"as":[173,175],"well":[174],"provides":[176],"basic":[178],"visualisation":[179],"model":[180],"assessing":[182],"validity":[185,212,224,264],"study.":[188],"In":[189],"conclusion,":[190],"current":[192],"indicates":[194],"future":[196],"should":[198],"address":[199],"more":[200,257],"detail":[202],"how":[203],"approaches":[207],"characterized":[208,220],"by":[209,221],"high":[210,227],"(and":[213,225],"correspondingly,":[214],"low":[215,222],"internal":[216,228],"validity)":[217,229],"those":[219],"normally":[226],"can":[230],"best":[231],"complement":[232],"each":[233],"other":[234],"order":[236],"better":[238],"understand":[239],"factors":[242],"influencing":[243],"various":[246],"types":[247],"applications,":[249],"segments,":[251],"settings.":[252],"Further,":[253],"we":[254],"argue":[255],"transparency":[258],"around":[259],"(sub)dimensions":[261],"with":[265],"respect":[266],"particular":[269],"study":[270],"or":[271],"set":[272],"necessary.":[276]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":4}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}