{"id":"https://openalex.org/W2027392422","doi":"https://doi.org/10.1080/01449290701682761","title":"Comparing of feedback-collection and think-aloud methods in program comprehension studies","display_name":"Comparing of feedback-collection and think-aloud methods in program comprehension studies","publication_year":2009,"publication_date":"2009-02-24","ids":{"openalex":"https://openalex.org/W2027392422","doi":"https://doi.org/10.1080/01449290701682761","mag":"2027392422"},"language":"en","primary_location":{"id":"doi:10.1080/01449290701682761","is_oa":false,"landing_page_url":"https://doi.org/10.1080/01449290701682761","pdf_url":null,"source":{"id":"https://openalex.org/S123849098","display_name":"Behaviour and Information Technology","issn_l":"0144-929X","issn":["0144-929X","1362-3001"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Behaviour & Information Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025896424","display_name":"Amela Karahasanovi\u0107","orcid":null},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]},{"id":"https://openalex.org/I2799829267","display_name":"Simula Research Laboratory","ror":"https://ror.org/00vn06n10","country_code":"NO","type":"facility","lineage":["https://openalex.org/I2799829267"]}],"countries":["NO"],"is_corresponding":true,"raw_author_name":"Amela Karahasanovi\u0107","raw_affiliation_strings":["Simula Research Laboratory , Norway ; Department of Informatics , University of Oslo , Norway","Simula Research Laboratory , Norway","Department of Informatics , University of Oslo , Norway"],"affiliations":[{"raw_affiliation_string":"Simula Research Laboratory , Norway ; Department of Informatics , University of Oslo , Norway","institution_ids":["https://openalex.org/I2799829267"]},{"raw_affiliation_string":"Simula Research Laboratory , Norway","institution_ids":["https://openalex.org/I2799829267"]},{"raw_affiliation_string":"Department of Informatics , University of Oslo , Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057932411","display_name":"Unni Nyhamar Hinkel","orcid":null},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Unni Nyhamar Hinkel","raw_affiliation_strings":["Department of Informatics , University of Oslo , Norway"],"affiliations":[{"raw_affiliation_string":"Department of Informatics , University of Oslo , Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021226784","display_name":"D Sjberg","orcid":null},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]},{"id":"https://openalex.org/I2799829267","display_name":"Simula Research Laboratory","ror":"https://ror.org/00vn06n10","country_code":"NO","type":"facility","lineage":["https://openalex.org/I2799829267"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Dag I.K. Sj\u2298berg","raw_affiliation_strings":["Simula Research Laboratory , Norway ; Department of Informatics , University of Oslo , Norway","Simula Research Laboratory , Norway","Department of Informatics , University of Oslo , Norway"],"affiliations":[{"raw_affiliation_string":"Simula Research Laboratory , Norway ; Department of Informatics , University of Oslo , Norway","institution_ids":["https://openalex.org/I2799829267"]},{"raw_affiliation_string":"Simula Research Laboratory , Norway","institution_ids":["https://openalex.org/I2799829267"]},{"raw_affiliation_string":"Department of Informatics , University of Oslo , Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5113807088","display_name":"Richard C. Thomas","orcid":null},"institutions":[{"id":"https://openalex.org/I177877127","display_name":"University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Richard Thomas","raw_affiliation_strings":["School of Computer Science and Software Engineering , The University of Western Australia , WA, Australia"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Software Engineering , The University of Western Australia , WA, Australia","institution_ids":["https://openalex.org/I177877127"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5025896424"],"corresponding_institution_ids":["https://openalex.org/I184942183","https://openalex.org/I2799829267"],"apc_list":null,"apc_paid":null,"fwci":5.4232,"has_fulltext":false,"cited_by_count":22,"citation_normalized_percentile":{"value":0.95788786,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"28","issue":"2","first_page":"139","last_page":"164"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/think-aloud-protocol","display_name":"Think aloud protocol","score":0.747821569442749},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7474960684776306},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.7440769076347351},{"id":"https://openalex.org/keywords/protocol-analysis","display_name":"Protocol analysis","score":0.728574275970459},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6344302892684937},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.6082429885864258},{"id":"https://openalex.org/keywords/program-comprehension","display_name":"Program comprehension","score":0.5801281332969666},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.5741285085678101},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.48660075664520264},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3800904452800751},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.34617382287979126},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.34561455249786377},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3253057599067688},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3245290517807007},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.22060716152191162},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.17789870500564575},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.16894063353538513},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.16807034611701965},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.16793116927146912},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.14827212691307068},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14213022589683533}],"concepts":[{"id":"https://openalex.org/C78821406","wikidata":"https://www.wikidata.org/wiki/Q391810","display_name":"Think aloud protocol","level":3,"score":0.747821569442749},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7474960684776306},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.7440769076347351},{"id":"https://openalex.org/C133112747","wikidata":"https://www.wikidata.org/wiki/Q7251931","display_name":"Protocol analysis","level":2,"score":0.728574275970459},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6344302892684937},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.6082429885864258},{"id":"https://openalex.org/C2777561058","wikidata":"https://www.wikidata.org/wiki/Q2652119","display_name":"Program comprehension","level":4,"score":0.5801281332969666},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.5741285085678101},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.48660075664520264},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3800904452800751},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.34617382287979126},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34561455249786377},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3253057599067688},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3245290517807007},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.22060716152191162},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.17789870500564575},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.16894063353538513},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.16807034611701965},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.16793116927146912},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.14827212691307068},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14213022589683533},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1080/01449290701682761","is_oa":false,"landing_page_url":"https://doi.org/10.1080/01449290701682761","pdf_url":null,"source":{"id":"https://openalex.org/S123849098","display_name":"Behaviour and Information Technology","issn_l":"0144-929X","issn":["0144-929X","1362-3001"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Behaviour & Information Technology","raw_type":"journal-article"},{"id":"pmh:oai:pure.atira.dk:publications/caa7c487-0ded-48af-8a67-129ee71a76b2","is_oa":false,"landing_page_url":"https://research-repository.uwa.edu.au/en/publications/caa7c487-0ded-48af-8a67-129ee71a76b2","pdf_url":null,"source":{"id":"https://openalex.org/S4306402523","display_name":"UWA Profiles and Research Repository (University of Western Australia)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I177877127","host_organization_name":"The University of Western Australia","host_organization_lineage":["https://openalex.org/I177877127"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Karahasanovic , A , Hinkel , U N , Sjoberg , D I K & Thomas , R 2009 , ' Comparing of feedback-collection and think-aloud methods in program comprehension studies ' , Behaviour & information technology , vol. 28 , no. 2 , pp. 139-164 . https://doi.org/10.1080/01449290701682761","raw_type":"article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5699999928474426}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":64,"referenced_works":["https://openalex.org/W30814344","https://openalex.org/W86337602","https://openalex.org/W101656945","https://openalex.org/W161206032","https://openalex.org/W603550078","https://openalex.org/W1490657313","https://openalex.org/W1501367599","https://openalex.org/W1508619279","https://openalex.org/W1538198148","https://openalex.org/W1538978591","https://openalex.org/W1548233051","https://openalex.org/W1563148606","https://openalex.org/W1584697775","https://openalex.org/W1601662243","https://openalex.org/W1741920700","https://openalex.org/W1774183706","https://openalex.org/W1967971235","https://openalex.org/W1975476104","https://openalex.org/W1979215109","https://openalex.org/W1985624742","https://openalex.org/W1994384613","https://openalex.org/W1997348232","https://openalex.org/W1999099079","https://openalex.org/W1999649023","https://openalex.org/W2000088790","https://openalex.org/W2011965236","https://openalex.org/W2024746708","https://openalex.org/W2025538861","https://openalex.org/W2030348613","https://openalex.org/W2033995789","https://openalex.org/W2036625316","https://openalex.org/W2039831248","https://openalex.org/W2041654089","https://openalex.org/W2050187629","https://openalex.org/W2060642394","https://openalex.org/W2066404961","https://openalex.org/W2075370946","https://openalex.org/W2077635626","https://openalex.org/W2079247256","https://openalex.org/W2086314066","https://openalex.org/W2089597321","https://openalex.org/W2090976204","https://openalex.org/W2099444170","https://openalex.org/W2106970664","https://openalex.org/W2120695302","https://openalex.org/W2125376432","https://openalex.org/W2126365753","https://openalex.org/W2133783450","https://openalex.org/W2135186339","https://openalex.org/W2135676535","https://openalex.org/W2135966267","https://openalex.org/W2137247190","https://openalex.org/W2156196376","https://openalex.org/W2160182024","https://openalex.org/W2163276528","https://openalex.org/W2165117198","https://openalex.org/W2178329527","https://openalex.org/W2180379667","https://openalex.org/W2331831281","https://openalex.org/W2333057202","https://openalex.org/W2333196491","https://openalex.org/W2342091124","https://openalex.org/W2442741683","https://openalex.org/W2521745931"],"related_works":["https://openalex.org/W2724855087","https://openalex.org/W2886906914","https://openalex.org/W2884362859","https://openalex.org/W1508879959","https://openalex.org/W2188860261","https://openalex.org/W4299529040","https://openalex.org/W2123179197","https://openalex.org/W3188118552","https://openalex.org/W2125259630","https://openalex.org/W3111869873"],"abstract_inverted_index":{"This":[0,203],"paper":[1],"reports":[2],"an":[3,9],"explorative":[4],"experimental":[5],"comparison":[6],"of":[7,37,80,88,94,111,163,180,189,228,234,250],"(i)":[8],"experience-sampling":[10],"method":[11,48],"called":[12],"feedback":[13,58],"collection":[14,97,182],"and":[15,72,78,92,153,185,196,221,252,276,279],"(ii)":[16],"the":[17,46,60,81,89,95,100,156,187,190,216,226,232],"think-aloud":[18,70,74],"methods":[19,31,98,159,179,258],"with":[20,68,192,272,288],"respect":[21,193],"to":[22,86,106,194,199,256,283],"their":[23],"usefulness":[24,79],"in":[25,35,45,132,150,212,243,286],"studies":[26,36,213,244,287],"on":[27,99,294],"program":[28,41],"comprehension.":[29,42],"Think-aloud":[30],"are":[32],"widely":[33],"used":[34,210,242,285],"cognitive":[38,50,217,261],"processes,":[39,262],"including":[40],"Alternatively,":[43],"as":[44,253,264],"feedback-collection":[47],"(FCM),":[49],"processes":[51,218],"can":[52,208],"be":[53,209,241,284],"traced":[54],"by":[55],"collecting":[56],"written":[57],"from":[59],"subjects":[61,191,251],"at":[62],"regular":[63],"intervals.":[64],"We":[65,268],"compare":[66],"FCM":[67,103,146,207,240,289],"concurrent":[69],"(CTA)":[71],"retrospective":[73],"(RTA)":[75],"regarding":[76],"type":[77],"collected":[82,90],"information,":[83],"costs":[84],"related":[85,224],"analysis":[87,151,164],"information":[91],"effects":[93],"data":[96,181],"subjects'":[101],"performance.":[102],"allowed":[104],"us":[105],"identify":[107,222],"a":[108,247,254,270,280],"greater":[109],"number":[110,249],"comprehension":[112,136,227],"problems":[113,223],"that":[114,176,206,214,239,245,292],"prevented":[115],"progress":[116],"or":[117,290],"caused":[118],"significant":[119],"delay":[120],"(FCM:":[121,160],"30":[122],"problems;":[123],"CTA:":[124,167],"5;":[125],"RTA:":[126,170],"15).":[127],"It":[128],"was":[129,147],"less":[130,148],"precise":[131],"identifying":[133],"strategies":[134],"for":[135,141,144,259],"than":[137,155],"CTA":[138,291],"(92%":[139],"correctness":[140,197],"FCM;":[142],"100%":[143],"CTA).":[145],"expensive":[149],"(transcription":[152],"coding)":[154],"other":[157,257],"two":[158,273],"0.7":[161],"h":[162],"per":[165],"protocol;":[166],"31":[168],"h;":[169],"7.9":[171],"h).":[172],"The":[173],"results":[174],"indicate":[175],"all":[177],"three":[178],"were":[183],"intrusive":[184],"affected":[186],"performance":[188],"time":[195],"(small":[198],"medium":[200],"effect":[201],"size).":[202],"research":[204],"confirms":[205],"beneficially":[211],"trace":[215],"involved":[219],"in,":[220],"to,":[225],"software":[229],"applications.":[230],"On":[231],"basis":[233],"our":[235],"experience,":[236],"we":[237],"recommend":[238,269],"have":[246],"large":[248],"complement":[255],"tracing":[260],"such":[263],"user":[265],"log":[266],"files.":[267],"design":[271],"groups":[274],"(verbalisation":[275],"silent":[277],"control)":[278],"pretest":[281],"task":[282],"focus":[293],"performances.":[295]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":1},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":3}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}