{"id":"https://openalex.org/W4296578795","doi":"https://doi.org/10.1119/perc.2022.pr.fussell","title":"Machine learning for automated content analysis: characteristics of training data impact reliability","display_name":"Machine learning for automated content analysis: characteristics of training data impact reliability","publication_year":2022,"publication_date":"2022-09-21","ids":{"openalex":"https://openalex.org/W4296578795","doi":"https://doi.org/10.1119/perc.2022.pr.fussell"},"language":"en","primary_location":{"id":"doi:10.1119/perc.2022.pr.fussell","doi":"https://doi.org/10.1119/perc.2022.pr.fussell","is_oa":true,"landing_page_url":"https://doi.org/10.1119/perc.2022.pr.fussell","pdf_url":"https://www.per-central.org/items/perc/5600.pdf","source":{"id":"https://openalex.org/S4306498225","display_name":"Physics Education Research Conference (PERC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Physics Education Research Conference Proceedings","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.per-central.org/items/perc/5600.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073661883","display_name":"Rebeckah K. Fussell","orcid":"https://orcid.org/0000-0002-5600-9309","observed_orcids":["https://orcid.org/0000-0002-5600-9309"]},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rebeckah K. Fussell","raw_affiliation_strings":["Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA"],"raw_orcid":"https://orcid.org/0000-0002-5600-9309","affiliations":[{"raw_affiliation_string":"Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038885754","display_name":"Al\u00ed A. Mazrui","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ali Mazrui","raw_affiliation_strings":["Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091637606","display_name":"Natasha G. Holmes","orcid":"https://orcid.org/0000-0001-9141-373X","observed_orcids":["https://orcid.org/0000-0001-9141-373X"]},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"N. G. Holmes","raw_affiliation_strings":["Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA"],"raw_orcid":"https://orcid.org/0000-0001-9141-373X","affiliations":[{"raw_affiliation_string":"Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA","institution_ids":["https://openalex.org/I205783295"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I205783295"],"apc_list":null,"apc_paid":null,"fwci":2.6961,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.88803089,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"194","last_page":"199"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.0032999999821186066,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-text-analysis","display_name":"computational text analysis","score":0.8659999966621399},{"id":"https://openalex.org/keywords/training-set-size","display_name":"training data size","score":0.9070000052452087},{"id":"https://openalex.org/keywords/coding-reliability","display_name":"coding reliability","score":0.7310000061988831},{"id":"https://openalex.org/keywords/student-response","display_name":"student response","score":0.5049999952316284},{"id":"https://openalex.org/keywords/physics-education-research","display_name":"physics education research","score":0.734000027179718},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.718999981880188},{"id":"https://openalex.org/keywords/neural-network","display_name":"artificial neural networks","score":0.7170000076293945}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6937999725341797},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.6535999774932861},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5706999897956848},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.529699981212616},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4991999864578247},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.46700000762939453},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4336000084877014},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.4293999969959259},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.4097000062465668},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4072999954223633},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.40139999985694885},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3587000072002411},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3463999927043915},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.33889999985694885},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.3336000144481659},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.3176000118255615},{"id":"https://openalex.org/C138958017","wikidata":"https://www.wikidata.org/wiki/Q190087","display_name":"Data type","level":2,"score":0.3165999948978424},{"id":"https://openalex.org/C77637269","wikidata":"https://www.wikidata.org/wiki/Q7002051","display_name":"Neural coding","level":2,"score":0.31060001254081726},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.3093000054359436},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3037000000476837},{"id":"https://openalex.org/C22680326","wikidata":"https://www.wikidata.org/wiki/Q7444867","display_name":"Secure coding","level":5,"score":0.2840000092983246},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C2780866740","wikidata":"https://www.wikidata.org/wiki/Q5227345","display_name":"Data retention","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C59656382","wikidata":"https://www.wikidata.org/wiki/Q191536","display_name":"Conjunction (astronomy)","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C138827492","wikidata":"https://www.wikidata.org/wiki/Q6661985","display_name":"Data processing","level":2,"score":0.25369998812675476},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.25220000743865967}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1119/perc.2022.pr.fussell","doi":"https://doi.org/10.1119/perc.2022.pr.fussell","is_oa":true,"landing_page_url":"https://doi.org/10.1119/perc.2022.pr.fussell","pdf_url":"https://www.per-central.org/items/perc/5600.pdf","source":{"id":"https://openalex.org/S4306498225","display_name":"Physics Education Research Conference (PERC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Physics Education Research Conference Proceedings","raw_type":"proceedings-article"},{"id":"pmh:oai:compadre.org:16231","doi":null,"is_oa":true,"landing_page_url":"https://www.compadre.org/Repository/document/ServeFile.cfm?ID=16231&DocID=5600","pdf_url":null,"source":{"id":"https://openalex.org/S4306400571","display_name":"The Physics Video Demonstration Database (Cornell University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Reference Material"}],"best_oa_location":{"id":"doi:10.1119/perc.2022.pr.fussell","doi":"https://doi.org/10.1119/perc.2022.pr.fussell","is_oa":true,"landing_page_url":"https://doi.org/10.1119/perc.2022.pr.fussell","pdf_url":"https://www.per-central.org/items/perc/5600.pdf","source":{"id":"https://openalex.org/S4306498225","display_name":"Physics Education Research Conference (PERC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Physics Education Research Conference Proceedings","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.975600004196167,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G773383243","display_name":"Collaborative Research: Investigating How to Better Prepare Undergraduate Students for Physics Labs that Focus on Experimental Science","funder_award_id":"2000739","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4296578795.pdf","grobid_xml":"https://content.openalex.org/works/W4296578795.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W2164777277","https://openalex.org/W2333095163","https://openalex.org/W2887624058","https://openalex.org/W2908891471","https://openalex.org/W3004067956","https://openalex.org/W3005577534","https://openalex.org/W3087297528","https://openalex.org/W3106660272","https://openalex.org/W3148053179","https://openalex.org/W3157984936","https://openalex.org/W4281779716","https://openalex.org/W4402271631","https://openalex.org/W6674325391"],"related_works":["https://openalex.org/W4387389633","https://openalex.org/W4392520336","https://openalex.org/W4293051874","https://openalex.org/W4393082402","https://openalex.org/W4391614693","https://openalex.org/W7018040360","https://openalex.org/W7171927288","https://openalex.org/W3106660272","https://openalex.org/W4281779716","https://openalex.org/W7170574787"],"abstract_inverted_index":{"Natural":[0],"language":[1],"processing":[2],"(NLP)":[3],"has":[4,40],"the":[5,9,25,77,92,109,159,199],"capacity":[6],"to":[7,95,125,176,226],"increase":[8],"scale":[10],"and":[11,171,221],"efficiency":[12],"of":[13,21,27,111,122,129,201,217],"content":[14],"analysis":[15],"in":[16,85,118,144,189,208],"Physics":[17],"Education":[18],"Research.One":[19],"promise":[20],"this":[22,88],"approach":[23],"is":[24,51,57,139,152,174],"possibility":[26],"implementing":[28],"coding":[29,81,128,179,218],"schemes":[30],"on":[31,108],"large":[32,46],"data":[33,49,64,69,157,182,222],"sets":[34],"taken":[35,71],"from":[36,72,76,183],"diverse":[37,215],"contexts.Applying":[38],"NLP":[39],"two":[41,98,133],"main":[42],"challenges,":[43],"however.First,":[44],"a":[45,73,101,145,184,214],"initial":[47],"human-coded":[48],"set":[50,161],"needed":[52,225],"for":[53,100,132,155,205],"training,":[54],"though":[55],"it":[56,173],"not":[58],"immediately":[59],"clear":[60],"how":[61],"much":[62],"training":[63,78,160,181],"are":[65,70,224],"needed.Second,":[66],"if":[67],"new":[68],"different":[74],"context":[75],"data,":[79],"automated":[80,127,178,209],"may":[82],"be":[83],"impacted":[84],"unpredictable":[86],"ways.In":[87],"study,":[89],"we":[90],"investigate":[91],"conditions":[93],"necessary":[94],"address":[96],"these":[97],"challenges":[99],"survey":[102],"question":[103],"that":[104,148],"probes":[105],"students'":[106],"perspectives":[107],"reliability":[110,200],"physics":[112],"experimental":[113],"results.We":[114],"use":[115],"neural":[116],"networks":[117],"conjunction":[119],"with":[120,165],"Bag":[121],"Words":[123],"embedding":[124],"perform":[126,177],"student":[130],"responses":[131,167],"binary":[134],"codes,":[135],"meaning":[136],"each":[137,169],"code":[138,170,190],"either":[140],"present":[141],"or":[142],"absent":[143],"response.We":[146],"find":[147],"i)":[149],"substantial":[150],"agreement":[151],"consistently":[153],"achieved":[154],"our":[156],"when":[158],"exceeds":[162],"600":[163],"responses,":[164],"80-100":[166],"containing":[168],"ii)":[172],"possible":[175],"using":[180],"disparate":[185],"context,":[186],"but":[187],"variation":[188],"frequencies":[191],"(outcome":[192],"balances)":[193],"across":[194,213],"specific":[195],"contexts":[196,223],"can":[197],"affect":[198],"coding.We":[202],"offer":[203],"suggestions":[204],"best":[206],"practices":[207],"coding.Other":[210],"smaller-scale":[211],"investigations":[212],"range":[216],"scheme":[219],"types":[220],"develop":[227],"generalized":[228],"principles.":[229]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2022-09-22T00:00:00"}