{"id":"https://openalex.org/W4392520336","doi":"https://doi.org/10.1103/physrevphyseducres.20.010113","title":"Method to assess the trustworthiness of machine coding at scale","display_name":"Method to assess the trustworthiness of machine coding at scale","publication_year":2024,"publication_date":"2024-03-06","ids":{"openalex":"https://openalex.org/W4392520336","doi":"https://doi.org/10.1103/physrevphyseducres.20.010113"},"language":"en","primary_location":{"id":"doi:10.1103/physrevphyseducres.20.010113","doi":"https://doi.org/10.1103/physrevphyseducres.20.010113","is_oa":true,"landing_page_url":"https://doi.org/10.1103/physrevphyseducres.20.010113","pdf_url":"http://link.aps.org/pdf/10.1103/PhysRevPhysEducRes.20.010113","source":{"id":"https://openalex.org/S4210190682","display_name":"Physical Review Physics Education Research","issn_l":"2469-9896","issn":["2469-9896"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-2","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320261","host_organization_name":"American Physical Society","host_organization_lineage":["https://openalex.org/P4310320261"],"host_organization_lineage_names":["American Physical Society"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Physical Review Physics Education Research","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://link.aps.org/pdf/10.1103/PhysRevPhysEducRes.20.010113","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073661883","display_name":"Rebeckah K. Fussell","orcid":"https://orcid.org/0000-0002-5600-9309","observed_orcids":["https://orcid.org/0000-0002-5600-9309"]},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rebeckah K. Fussell","raw_affiliation_strings":["Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA"],"raw_orcid":"https://orcid.org/0000-0002-5600-9309","affiliations":[{"raw_affiliation_string":"Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081626741","display_name":"Emily M. Stump","orcid":"https://orcid.org/0000-0002-4887-4327","observed_orcids":["https://orcid.org/0000-0002-4887-4327"]},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Emily M. Stump","raw_affiliation_strings":["Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA"],"raw_orcid":"https://orcid.org/0000-0002-4887-4327","affiliations":[{"raw_affiliation_string":"Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091637606","display_name":"Natasha G. Holmes","orcid":"https://orcid.org/0000-0001-9141-373X","observed_orcids":["https://orcid.org/0000-0001-9141-373X"]},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"N.\u2009G. Holmes","raw_affiliation_strings":["Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA"],"raw_orcid":"https://orcid.org/0000-0001-9141-373X","affiliations":[{"raw_affiliation_string":"Laboratory of Atomic and Solid State Physics, Cornell University, Ithaca, New York 14853, USA","institution_ids":["https://openalex.org/I205783295"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I205783295"],"apc_list":{"value":2000,"currency":"USD","value_usd":2000},"apc_paid":{"value":2000,"currency":"USD","value_usd":2000},"fwci":4.3738,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.95661765,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"20","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.0015999999595806003,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/trustworthiness","display_name":"trustworthiness","score":0.8600000143051147},{"id":"https://openalex.org/keywords/systematic-uncertainties","display_name":"systematic uncertainties","score":0.7540000081062317},{"id":"https://openalex.org/keywords/statistical-uncertainty","display_name":"statistical uncertainty","score":0.9890000224113464},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.574999988079071},{"id":"https://openalex.org/keywords/physics-education-research","display_name":"physics education research","score":0.6909999847412109}],"concepts":[{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.7632585763931274},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5521845817565918},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5266451835632324},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.47778046131134033},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.3762235641479492},{"id":"https://openalex.org/C185544564","wikidata":"https://www.wikidata.org/wiki/Q81197","display_name":"Nuclear physics","level":1,"score":0.3455919027328491},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.3218909502029419},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.20667991042137146},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.20212215185165405},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.1577429473400116},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14612042903900146}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1103/physrevphyseducres.20.010113","doi":"https://doi.org/10.1103/physrevphyseducres.20.010113","is_oa":true,"landing_page_url":"https://doi.org/10.1103/physrevphyseducres.20.010113","pdf_url":"http://link.aps.org/pdf/10.1103/PhysRevPhysEducRes.20.010113","source":{"id":"https://openalex.org/S4210190682","display_name":"Physical Review Physics Education Research","issn_l":"2469-9896","issn":["2469-9896"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-2","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320261","host_organization_name":"American Physical Society","host_organization_lineage":["https://openalex.org/P4310320261"],"host_organization_lineage_names":["American Physical Society"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Physical Review Physics Education Research","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:40815cb1dca849c0b5d38d1b7a9d80db","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/40815cb1dca849c0b5d38d1b7a9d80db","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Physical Review Physics Education Research, Vol 20, Iss 1, p 010113 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1103/physrevphyseducres.20.010113","doi":"https://doi.org/10.1103/physrevphyseducres.20.010113","is_oa":true,"landing_page_url":"https://doi.org/10.1103/physrevphyseducres.20.010113","pdf_url":"http://link.aps.org/pdf/10.1103/PhysRevPhysEducRes.20.010113","source":{"id":"https://openalex.org/S4210190682","display_name":"Physical Review Physics Education Research","issn_l":"2469-9896","issn":["2469-9896"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-2","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320261","host_organization_name":"American Physical Society","host_organization_lineage":["https://openalex.org/P4310320261"],"host_organization_lineage_names":["American Physical Society"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Physical Review Physics Education Research","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality education","id":"https://metadata.un.org/sdg/4","score":0.6164000034332275}],"study_designs":[],"awards":[{"id":"https://openalex.org/G6174244922","display_name":"Graduate Research Fellowship Program (GRFP)","funder_award_id":"2139899","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6849693606","display_name":"Collaborative Research: Student Thinking About Measurements Across the Physics Curriculum","funder_award_id":"1808945","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G773383243","display_name":"Collaborative Research: Investigating How to Better Prepare Undergraduate Students for Physics Labs that Focus on Experimental Science","funder_award_id":"2000739","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392520336.pdf","grobid_xml":"https://content.openalex.org/works/W4392520336.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1521626219","https://openalex.org/W1993220166","https://openalex.org/W2095371090","https://openalex.org/W2102611807","https://openalex.org/W2144211451","https://openalex.org/W2164777277","https://openalex.org/W2333095163","https://openalex.org/W2763750328","https://openalex.org/W2769563773","https://openalex.org/W2805117413","https://openalex.org/W2887624058","https://openalex.org/W2911994647","https://openalex.org/W2954644225","https://openalex.org/W2981679558","https://openalex.org/W2981686111","https://openalex.org/W3004067956","https://openalex.org/W3033185959","https://openalex.org/W3087297528","https://openalex.org/W3091838623","https://openalex.org/W3121705224","https://openalex.org/W3128553165","https://openalex.org/W3148053179","https://openalex.org/W3160589955","https://openalex.org/W3186567754","https://openalex.org/W3212789071","https://openalex.org/W4210947788","https://openalex.org/W4214819927","https://openalex.org/W4246304117","https://openalex.org/W4281779716","https://openalex.org/W4289519276","https://openalex.org/W4383815588","https://openalex.org/W4386365948","https://openalex.org/W4387773534","https://openalex.org/W4388692147","https://openalex.org/W4388822422"],"related_works":["https://openalex.org/W4387389633","https://openalex.org/W4296578795","https://openalex.org/W4402470540","https://openalex.org/W4281779716","https://openalex.org/W4417079106","https://openalex.org/W7160944489","https://openalex.org/W7167233472","https://openalex.org/W2562763223","https://openalex.org/W7096433207","https://openalex.org/W7018040360"],"abstract_inverted_index":{"Physics":[0],"education":[1,182],"researchers":[2,183],"are":[3],"interested":[4],"in":[5,189],"using":[6,96,143,164],"the":[7,93,126,190,194],"tools":[8],"of":[9,39,95,163],"machine":[10,40,168],"learning":[11],"and":[12,23,46,54,129],"natural":[13,21,112],"language":[14,22,113],"processing":[15],"to":[16,30,57,167,179],"make":[17],"quantitative":[18,81],"claims":[19,82,109],"from":[20,125,133,145],"text":[24],"data,":[25],"such":[26,108],"as":[27],"open-ended":[28],"responses":[29],"survey":[31,150],"questions.":[32],"The":[33],"aspiration":[34],"is":[35,61],"that":[36,68,77],"this":[37,141],"form":[38],"coding":[41,155],"may":[42,185],"be":[43,58],"more":[44],"efficient":[45],"consistent":[47],"than":[48,60],"human":[49,64,89,174],"coding,":[50,90],"allowing":[51],"much":[52],"larger":[53],"broader":[55],"datasets":[56],"analyzed":[59],"practical":[62],"with":[63,88,110,152],"coders.":[65,175],"Existing":[66],"work":[67],"uses":[69],"these":[70,97,165],"tools,":[71],"however,":[72],"does":[73],"not":[74],"investigate":[75],"norms":[76],"allow":[78],"for":[79,106,140],"trustworthy":[80],"without":[83],"full":[84],"reliance":[85],"on":[86],"cross-checking":[87],"which":[91],"defeats":[92],"purpose":[94],"automated":[98],"tools.":[99],"Here":[100],"we":[101],"propose":[102],"a":[103,116,160,170],"four-part":[104],"method":[105,142],"making":[107],"supervised":[111],"processing:":[114],"evaluating":[115],"trained":[117,127],"model,":[118],"calculating":[119,122,130],"statistical":[120],"uncertainty,":[121],"systematic":[123,131],"uncertainty":[124,132],"algorithm,":[128],"novel":[134],"data":[135,144],"sources.":[136],"We":[137,157,176],"provide":[138,159],"evidence":[139],"two":[146,153],"distinct":[147,154],"short":[148],"response":[149],"questions":[151],"schemes.":[156],"also":[158],"real-world":[161],"example":[162],"practices":[166],"code":[169],"dataset":[171],"unseen":[172],"by":[173,193],"offer":[177],"recommendations":[178],"guide":[180],"physics":[181],"who":[184],"use":[186],"machine-coding":[187],"methods":[188],"future.":[191],"Published":[192],"American":[195],"Physical":[196],"Society":[197],"2024":[198]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2024-03-07T00:00:00"}