{"id":"https://openalex.org/W4386869735","doi":"https://doi.org/10.1109/lra.2023.3316896","title":"Improving Robot Proficiency Self-Assessment Via Meta-Assessment","display_name":"Improving Robot Proficiency Self-Assessment Via Meta-Assessment","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4386869735","doi":"https://doi.org/10.1109/lra.2023.3316896"},"language":"en","primary_location":{"id":"doi:10.1109/lra.2023.3316896","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2023.3316896","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066624236","display_name":"Xuan Cao","orcid":"https://orcid.org/0000-0002-7211-6999"},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xuan Cao","raw_affiliation_strings":["Computer Science Department, Brigham Young University, Provo, UT, USA"],"raw_orcid":"https://orcid.org/0000-0002-7211-6999","affiliations":[{"raw_affiliation_string":"Computer Science Department, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059420501","display_name":"Jacob W. Crandall","orcid":"https://orcid.org/0000-0002-5602-4146"},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jacob W. Crandall","raw_affiliation_strings":["Computer Science Department, Brigham Young University, Provo, UT, USA"],"raw_orcid":"https://orcid.org/0000-0002-5602-4146","affiliations":[{"raw_affiliation_string":"Computer Science Department, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077345598","display_name":"Michael A. Goodrich","orcid":"https://orcid.org/0000-0002-2489-5705"},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael A. Goodrich","raw_affiliation_strings":["Computer Science Department, Brigham Young University, Provo, UT, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I100005738"],"apc_list":null,"apc_paid":null,"fwci":0.5364,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.73433735,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"7"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10876","display_name":"Fault Detection and Control Systems","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6389694809913635},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.573077917098999},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5175727605819702},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5127092599868774},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36323055624961853},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11019954085350037},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10905405879020691}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6389694809913635},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.573077917098999},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5175727605819702},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5127092599868774},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36323055624961853},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11019954085350037},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10905405879020691},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lra.2023.3316896","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2023.3316896","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.4300000071525574,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G3336173364","display_name":null,"funder_award_id":"N00014-16-1-3025","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G7953641807","display_name":null,"funder_award_id":"N00014-18-1-2503","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":60,"referenced_works":["https://openalex.org/W582134693","https://openalex.org/W612372977","https://openalex.org/W1598033630","https://openalex.org/W1618905105","https://openalex.org/W1719489212","https://openalex.org/W1963934847","https://openalex.org/W1988790447","https://openalex.org/W1990212011","https://openalex.org/W2095705004","https://openalex.org/W2098824882","https://openalex.org/W2101234009","https://openalex.org/W2130656620","https://openalex.org/W2164411961","https://openalex.org/W2254249950","https://openalex.org/W2531327146","https://openalex.org/W2548077621","https://openalex.org/W2626967530","https://openalex.org/W2765407302","https://openalex.org/W2786712888","https://openalex.org/W2792955190","https://openalex.org/W2911964244","https://openalex.org/W2933254221","https://openalex.org/W2956805085","https://openalex.org/W2963238274","https://openalex.org/W2964339591","https://openalex.org/W2970121940","https://openalex.org/W2970565181","https://openalex.org/W2979585146","https://openalex.org/W3004865641","https://openalex.org/W3035059588","https://openalex.org/W3035600444","https://openalex.org/W3038235410","https://openalex.org/W3102961490","https://openalex.org/W3111980464","https://openalex.org/W4223977507","https://openalex.org/W4224057949","https://openalex.org/W4232919069","https://openalex.org/W4285102526","https://openalex.org/W4287865413","https://openalex.org/W4366492961","https://openalex.org/W4383108402","https://openalex.org/W6617145748","https://openalex.org/W6635717444","https://openalex.org/W6636501900","https://openalex.org/W6674330103","https://openalex.org/W6675354045","https://openalex.org/W6684488266","https://openalex.org/W6691692454","https://openalex.org/W6728622933","https://openalex.org/W6729106125","https://openalex.org/W6730042731","https://openalex.org/W6739651123","https://openalex.org/W6745136726","https://openalex.org/W6748163547","https://openalex.org/W6761586423","https://openalex.org/W6772650050","https://openalex.org/W6773813882","https://openalex.org/W6780663757","https://openalex.org/W6786212429","https://openalex.org/W6810300553"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4313488044","https://openalex.org/W3209574120","https://openalex.org/W4312192474","https://openalex.org/W4210805261"],"abstract_inverted_index":{"Proficiency":[0],"self-assessment":[1],"(PSA),":[2],"which":[3,35],"is":[4,16,78,86,88,148,160,236],"the":[5,26,37,51,57,75,83,96,108,112,119,122,125,135,138,140,151,171,174,203,209,231,242,247,252],"ability":[6],"to":[7,50,103],"estimate":[8],"how":[9],"likely":[10],"one":[11],"can":[12],"complete":[13,44],"a":[14,17,40,45,66,100,155,188,193,197],"task,":[15],"beneficial":[18],"property":[19],"for":[20,33],"autonomous":[21],"robots.":[22],"Prior":[23],"work":[24],"developed":[25],"<italic":[27,58,67,92,165,178],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[28,59,68,93,166,179],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">assumption-alignment":[29],"tracking</i>":[30],"(AAT)":[31],"method":[32],"PSA,":[34],"estimates":[36],"probability":[38,81,114],"that":[39,72,82,202],"robot":[41,141],"will":[42],"successfully":[43],"task.":[46],"This":[47],"paper":[48],"refers":[49],"prediction":[52,77,85],"made":[53,117],"by":[54,118,162,170,196,251],"AAT":[55],"as":[56,212,214],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">first-level":[60],"assessment</i>":[61,70],"(FLA),":[62],"and":[63,129,133,177,192,220],"further":[64],"proposes":[65],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">second-level":[69],"(SLA)":[71],"determines":[73],"whether":[74],"FLA":[76,84,210,232],"correct.":[79],"The":[80,145],"correct":[87],"conditioned":[89],"on":[90,150],"four":[91,152,248],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">features:</i>":[94],"(1)":[95],"mean":[97],"distance":[98],"from":[99],"test":[101],"sample":[102],"its":[104,130],"nearest":[105],"neighbors":[106],"in":[107],"training":[109],"set;":[110],"(2)":[111],"predicted":[113],"of":[115,137,244,246],"success":[116],"FLA;":[120],"(3)":[121],"ratio":[123],"between":[124],"robot's":[126],"current":[127],"performance":[128,131],"standard;":[132],"(4)":[134],"percentage":[136],"task":[139,191,195],"has":[142],"already":[143],"completed.":[144],"SLA":[146,204,253],"model":[147,205,211],"trained":[149],"features":[153,249],"using":[154,183],"Random":[156],"Forest":[157],"algorithm.":[158],"It":[159],"evaluated":[161],"two":[163],"metrics:":[164],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">discriminability</i>":[167],",":[168,181],"measured":[169,182],"area":[172],"under":[173],"ROC":[175],"curve,":[176],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">calibration</i>":[180],"expected":[184],"calibration":[185,216,219],"error.":[186],"On":[187],"simulated":[189],"navigation":[190],"manipulation":[194],"Sawyer":[198],"robot,":[199],"results":[200],"demonstrate":[201],"not":[206],"only":[207],"calibrates":[208],"well":[213],"existing":[215],"methods":[217],"(Platt":[218],"isotonic":[221],"regression),":[222],"but":[223],"also":[224,240],"produces":[225],"very":[226],"high":[227],"discriminability":[228,235],"even":[229],"if":[230],"model's":[233],"original":[234],"much":[237],"lower.":[238],"Results":[239],"indicate":[241],"usefulness":[243],"each":[245],"used":[250],"model.":[254]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}