{"id":"https://openalex.org/W1966495972","doi":"https://doi.org/10.1145/1660877.1660901","title":"Evaluation of an integrated multi-task machine learning system with humans in the loop","display_name":"Evaluation of an integrated multi-task machine learning system with humans in the loop","publication_year":2007,"publication_date":"2007-08-28","ids":{"openalex":"https://openalex.org/W1966495972","doi":"https://doi.org/10.1145/1660877.1660901","mag":"1966495972"},"language":"en","primary_location":{"id":"doi:10.1145/1660877.1660901","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1660877.1660901","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2007 Workshop on Performance Metrics for Intelligent Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108044002","display_name":"Aaron Steinfeld","orcid":"https://orcid.org/0000-0003-2274-0053"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaron Steinfeld","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA","Carnegie-Mellon University, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108774336","display_name":"Susi Bennett","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"S. Rachael Bennett","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA","Carnegie-Mellon University, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064322102","display_name":"Kyle Cunningham","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kyle Cunningham","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA","Carnegie-Mellon University, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006862141","display_name":"Matt Lahut","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matt Lahut","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA","Carnegie-Mellon University, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037938294","display_name":"Pablo-Alejandro Quinones","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pablo-Alejandro Quinones","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA","Carnegie-Mellon University, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023570206","display_name":"Django Wexler","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Django Wexler","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA","Carnegie-Mellon University, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024012980","display_name":"Daniel P. Siewiorek","orcid":"https://orcid.org/0000-0001-6644-3929"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Siewiorek","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA","Carnegie-Mellon University, Pittsburgh, PA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie-Mellon University, Pittsburgh, PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083355591","display_name":"Jordan Hayes","orcid":"https://orcid.org/0000-0002-7604-1453"},"institutions":[{"id":"https://openalex.org/I190533189","display_name":"Amway (United States)","ror":"https://ror.org/0255w3r50","country_code":"US","type":"company","lineage":["https://openalex.org/I190533189"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jordan Hayes","raw_affiliation_strings":["Bitway, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bitway, Inc","institution_ids":["https://openalex.org/I190533189"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108230279","display_name":"Paul R. Cohen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul Cohen","raw_affiliation_strings":["U. of Southern California","U. of Southern California;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"U. of Southern California","institution_ids":[]},{"raw_affiliation_string":"U. of Southern California;","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088663182","display_name":"Julie C. Fitzgerald","orcid":"https://orcid.org/0000-0001-9936-9569"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Julie Fitzgerald","raw_affiliation_strings":["JSF Consulting"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JSF Consulting","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078277424","display_name":"Othar Hansson","orcid":null},"institutions":[{"id":"https://openalex.org/I190533189","display_name":"Amway (United States)","ror":"https://ror.org/0255w3r50","country_code":"US","type":"company","lineage":["https://openalex.org/I190533189"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Othar Hansson","raw_affiliation_strings":["Bitway, Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bitway, Inc","institution_ids":["https://openalex.org/I190533189"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077731000","display_name":"Mike Pool","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mike Pool","raw_affiliation_strings":["IET"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IET","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109052811","display_name":"Mark Drummond","orcid":null},"institutions":[{"id":"https://openalex.org/I1298353152","display_name":"SRI International","ror":"https://ror.org/05s570m15","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1298353152"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Drummond","raw_affiliation_strings":["SRI International","SRI, International;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SRI International","institution_ids":["https://openalex.org/I1298353152"]},{"raw_affiliation_string":"SRI, International;","institution_ids":["https://openalex.org/I1298353152"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":13,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.6403,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.91974772,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"168","last_page":"174"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7211080193519592},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6221632361412048},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.6004512906074524},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4635184705257416},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.4376927614212036},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4136536419391632},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.14868256449699402},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.143724262714386}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7211080193519592},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6221632361412048},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.6004512906074524},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4635184705257416},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.4376927614212036},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4136536419391632},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.14868256449699402},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.143724262714386},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/1660877.1660901","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1660877.1660901","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2007 Workshop on Performance Metrics for Intelligent Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.471.1532","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.471.1532","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.dtic.mil/dtic/tr/fulltext/u2/a534467.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.73.1028","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.73.1028","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.ri.cmu.edu/pub_files/pub4/steinfeld_aaron_2007_1/steinfeld_aaron_2007_1.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7900000214576721,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G2857030524","display_name":null,"funder_award_id":"NBCHD030010","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W170253358","https://openalex.org/W178903071","https://openalex.org/W1845016054","https://openalex.org/W2052755263","https://openalex.org/W2078786834","https://openalex.org/W2099192642","https://openalex.org/W2107443596","https://openalex.org/W2125151207","https://openalex.org/W2145657314","https://openalex.org/W2146447174","https://openalex.org/W6681549436"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W2791504333","https://openalex.org/W4224009465","https://openalex.org/W4286629047","https://openalex.org/W4306321456","https://openalex.org/W3046775127","https://openalex.org/W2081647779","https://openalex.org/W4205958290","https://openalex.org/W176937503"],"abstract_inverted_index":{"Abstract":[0],"\u2014":[1],"Performance":[2],"of":[3,10,31],"a":[4,23,38,42,60,73],"cognitive":[5],"personal":[6],"assistant,":[7],"RADAR,":[8],"consisting":[9],"multiple":[11],"machine":[12,33,69,85],"learning":[13,34,70,86],"components,":[14],"natural":[15],"language":[16],"processing,":[17],"and":[18,53],"optimization":[19],"was":[20],"examined":[21],"with":[22,55],"test":[24],"explicitly":[25],"developed":[26],"to":[27,89],"measure":[28],"the":[29],"impact":[30,75],"integrated":[32,68],"when":[35],"used":[36],"by":[37],"human":[39],"user":[40],"in":[41,59],"real":[43],"world":[44],"setting.":[45],"Three":[46],"conditions":[47],"(conventional":[48],"tools,":[49],"Radar":[50,54],"without":[51],"learning,":[52],"learning)":[56],"were":[57],"evaluated":[58],"large-scale,":[61],"between-subjects":[62],"study.":[63],"The":[64],"study":[65],"revealed":[66],"that":[67],"does":[71],"produce":[72],"positive":[74],"on":[76],"overall":[77],"performance.":[78,91],"This":[79],"paper":[80],"also":[81],"discusses":[82],"how":[83],"specific":[84],"components":[87],"contributed":[88],"human-system":[90]},"counts_by_year":[{"year":2018,"cited_by_count":5},{"year":2016,"cited_by_count":1},{"year":2012,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}