{"id":"https://openalex.org/W2117733879","doi":"https://doi.org/10.1145/1216295.1216316","title":"Toward harnessing user feedback for machine learning","display_name":"Toward harnessing user feedback for machine learning","publication_year":2007,"publication_date":"2007-01-28","ids":{"openalex":"https://openalex.org/W2117733879","doi":"https://doi.org/10.1145/1216295.1216316","mag":"2117733879"},"language":"en","primary_location":{"id":"doi:10.1145/1216295.1216316","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1216295.1216316","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 12th international conference on Intelligent user interfaces","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014846205","display_name":"Simone Stumpf","orcid":"https://orcid.org/0000-0001-6482-1973"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Simone Stumpf","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074311877","display_name":"Vidya Rajaram","orcid":null},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vidya Rajaram","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035689543","display_name":"Lida Li","orcid":null},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lida Li","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011173372","display_name":"Margaret Burnett","orcid":"https://orcid.org/0000-0001-6536-7629"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Margaret Burnett","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074537073","display_name":"Thomas G. Dietterich","orcid":"https://orcid.org/0000-0002-8223-8586"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thomas Dietterich","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038474957","display_name":"Erin Sullivan","orcid":"https://orcid.org/0000-0001-9574-9716"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Erin Sullivan","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110583339","display_name":"Russell Drummond","orcid":null},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Russell Drummond","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109877919","display_name":"Jonathan L. Herlocker","orcid":null},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jonathan Herlocker","raw_affiliation_strings":["Oregon State University, Corvallis, OR","[Oregon State University, Corvallis, OR]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR","institution_ids":["https://openalex.org/I131249849"]},{"raw_affiliation_string":"[Oregon State University, Corvallis, OR]","institution_ids":["https://openalex.org/I131249849"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5014846205"],"corresponding_institution_ids":["https://openalex.org/I131249849"],"apc_list":null,"apc_paid":null,"fwci":12.5894,"has_fulltext":false,"cited_by_count":128,"citation_normalized_percentile":{"value":0.98590463,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"82","last_page":"91"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6993011236190796},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5301331877708435}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6993011236190796},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5301331877708435}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/1216295.1216316","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1216295.1216316","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 12th international conference on Intelligent user interfaces","raw_type":"proceedings-article"},{"id":"pmh:ADA471484","is_oa":false,"landing_page_url":"http://oai.dtic.mil/oai/oai?&verb=getRecord&metadataPrefix=html&identifier=ADA471484","pdf_url":null,"source":{"id":"https://openalex.org/S4406923043","display_name":"Defense Technical Information Center (DTIC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"DTIC","raw_type":"Text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.129.2393","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.129.2393","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://web.engr.oregonstate.edu/~tgd/publications/IUI07-stumpf.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.86.140","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.86.140","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://web.engr.oregonstate.edu/~stumpf/pubs/iui10607f-stumpf.pdf","raw_type":"text"},{"id":"pmh:oai:eprints.gla.ac.uk:261430","is_oa":false,"landing_page_url":"https://eprints.gla.ac.uk/261430/","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":false,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Proceedings"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W131458779","https://openalex.org/W148103248","https://openalex.org/W1562482579","https://openalex.org/W1570448133","https://openalex.org/W1603920809","https://openalex.org/W1768482326","https://openalex.org/W1880262756","https://openalex.org/W1978372311","https://openalex.org/W1993680470","https://openalex.org/W1995445252","https://openalex.org/W2025018877","https://openalex.org/W2030676502","https://openalex.org/W2056655156","https://openalex.org/W2078786834","https://openalex.org/W2080301984","https://openalex.org/W2081580037","https://openalex.org/W2098162425","https://openalex.org/W2121906195","https://openalex.org/W2123512824","https://openalex.org/W2126185296","https://openalex.org/W2134584261","https://openalex.org/W2136548598","https://openalex.org/W2144742206","https://openalex.org/W2150662287","https://openalex.org/W2157289187","https://openalex.org/W2163979379","https://openalex.org/W2293768274","https://openalex.org/W2541066917","https://openalex.org/W2963146817","https://openalex.org/W2966207845","https://openalex.org/W6679102489"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W2130043461","https://openalex.org/W2530322880","https://openalex.org/W1596801655"],"abstract_inverted_index":{"There":[0],"has":[1,138],"been":[2],"little":[3],"research":[4],"into":[5],"how":[6,59],"end":[7],"users":[8,22,61,73,96],"might":[9],"be":[10,37,153,159],"able":[11,160],"to":[12,15,57,63,67,86,89,116,128,141,152,158,161],"communicate":[13],"advice":[14],"machine":[16,28,81,144],"learning":[17,29,34,82,130,145,149],"systems.":[18],"If":[19],"this":[20,163],"resource--the":[21],"themselves--could":[23],"somehow":[24],"work":[25],"hand-in-hand":[26],"with":[27],"systems,":[30,146],"the":[31,40,46,91,129,139],"accuracy":[32],"of":[33,45,71,80,103,107,114],"systems":[35],"could":[36,48,74],"improved":[38],"and":[39,43,66,84,125],"users'":[41],"understanding":[42],"trust":[44],"system":[47],"improve":[49,90,143],"as":[50],"well.":[51],"We":[52,93],"conducted":[53],"a":[54],"think-aloud":[55],"study":[56],"see":[58],"willing":[60],"were":[62,77],"provide":[64,87],"feedback":[65,72,88,108,137],"understand":[68],"what":[69],"kinds":[70,106],"give.":[75],"Users":[76],"shown":[78],"explanations":[79],"predictions":[83],"asked":[85],"predictions.":[92],"found":[94],"that":[95,135,148],"had":[97],"no":[98],"difficulty":[99],"providing":[100],"generous":[101],"amounts":[102],"feedback.":[104,164],"The":[105,132],"ranged":[109],"from":[110],"suggestions":[111],"for":[112,118],"reweighting":[113],"features":[115],"proposals":[117],"new":[119],"features,":[120,124],"feature":[121],"combinations,":[122],"relational":[123],"wholesale":[126],"changes":[127],"algorithm.":[131],"results":[133],"show":[134],"user":[136],"potential":[140],"significantly":[142],"but":[147],"algorithms":[150],"need":[151],"extended":[154],"in":[155],"several":[156],"ways":[157],"assimilate":[162]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":14},{"year":2019,"cited_by_count":12},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":8},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":2},{"year":2012,"cited_by_count":5}],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}