{"id":"https://openalex.org/W4388007964","doi":"https://doi.org/10.1145/3617694.3623239","title":"FeedbackLogs: Recording and Incorporating Stakeholder Feedback into Machine Learning Pipelines","display_name":"FeedbackLogs: Recording and Incorporating Stakeholder Feedback into Machine Learning Pipelines","publication_year":2023,"publication_date":"2023-10-29","ids":{"openalex":"https://openalex.org/W4388007964","doi":"https://doi.org/10.1145/3617694.3623239"},"language":"en","primary_location":{"id":"doi:10.1145/3617694.3623239","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3617694.3623239","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3617694.3623239","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Equity and Access in Algorithms, Mechanisms, and Optimization","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3617694.3623239","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082151748","display_name":"Matthew L Barker","orcid":"https://orcid.org/0000-0003-4910-1825"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Matthew Barker","raw_affiliation_strings":["University of Cambridge, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Cambridge, United Kingdom","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027539799","display_name":"Emma Kallina","orcid":"https://orcid.org/0000-0003-4912-7216"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emma Kallina","raw_affiliation_strings":["University of Cambridge, United Kingdom and Responsible AI Institute, USA"],"affiliations":[{"raw_affiliation_string":"University of Cambridge, United Kingdom and Responsible AI Institute, USA","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031475293","display_name":"Dhananjay Ashok","orcid":"https://orcid.org/0000-0001-7500-3730"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dhananjay Ashok","raw_affiliation_strings":["Carnegie Mellon University, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024828997","display_name":"Katherine M. Collins","orcid":"https://orcid.org/0000-0002-7032-716X"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Katherine Collins","raw_affiliation_strings":["University of Cambridge, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Cambridge, United Kingdom","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006816112","display_name":"Ashley Casovan","orcid":"https://orcid.org/0009-0001-4090-3461"},"institutions":[{"id":"https://openalex.org/I2799847335","display_name":"Art Institute of Portland","ror":"https://ror.org/01cb0jg64","country_code":"US","type":"education","lineage":["https://openalex.org/I2799847335","https://openalex.org/I2799969541"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ashley Casovan","raw_affiliation_strings":["Responsible AI Institute, USA"],"affiliations":[{"raw_affiliation_string":"Responsible AI Institute, USA","institution_ids":["https://openalex.org/I2799847335"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042278493","display_name":"Adrian Weller","orcid":"https://orcid.org/0000-0003-1915-7158"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210128584","display_name":"The Alan Turing Institute","ror":"https://ror.org/035dkdb55","country_code":"GB","type":"facility","lineage":["https://openalex.org/I4210128584"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Adrian Weller","raw_affiliation_strings":["University of Cambridge, United Kingdom and The Alan Turing Institute, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Cambridge, United Kingdom and The Alan Turing Institute, United Kingdom","institution_ids":["https://openalex.org/I4210128584","https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029768722","display_name":"Ameet Talwalkar","orcid":"https://orcid.org/0000-0001-6650-1893"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ameet Talwalkar","raw_affiliation_strings":["Carnegie Mellon University, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088847857","display_name":"Valerie Chen","orcid":"https://orcid.org/0009-0007-2783-0265"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Valerie Chen","raw_affiliation_strings":["Carnegie Mellon University, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016469734","display_name":"Umang Bhatt","orcid":"https://orcid.org/0000-0002-4611-1668"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210128584","display_name":"The Alan Turing Institute","ror":"https://ror.org/035dkdb55","country_code":"GB","type":"facility","lineage":["https://openalex.org/I4210128584"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Umang Bhatt","raw_affiliation_strings":["University of Cambridge, United Kingdom and The Alan Turing Institute, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Cambridge, United Kingdom and The Alan Turing Institute, United Kingdom","institution_ids":["https://openalex.org/I4210128584","https://openalex.org/I241749"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5082151748"],"corresponding_institution_ids":["https://openalex.org/I241749"],"apc_list":null,"apc_paid":null,"fwci":1.0332,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.81669929,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"15"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/documentation","display_name":"Documentation","score":0.8454858064651489},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.748903751373291},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7427288889884949},{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.7339366674423218},{"id":"https://openalex.org/keywords/stakeholder","display_name":"Stakeholder","score":0.6720727682113647},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6464255452156067},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.5260018110275269},{"id":"https://openalex.org/keywords/track","display_name":"Track (disk drive)","score":0.41706743836402893},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.16456115245819092},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.1028527021408081},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08433258533477783}],"concepts":[{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.8454858064651489},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.748903751373291},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7427288889884949},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.7339366674423218},{"id":"https://openalex.org/C201305675","wikidata":"https://www.wikidata.org/wiki/Q852998","display_name":"Stakeholder","level":2,"score":0.6720727682113647},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6464255452156067},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.5260018110275269},{"id":"https://openalex.org/C89992363","wikidata":"https://www.wikidata.org/wiki/Q5961558","display_name":"Track (disk drive)","level":2,"score":0.41706743836402893},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16456115245819092},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.1028527021408081},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08433258533477783},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3617694.3623239","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3617694.3623239","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3617694.3623239","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Equity and Access in Algorithms, Mechanisms, and Optimization","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3617694.3623239","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3617694.3623239","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3617694.3623239","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Equity and Access in Algorithms, Mechanisms, and Optimization","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1093520412","display_name":null,"funder_award_id":"Scholarship","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G1196033166","display_name":"FAIR: Framework for responsible adoption of Artificial Intelligence in the financial seRvices industry","funder_award_id":"EP/V056883/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G1361938442","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G136518982","display_name":null,"funder_award_id":"IIS1705121","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2126551001","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320319993","funder_display_name":"Leverhulme Trust"},{"id":"https://openalex.org/G3330119427","display_name":null,"funder_award_id":"EPSRC","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4740676945","display_name":null,"funder_award_id":"1705121","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6894402473","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6970565763","display_name":null,"funder_award_id":"EP/V056522/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G7196478567","display_name":"Advancing Probabilistic Machine Learning to Deliver Safer, More Efficient, and Predictable Air Traffic Control","funder_award_id":"EP/V056522/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G8233204778","display_name":"Turing AI Fellowship: Trustworthy Machine Learning","funder_award_id":"EP/V025279/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320317153","display_name":"DeepMind","ror":"https://ror.org/00971b260"},{"id":"https://openalex.org/F4320319993","display_name":"Leverhulme Trust","ror":"https://ror.org/012mzw131"},{"id":"https://openalex.org/F4320321848","display_name":"Cambridge Trust","ror":"https://ror.org/05sprqy15"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4388007964.pdf","grobid_xml":"https://content.openalex.org/works/W4388007964.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W1501005121","https://openalex.org/W1995502643","https://openalex.org/W2002454301","https://openalex.org/W2003238113","https://openalex.org/W2027313955","https://openalex.org/W2117733879","https://openalex.org/W2367397349","https://openalex.org/W2398344594","https://openalex.org/W2769041395","https://openalex.org/W2792417479","https://openalex.org/W2897042519","https://openalex.org/W2904095679","https://openalex.org/W2934636919","https://openalex.org/W2953923985","https://openalex.org/W2959716049","https://openalex.org/W2962059918","https://openalex.org/W2963290659","https://openalex.org/W2963466845","https://openalex.org/W2974071289","https://openalex.org/W3001807593","https://openalex.org/W3004553109","https://openalex.org/W3018911758","https://openalex.org/W3035403907","https://openalex.org/W3046786385","https://openalex.org/W3052825600","https://openalex.org/W3100279624","https://openalex.org/W3104128335","https://openalex.org/W3123497755","https://openalex.org/W3126632784","https://openalex.org/W3130234976","https://openalex.org/W3144277077","https://openalex.org/W3158479996","https://openalex.org/W3160037564","https://openalex.org/W3160920566","https://openalex.org/W3163696583","https://openalex.org/W3183398589","https://openalex.org/W3185334749","https://openalex.org/W3190995216","https://openalex.org/W3191922428","https://openalex.org/W3198675127","https://openalex.org/W3212368439","https://openalex.org/W4221143046","https://openalex.org/W4226278401","https://openalex.org/W4283166427","https://openalex.org/W4283168787","https://openalex.org/W4287991409","https://openalex.org/W4288058238","https://openalex.org/W4288083797","https://openalex.org/W4288414189","https://openalex.org/W4306313145","https://openalex.org/W4309618895","https://openalex.org/W4315881234"],"related_works":["https://openalex.org/W4380433113","https://openalex.org/W4386072068","https://openalex.org/W252339960","https://openalex.org/W2390529043","https://openalex.org/W2378320433","https://openalex.org/W2358343511","https://openalex.org/W2051877971","https://openalex.org/W1970117064","https://openalex.org/W1787170397","https://openalex.org/W4292347844"],"abstract_inverted_index":{"As":[0],"machine":[1],"learning":[2],"(ML)":[3],"pipelines":[4],"affect":[5],"an":[6],"increasing":[7],"array":[8],"of":[9,33,40],"stakeholders,":[10],"there":[11],"is":[12,22,60],"a":[13,74,78,98],"growing":[14],"need":[15],"for":[16,76,93],"documenting":[17],"how":[18,57],"input":[19,39],"from":[20],"stakeholders":[21],"recorded":[23],"and":[24,56,72,96],"incorporated.":[25],"We":[26,80],"propose":[27],"FeedbackLogs,":[28],"addenda":[29],"to":[30,36,62,100],"existing":[31],"documentation":[32],"ML":[34,65],"pipelines,":[35],"track":[37],"the":[38,49,53,58,64],"multiple":[41],"stakeholders.":[42],"Each":[43],"log":[44],"records":[45],"important":[46],"details":[47],"about":[48],"feedback":[50,54,59],"collection":[51],"process,":[52],"itself,":[55],"used":[61],"update":[63],"pipeline.":[66],"In":[67],"this":[68],"paper,":[69],"we":[70],"introduce":[71],"formalise":[73],"process":[75],"collecting":[77],"FeedbackLog.":[79],"also":[81],"provide":[82],"concrete":[83],"use":[84],"cases":[85],"where":[86],"FeedbackLogs":[87],"can":[88],"be":[89],"employed":[90],"as":[91,97],"evidence":[92],"algorithmic":[94],"auditing":[95],"tool":[99],"record":[101],"updates":[102],"based":[103],"on":[104],"stakeholder":[105],"feedback.":[106]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}