{"id":"https://openalex.org/W4393153298","doi":"https://doi.org/10.1609/aaai.v38i9.28878","title":"When to Show a Suggestion? Integrating Human Feedback in AI-Assisted Programming","display_name":"When to Show a Suggestion? Integrating Human Feedback in AI-Assisted Programming","publication_year":2024,"publication_date":"2024-03-24","ids":{"openalex":"https://openalex.org/W4393153298","doi":"https://doi.org/10.1609/aaai.v38i9.28878"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v38i9.28878","doi":"https://doi.org/10.1609/aaai.v38i9.28878","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i9.28878","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/28878/29669","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/28878/29669","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004148354","display_name":"Hussein Mozannar","orcid":"https://orcid.org/0009-0001-9660-9878","observed_orcids":["https://orcid.org/0009-0001-9660-9878"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hussein Mozannar","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Gagan Bansal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gagan Bansal","raw_affiliation_strings":["MSR","Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MSR","institution_ids":[]},{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071293344","display_name":"Adam Fourney","orcid":"https://orcid.org/0000-0002-4986-7794","observed_orcids":["https://orcid.org/0000-0002-4986-7794"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adam Fourney","raw_affiliation_strings":["MSR","Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MSR","institution_ids":[]},{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043228682","display_name":"Eric Horvitz","orcid":"https://orcid.org/0000-0002-8823-0614","observed_orcids":["https://orcid.org/0000-0002-8823-0614"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eric Horvitz","raw_affiliation_strings":["MSR","Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MSR","institution_ids":[]},{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.8098,"has_fulltext":true,"cited_by_count":33,"citation_normalized_percentile":{"value":0.97988015,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":"38","issue":"9","first_page":"10137","last_page":"10144"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0015999999595806003,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-assisted-programming","display_name":"AI-assisted programming","score":0.9409999847412109},{"id":"https://openalex.org/keywords/code-recommendation","display_name":"code recommendation","score":0.8090000152587891},{"id":"https://openalex.org/keywords/human-feedback","display_name":"human feedback","score":0.8339999914169312},{"id":"https://openalex.org/keywords/github-copilot","display_name":"GitHub Copilot","score":0.8040000200271606},{"id":"https://openalex.org/keywords/developer-productivity","display_name":"developer productivity","score":0.6830000281333923},{"id":"https://openalex.org/keywords/reward-signals","display_name":"reward signals","score":0.531000018119812}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5830503702163696},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39573174715042114},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3607720732688904},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3327528238296509},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3276204466819763},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3263995349407196}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v38i9.28878","doi":"https://doi.org/10.1609/aaai.v38i9.28878","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i9.28878","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/28878/29669","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2306.04930","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.04930","pdf_url":"https://arxiv.org/pdf/2306.04930","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/28878","doi":null,"is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/28878","pdf_url":null,"source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.2306.04930","doi":"https://doi.org/10.48550/arxiv.2306.04930","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2306.04930","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v38i9.28878","doi":"https://doi.org/10.1609/aaai.v38i9.28878","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v38i9.28878","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/28878/29669","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393153298.pdf","grobid_xml":"https://content.openalex.org/works/W4393153298.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W24931679","https://openalex.org/W1778770750","https://openalex.org/W2059216172","https://openalex.org/W2110064869","https://openalex.org/W2130259926","https://openalex.org/W2254249950","https://openalex.org/W2403250881","https://openalex.org/W2478911292","https://openalex.org/W2580300496","https://openalex.org/W2973379954","https://openalex.org/W2984353433","https://openalex.org/W3008088841","https://openalex.org/W3033575144","https://openalex.org/W3098605233","https://openalex.org/W3177813494","https://openalex.org/W4223908421","https://openalex.org/W4226161957","https://openalex.org/W4226287673","https://openalex.org/W4235776839","https://openalex.org/W4281669078","https://openalex.org/W4283799808","https://openalex.org/W4302797994","https://openalex.org/W4367672983","https://openalex.org/W4375859932","https://openalex.org/W4396833115","https://openalex.org/W4399647672","https://openalex.org/W4401635661","https://openalex.org/W6637875753","https://openalex.org/W7133208539"],"related_works":["https://openalex.org/W7166662038","https://openalex.org/W7126254537","https://openalex.org/W4396833115","https://openalex.org/W4407185809","https://openalex.org/W4312438588","https://openalex.org/W4383888921","https://openalex.org/W4406866173","https://openalex.org/W7212260335","https://openalex.org/W7124994603","https://openalex.org/W4415156714"],"abstract_inverted_index":{"AI":[0],"powered":[1],"code-recommendation":[2],"systems,":[3],"such":[4],"as":[5,180],"Copilot":[6],"and":[7,34,117,133],"CodeWhisperer,":[8],"provide":[9,97],"code":[10,37,102],"suggestions":[11,38,76,144,168,189,193],"inside":[12],"a":[13,51,69,92,128,140,181],"programmer's":[14,158],"environment":[15],"(e.g.,":[16],"an":[17,170,198],"IDE)":[18],"with":[19,48],"the":[20,98,153,157,186],"aim":[21],"of":[22,36,56,94,131,143,155,188,194],"improving":[23],"productivity.":[24],"We":[25,42,67,150],"pursue":[26],"mechanisms":[27],"for":[28,65,184],"leveraging":[29],"signals":[30],"about":[31,75,164],"programmers'":[32],"acceptance":[33,179],"rejection":[35],"to":[39,58,72,77,110,166,192],"guide":[40],"recommendations.":[41],"harness":[43],"data":[44,122],"drawn":[45],"from":[46,86,123],"interactions":[47],"GitHub":[49],"Copilot,":[50],"system":[52],"used":[53,109],"by":[54],"millions":[55],"programmers,":[57,125],"develop":[59],"interventions":[60],"that":[61,96,100,135,145],"can":[62,137,190],"save":[63],"time":[64],"programmers.":[66],"introduce":[68],"utility-theoretic":[70],"framework":[71],"drive":[73],"decisions":[74,163],"display":[78,85,167,187],"versus":[79],"withhold.":[80],"The":[81],"approach,":[82],"conditional":[83],"suggestion":[84,178],"human":[87],"feedback":[88],"(CDHF),":[89],"relies":[90],"on":[91],"cascade":[93],"models":[95],"likelihood":[99],"recommended":[101],"will":[103],"be":[104],"accepted.":[105],"These":[106],"likelihoods":[107],"are":[108],"selectively":[111],"hide":[112],"suggestions,":[113],"reducing":[114],"both":[115],"latency":[116],"programmer":[118],"verification":[119],"time.":[120],"Using":[121],"535":[124],"we":[126,136,174],"perform":[127],"retrospective":[129],"evaluation":[130],"CDHF":[132],"show":[134],"avoid":[138],"displaying":[139],"significant":[141],"fraction":[142],"would":[146],"have":[147],"been":[148],"rejected.":[149],"further":[151],"demonstrate":[152],"importance":[154],"incorporating":[156],"latent":[159],"unobserved":[160],"state":[161],"in":[162],"when":[165],"through":[169],"ablation":[171],"study.":[172],"Finally,":[173],"showcase":[175],"how":[176],"using":[177],"reward":[182],"signal":[183],"guiding":[185],"lead":[191],"reduced":[195],"quality,":[196],"indicating":[197],"unexpected":[199],"pitfall.":[200]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-03-26T00:00:00"}