{"id":"https://openalex.org/W3111275476","doi":"https://doi.org/10.1109/tse.2021.3078384","title":"A Comparison of Natural Language Understanding Platforms for Chatbots in Software Engineering","display_name":"A Comparison of Natural Language Understanding Platforms for Chatbots in Software Engineering","publication_year":2021,"publication_date":"2021-05-08","ids":{"openalex":"https://openalex.org/W3111275476","doi":"https://doi.org/10.1109/tse.2021.3078384","mag":"3111275476"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2021.3078384","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2021.3078384","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2012.02640","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033095642","display_name":"Ahmad Abdellatif","orcid":"https://orcid.org/0000-0003-1863-9147"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ahmad Abdellatif","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1863-9147","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102847449","display_name":"Khaled Badran","orcid":"https://orcid.org/0000-0003-2293-3656"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Khaled Badran","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023951345","display_name":"Diego Elias Costa","orcid":"https://orcid.org/0000-0001-7084-2594"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Diego Elias Costa","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada"],"raw_orcid":"https://orcid.org/0000-0001-7084-2594","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049727493","display_name":"Emad Shihab","orcid":"https://orcid.org/0000-0003-1285-9878"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Emad Shihab","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1285-9878","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Data-Driven Analysis of Software (DAS) Lab, Concordia University, Montr&#x00E9;al, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I60158472"],"apc_list":null,"apc_paid":null,"fwci":21.4479,"has_fulltext":false,"cited_by_count":107,"citation_normalized_percentile":{"value":0.99453657,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"48","issue":"8","first_page":"3087","last_page":"3102"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatbot","display_name":"Chatbot","score":0.8154576420783997},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.768119215965271},{"id":"https://openalex.org/keywords/ibm","display_name":"IBM","score":0.5433678030967712},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.49884843826293945},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.48978662490844727},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.47742021083831787},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4518534243106842},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36477118730545044},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3511233329772949},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3438590168952942},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.34135305881500244},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3367277681827545},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11676636338233948},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10157862305641174}],"concepts":[{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.8154576420783997},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.768119215965271},{"id":"https://openalex.org/C70388272","wikidata":"https://www.wikidata.org/wiki/Q5968558","display_name":"IBM","level":2,"score":0.5433678030967712},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.49884843826293945},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.48978662490844727},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.47742021083831787},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4518534243106842},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36477118730545044},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3511233329772949},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3438590168952942},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.34135305881500244},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3367277681827545},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11676636338233948},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10157862305641174},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C171250308","wikidata":"https://www.wikidata.org/wiki/Q11468","display_name":"Nanotechnology","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tse.2021.3078384","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2021.3078384","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2012.02640","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2012.02640","pdf_url":"https://arxiv.org/pdf/2012.02640","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2012.02640","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2012.02640","pdf_url":"https://arxiv.org/pdf/2012.02640","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.44999998807907104,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":65,"referenced_works":["https://openalex.org/W1969939902","https://openalex.org/W1979290264","https://openalex.org/W1990334093","https://openalex.org/W2017581646","https://openalex.org/W2053154970","https://openalex.org/W2074134415","https://openalex.org/W2082468228","https://openalex.org/W2098046929","https://openalex.org/W2099769844","https://openalex.org/W2133012565","https://openalex.org/W2140190241","https://openalex.org/W2160715838","https://openalex.org/W2163004671","https://openalex.org/W2169818249","https://openalex.org/W2347054793","https://openalex.org/W2406204547","https://openalex.org/W2546705944","https://openalex.org/W2603990485","https://openalex.org/W2725479318","https://openalex.org/W2732909020","https://openalex.org/W2745156626","https://openalex.org/W2746791238","https://openalex.org/W2767532479","https://openalex.org/W2767782162","https://openalex.org/W2781728735","https://openalex.org/W2786860129","https://openalex.org/W2787096751","https://openalex.org/W2795206714","https://openalex.org/W2795319782","https://openalex.org/W2808438073","https://openalex.org/W2889215087","https://openalex.org/W2896121998","https://openalex.org/W2899117127","https://openalex.org/W2902255619","https://openalex.org/W2902287847","https://openalex.org/W2904954891","https://openalex.org/W2911420856","https://openalex.org/W2944997256","https://openalex.org/W2945826489","https://openalex.org/W2949945492","https://openalex.org/W2953543126","https://openalex.org/W2954554621","https://openalex.org/W2954736564","https://openalex.org/W2955449396","https://openalex.org/W2967531795","https://openalex.org/W3088041916","https://openalex.org/W3088195551","https://openalex.org/W3088643038","https://openalex.org/W3091658513","https://openalex.org/W3098348021","https://openalex.org/W3099538536","https://openalex.org/W3102474786","https://openalex.org/W4210997624","https://openalex.org/W4249731076","https://openalex.org/W4287552728","https://openalex.org/W4323317604","https://openalex.org/W4389891951","https://openalex.org/W6655010776","https://openalex.org/W6742643542","https://openalex.org/W6748584161","https://openalex.org/W6750034460","https://openalex.org/W6752710737","https://openalex.org/W6754382531","https://openalex.org/W6756856409","https://openalex.org/W6776350323"],"related_works":["https://openalex.org/W2750213978","https://openalex.org/W3200957432","https://openalex.org/W4212804285","https://openalex.org/W4223576922","https://openalex.org/W4390449165","https://openalex.org/W4386790411","https://openalex.org/W4289529888","https://openalex.org/W4397050657","https://openalex.org/W2955076648","https://openalex.org/W4288263119"],"abstract_inverted_index":{"Chatbots":[0],"are":[1],"envisioned":[2],"to":[3,13,46,57,101,156,176,272,280],"dramatically":[4],"change":[5],"the":[6,30,44,68,87,118,132,149,154,163,182,188,199,213,264],"future":[7],"of":[8,32,86,151,165],"Software":[9,72,111,145],"Engineering,":[10],"allowing":[11],"practitioners":[12,275],"chat":[14],"and":[15,21,98,127,195,257],"inquire":[16],"about":[17,159],"their":[18,283],"software":[19,160,273],"projects":[20],"interact":[22],"with":[23,153,215,229],"different":[24],"services":[25],"using":[26],"natural":[27,48],"language.":[28],"At":[29],"heart":[31],"every":[33],"chatbot":[34,45,155],"is":[35,181],"a":[36,230],"Natural":[37],"Language":[38],"Understanding":[39],"(NLU)":[40],"component":[41,63],"that":[42,138,241],"enables":[43],"understand":[47],"language":[49],"input.":[50],"Recently,":[51],"many":[52],"NLU":[53,62,70,106,185,279],"platforms":[54],"were":[55],"provided":[56],"serve":[58],"as":[59],"an":[60,76,216],"off-the-shelf":[61],"for":[64,71,245],"chatbots,":[65],"however,":[66],"selecting":[67],"best":[69,183,214],"Engineering":[73,112,146],"chatbots":[74],"remains":[75],"open":[77],"challenge.":[78],"Therefore,":[79],"in":[80,110,121,207,222,263,282],"this":[81],"paper,":[82],"we":[83,116,134],"evaluate":[84,131],"four":[85],"most":[88],"commonly":[89],"used":[90,109],"NLUs,":[91,133,243],"namely":[92],"IBM":[93,179,210,258],"Watson,":[94],"Google":[95],"Dialogflow,":[96,246],"Rasa,":[97],"Microsoft":[99,255],"LUIS":[100,256],"shed":[102],"light":[103],"on":[104,169,227],"which":[105,278],"should":[107],"be":[108],"based":[113],"chatbots.":[114,284],"Specifically,":[115],"examine":[117],"NLUs\u2019":[119],"performance":[120],"classifying":[122],"intents,":[123],"confidence":[124,193,223,232,250],"scores":[125],"stability,":[126],"extracting":[128],"entities.":[129],"To":[130],"use":[135,281],"two":[136,140,265],"datasets":[137],"reflect":[139],"common":[141],"tasks":[142],"performed":[143],"by":[144],"practitioners,":[147],"1)":[148],"task":[150,164],"chatting":[152],"ask":[157],"questions":[158,168],"repositories":[161],"2)":[162],"asking":[166],"development":[167],"Q&A":[170],"forums":[171],"(e.g.,":[172],"Stack":[173],"Overflow).":[174],"According":[175],"our":[177],"findings,":[178],"Watson":[180,211,259],"performing":[184],"when":[186,276],"considering":[187],"three":[189],"aspects":[190],"(intents":[191],"classification,":[192,209],"scores,":[194,224],"entity":[196,253],"extraction).":[197],"However,":[198],"results":[200,238,269],"from":[201],"each":[202],"individual":[203],"aspect":[204],"show":[205,240],"that,":[206],"intents":[208],"performs":[212],"F1-measure":[217],"<inline-formula><tex-math":[218],"notation=\"LaTeX\">$&gt;$</tex-math></inline-formula>":[219],"84%,":[220],"but":[221],"Rasa":[225],"comes":[226],"top":[228],"median":[231],"score":[233],"higher":[234],"than":[235],"0.91.":[236],"Our":[237,268],"also":[239],"all":[242],"except":[244],"generally":[247],"provide":[248,270],"trustable":[249],"scores.":[251],"For":[252],"extraction,":[254],"outperform":[260],"other":[261],"NLUs":[262],"SE":[266],"tasks.":[267],"guidance":[271],"engineering":[274],"deciding":[277]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":24},{"year":2024,"cited_by_count":23},{"year":2023,"cited_by_count":33},{"year":2022,"cited_by_count":16},{"year":2021,"cited_by_count":4}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2020-12-21T00:00:00"}