{"id":"https://openalex.org/W4417447608","doi":"https://doi.org/10.1016/j.jss.2026.112915","title":"Benchmarking contextual understanding for in-car conversational systems","display_name":"Benchmarking contextual understanding for in-car conversational systems","publication_year":2026,"publication_date":"2026-05-05","ids":{"openalex":"https://openalex.org/W4417447608","doi":"https://doi.org/10.1016/j.jss.2026.112915"},"language":"en","primary_location":{"id":"doi:10.1016/j.jss.2026.112915","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2026.112915","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.jss.2026.112915","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120839879","display_name":"Philipp Habicht","orcid":null},"institutions":[{"id":"https://openalex.org/I1283382300","display_name":"BMW (Germany)","ror":"https://ror.org/05vs9tj88","country_code":"DE","type":"company","lineage":["https://openalex.org/I1283382300","https://openalex.org/I4210156768"]},{"id":"https://openalex.org/I192389796","display_name":"Cal Poly Humboldt","ror":"https://ror.org/02qt0xs84","country_code":"US","type":"education","lineage":["https://openalex.org/I192389796"]},{"id":"https://openalex.org/I39343248","display_name":"Humboldt-Universit\u00e4t zu Berlin","ror":"https://ror.org/01hcx6992","country_code":"DE","type":"education","lineage":["https://openalex.org/I39343248"]},{"id":"https://openalex.org/I4210156768","display_name":"BMW Group (Germany)","ror":"https://ror.org/044kkbh92","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210156768"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Philipp Habicht","raw_affiliation_strings":["BMW Group, Petuelring 130, Munich, 80809, Germany","Humboldt University of Berlin, Unter den Linden 6, Berlin, 10099, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"BMW Group, Petuelring 130, Munich, 80809, Germany","institution_ids":["https://openalex.org/I1283382300","https://openalex.org/I4210156768"]},{"raw_affiliation_string":"Humboldt University of Berlin, Unter den Linden 6, Berlin, 10099, Germany","institution_ids":["https://openalex.org/I39343248","https://openalex.org/I192389796"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003444569","display_name":"Lev Sorokin","orcid":"https://orcid.org/0009-0003-1162-6252"},"institutions":[{"id":"https://openalex.org/I1283382300","display_name":"BMW (Germany)","ror":"https://ror.org/05vs9tj88","country_code":"DE","type":"company","lineage":["https://openalex.org/I1283382300","https://openalex.org/I4210156768"]},{"id":"https://openalex.org/I4210156768","display_name":"BMW Group (Germany)","ror":"https://ror.org/044kkbh92","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210156768"]},{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Lev Sorokin","raw_affiliation_strings":["BMW Group, Petuelring 130, Munich, 80809, Germany","Technical University of Munich, Boltzmannstra\u00dfe 3, Garching, 85748, Germany"],"raw_orcid":"https://orcid.org/0009-0003-1162-6252","affiliations":[{"raw_affiliation_string":"BMW Group, Petuelring 130, Munich, 80809, Germany","institution_ids":["https://openalex.org/I1283382300","https://openalex.org/I4210156768"]},{"raw_affiliation_string":"Technical University of Munich, Boltzmannstra\u00dfe 3, Garching, 85748, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120839880","display_name":"Abdullah Saydemir","orcid":null},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Abdullah Saydemir","raw_affiliation_strings":["Technical University of Munich, Boltzmannstra\u00dfe 3, Garching, 85748, Germany"],"raw_orcid":"https://orcid.org/0000-0003-4185-3504","affiliations":[{"raw_affiliation_string":"Technical University of Munich, Boltzmannstra\u00dfe 3, Garching, 85748, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082902799","display_name":"Ken E. Friedl","orcid":null},"institutions":[{"id":"https://openalex.org/I1283382300","display_name":"BMW (Germany)","ror":"https://ror.org/05vs9tj88","country_code":"DE","type":"company","lineage":["https://openalex.org/I1283382300","https://openalex.org/I4210156768"]},{"id":"https://openalex.org/I4210156768","display_name":"BMW Group (Germany)","ror":"https://ror.org/044kkbh92","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210156768"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ken Friedl","raw_affiliation_strings":["BMW Group, Petuelring 130, Munich, 80809, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"BMW Group, Petuelring 130, Munich, 80809, Germany","institution_ids":["https://openalex.org/I1283382300","https://openalex.org/I4210156768"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058345649","display_name":"Andrea Stocco","orcid":"https://orcid.org/0000-0001-8956-3894"},"institutions":[{"id":"https://openalex.org/I4210134795","display_name":"Fortiss","ror":"https://ror.org/042x1tp92","country_code":"DE","type":"nonprofit","lineage":["https://openalex.org/I4210134795"]},{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Andrea Stocco","raw_affiliation_strings":["Technical University of Munich, Boltzmannstra\u00dfe 3, Garching, 85748, Germany","fortiss GmbH, Guerickestra\u00dfe 21, Munich, 80805, Germany"],"raw_orcid":"https://orcid.org/0000-0001-8956-3894","affiliations":[{"raw_affiliation_string":"Technical University of Munich, Boltzmannstra\u00dfe 3, Garching, 85748, Germany","institution_ids":["https://openalex.org/I62916508"]},{"raw_affiliation_string":"fortiss GmbH, Guerickestra\u00dfe 21, Munich, 80805, Germany","institution_ids":["https://openalex.org/I4210134795"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5003444569"],"corresponding_institution_ids":["https://openalex.org/I1283382300","https://openalex.org/I4210156768","https://openalex.org/I62916508"],"apc_list":{"value":3560,"currency":"USD","value_usd":3560},"apc_paid":{"value":3560,"currency":"USD","value_usd":3560},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.01476859,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"240","issue":null,"first_page":"112915","last_page":"112915"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3756999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3756999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.1712999939918518,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.13740000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8865000009536743},{"id":"https://openalex.org/keywords/situational-ethics","display_name":"Situational ethics","score":0.5551000237464905},{"id":"https://openalex.org/keywords/coherence","display_name":"Coherence (philosophical gambling strategy)","score":0.4830999970436096},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4668000042438507},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4253999888896942},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.420199990272522},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.41040000319480896},{"id":"https://openalex.org/keywords/user-modeling","display_name":"User modeling","score":0.39739999175071716}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8865000009536743},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7612000107765198},{"id":"https://openalex.org/C9114305","wikidata":"https://www.wikidata.org/wiki/Q1428317","display_name":"Situational ethics","level":2,"score":0.5551000237464905},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.4830999970436096},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4699999988079071},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4683000147342682},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4668000042438507},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4253999888896942},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.420199990272522},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.41040000319480896},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.39739999175071716},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.3407000005245209},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.32339999079704285},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3174000084400177},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.31439998745918274},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3100000023841858},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.2964000105857849},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.29179999232292175},{"id":"https://openalex.org/C14911803","wikidata":"https://www.wikidata.org/wiki/Q7532148","display_name":"Situation analysis","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C145804949","wikidata":"https://www.wikidata.org/wiki/Q478123","display_name":"Situation awareness","level":2,"score":0.2775000035762787},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.2759000062942505},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C201025465","wikidata":"https://www.wikidata.org/wiki/Q11248500","display_name":"User experience design","level":2,"score":0.26510000228881836}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1016/j.jss.2026.112915","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2026.112915","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2512.12042","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2512.12042","pdf_url":"https://arxiv.org/pdf/2512.12042","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2512.12042","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2512.12042","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.1016/j.jss.2026.112915","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2026.112915","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In-Car":[0],"Conversational":[1],"Question":[2],"Answering":[3],"(ConvQA)":[4],"systems":[5],"significantly":[6,170],"enhance":[7],"user":[8,52,69,85,140],"experience":[9],"by":[10,88,172],"enabling":[11],"seamless":[12],"voice":[13],"interactions.":[14],"However,":[15,166],"assessing":[16],"their":[17],"accuracy":[18],"and":[19,38,71,113,120,127,210,229],"reliability":[20],"remains":[21],"a":[22,134,139,197,227],"challenge.":[23],"This":[24],"paper":[25],"explores":[26],"the":[27,43,60,68,76,176,187,191,205,216],"use":[28,97],"of":[29,100,194,199],"Large":[30],"Language":[31],"Models":[32],"(LLMs)":[33],"alongside":[34],"advanced":[35,157],"prompting":[36,108,158,183],"techniques":[37],"agent-based":[39],"methods":[40],"to":[41,45,51,62,232],"evaluate":[42,75,130],"extent":[44],"which":[46],"ConvQA":[47,241],"system":[48,94],"responses":[49],"adhere":[50],"utterances.":[53],"The":[54,145],"focus":[55],"lies":[56],"on":[57,133],"contextual":[58,238],"understanding,":[59],"ability":[61],"provide":[63],"accurate":[64,230],"venue":[65],"recommendations":[66],"considering":[67],"constraints":[70],"situational":[72],"context.":[73],"To":[74],"utterance/response":[77],"coherence":[78],"using":[79],"an":[80],"LLM,":[81],"we":[82],"synthetically":[83],"generate":[84],"utterances":[86],"accompanied":[87],"correct":[89],"but":[90],"also":[91],"modified":[92],"failure-containing":[93],"responses.":[95],"We":[96,129],"input\u2013output,":[98],"chain":[99],"thought,":[101],"self-consistency":[102],"prompting,":[103],"as":[104,106],"well":[105],"multi-agent":[107,164],"techniques,":[109,159],"with":[110,181,215],"13":[111],"reasoning":[112,173],"non-reasoning":[114,153,167,217],"LLMs,":[115],"varying":[116],"in":[117,160,240],"model":[118,189,218],"size":[119],"providers,":[121],"from":[122],"OpenAI,":[123],"DeepSeek,":[124],"Mistral":[125],"AI,":[126],"Meta.":[128],"our":[131],"approach":[132],"case":[135],"study":[136],"that":[137,223],"involves":[138],"asking":[141],"for":[142,151,236],"restaurant":[143],"recommendations.":[144],"most":[146],"substantial":[147],"improvements":[148],"are":[149,169],"observed":[150],"small":[152],"models":[154,168],"when":[155,162],"applying":[156,163],"particular,":[161],"prompting.":[165],"surpassed":[171],"models,":[174],"where":[175],"best":[177,206],"result":[178],"is":[179,213],"achieved":[180,214],"single-agent":[182],"incorporating":[184],"self-consistency.":[185],"Notably,":[186],"DeepSeek-R1":[188],"achieves":[190],"highest":[192],"F1-score":[193],"0.99":[195],"at":[196],"cost":[198],"0.002":[200],"USD":[201],"per":[202],"request.":[203],"Overall,":[204],"tradeoff":[207],"between":[208],"effectiveness":[209],"cost/time":[211],"efficiency":[212],"DeepSeek-V3.":[219],"Our":[220],"results":[221],"demonstrate":[222],"LLM-based":[224],"evaluations":[225,235],"offer":[226],"scalable":[228],"alternative":[231],"traditional":[233],"human-based":[234],"benchmarking":[237],"understanding":[239],"systems.":[242]},"counts_by_year":[],"updated_date":"2026-05-14T06:09:40.864956","created_date":"2025-12-17T00:00:00"}