{"id":"https://openalex.org/W7163367737","doi":"https://doi.org/10.48550/arxiv.2606.03331","title":"Evaluating LLMs' Effectiveness on Real-World Consumer Device Repair Questions","display_name":"Evaluating LLMs' Effectiveness on Real-World Consumer Device Repair Questions","publication_year":2026,"publication_date":"2026-06-02","ids":{"openalex":"https://openalex.org/W7163367737","doi":"https://doi.org/10.48550/arxiv.2606.03331"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.03331","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03331","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.03331","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035402766","display_name":"Atm Mizanur Rahman","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rahman, Atm Mizanur","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137738498","display_name":"Md Arid Hasan","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Hasan, Md Arid","raw_affiliation_strings":["University of Toronto"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137751121","display_name":"Syed Ishtiaque Ahmed","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ahmed, Syed Ishtiaque","raw_affiliation_strings":["University of Toronto"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103026409","display_name":"Sharifa Sultana","orcid":"https://orcid.org/0000-0003-2906-7391"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sultana, Sharifa","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1720000058412552,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1720000058412552,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.12720000743865967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.08079999685287476,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.541100025177002},{"id":"https://openalex.org/keywords/mobile-phone","display_name":"Mobile phone","score":0.375900000333786},{"id":"https://openalex.org/keywords/bengali","display_name":"Bengali","score":0.37529999017715454},{"id":"https://openalex.org/keywords/phone","display_name":"Phone","score":0.36320000886917114},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.3497999906539917}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5703999996185303},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.541100025177002},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.40310001373291016},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3935000002384186},{"id":"https://openalex.org/C2777421447","wikidata":"https://www.wikidata.org/wiki/Q17517","display_name":"Mobile phone","level":2,"score":0.375900000333786},{"id":"https://openalex.org/C19235068","wikidata":"https://www.wikidata.org/wiki/Q9610","display_name":"Bengali","level":2,"score":0.37529999017715454},{"id":"https://openalex.org/C2778707766","wikidata":"https://www.wikidata.org/wiki/Q202064","display_name":"Phone","level":2,"score":0.36320000886917114},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.3497999906539917},{"id":"https://openalex.org/C186967261","wikidata":"https://www.wikidata.org/wiki/Q5082128","display_name":"Mobile device","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3167000114917755},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2906000018119812},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2574999928474426},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.03331","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03331","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.03331","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.03331","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5921527147293091,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Consumer":[0],"device":[1,34],"repair":[2,49,102,110,120,137],"is":[3,121],"an":[4],"important":[5],"but":[6],"underexplored":[7],"testbed":[8],"for":[9,107],"large":[10],"language":[11],"models":[12,130],"(LLMs).":[13],"Repair":[14],"tasks":[15,111],"require":[16],"reasoning":[17],"over":[18],"incomplete":[19],"problem":[20],"descriptions,":[21],"hardware-specific":[22],"diagnostics,":[23],"actionable":[24],"troubleshooting,":[25],"and":[26,58,67,82,91,115,125,128,139,145],"safety-critical":[27],"decisions,":[28],"where":[29],"incorrect":[30],"advice":[31],"can":[32,99],"cause":[33],"damage,":[35],"battery":[36],"hazards,":[37],"or":[38],"permanent":[39],"data":[40,59],"loss.":[41],"We":[42,75],"introduce":[43],"a":[44],"benchmark":[45],"of":[46],"991":[47],"real-world":[48,109],"questions":[50],"from":[51],"Reddit":[52],"spanning":[53],"phone":[54],"repair,":[55,57],"computer":[56],"recovery,":[60],"each":[61],"paired":[62],"with":[63],"technician-written":[64],"reference":[65],"solutions,":[66],"provide":[68,100],"Bangla":[69,83,147],"translations":[70],"to":[71],"evaluate":[72,76],"cross-lingual":[73],"performance.":[74],"six":[77],"state-of-the-art":[78],"LLMs":[79,98],"in":[80,134],"English":[81,153],"using":[84],"four":[85],"repair-specific":[86],"criteria:":[87],"correctness,":[88],"completeness,":[89],"practicality,":[90],"safety.":[92],"Our":[93],"results":[94],"show":[95],"that":[96],"while":[97],"useful":[101],"assistance,":[103],"they":[104],"remain":[105],"unreliable":[106],"high-risk":[108],"without":[112],"rigorous":[113],"evaluation":[114],"explicit":[116],"safety":[117],"safeguards.":[118],"Phone":[119],"the":[122,156],"most":[123],"difficult":[124],"safety-sensitive":[126],"domain,":[127],"all":[129],"make":[131],"substantial":[132],"errors":[133],"board-level":[135],"diagnosis,":[136],"prioritization,":[138],"safe":[140],"recovery":[141],"procedures.":[142],"Across":[143],"domains":[144],"models,":[146,158],"responses":[148],"consistently":[149],"perform":[150],"worse":[151],"than":[152],"responses.":[154],"Among":[155],"evaluated":[157],"GPT-5.4":[159],"performs":[160],"best":[161],"overall.":[162]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-04T00:00:00"}