{"id":"https://openalex.org/W1999911911","doi":"https://doi.org/10.1145/2702613.2732851","title":"Evaluation for Evaluation","display_name":"Evaluation for Evaluation","publication_year":2015,"publication_date":"2015-04-17","ids":{"openalex":"https://openalex.org/W1999911911","doi":"https://doi.org/10.1145/2702613.2732851","mag":"1999911911"},"language":"en","primary_location":{"id":"doi:10.1145/2702613.2732851","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2702613.2732851","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd Annual ACM Conference Extended Abstracts on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.utupub.fi/bitstream/10024/156155/1/Evaluation_for_evaluation_copy.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047518655","display_name":"Kimmo Tarkkanen","orcid":"https://orcid.org/0000-0002-3576-2780"},"institutions":[{"id":"https://openalex.org/I155660961","display_name":"University of Turku","ror":"https://ror.org/05vghhr25","country_code":"FI","type":"education","lineage":["https://openalex.org/I155660961"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Kimmo Tarkkanen","raw_affiliation_strings":["University of Turku, Turku, Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Turku, Turku, Finland","institution_ids":["https://openalex.org/I155660961"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013070991","display_name":"Ville Harkke","orcid":"https://orcid.org/0000-0003-3743-2686"},"institutions":[{"id":"https://openalex.org/I155660961","display_name":"University of Turku","ror":"https://ror.org/05vghhr25","country_code":"FI","type":"education","lineage":["https://openalex.org/I155660961"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Ville Harkke","raw_affiliation_strings":["University of Turku, Turku, Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Turku, Turku, Finland","institution_ids":["https://openalex.org/I155660961"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I155660961"],"apc_list":null,"apc_paid":null,"fwci":0.9433,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.69984488,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"2289","last_page":"2294"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11572","display_name":"Information Technology Governance and Strategy","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11572","display_name":"Information Technology Governance and Strategy","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11938","display_name":"ERP Systems Implementation and Impact","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11912","display_name":"Outsourcing and Supply Chain Management","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.9390026330947876},{"id":"https://openalex.org/keywords/vendor","display_name":"Vendor","score":0.7379703521728516},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7071138620376587},{"id":"https://openalex.org/keywords/system-usability-scale","display_name":"System usability scale","score":0.6933028101921082},{"id":"https://openalex.org/keywords/usability-goals","display_name":"Usability goals","score":0.6331836581230164},{"id":"https://openalex.org/keywords/procurement","display_name":"Procurement","score":0.6204343438148499},{"id":"https://openalex.org/keywords/cognitive-walkthrough","display_name":"Cognitive walkthrough","score":0.5398629903793335},{"id":"https://openalex.org/keywords/usability-inspection","display_name":"Usability inspection","score":0.5316441655158997},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5298713445663452},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.5259501934051514},{"id":"https://openalex.org/keywords/heuristic-evaluation","display_name":"Heuristic evaluation","score":0.4670601785182953},{"id":"https://openalex.org/keywords/web-usability","display_name":"Web usability","score":0.4602481424808502},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.45417237281799316},{"id":"https://openalex.org/keywords/usability-engineering","display_name":"Usability engineering","score":0.4491522014141083},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.42039602994918823},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.34650176763534546},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.30326008796691895},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.2179585099220276},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.19420090317726135},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.13809382915496826}],"concepts":[{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.9390026330947876},{"id":"https://openalex.org/C2777338717","wikidata":"https://www.wikidata.org/wiki/Q1762621","display_name":"Vendor","level":2,"score":0.7379703521728516},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7071138620376587},{"id":"https://openalex.org/C139225968","wikidata":"https://www.wikidata.org/wiki/Q17146354","display_name":"System usability scale","level":4,"score":0.6933028101921082},{"id":"https://openalex.org/C62993174","wikidata":"https://www.wikidata.org/wiki/Q2928808","display_name":"Usability goals","level":4,"score":0.6331836581230164},{"id":"https://openalex.org/C201650216","wikidata":"https://www.wikidata.org/wiki/Q829492","display_name":"Procurement","level":2,"score":0.6204343438148499},{"id":"https://openalex.org/C87105883","wikidata":"https://www.wikidata.org/wiki/Q1107002","display_name":"Cognitive walkthrough","level":4,"score":0.5398629903793335},{"id":"https://openalex.org/C23456302","wikidata":"https://www.wikidata.org/wiki/Q7901668","display_name":"Usability inspection","level":4,"score":0.5316441655158997},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5298713445663452},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.5259501934051514},{"id":"https://openalex.org/C3255780","wikidata":"https://www.wikidata.org/wiki/Q1616517","display_name":"Heuristic evaluation","level":3,"score":0.4670601785182953},{"id":"https://openalex.org/C4237393","wikidata":"https://www.wikidata.org/wiki/Q1636686","display_name":"Web usability","level":3,"score":0.4602481424808502},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.45417237281799316},{"id":"https://openalex.org/C100302975","wikidata":"https://www.wikidata.org/wiki/Q1642623","display_name":"Usability engineering","level":3,"score":0.4491522014141083},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.42039602994918823},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.34650176763534546},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.30326008796691895},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2179585099220276},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.19420090317726135},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.13809382915496826},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2702613.2732851","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2702613.2732851","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd Annual ACM Conference Extended Abstracts on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:www.utupub.fi:10024/156155","is_oa":true,"landing_page_url":"https://www.utupub.fi/handle/10024/156155","pdf_url":"https://www.utupub.fi/bitstream/10024/156155/1/Evaluation_for_evaluation_copy.pdf","source":{"id":"https://openalex.org/S4306402470","display_name":"UTUPub (University of Turku)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I155660961","host_organization_name":"University of Turku","host_organization_lineage":["https://openalex.org/I155660961"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:www.utupub.fi:10024/156155","is_oa":true,"landing_page_url":"https://www.utupub.fi/handle/10024/156155","pdf_url":"https://www.utupub.fi/bitstream/10024/156155/1/Evaluation_for_evaluation_copy.pdf","source":{"id":"https://openalex.org/S4306402470","display_name":"UTUPub (University of Turku)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I155660961","host_organization_name":"University of Turku","host_organization_lineage":["https://openalex.org/I155660961"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"sustainable_development_goals":[{"score":0.41999998688697815,"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1999911911.pdf","grobid_xml":"https://content.openalex.org/works/W1999911911.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W34094619","https://openalex.org/W627719897","https://openalex.org/W1765266268","https://openalex.org/W2042709572","https://openalex.org/W2085532840","https://openalex.org/W2115121100","https://openalex.org/W2116203860","https://openalex.org/W2129226033","https://openalex.org/W2395785659","https://openalex.org/W2536585477","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2150599767","https://openalex.org/W2155602934","https://openalex.org/W2016022053","https://openalex.org/W2085749886","https://openalex.org/W147331940","https://openalex.org/W2772000347","https://openalex.org/W2130660484","https://openalex.org/W2075732570","https://openalex.org/W4206743060","https://openalex.org/W2580784735"],"abstract_inverted_index":{"In":[0,21],"large-scale":[1,108],"IT":[2,37,81,109],"implementations":[3],"the":[4,22,55,65,73,76,80,93,101],"iterative":[5],"system":[6,9],"development":[7],"and":[8,33,40,42,88,122],"tailoring":[10],"to":[11,71,117],"organization-specific":[12],"needs":[13],"begins":[14],"after":[15],"a":[16,25],"tendering":[17,23,94],"process":[18,24,95],"is":[19],"over.":[20],"procuring":[26,56],"organization":[27],"defines":[28],"usability":[29,51,60,74,90,120],"requirements,":[30],"evaluation":[31],"procedures":[32],"selection":[34],"criteria,":[35],"while":[36],"vendors":[38,118],"try":[39],"build":[41],"propose":[43],"solutions":[44],"that":[45],"best":[46],"fulfill":[47],"these":[48],"requirements.":[49],"Existing":[50],"research":[52],"has":[53],"instructed":[54],"organizations":[57],"on":[58,84,107],"how":[59],"should":[61],"be":[62],"included":[63],"in":[64,69],"request":[66,85,125],"for":[67,86,126],"proposals,":[68],"order":[70],"ensure":[72],"of":[75],"selected":[77],"system.":[78],"However,":[79],"vendor":[82],"perspective":[83],"proposals":[87],"related":[89],"work":[91,121],"during":[92],"remain":[96],"unexplored.":[97],"This":[98],"paper":[99],"grasps":[100],"issue":[102],"with":[103],"an":[104],"empirical":[105],"study":[106],"procurement.":[110],"The":[111],"preliminary":[112],"findings":[113],"give":[114],"practical":[115],"advice":[116],"planning":[119],"buyers":[123],"writing":[124],"proposals.":[127]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2016,"cited_by_count":2}],"updated_date":"2026-08-14T07:06:38.338062","created_date":"2025-10-10T00:00:00"}