{"id":"https://openalex.org/W2554071524","doi":"https://doi.org/10.1145/2935750","title":"Distributed Algorithms for Computing Very Large Thresholded Covariance Matrices","display_name":"Distributed Algorithms for Computing Very Large Thresholded Covariance Matrices","publication_year":2016,"publication_date":"2016-11-19","ids":{"openalex":"https://openalex.org/W2554071524","doi":"https://doi.org/10.1145/2935750","mag":"2554071524"},"language":"en","primary_location":{"id":"doi:10.1145/2935750","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2935750","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2935750&type=pdf","source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=2935750&type=pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058252467","display_name":"Zekai J. Gao","orcid":"https://orcid.org/0000-0002-1886-1912"},"institutions":[{"id":"https://openalex.org/I74775410","display_name":"Rice University","ror":"https://ror.org/008zs3103","country_code":"US","type":"education","lineage":["https://openalex.org/I74775410"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Zekai J. Gao","raw_affiliation_strings":["Rice University, Houston, TX"],"affiliations":[{"raw_affiliation_string":"Rice University, Houston, TX","institution_ids":["https://openalex.org/I74775410"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5002518742","display_name":"Chris Jermaine","orcid":null},"institutions":[{"id":"https://openalex.org/I74775410","display_name":"Rice University","ror":"https://ror.org/008zs3103","country_code":"US","type":"education","lineage":["https://openalex.org/I74775410"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Jermaine","raw_affiliation_strings":["Rice University, Houston, TX"],"affiliations":[{"raw_affiliation_string":"Rice University, Houston, TX","institution_ids":["https://openalex.org/I74775410"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5058252467"],"corresponding_institution_ids":["https://openalex.org/I74775410"],"apc_list":null,"apc_paid":null,"fwci":0.3637,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.70903993,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"11","issue":"2","first_page":"1","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11716","display_name":"Random Matrices and Applications","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11716","display_name":"Random Matrices and Applications","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10057","display_name":"Face and Expression Recognition","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/covariance-matrix","display_name":"Covariance matrix","score":0.7589261531829834},{"id":"https://openalex.org/keywords/covariance-intersection","display_name":"Covariance intersection","score":0.7285233736038208},{"id":"https://openalex.org/keywords/covariance-function","display_name":"Covariance function","score":0.6912936568260193},{"id":"https://openalex.org/keywords/covariance","display_name":"Covariance","score":0.6825171113014221},{"id":"https://openalex.org/keywords/rational-quadratic-covariance-function","display_name":"Rational quadratic covariance function","score":0.6374377608299255},{"id":"https://openalex.org/keywords/estimation-of-covariance-matrices","display_name":"Estimation of covariance matrices","score":0.6372836828231812},{"id":"https://openalex.org/keywords/mat\u00e9rn-covariance-function","display_name":"Mat\u00e9rn covariance function","score":0.5728952288627625},{"id":"https://openalex.org/keywords/cma-es","display_name":"CMA-ES","score":0.5613954067230225},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.5338612198829651},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.49686363339424133},{"id":"https://openalex.org/keywords/law-of-total-covariance","display_name":"Law of total covariance","score":0.4858822226524353},{"id":"https://openalex.org/keywords/matrix","display_name":"Matrix (chemical analysis)","score":0.4348367154598236},{"id":"https://openalex.org/keywords/scatter-matrix","display_name":"Scatter matrix","score":0.4261828362941742},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.38825011253356934},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.2697456479072571}],"concepts":[{"id":"https://openalex.org/C185142706","wikidata":"https://www.wikidata.org/wiki/Q1134404","display_name":"Covariance matrix","level":2,"score":0.7589261531829834},{"id":"https://openalex.org/C83042196","wikidata":"https://www.wikidata.org/wiki/Q5178898","display_name":"Covariance intersection","level":4,"score":0.7285233736038208},{"id":"https://openalex.org/C137250428","wikidata":"https://www.wikidata.org/wiki/Q5178897","display_name":"Covariance function","level":3,"score":0.6912936568260193},{"id":"https://openalex.org/C178650346","wikidata":"https://www.wikidata.org/wiki/Q201984","display_name":"Covariance","level":2,"score":0.6825171113014221},{"id":"https://openalex.org/C148893098","wikidata":"https://www.wikidata.org/wiki/Q7295778","display_name":"Rational quadratic covariance function","level":5,"score":0.6374377608299255},{"id":"https://openalex.org/C180877172","wikidata":"https://www.wikidata.org/wiki/Q5401390","display_name":"Estimation of covariance matrices","level":3,"score":0.6372836828231812},{"id":"https://openalex.org/C118006245","wikidata":"https://www.wikidata.org/wiki/Q6792079","display_name":"Mat\u00e9rn covariance function","level":5,"score":0.5728952288627625},{"id":"https://openalex.org/C205555498","wikidata":"https://www.wikidata.org/wiki/Q505588","display_name":"CMA-ES","level":4,"score":0.5613954067230225},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5338612198829651},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.49686363339424133},{"id":"https://openalex.org/C126372606","wikidata":"https://www.wikidata.org/wiki/Q6503511","display_name":"Law of total covariance","level":5,"score":0.4858822226524353},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.4348367154598236},{"id":"https://openalex.org/C176917957","wikidata":"https://www.wikidata.org/wiki/Q7430596","display_name":"Scatter matrix","level":4,"score":0.4261828362941742},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.38825011253356934},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2697456479072571},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2935750","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2935750","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2935750&type=pdf","source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},{"id":"pmh:oai:scholarship.rice.edu:1911/87863","is_oa":false,"landing_page_url":"http://hdl.handle.net/1911/87863","pdf_url":null,"source":{"id":"https://openalex.org/S4306401268","display_name":"Rice Digital Scholarship Archive (Rice University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I74775410","host_organization_name":"Rice University","host_organization_lineage":["https://openalex.org/I74775410"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Thesis"}],"best_oa_location":{"id":"doi:10.1145/2935750","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2935750","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2935750&type=pdf","source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.6899999976158142,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G5921281487","display_name":null,"funder_award_id":"number","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8686621231","display_name":null,"funder_award_id":"1409543","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2554071524.pdf","grobid_xml":"https://content.openalex.org/works/W2554071524.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W153918522","https://openalex.org/W1510543252","https://openalex.org/W1511986666","https://openalex.org/W1520752838","https://openalex.org/W1535602073","https://openalex.org/W1888464276","https://openalex.org/W1968829657","https://openalex.org/W1972276842","https://openalex.org/W1981638497","https://openalex.org/W1989727964","https://openalex.org/W2018666252","https://openalex.org/W2046649434","https://openalex.org/W2048779798","https://openalex.org/W2052386156","https://openalex.org/W2057535756","https://openalex.org/W2059606882","https://openalex.org/W2060581589","https://openalex.org/W2073681337","https://openalex.org/W2079976213","https://openalex.org/W2081746825","https://openalex.org/W2084913894","https://openalex.org/W2095243891","https://openalex.org/W2098056745","https://openalex.org/W2102489964","https://openalex.org/W2106116051","https://openalex.org/W2112545207","https://openalex.org/W2113968881","https://openalex.org/W2116780029","https://openalex.org/W2116805437","https://openalex.org/W2118229812","https://openalex.org/W2120695425","https://openalex.org/W2122686681","https://openalex.org/W2123499997","https://openalex.org/W2130351130","https://openalex.org/W2132664995","https://openalex.org/W2133396774","https://openalex.org/W2149659421","https://openalex.org/W2164507334","https://openalex.org/W2173213060","https://openalex.org/W2964311145","https://openalex.org/W2998237634","https://openalex.org/W3098826229","https://openalex.org/W3098880893","https://openalex.org/W3099514962","https://openalex.org/W3099609308","https://openalex.org/W3101788651","https://openalex.org/W3103917751","https://openalex.org/W3122029670","https://openalex.org/W4232849517","https://openalex.org/W4247236552","https://openalex.org/W4300541453","https://openalex.org/W4406726441"],"related_works":["https://openalex.org/W2887132723","https://openalex.org/W4311761947","https://openalex.org/W3211883524","https://openalex.org/W2060816402","https://openalex.org/W341081156","https://openalex.org/W2477744635","https://openalex.org/W2164924639","https://openalex.org/W2250095701","https://openalex.org/W2179028626","https://openalex.org/W1976318097"],"abstract_inverted_index":{"Computation":[0],"of":[1,35,106,116,131,139,212,268,300,321,341,349,358,387],"covariance":[2,43,76,91,157,167,180,226,241,274],"matrices":[3,13],"from":[4,62],"observed":[5],"data":[6,388],"is":[7,45,70,109,121,159,176,203,228,333,344],"an":[8,41,48,254,374],"important":[9,279,323,360],"problem,":[10],"as":[11,19,262],"such":[12,18,59],"are":[14,52,79,144,243,277,403],"used":[15],"in":[16,30,88,124,127,231,239,272,280,293,307,328],"applications":[17],"principal":[20],"component":[21],"analysis":[22,26],"(PCA),":[23],"linear":[24],"discriminant":[25],"(LDA),":[27],"and":[28,33,195,246,291,318,347,370,390,406],"increasingly":[29],"the":[31,89,103,107,114,129,137,140,165,170,233,240,249,259,266,269,273,281,298,319,322,345,356,359,400],"learning":[32],"application":[34],"probabilistic":[36],"graphical":[37],"models.":[38],"However,":[39],"computing":[40,58],"empirical":[42,90,156],"matrix":[44,61,92,158,168,181,242,275],"not":[46,160,286],"always":[47],"easy":[49],"problem.":[50],"There":[51],"two":[53,350],"key":[54,221,338],"difficulties":[55,214],"associated":[56],"with":[57,248],"a":[60,63,73,122,161,179,220,294,334],"very":[64],"high-dimensional":[65,152,225],"dataset.":[66,141],"The":[67,173,337],"first":[68],"problem":[69,175],"over-fitting.":[71],"For":[72,189],"p":[74,80,82,96,98,191,200,316],"-dimensional":[75],"matrix,":[77],"there":[78,143],"(":[81,184,309],"\u2212":[83],"1)/2":[84],"unique,":[85],"off-diagonal":[86],"entries":[87,238,271,324,361],"\u015c":[93],"for":[94,151,164,224,354],"large":[95,190],"(say,":[97],">":[99],"10":[100],"5":[101],"),":[102],"size":[104,138],"n":[105,196],"dataset":[108],"often":[110],"much":[111,197],"smaller":[112],"than":[113,193,199],"number":[115,130],"covariances":[117],"to":[118,148,257,392],"compute.":[119],"Over-fitting":[120],"concern":[123],"any":[125],"situation":[126],"which":[128,232],"parameters":[132],"learned":[133],"can":[134,215,304,325],"greatly":[135],"exceed":[136],"Thus,":[142],"strong":[145],"theoretical":[146],"reasons":[147],"expect":[149],"that":[150,276,283,399],"data\u2014even":[153],"Gaussian":[154],"data\u2014the":[155],"good":[162],"estimate":[163],"true":[166],"underlying":[169],"generative":[171],"process.":[172],"second":[174,295],"computational.":[177],"Computing":[178],"takes":[182],"O":[183,308],"np":[185],"2":[186,317],")":[187,311],"time.":[188,312],"(greater":[192],"10,000)":[194],"greater":[198],",":[201,230],"this":[202,206,332,342],"debilitating.":[204],"In":[205],"article,":[207],"we":[208],"consider":[209],"how":[210],"both":[211,404],"these":[213,368],"be":[216,287,305,326],"handled":[217],"simultaneously.":[218],"Specifically,":[219],"regularization":[222],"technique":[223],"estimation":[227],"thresholding":[229],"smallest":[234],"or":[235],"least":[236],"significant":[237],"simply":[244],"dropped":[245],"replaced":[247],"value":[250],"0.":[251],"This":[252,303],"suggests":[253],"obvious":[255],"way":[256],"address":[258],"computational":[260],"difficulty":[261],"well:":[263],"First,":[264],"compute":[265,297,376],"identities":[267,320,357],"K":[270,314],"actually":[278],"sense":[282],"they":[284],"will":[285],"removed":[288],"during":[289],"thresholding,":[290],"then":[292,331],"step,":[296],"values":[299],"those":[301],"entries.":[302],"done":[306],"Kn":[310],"If":[313],"\u226a":[315],"computed":[327],"reasonable":[329],"time,":[330],"big":[335],"win.":[336],"technical":[339],"contribution":[340],"article":[343],"design":[346],"implementation":[348],"different":[351],"distributed":[352],"algorithms":[353],"approximating":[355],"quickly,":[362],"using":[363,373,382],"sampling.":[364],"We":[365],"have":[366,379],"implemented":[367],"methods":[369,402],"tested":[371],"them":[372],"800-core":[375],"cluster.":[377],"Experiments":[378],"been":[380],"run":[381],"real":[383],"datasets":[384],"having":[385],"millions":[386],"points":[389],"up":[391],"40,":[393],"000":[394],"dimensions.":[395],"These":[396],"experiments":[397],"show":[398],"proposed":[401],"accurate":[405],"efficient.":[407]},"counts_by_year":[{"year":2019,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}