From 90a92d770a9beae1ad2c9f064ed8d9079208219a Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Sun, 29 Sep 2024 00:12:28 +0200 Subject: [PATCH] Upgrade PartitionedArrays from 0.3.x to 0.5.x. (#18) This patch upgrades the PartitionedArrays dependency from the 0.3.x release series to 0.5.x (closes #17). Also updates documentation build dependencies. --- Project.toml | 4 +- docs/Manifest.toml | 154 +++++++++++++++++++++++++++++++---------- docs/src/api.md | 5 ++ src/HYPRE.jl | 169 +++++++++++++++++++++++++++------------------ test/runtests.jl | 17 ++--- 5 files changed, 232 insertions(+), 117 deletions(-) diff --git a/Project.toml b/Project.toml index e5878b2..9563442 100644 --- a/Project.toml +++ b/Project.toml @@ -12,9 +12,9 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SparseMatricesCSR = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1" [compat] -CEnum = "0.4" +CEnum = "0.4, 0.5" MPI = "0.19, 0.20" -PartitionedArrays = "0.3" +PartitionedArrays = "0.5" SparseMatricesCSR = "0.6" julia = "1.6" diff --git a/docs/Manifest.toml b/docs/Manifest.toml index dbcd91f..4a37bc1 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.0" +julia_version = "1.10.5" manifest_format = "2.0" project_hash = "7c98a97551e318432a6ba3bc3fd4758623a247ac" @@ -19,23 +19,33 @@ deps = ["LinearAlgebra", "Requires"] git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" version = "3.6.2" +weakdeps = ["StaticArrays"] [deps.Adapt.extensions] AdaptStaticArraysExt = "StaticArrays" - [deps.Adapt.weakdeps] - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" version = "1.1.1" +[[deps.ArrayLayouts]] +deps = ["FillArrays", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "1d9e98721e71dcf4db5a7d34f55d8aa07c43468f" +uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" +version = "1.0.6" + [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +[[deps.BlockArrays]] +deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra"] +git-tree-sha1 = "bed8cfec0c348753a79d915cc82999c395299297" +uuid = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" +version = "0.16.33" + [[deps.CEnum]] git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" @@ -47,10 +57,16 @@ git-tree-sha1 = "61bc114e595167090b4cbcb7305ddeacd4274f16" uuid = "7a955b69-7140-5f4e-a0ed-f168c5e2e749" version = "1.3.2" +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.6" + [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.2+0" +version = "1.1.1+0" [[deps.Dates]] deps = ["Printf"] @@ -73,18 +89,22 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.3" [[deps.Documenter]] -deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "REPL", "SnoopPrecompile", "Test", "Unicode"] -git-tree-sha1 = "2afe1f1706b90fd4a8593d70b5324d04ddefed69" -repo-rev = "master" -repo-url = "https://github.com/JuliaDocs/Documenter.jl.git" +deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] +git-tree-sha1 = "5a1ee886566f2fa9318df1273d8b778b9d42712d" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.28.0-DEV" +version = "1.7.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" version = "1.6.0" +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.6.2+0" + [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" @@ -94,6 +114,18 @@ git-tree-sha1 = "ed569cb9e7e3590d5ba884da7edc50216aac5811" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" version = "1.1.0" +[[deps.Git]] +deps = ["Git_jll"] +git-tree-sha1 = "04eff47b1354d702c3a85e8ab23d539bb7d5957e" +uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +version = "1.3.1" + +[[deps.Git_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "ea372033d09e4552a04fd38361cd019f9003f4f4" +uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" +version = "2.46.2+0" + [[deps.HYPRE]] deps = ["CEnum", "HYPRE_jll", "Libdl", "MPI", "PartitionedArrays", "SparseArrays", "SparseMatricesCSR"] path = ".." @@ -140,6 +172,11 @@ git-tree-sha1 = "cc6ca91b2ca8d3600478d40097d30a6f046d0759" uuid = "51474c39-65e3-53ba-86ba-03b1b862ec14" version = "3.11.0+0" +[[deps.LazilyInitializedFields]] +git-tree-sha1 = "8f7f3cabab0fd1800699663533b6d5cb3fc0e612" +uuid = "0e77f7df-68c5-4e49-93ce-4cd80f5598bf" +version = "1.2.2" + [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" @@ -147,25 +184,36 @@ uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.3" +version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "7.84.0+0" +version = "8.4.0+0" [[deps.LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.6.4+0" + [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.10.2+0" +version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+0" + [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -218,7 +266,7 @@ version = "0.1.1" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+0" +version = "2.28.2+1" [[deps.MicrosoftMPI_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -231,7 +279,7 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2022.10.11" +version = "2023.1.10" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -246,7 +294,7 @@ version = "1.12.9" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.21+4" +version = "0.3.23+4" [[deps.OpenMPI_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] @@ -254,6 +302,17 @@ git-tree-sha1 = "f3080f4212a8ba2ceb10a34b938601b862094314" uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" version = "4.1.5+0" +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] git-tree-sha1 = "a5aef8d4a6e8d81f171b2bd4be5265b01384c74c" @@ -261,15 +320,15 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "2.5.10" [[deps.PartitionedArrays]] -deps = ["CircularArrays", "Distances", "FillArrays", "IterativeSolvers", "LinearAlgebra", "MPI", "Printf", "Random", "SparseArrays", "SparseMatricesCSR"] -git-tree-sha1 = "efdde29f1a7def5f81d1ed0265b3c7bc1c3ee00f" +deps = ["BlockArrays", "CircularArrays", "Distances", "FillArrays", "IterativeSolvers", "LinearAlgebra", "MPI", "Printf", "Random", "SparseArrays", "SparseMatricesCSR", "StaticArrays"] +git-tree-sha1 = "912e48995e001f7b24d425fb1eebbb97e8c3cb09" uuid = "5a9dfac6-5c52-46f7-8278-5e2210713be9" -version = "0.3.2" +version = "0.5.4" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.9.0" +version = "1.10.0" [[deps.PrecompileTools]] deps = ["Preferences"] @@ -292,7 +351,7 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[deps.Random]] -deps = ["SHA", "Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.RecipesBase]] @@ -301,6 +360,12 @@ git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" version = "1.3.4" +[[deps.RegistryInstances]] +deps = ["LazilyInitializedFields", "Pkg", "TOML", "Tar"] +git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51" +uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3" +version = "0.1.0" + [[deps.Requires]] deps = ["UUIDs"] git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" @@ -314,18 +379,13 @@ version = "0.7.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -[[deps.SnoopPrecompile]] -deps = ["Preferences"] -git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" -uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" -version = "1.0.3" - [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.10.0" [[deps.SparseMatricesCSR]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] @@ -333,10 +393,29 @@ git-tree-sha1 = "38677ca58e80b5cad2382e5a1848f93b054ad28d" uuid = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1" version = "0.6.7" +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "eeafab08ae20c62c44c8399ccb9354a04b80db50" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.7" + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.3" + [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.9.0" +version = "1.10.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] @@ -349,9 +428,9 @@ deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" [[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "5.10.1+6" +version = "7.2.1+1" [[deps.TOML]] deps = ["Dates"] @@ -367,6 +446,11 @@ version = "1.10.0" deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[[deps.TranscodingStreams]] +git-tree-sha1 = "e84b3a11b9bece70d14cce63406bbc79ed3464d2" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.2" + [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -377,19 +461,19 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+0" +version = "1.2.13+1" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.7.0+0" +version = "5.11.0+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.48.0+0" +version = "1.52.0+1" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+0" +version = "17.4.0+2" diff --git a/docs/src/api.md b/docs/src/api.md index 75e12fb..ca6b8fb 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -32,3 +32,8 @@ HYPRE.ILU HYPRE.PCG HYPRE.ParaSails ``` + +```@docs +HYPRE.GetNumIterations +HYPRE.GetFinalRelativeResidualNorm +``` diff --git a/src/HYPRE.jl b/src/HYPRE.jl index 1da8fe6..142d5f6 100644 --- a/src/HYPRE.jl +++ b/src/HYPRE.jl @@ -3,10 +3,9 @@ module HYPRE using MPI: MPI -using PartitionedArrays: own_length, tuple_of_arrays, own_to_global, global_length, - own_to_local, local_to_global, global_to_own, global_to_local, - MPIArray, PSparseMatrix, PVector, PartitionedArrays, AbstractLocalIndices, - local_values, own_values, partition +using PartitionedArrays: PartitionedArrays, AbstractLocalIndices, MPIArray, PSparseMatrix, + PVector, SplitMatrix, ghost_to_global, local_values, own_to_global, own_values, + partition using SparseArrays: SparseArrays, SparseMatrixCSC, nnz, nonzeros, nzrange, rowvals using SparseMatricesCSR: SparseMatrixCSR, colvals, getrowptr @@ -344,36 +343,48 @@ function subarray_unsafe_supported() return VERSION >= v"1.9.0" end -# TODO: This has some duplicated code with to_hypre_data(::SparseMatrixCSC, ilower, iupper) -function Internals.to_hypre_data(A::SparseMatrixCSC, r::AbstractLocalIndices, c::AbstractLocalIndices) - g_to_l_rows = global_to_local(r) # Not sure about this assert - l_to_g_rows = local_to_global(r) - @assert g_to_l_rows.own_to_local isa UnitRange && g_to_l_rows.own_to_local.start == 1 - - n_local_rows = own_length(r) - n_local_cols = own_length(c) - ilower = l_to_g_rows[1] - iupper = l_to_g_rows[own_length(r)] - a_rows = rowvals(A) - a_vals = nonzeros(A) +function Internals.to_hypre_data( + A::SplitMatrix{<:SparseMatrixCSC}, r::AbstractLocalIndices, c::AbstractLocalIndices + ) + # Own/ghost to global index mappings + own_to_global_row = own_to_global(r) + own_to_global_col = own_to_global(c) + ghost_to_global_col = ghost_to_global(c) + + # HYPRE requires contiguous row indices + ilower = own_to_global_row[1] + iupper = own_to_global_row[end] + @assert iupper - ilower + 1 == length(own_to_global_row) + + # Extract sparse matrices from the SplitMatrix. We are only interested in the owned + # rows, so only consider own-own and own-ghost blocks. + Aoo = A.blocks.own_own::SparseMatrixCSC + Aoo_rows = rowvals(Aoo) + Aoo_vals = nonzeros(Aoo) + Aog = A.blocks.own_ghost::SparseMatrixCSC + Aog_rows = rowvals(Aog) + Aog_vals = nonzeros(Aog) + @assert size(Aoo, 1) == size(Aog, 1) == length(own_to_global_row) # Initialize the data buffers HYPRE wants - nrows = HYPRE_Int(iupper - ilower + 1) # Total number of rows - ncols = zeros(HYPRE_Int, nrows) # Number of colums for each row - rows = collect(HYPRE_BigInt, ilower:iupper) # The row indices + nrows = HYPRE_Int(length(own_to_global_row)) # Total number of rows + ncols = zeros(HYPRE_Int, nrows) # Number of colums for each row + rows = collect(HYPRE_BigInt, ilower:iupper) # The row indices # cols = Vector{HYPRE_BigInt}(undef, nnz) # The column indices # values = Vector{HYPRE_Complex}(undef, nnz) # The values - # First pass to count nnz per row (note that the fact that columns are permuted - # doesn't matter for this pass) - a_rows = rowvals(A) - a_vals = nonzeros(A) - @inbounds for j in 1:size(A, 2) - for i in nzrange(A, j) - row = a_rows[i] - row > n_local_rows && continue # Skip ghost rows - # grow = r.lid_to_gid[lrow] - ncols[row] += 1 + # First pass to count nnz per row (note that global column indices and column + # permutation doesn't matter for this pass) + @inbounds for own_col in 1:size(Aoo, 2) + for k in nzrange(Aoo, own_col) + own_row = Aoo_rows[k] + ncols[own_row] += 1 + end + end + @inbounds for ghost_col in 1:size(Aog, 2) + for k in nzrange(Aog, ghost_col) + own_row = Aog_rows[k] + ncols[own_row] += 1 end end @@ -386,61 +397,83 @@ function Internals.to_hypre_data(A::SparseMatrixCSC, r::AbstractLocalIndices, c: lastinds = zeros(Int, nrows) cumsum!((@view lastinds[2:end]), (@view ncols[1:end-1])) - # Second pass to populate the output -- here we need to take care of the permutation - # of columns. TODO: Problem that they are not sorted? - l_to_g_cols = local_to_global(c) - @inbounds for j in 1:size(A, 2) - for i in nzrange(A, j) - row = a_rows[i] - row > n_local_cols && continue # Skip ghost rows - k = lastinds[row] += 1 - val = a_vals[i] - cols[k] = l_to_g_cols[j] - values[k] = val + # Second pass to populate the output. Here we need to map column + # indices from own/ghost to global + @inbounds for own_col in 1:size(Aoo, 2) + for k in nzrange(Aoo, own_col) + own_row = Aoo_rows[k] + i = lastinds[own_row] += 1 + values[i] = Aoo_vals[k] + cols[i] = own_to_global_col[own_col] end end + @inbounds for ghost_col in 1:size(Aog, 2) + for k in nzrange(Aog, ghost_col) + own_row = Aog_rows[k] + i = lastinds[own_row] += 1 + values[i] = Aog_vals[k] + cols[i] = ghost_to_global_col[ghost_col] + end + end + + # Sanity checks and return @assert nrows == length(ncols) == length(rows) return nrows, ncols, rows, cols, values end -# TODO: Possibly this can be optimized if it is possible to pass overlong vectors to HYPRE. -# At least values should be possible to directly share, but cols needs to translated -# to global ids. -function Internals.to_hypre_data(A::SparseMatrixCSR, r::AbstractLocalIndices, c::AbstractLocalIndices) - g_to_l_rows = global_to_local(r) - l_to_g_rows = local_to_global(r) - @assert g_to_l_rows.own_to_local isa UnitRange && g_to_l_rows.own_to_local.start == 1 - - n_local_rows = own_length(r) - ilower = l_to_g_rows[1] - iupper = l_to_g_rows[n_local_rows] - - a_cols = colvals(A) - a_vals = nonzeros(A) - nnz = getrowptr(A)[n_local_rows + 1] - 1 +function Internals.to_hypre_data( + A::SplitMatrix{<:SparseMatrixCSR}, r::AbstractLocalIndices, c::AbstractLocalIndices + ) + # Own/ghost to global index mappings + own_to_global_row = own_to_global(r) + own_to_global_col = own_to_global(c) + ghost_to_global_col = ghost_to_global(c) + + # HYPRE requires contiguous row indices + ilower = own_to_global_row[1] + iupper = own_to_global_row[end] + @assert iupper - ilower + 1 == length(own_to_global_row) + + # Extract sparse matrices from the SplitMatrix. We are only interested in the owned + # rows, so only consider own-own and own-ghost blocks. + Aoo = A.blocks.own_own::SparseMatrixCSR + Aoo_cols = colvals(Aoo) + Aoo_vals = nonzeros(Aoo) + Aog = A.blocks.own_ghost::SparseMatrixCSR + Aog_cols = colvals(Aog) + Aog_vals = nonzeros(Aog) + @assert size(Aoo, 1) == size(Aog, 1) == length(own_to_global_row) # Initialize the data buffers HYPRE wants + nnz = SparseArrays.nnz(Aoo) + SparseArrays.nnz(Aog) nrows = HYPRE_Int(iupper - ilower + 1) # Total number of rows ncols = zeros(HYPRE_Int, nrows) # Number of columns for each row rows = collect(HYPRE_BigInt, ilower:iupper) # The row indices cols = Vector{HYPRE_BigInt}(undef, nnz) # The column indices values = Vector{HYPRE_Complex}(undef, nnz) # The values - # Loop over the (owned) rows and collect all values - l_to_g_cols = local_to_global(c) - k = 0 - @inbounds for i in own_to_local(r) - nzr = nzrange(A, i) - ncols[i] = length(nzr) - for j in nzr - k += 1 - col = a_cols[j] - val = a_vals[j] - cols[k] = l_to_g_cols[col] - values[k] = val + # For CSR we only need a single pass to over the owned rows to collect everything + i = 0 + for own_row in 1:size(Aoo, 1) + nzro = nzrange(Aoo, own_row) + nzrg = nzrange(Aog, own_row) + ncols[own_row] = length(nzro) + length(nzrg) + for k in nzro + i += 1 + own_col = Aoo_cols[k] + cols[i] = own_to_global_col[own_col] + values[i] = Aoo_vals[k] + end + for k in nzrg + i += 1 + ghost_col = Aog_cols[k] + cols[i] = ghost_to_global_col[ghost_col] + values[i] = Aog_vals[k] end end - @assert nnz == k + + # Sanity checks and return + @assert nnz == i @assert nrows == length(ncols) == length(rows) return nrows, ncols, rows, cols, values end diff --git a/test/runtests.jl b/test/runtests.jl index d3e7290..df67647 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -91,13 +91,6 @@ end @test H.iupper == H.jupper == 10 end -function default_local_values_csr(I,J,V,row_indices,col_indices) - # Adapted from p_sparse_matrix.jl line 487 - m = local_length(row_indices) - n = local_length(col_indices) - sparsecsr(I,J,V,m,n) -end - function distribute_as_parray(parts, backend) if backend == :debug parts = DebugArray(parts) @@ -151,8 +144,8 @@ end parts = 1:2 end parts = distribute_as_parray(parts, backend) - CSC = psparse!(diag_data(parts)...) |> fetch - CSR = psparse!(default_local_values_csr, diag_data(parts)...) |> fetch + CSC = psparse(diag_data(parts)...) |> fetch + CSR = psparse(sparsecsr, diag_data(parts)...) |> fetch for A in [CSC, CSR] map(local_values(A), A.row_partition, A.col_partition, parts) do values, rows, cols, p @@ -182,7 +175,7 @@ end @test hypre_data[2]::Vector{HYPRE_Int} == ncols @test hypre_data[3]::Vector{HYPRE_BigInt} == rows @test hypre_data[4]::Vector{HYPRE_BigInt} == cols - @test hypre_data[5]::Vector{HYPRE_Complex} == values + @test hypre_data[5]::Vector{HYPRE_Complex} == values end end end @@ -276,7 +269,7 @@ end return collect(row_indices), values end I, V = tuple_of_arrays(IV) - pb = pvector!(I, V, rows) |> fetch + pb = pvector(I, V, rows) |> fetch H = HYPREVector(pb) # Check for valid vector @test H.ijvector != HYPRE_IJVector(C_NULL) @@ -694,7 +687,7 @@ function topartitioned(x::Vector, A::SparseMatrixCSC, b::Vector, backend) return findnz(A)..., b, x end II, JJ, VV, bb, xx = tuple_of_arrays(tmp) - A_p = psparse!(II, JJ, VV, rows, cols) |> fetch + A_p = psparse(II, JJ, VV, rows, cols) |> fetch b_p = PVector(bb, rows) x_p = PVector(xx, cols) return x_p, A_p, b_p