|
|
|
@ -91,104 +91,105 @@ end |
|
|
|
@test H.iupper == H.jupper == 10 |
|
|
|
@test H.iupper == H.jupper == 10 |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
function tomain(x) |
|
|
|
function default_local_values_csr(I,J,V,row_indices,col_indices) |
|
|
|
g = gather(copy(x)) |
|
|
|
# Adapted from p_sparse_matrix.jl line 487 |
|
|
|
be = get_backend(g.values) |
|
|
|
m = local_length(row_indices) |
|
|
|
if be isa SequentialBackend |
|
|
|
n = local_length(col_indices) |
|
|
|
return g.values.parts[1] |
|
|
|
sparsecsr(I,J,V,m,n) |
|
|
|
else # if be isa MPIBackend |
|
|
|
|
|
|
|
return g.values.part |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function distribute_as_parray(parts, backend) |
|
|
|
|
|
|
|
if backend == :debug |
|
|
|
|
|
|
|
parts = DebugArray(parts) |
|
|
|
|
|
|
|
elseif backend == :mpi |
|
|
|
|
|
|
|
parts = distribute_with_mpi(parts) |
|
|
|
|
|
|
|
else |
|
|
|
|
|
|
|
@assert backend == :native |
|
|
|
|
|
|
|
parts = collect(parts) |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
return parts |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@testset "HYPREMatrix(::PSparseMatrix)" begin |
|
|
|
@testset "HYPREMatrix(::PSparseMatrix)" begin |
|
|
|
# Sequential backend |
|
|
|
function diag_data(parts) |
|
|
|
function diag_data(backend, parts) |
|
|
|
rows = uniform_partition(parts, 10) |
|
|
|
is_seq = backend isa SequentialBackend |
|
|
|
cols = uniform_partition(parts, 10) |
|
|
|
rows = PRange(parts, 10) |
|
|
|
np = length(parts) |
|
|
|
cols = PRange(parts, 10) |
|
|
|
IJV = map(parts) do p |
|
|
|
I, J, V = map_parts(parts) do p |
|
|
|
|
|
|
|
i = Int[] |
|
|
|
i = Int[] |
|
|
|
j = Int[] |
|
|
|
j = Int[] |
|
|
|
v = Float64[] |
|
|
|
v = Float64[] |
|
|
|
if (is_seq && p == 1) || !is_seq |
|
|
|
if np == 1 |
|
|
|
|
|
|
|
# MPI case is special, we only have one MPI process. |
|
|
|
|
|
|
|
@assert p == 1 |
|
|
|
|
|
|
|
append!(i, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) |
|
|
|
|
|
|
|
append!(j, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) |
|
|
|
|
|
|
|
append!(v, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) |
|
|
|
|
|
|
|
elseif p == 1 |
|
|
|
|
|
|
|
@assert np == 2 |
|
|
|
append!(i, [1, 2, 3, 4, 5, 6]) |
|
|
|
append!(i, [1, 2, 3, 4, 5, 6]) |
|
|
|
append!(j, [1, 2, 3, 4, 5, 6]) |
|
|
|
append!(j, [1, 2, 3, 4, 5, 6]) |
|
|
|
append!(v, [1, 2, 3, 4, 5, 6]) |
|
|
|
append!(v, [1, 2, 3, 4, 5, 6]) |
|
|
|
end |
|
|
|
else |
|
|
|
if (is_seq && p == 2) || !is_seq |
|
|
|
@assert np == 2 |
|
|
|
|
|
|
|
@assert p == 2 |
|
|
|
append!(i, [4, 5, 6, 7, 8, 9, 10]) |
|
|
|
append!(i, [4, 5, 6, 7, 8, 9, 10]) |
|
|
|
append!(j, [4, 5, 6, 7, 8, 9, 10]) |
|
|
|
append!(j, [4, 5, 6, 7, 8, 9, 10]) |
|
|
|
append!(v, [4, 5, 6, 7, 8, 9, 10]) |
|
|
|
append!(v, [4, 5, 6, 7, 8, 9, 10]) |
|
|
|
end |
|
|
|
end |
|
|
|
return i, j, v |
|
|
|
return i, j, v |
|
|
|
end |
|
|
|
end |
|
|
|
add_gids!(rows, I) |
|
|
|
I, J, V = tuple_of_arrays(IJV) |
|
|
|
assemble!(I, J, V, rows) |
|
|
|
|
|
|
|
add_gids!(cols, J) |
|
|
|
|
|
|
|
return I, J, V, rows, cols |
|
|
|
return I, J, V, rows, cols |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
backend = SequentialBackend() |
|
|
|
for backend in [:native, :debug, :mpi] |
|
|
|
parts = get_part_ids(backend, 2) |
|
|
|
@testset "Backend=$backend" begin |
|
|
|
CSC = PSparseMatrix(diag_data(backend, parts)...; ids=:global) |
|
|
|
if backend == :mpi |
|
|
|
CSR = PSparseMatrix(sparsecsr, diag_data(backend, parts)...; ids=:global) |
|
|
|
parts = 1:1 |
|
|
|
|
|
|
|
else |
|
|
|
@test tomain(CSC) == tomain(CSR) == |
|
|
|
parts = 1:2 |
|
|
|
Diagonal([1, 2, 3, 8, 10, 12, 7, 8, 9, 10]) |
|
|
|
end |
|
|
|
|
|
|
|
parts = distribute_as_parray(parts, backend) |
|
|
|
|
|
|
|
CSC = psparse!(diag_data(parts)...) |> fetch |
|
|
|
|
|
|
|
CSR = psparse!(default_local_values_csr, diag_data(parts)...) |> fetch |
|
|
|
|
|
|
|
|
|
|
|
map_parts(CSC.values, CSC.rows.partition, CSC.cols.partition, |
|
|
|
for A in [CSC, CSR] |
|
|
|
CSR.values, CSR.rows.partition, CSR.cols.partition, parts) do args... |
|
|
|
map(local_values(A), A.row_partition, A.col_partition, parts) do values, rows, cols, p |
|
|
|
cscvalues, cscrows, csccols, csrvalues, csrrows, csrcols, p = args |
|
|
|
hypre_data = Internals.to_hypre_data(values, rows, cols) |
|
|
|
csc = Internals.to_hypre_data(cscvalues, cscrows, csccols) |
|
|
|
if backend == :mpi |
|
|
|
csr = Internals.to_hypre_data(csrvalues, csrrows, csrcols) |
|
|
|
@assert p == 1 |
|
|
|
if p == 1 |
|
|
|
nrows = 10 |
|
|
|
|
|
|
|
ncols = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] |
|
|
|
|
|
|
|
rows = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] |
|
|
|
|
|
|
|
cols = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] |
|
|
|
|
|
|
|
values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] |
|
|
|
|
|
|
|
elseif p == 1 |
|
|
|
nrows = 5 |
|
|
|
nrows = 5 |
|
|
|
ncols = [1, 1, 1, 1, 1] |
|
|
|
ncols = [1, 1, 1, 1, 1] |
|
|
|
rows = [1, 2, 3, 4, 5] |
|
|
|
rows = [1, 2, 3, 4, 5] |
|
|
|
cols = [1, 2, 3, 4, 5] |
|
|
|
cols = [1, 2, 3, 4, 5] |
|
|
|
values = [1, 2, 3, 8, 10] |
|
|
|
values = [1, 2, 3, 8, 10] |
|
|
|
else # if p == 1 |
|
|
|
else |
|
|
|
|
|
|
|
@assert p == 2 |
|
|
|
nrows = 5 |
|
|
|
nrows = 5 |
|
|
|
ncols = [1, 1, 1, 1, 1] |
|
|
|
ncols = [1, 1, 1, 1, 1] |
|
|
|
rows = [6, 7, 8, 9, 10] |
|
|
|
rows = [6, 7, 8, 9, 10] |
|
|
|
cols = [6, 7, 8, 9, 10] |
|
|
|
cols = [6, 7, 8, 9, 10] |
|
|
|
values = [12, 7, 8, 9, 10] |
|
|
|
values = [12, 7, 8, 9, 10] |
|
|
|
end |
|
|
|
end |
|
|
|
@test csc[1]::HYPRE_Int == csr[1]::HYPRE_Int == nrows |
|
|
|
@test hypre_data[1]::HYPRE_Int == nrows |
|
|
|
@test csc[2]::Vector{HYPRE_Int} == csr[2]::Vector{HYPRE_Int} == ncols |
|
|
|
@test hypre_data[2]::Vector{HYPRE_Int} == ncols |
|
|
|
@test csc[3]::Vector{HYPRE_BigInt} == csr[3]::Vector{HYPRE_BigInt} == rows |
|
|
|
@test hypre_data[3]::Vector{HYPRE_BigInt} == rows |
|
|
|
@test csc[4]::Vector{HYPRE_BigInt} == csr[4]::Vector{HYPRE_BigInt} == cols |
|
|
|
@test hypre_data[4]::Vector{HYPRE_BigInt} == cols |
|
|
|
@test csc[5]::Vector{HYPRE_Complex} == csr[5]::Vector{HYPRE_Complex} == values |
|
|
|
@test hypre_data[5]::Vector{HYPRE_Complex} == values |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
# MPI backend |
|
|
|
|
|
|
|
backend = MPIBackend() |
|
|
|
|
|
|
|
parts = MPIData(1, MPI.COMM_WORLD, (1,)) # get_part_ids duplicates the comm |
|
|
|
|
|
|
|
CSC = PSparseMatrix(diag_data(backend, parts)...; ids=:global) |
|
|
|
|
|
|
|
CSR = PSparseMatrix(sparsecsr, diag_data(backend, parts)...; ids=:global) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@test tomain(CSC) == tomain(CSR) == |
|
|
|
|
|
|
|
Diagonal([1, 2, 3, 8, 10, 12, 7, 8, 9, 10]) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
map_parts(CSC.values, CSC.rows.partition, CSC.cols.partition, |
|
|
|
|
|
|
|
CSR.values, CSR.rows.partition, CSR.cols.partition, parts) do args... |
|
|
|
|
|
|
|
cscvalues, cscrows, csccols, csrvalues, csrrows, csrcols, p = args |
|
|
|
|
|
|
|
csc = Internals.to_hypre_data(cscvalues, cscrows, csccols) |
|
|
|
|
|
|
|
csr = Internals.to_hypre_data(csrvalues, csrrows, csrcols) |
|
|
|
|
|
|
|
nrows = 10 |
|
|
|
|
|
|
|
ncols = fill(1, 10) |
|
|
|
|
|
|
|
rows = collect(1:10) |
|
|
|
|
|
|
|
cols = collect(1:10) |
|
|
|
|
|
|
|
values = [1, 2, 3, 8, 10, 12, 7, 8, 9, 10] |
|
|
|
|
|
|
|
@test csc[1]::HYPRE_Int == csr[1]::HYPRE_Int == nrows |
|
|
|
|
|
|
|
@test csc[2]::Vector{HYPRE_Int} == csr[2]::Vector{HYPRE_Int} == ncols |
|
|
|
|
|
|
|
@test csc[3]::Vector{HYPRE_BigInt} == csr[3]::Vector{HYPRE_BigInt} == rows |
|
|
|
|
|
|
|
@test csc[4]::Vector{HYPRE_BigInt} == csr[4]::Vector{HYPRE_BigInt} == cols |
|
|
|
|
|
|
|
@test csc[5]::Vector{HYPRE_Complex} == csr[5]::Vector{HYPRE_Complex} == values |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@testset "HYPREVector" begin |
|
|
|
@testset "HYPREVector" begin |
|
|
|
h = HYPREVector(MPI.COMM_WORLD, 1, 5) |
|
|
|
h = HYPREVector(MPI.COMM_WORLD, 1, 5) |
|
|
|
@ -250,52 +251,47 @@ end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@testset "HYPREVector(::PVector)" begin |
|
|
|
@testset "HYPREVector(::PVector)" begin |
|
|
|
# Sequential backend |
|
|
|
for backend in [:native, :debug, :mpi] |
|
|
|
backend = SequentialBackend() |
|
|
|
if backend == :mpi |
|
|
|
parts = get_part_ids(backend, 2) |
|
|
|
parts = distribute_as_parray(1:1, backend) |
|
|
|
rows = PRange(parts, 10) |
|
|
|
else |
|
|
|
|
|
|
|
parts = distribute_as_parray(1:2, backend) |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
rows = uniform_partition(parts, 10) |
|
|
|
b = rand(10) |
|
|
|
b = rand(10) |
|
|
|
I, V = map_parts(parts) do p |
|
|
|
IV = map(parts, rows) do p, owned |
|
|
|
if p == 1 |
|
|
|
if backend == :mpi |
|
|
|
return collect(1:6), b[1:6] |
|
|
|
row_indices = 1:10 |
|
|
|
|
|
|
|
elseif p == 1 |
|
|
|
|
|
|
|
row_indices = 1:6 |
|
|
|
else # p == 2 |
|
|
|
else # p == 2 |
|
|
|
return collect(4:10), b[4:10] |
|
|
|
row_indices = 4:10 |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
values = zeros(length(row_indices)) |
|
|
|
|
|
|
|
for (i, row) in enumerate(row_indices) |
|
|
|
|
|
|
|
if row in owned |
|
|
|
|
|
|
|
values[i] = b[row] |
|
|
|
end |
|
|
|
end |
|
|
|
add_gids!(rows, I) |
|
|
|
end |
|
|
|
pb = PVector(I, V, rows; ids=:global) |
|
|
|
return collect(row_indices), values |
|
|
|
assemble!(pb) |
|
|
|
end |
|
|
|
@test tomain(pb) == [i in 4:6 ? 2x : x for (i, x) in zip(eachindex(b), b)] |
|
|
|
I, V = tuple_of_arrays(IV) |
|
|
|
|
|
|
|
pb = pvector!(I, V, rows) |> fetch |
|
|
|
H = HYPREVector(pb) |
|
|
|
H = HYPREVector(pb) |
|
|
|
|
|
|
|
# Check for valid vector |
|
|
|
@test H.ijvector != HYPRE_IJVector(C_NULL) |
|
|
|
@test H.ijvector != HYPRE_IJVector(C_NULL) |
|
|
|
@test H.parvector != HYPRE_ParVector(C_NULL) |
|
|
|
@test H.parvector != HYPRE_ParVector(C_NULL) |
|
|
|
pbc = fill!(copy(pb), 0) |
|
|
|
# Copy back, check if identical |
|
|
|
copy!(pbc, H) |
|
|
|
b_copy = copy!(similar(b), H) |
|
|
|
@test tomain(pbc) == tomain(pb) |
|
|
|
@test b_copy == b |
|
|
|
|
|
|
|
# Test copy to and from HYPREVector |
|
|
|
pb2 = 2 * pb |
|
|
|
pb2 = 2 * pb |
|
|
|
H′ = copy!(H, pb2) |
|
|
|
H′ = copy!(H, pb2) |
|
|
|
@test H === H′ |
|
|
|
@test H === H′ |
|
|
|
|
|
|
|
pbc = similar(pb) |
|
|
|
copy!(pbc, H) |
|
|
|
copy!(pbc, H) |
|
|
|
@test tomain(pbc) == 2 * tomain(pb) |
|
|
|
@test pbc == 2*pb |
|
|
|
|
|
|
|
end |
|
|
|
# MPI backend |
|
|
|
|
|
|
|
backend = MPIBackend() |
|
|
|
|
|
|
|
parts = get_part_ids(backend, 1) |
|
|
|
|
|
|
|
rows = PRange(parts, 10) |
|
|
|
|
|
|
|
I, V = map_parts(parts) do p |
|
|
|
|
|
|
|
return collect(1:10), b |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
add_gids!(rows, I) |
|
|
|
|
|
|
|
pb = PVector(I, V, rows; ids=:global) |
|
|
|
|
|
|
|
assemble!(pb) |
|
|
|
|
|
|
|
@test tomain(pb) == b |
|
|
|
|
|
|
|
H = HYPREVector(pb) |
|
|
|
|
|
|
|
@test H.ijvector != HYPRE_IJVector(C_NULL) |
|
|
|
|
|
|
|
@test H.parvector != HYPRE_ParVector(C_NULL) |
|
|
|
|
|
|
|
pbc = fill!(copy(pb), 0) |
|
|
|
|
|
|
|
copy!(pbc, H) |
|
|
|
|
|
|
|
@test tomain(pbc) == tomain(pb) |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@testset "HYPRE(Matrix|Vector)?Assembler" begin |
|
|
|
@testset "HYPRE(Matrix|Vector)?Assembler" begin |
|
|
|
@ -689,40 +685,55 @@ end |
|
|
|
@test x ≈ A \ b atol=tol |
|
|
|
@test x ≈ A \ b atol=tol |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
function topartitioned(x::Vector, A::SparseMatrixCSC, b::Vector) |
|
|
|
function topartitioned(x::Vector, A::SparseMatrixCSC, b::Vector, backend) |
|
|
|
parts = get_part_ids(SequentialBackend(), 1) |
|
|
|
parts = distribute_as_parray(1:1, backend) |
|
|
|
rows = PRange(parts, size(A, 1)) |
|
|
|
n = size(A, 1) |
|
|
|
cols = PRange(parts, size(A, 2)) |
|
|
|
rows = uniform_partition(parts, n) |
|
|
|
II, JJ, VV, bb, xx = map_parts(parts) do _ |
|
|
|
cols = uniform_partition(parts, n) |
|
|
|
|
|
|
|
tmp = map(parts) do _ |
|
|
|
return findnz(A)..., b, x |
|
|
|
return findnz(A)..., b, x |
|
|
|
end |
|
|
|
end |
|
|
|
add_gids!(rows, II) |
|
|
|
II, JJ, VV, bb, xx = tuple_of_arrays(tmp) |
|
|
|
assemble!(II, JJ, VV, rows) |
|
|
|
A_p = psparse!(II, JJ, VV, rows, cols) |> fetch |
|
|
|
add_gids!(cols, JJ) |
|
|
|
|
|
|
|
A_p = PSparseMatrix(II, JJ, VV, rows, cols; ids = :global) |
|
|
|
|
|
|
|
b_p = PVector(bb, rows) |
|
|
|
b_p = PVector(bb, rows) |
|
|
|
x_p = PVector(xx, cols) |
|
|
|
x_p = PVector(xx, cols) |
|
|
|
return x_p, A_p, b_p |
|
|
|
return x_p, A_p, b_p |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@testset "solve with PartitionedArrays" begin |
|
|
|
@testset "solve with PartitionedArrays" begin |
|
|
|
|
|
|
|
for backend in [:native, :debug, :mpi] |
|
|
|
# Setup |
|
|
|
# Setup |
|
|
|
A = sprand(100, 100, 0.05); A = A'A + 5I |
|
|
|
A = sprand(100, 100, 0.05); A = A'A + 5I |
|
|
|
b = rand(100) |
|
|
|
b = rand(100) |
|
|
|
x = zeros(100) |
|
|
|
x = zeros(100) |
|
|
|
x_p, A_p, b_p = topartitioned(x, A, b) |
|
|
|
x_p, A_p, b_p = topartitioned(x, A, b, :native) |
|
|
|
@test A == tomain(A_p) |
|
|
|
# Data is distributed over a single process. We can then check the following |
|
|
|
@test b == tomain(b_p) |
|
|
|
# as local_values is the entire matrix/vector. |
|
|
|
@test x == tomain(x_p) |
|
|
|
map(local_values(x_p)) do x_l |
|
|
|
|
|
|
|
@test x_l == x |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
map(local_values(b_p)) do b_l |
|
|
|
|
|
|
|
@test b_l == b |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
map(local_values(A_p)) do A_l |
|
|
|
|
|
|
|
@test A_l == A |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
# Solve |
|
|
|
# Solve |
|
|
|
tol = 1e-9 |
|
|
|
tol = 1e-9 |
|
|
|
pcg = HYPRE.PCG(; Tol = tol) |
|
|
|
pcg = HYPRE.PCG(; Tol = tol) |
|
|
|
## solve! |
|
|
|
## solve! |
|
|
|
HYPRE.solve!(pcg, x_p, A_p, b_p) |
|
|
|
HYPRE.solve!(pcg, x_p, A_p, b_p) |
|
|
|
@test tomain(x_p) ≈ A \ b atol=tol |
|
|
|
ref = A\b |
|
|
|
|
|
|
|
map(local_values(x_p)) do x |
|
|
|
|
|
|
|
@test x ≈ ref atol=tol |
|
|
|
|
|
|
|
end |
|
|
|
## solve |
|
|
|
## solve |
|
|
|
x_p = HYPRE.solve(pcg, A_p, b_p) |
|
|
|
x_p = HYPRE.solve(pcg, A_p, b_p) |
|
|
|
@test tomain(x_p) ≈ A \ b atol=tol |
|
|
|
map(local_values(x_p)) do x |
|
|
|
|
|
|
|
@test x ≈ ref atol=tol |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@testset "solve with SparseMatrixCS(C|R)" begin |
|
|
|
@testset "solve with SparseMatrixCS(C|R)" begin |
|
|
|
|
Olav Møyner
3 years ago