|
|
|
@ -7,14 +7,15 @@ using HTTP: HTTP |
|
|
|
using SimpleBufferStream: BufferStream |
|
|
|
using SimpleBufferStream: BufferStream |
|
|
|
|
|
|
|
|
|
|
|
if VERSION >= v"1.11.0-DEV.469" |
|
|
|
if VERSION >= v"1.11.0-DEV.469" |
|
|
|
eval(Meta.parse(""" |
|
|
|
let str = """ |
|
|
|
public CollectorRegistry, register, unregister, |
|
|
|
public CollectorRegistry, register, unregister, |
|
|
|
Counter, Gauge, Histogram, Summary, GCCollector, ProcessCollector, |
|
|
|
Counter, Gauge, Histogram, Summary, GCCollector, ProcessCollector, |
|
|
|
inc, dec, set, set_to_current_time, observe, @inprogress, @time, |
|
|
|
inc, dec, set, set_to_current_time, observe, @inprogress, @time, |
|
|
|
Family, labels, remove, clear, |
|
|
|
Family, labels, remove, clear, |
|
|
|
expose |
|
|
|
expose |
|
|
|
""" |
|
|
|
""" |
|
|
|
)) |
|
|
|
eval(Meta.parse(str)) |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
abstract type Collector end |
|
|
|
abstract type Collector end |
|
|
|
@ -30,6 +31,7 @@ struct ArgumentError <: PrometheusException |
|
|
|
end |
|
|
|
end |
|
|
|
function Base.showerror(io::IO, err::ArgumentError) |
|
|
|
function Base.showerror(io::IO, err::ArgumentError) |
|
|
|
print(io, "Prometheus.", nameof(typeof(err)), ": ", err.msg) |
|
|
|
print(io, "Prometheus.", nameof(typeof(err)), ": ", err.msg) |
|
|
|
|
|
|
|
return |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
struct AssertionError <: PrometheusException |
|
|
|
struct AssertionError <: PrometheusException |
|
|
|
@ -46,6 +48,7 @@ function Base.showerror(io::IO, err::AssertionError) |
|
|
|
"Prometheus.AssertionError: `", err.msg, "`. This is unexpected, please file an " * |
|
|
|
"Prometheus.AssertionError: `", err.msg, "`. This is unexpected, please file an " * |
|
|
|
"issue at https://github.com/fredrikekre/Prometheus.jl/issues/new.", |
|
|
|
"issue at https://github.com/fredrikekre/Prometheus.jl/issues/new.", |
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
return |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
# https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels |
|
|
|
# https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels |
|
|
|
@ -115,9 +118,7 @@ function register(reg::CollectorRegistry, collector::Collector) |
|
|
|
end |
|
|
|
end |
|
|
|
for metric_name in metric_names(collector) |
|
|
|
for metric_name in metric_names(collector) |
|
|
|
if metric_name in existing_names |
|
|
|
if metric_name in existing_names |
|
|
|
throw(ArgumentError( |
|
|
|
throw(ArgumentError("collector already contains a metric with the name \"$(metric_name)\"")) |
|
|
|
"collector already contains a metric with the name \"$(metric_name)\"" |
|
|
|
|
|
|
|
)) |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
push!(reg.collectors, collector) |
|
|
|
push!(reg.collectors, collector) |
|
|
|
@ -199,21 +200,18 @@ Throw a `Prometheus.ArgumentError` if `v < 0` (a counter must not decrease). |
|
|
|
""" |
|
|
|
""" |
|
|
|
function inc(counter::Counter, v::Real = 1.0) |
|
|
|
function inc(counter::Counter, v::Real = 1.0) |
|
|
|
if v < 0 |
|
|
|
if v < 0 |
|
|
|
throw(ArgumentError( |
|
|
|
throw(ArgumentError("invalid value $v: a counter must not decrease")) |
|
|
|
"invalid value $v: a counter must not decrease" |
|
|
|
|
|
|
|
)) |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
@atomic counter.value += convert(Float64, v) |
|
|
|
@atomic counter.value += convert(Float64, v) |
|
|
|
return nothing |
|
|
|
return nothing |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
function collect!(metrics::Vector, counter::Counter) |
|
|
|
function collect!(metrics::Vector, counter::Counter) |
|
|
|
push!(metrics, |
|
|
|
metric = Metric( |
|
|
|
Metric( |
|
|
|
|
|
|
|
"counter", counter.metric_name, counter.help, |
|
|
|
"counter", counter.metric_name, counter.help, |
|
|
|
Sample(nothing, nothing, nothing, @atomic(counter.value)), |
|
|
|
Sample(nothing, nothing, nothing, @atomic(counter.value)) |
|
|
|
), |
|
|
|
|
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
push!(metrics, metric) |
|
|
|
return metrics |
|
|
|
return metrics |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@ -316,12 +314,11 @@ function set_to_current_time(gauge::Gauge) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
function collect!(metrics::Vector, gauge::Gauge) |
|
|
|
function collect!(metrics::Vector, gauge::Gauge) |
|
|
|
push!(metrics, |
|
|
|
metric = Metric( |
|
|
|
Metric( |
|
|
|
|
|
|
|
"gauge", gauge.metric_name, gauge.help, |
|
|
|
"gauge", gauge.metric_name, gauge.help, |
|
|
|
Sample(nothing, nothing, nothing, @atomic(gauge.value)), |
|
|
|
Sample(nothing, nothing, nothing, @atomic(gauge.value)), |
|
|
|
), |
|
|
|
|
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
push!(metrics, metric) |
|
|
|
return metrics |
|
|
|
return metrics |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@ -334,7 +331,7 @@ end |
|
|
|
# A histogram SHOULD have the same default buckets as other client libraries. |
|
|
|
# A histogram SHOULD have the same default buckets as other client libraries. |
|
|
|
# https://github.com/prometheus/client_python/blob/d8306b7b39ed814f3ec667a7901df249cee8a956/prometheus_client/metrics.py#L565 |
|
|
|
# https://github.com/prometheus/client_python/blob/d8306b7b39ed814f3ec667a7901df249cee8a956/prometheus_client/metrics.py#L565 |
|
|
|
const DEFAULT_BUCKETS = [ |
|
|
|
const DEFAULT_BUCKETS = [ |
|
|
|
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, Inf, |
|
|
|
0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, Inf, |
|
|
|
] |
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
mutable struct Histogram <: Collector |
|
|
|
mutable struct Histogram <: Collector |
|
|
|
@ -420,22 +417,18 @@ end |
|
|
|
|
|
|
|
|
|
|
|
function collect!(metrics::Vector, histogram::Histogram) |
|
|
|
function collect!(metrics::Vector, histogram::Histogram) |
|
|
|
label_names = LabelNames(("le",)) |
|
|
|
label_names = LabelNames(("le",)) |
|
|
|
push!(metrics, |
|
|
|
samples = Vector{Sample}(undef, 2 + length(histogram.buckets)) |
|
|
|
Metric( |
|
|
|
samples[1] = Sample("_count", nothing, nothing, @atomic(histogram._count)) |
|
|
|
"histogram", histogram.metric_name, histogram.help, |
|
|
|
samples[2] = Sample("_sum", nothing, nothing, @atomic(histogram._sum)) |
|
|
|
[ |
|
|
|
for i in 1:length(histogram.buckets) |
|
|
|
Sample("_count", nothing, nothing, @atomic(histogram._count)), |
|
|
|
sample = Sample( |
|
|
|
Sample("_sum", nothing, nothing, @atomic(histogram._sum)), |
|
|
|
nothing, label_names, make_label_values(label_names, (histogram.buckets[i],)), |
|
|
|
( |
|
|
|
|
|
|
|
Sample( |
|
|
|
|
|
|
|
nothing, label_names, |
|
|
|
|
|
|
|
make_label_values(label_names, (histogram.buckets[i],)), |
|
|
|
|
|
|
|
histogram.bucket_counters[i][], |
|
|
|
histogram.bucket_counters[i][], |
|
|
|
) for i in 1:length(histogram.buckets) |
|
|
|
|
|
|
|
)..., |
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
), |
|
|
|
|
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
samples[2 + i] = sample |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
metric = Metric("histogram", histogram.metric_name, histogram.help, samples) |
|
|
|
|
|
|
|
push!(metrics, metric) |
|
|
|
return metrics |
|
|
|
return metrics |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@ -503,15 +496,14 @@ function observe(summary::Summary, v::Real) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
function collect!(metrics::Vector, summary::Summary) |
|
|
|
function collect!(metrics::Vector, summary::Summary) |
|
|
|
push!(metrics, |
|
|
|
metric = Metric( |
|
|
|
Metric( |
|
|
|
|
|
|
|
"summary", summary.metric_name, summary.help, |
|
|
|
"summary", summary.metric_name, summary.help, |
|
|
|
[ |
|
|
|
[ |
|
|
|
Sample("_count", nothing, nothing, @atomic(summary._count)), |
|
|
|
Sample("_count", nothing, nothing, @atomic(summary._count)), |
|
|
|
Sample("_sum", nothing, nothing, @atomic(summary._sum)), |
|
|
|
Sample("_sum", nothing, nothing, @atomic(summary._sum)), |
|
|
|
] |
|
|
|
] |
|
|
|
), |
|
|
|
|
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
push!(metrics, metric) |
|
|
|
return metrics |
|
|
|
return metrics |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@ -600,7 +592,7 @@ function expr_gen(macroname, collector, code) |
|
|
|
] |
|
|
|
] |
|
|
|
postamble = Expr[ |
|
|
|
postamble = Expr[ |
|
|
|
Expr(:(=), val, Expr(:call, max, Expr(:call, -, Expr(:call, time), t0), 0.0)), |
|
|
|
Expr(:(=), val, Expr(:call, max, Expr(:call, -, Expr(:call, time), t0), 0.0)), |
|
|
|
Expr(:call, at_time, cllctr, val) |
|
|
|
Expr(:call, at_time, cllctr, val), |
|
|
|
] |
|
|
|
] |
|
|
|
elseif macroname === :inprogress |
|
|
|
elseif macroname === :inprogress |
|
|
|
local cllctr |
|
|
|
local cllctr |
|
|
|
@ -610,7 +602,7 @@ function expr_gen(macroname, collector, code) |
|
|
|
Expr(:call, at_inprogress_enter, cllctr), |
|
|
|
Expr(:call, at_inprogress_enter, cllctr), |
|
|
|
] |
|
|
|
] |
|
|
|
postamble = Expr[ |
|
|
|
postamble = Expr[ |
|
|
|
Expr(:call, at_inprogress_exit, cllctr) |
|
|
|
Expr(:call, at_inprogress_exit, cllctr), |
|
|
|
] |
|
|
|
] |
|
|
|
else |
|
|
|
else |
|
|
|
throw(ArgumentError("unknown macro name $(repr(macroname))")) |
|
|
|
throw(ArgumentError("unknown macro name $(repr(macroname))")) |
|
|
|
@ -630,7 +622,7 @@ function expr_gen(macroname, collector, code) |
|
|
|
Expr( |
|
|
|
Expr( |
|
|
|
:tryfinally, |
|
|
|
:tryfinally, |
|
|
|
Expr(:(=), ret, fbody), |
|
|
|
Expr(:(=), ret, fbody), |
|
|
|
Expr(:block, postamble...,), |
|
|
|
Expr(:block, postamble...), |
|
|
|
), |
|
|
|
), |
|
|
|
ret, |
|
|
|
ret, |
|
|
|
), |
|
|
|
), |
|
|
|
@ -642,7 +634,7 @@ function expr_gen(macroname, collector, code) |
|
|
|
Expr( |
|
|
|
Expr( |
|
|
|
:tryfinally, |
|
|
|
:tryfinally, |
|
|
|
Expr(:(=), ret, esc(code)), |
|
|
|
Expr(:(=), ret, esc(code)), |
|
|
|
Expr(:block, postamble...,), |
|
|
|
Expr(:block, postamble...), |
|
|
|
), |
|
|
|
), |
|
|
|
ret, |
|
|
|
ret, |
|
|
|
) |
|
|
|
) |
|
|
|
@ -668,7 +660,7 @@ end |
|
|
|
|
|
|
|
|
|
|
|
struct LabelNames{N} |
|
|
|
struct LabelNames{N} |
|
|
|
label_names::NTuple{N, Symbol} |
|
|
|
label_names::NTuple{N, Symbol} |
|
|
|
function LabelNames(label_names::NTuple{N, Symbol}) where N |
|
|
|
function LabelNames(label_names::NTuple{N, Symbol}) where {N} |
|
|
|
for label_name in label_names |
|
|
|
for label_name in label_names |
|
|
|
verify_label_name(String(label_name)) |
|
|
|
verify_label_name(String(label_name)) |
|
|
|
end |
|
|
|
end |
|
|
|
@ -677,12 +669,12 @@ struct LabelNames{N} |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
# Tuple of strings |
|
|
|
# Tuple of strings |
|
|
|
function LabelNames(label_names::NTuple{N, String}) where N |
|
|
|
function LabelNames(label_names::NTuple{N, String}) where {N} |
|
|
|
return LabelNames(map(Symbol, label_names)) |
|
|
|
return LabelNames(map(Symbol, label_names)) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
# NamedTuple-type or a (user defined) struct |
|
|
|
# NamedTuple-type or a (user defined) struct |
|
|
|
function LabelNames(::Type{T}) where T |
|
|
|
function LabelNames(::Type{T}) where {T} |
|
|
|
return LabelNames(fieldnames(T)) |
|
|
|
return LabelNames(fieldnames(T)) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@ -690,7 +682,7 @@ struct LabelValues{N} |
|
|
|
label_values::NTuple{N, String} |
|
|
|
label_values::NTuple{N, String} |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
function make_label_values(::LabelNames{N}, label_values::NTuple{N, String}) where N |
|
|
|
function make_label_values(::LabelNames{N}, label_values::NTuple{N, String}) where {N} |
|
|
|
return LabelValues(label_values) |
|
|
|
return LabelValues(label_values) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@ -698,12 +690,12 @@ stringify(str::String) = str |
|
|
|
stringify(str) = String(string(str))::String |
|
|
|
stringify(str) = String(string(str))::String |
|
|
|
|
|
|
|
|
|
|
|
# Heterogeneous tuple |
|
|
|
# Heterogeneous tuple |
|
|
|
function make_label_values(::LabelNames{N}, label_values::Tuple{Vararg{Any, N}}) where N |
|
|
|
function make_label_values(::LabelNames{N}, label_values::Tuple{Vararg{Any, N}}) where {N} |
|
|
|
return LabelValues(map(stringify, label_values)::NTuple{N, String}) |
|
|
|
return LabelValues(map(stringify, label_values)::NTuple{N, String}) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
# NamedTuple or a (user defined) struct |
|
|
|
# NamedTuple or a (user defined) struct |
|
|
|
function make_label_values(label_names::LabelNames{N}, label_values) where N |
|
|
|
function make_label_values(label_names::LabelNames{N}, label_values) where {N} |
|
|
|
t::NTuple{N, String} = ntuple(N) do i |
|
|
|
t::NTuple{N, String} = ntuple(N) do i |
|
|
|
stringify(getfield(label_values, label_names.label_names[i]))::String |
|
|
|
stringify(getfield(label_values, label_names.label_names[i]))::String |
|
|
|
end |
|
|
|
end |
|
|
|
@ -735,7 +727,7 @@ struct Family{C, N, F} <: Collector |
|
|
|
registry::Union{CollectorRegistry, Nothing} = DEFAULT_REGISTRY, kwargs..., |
|
|
|
registry::Union{CollectorRegistry, Nothing} = DEFAULT_REGISTRY, kwargs..., |
|
|
|
) where {C} |
|
|
|
) where {C} |
|
|
|
# Support ... on non-final argument |
|
|
|
# Support ... on non-final argument |
|
|
|
args_all = (args_first, args_tail...,) |
|
|
|
args_all = (args_first, args_tail...) |
|
|
|
label_names = last(args_all) |
|
|
|
label_names = last(args_all) |
|
|
|
args = Base.front(args_all) |
|
|
|
args = Base.front(args_all) |
|
|
|
@assert(isempty(args)) |
|
|
|
@assert(isempty(args)) |
|
|
|
@ -810,7 +802,7 @@ counter_family = Prometheus.Family{Counter}( |
|
|
|
Prometheus.inc(Prometheus.labels(counter_family, (target="/api", status_code=200))) |
|
|
|
Prometheus.inc(Prometheus.labels(counter_family, (target="/api", status_code=200))) |
|
|
|
``` |
|
|
|
``` |
|
|
|
""" |
|
|
|
""" |
|
|
|
Family{C}(::String, ::String, ::Any; kwargs...) where C |
|
|
|
Family{C}(::String, ::String, ::Any; kwargs...) where {C} |
|
|
|
|
|
|
|
|
|
|
|
function metric_names(family::Family) |
|
|
|
function metric_names(family::Family) |
|
|
|
return (family.metric_name,) |
|
|
|
return (family.metric_name,) |
|
|
|
@ -870,7 +862,7 @@ Refer to [`Prometheus.labels`](@ref) for how to specify `label_values`. |
|
|
|
!!! note |
|
|
|
!!! note |
|
|
|
This method invalidates cached collectors for the label names. |
|
|
|
This method invalidates cached collectors for the label names. |
|
|
|
""" |
|
|
|
""" |
|
|
|
function remove(family::Family{<:Any, N}, label_values) where N |
|
|
|
function remove(family::Family{<:Any, N}, label_values) where {N} |
|
|
|
labels = make_label_values(family.label_names, label_values)::LabelValues{N} |
|
|
|
labels = make_label_values(family.label_names, label_values)::LabelValues{N} |
|
|
|
@lock family.lock delete!(family.children, labels) |
|
|
|
@lock family.lock delete!(family.children, labels) |
|
|
|
return |
|
|
|
return |
|
|
|
@ -895,7 +887,7 @@ prometheus_type(::Type{Gauge}) = "gauge" |
|
|
|
prometheus_type(::Type{Histogram}) = "histogram" |
|
|
|
prometheus_type(::Type{Histogram}) = "histogram" |
|
|
|
prometheus_type(::Type{Summary}) = "summary" |
|
|
|
prometheus_type(::Type{Summary}) = "summary" |
|
|
|
|
|
|
|
|
|
|
|
function collect!(metrics::Vector, family::Family{C}) where C |
|
|
|
function collect!(metrics::Vector, family::Family{C}) where {C} |
|
|
|
type = prometheus_type(C) |
|
|
|
type = prometheus_type(C) |
|
|
|
samples = Sample[] |
|
|
|
samples = Sample[] |
|
|
|
buf = Metric[] |
|
|
|
buf = Metric[] |
|
|
|
@ -922,14 +914,8 @@ function collect!(metrics::Vector, family::Family{C}) where C |
|
|
|
length(child_sample.label_values.label_values) |
|
|
|
length(child_sample.label_values.label_values) |
|
|
|
) |
|
|
|
) |
|
|
|
# TODO: Bypass constructor verifications |
|
|
|
# TODO: Bypass constructor verifications |
|
|
|
merged_names = LabelNames(( |
|
|
|
merged_names = LabelNames((label_names.label_names..., child_sample.label_names.label_names...)) |
|
|
|
label_names.label_names..., |
|
|
|
merged_values = LabelValues((label_values.label_values..., child_sample.label_values.label_values...)) |
|
|
|
child_sample.label_names.label_names..., |
|
|
|
|
|
|
|
)) |
|
|
|
|
|
|
|
merged_values = LabelValues(( |
|
|
|
|
|
|
|
label_values.label_values..., |
|
|
|
|
|
|
|
child_sample.label_values.label_values..., |
|
|
|
|
|
|
|
)) |
|
|
|
|
|
|
|
push!(samples, Sample(child_sample.suffix, merged_names, merged_values, child_sample.value)) |
|
|
|
push!(samples, Sample(child_sample.suffix, merged_names, merged_values, child_sample.value)) |
|
|
|
else |
|
|
|
else |
|
|
|
@assert( |
|
|
|
@assert( |
|
|
|
@ -943,11 +929,13 @@ function collect!(metrics::Vector, family::Family{C}) where C |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
# Sort samples lexicographically by the labels |
|
|
|
# Sort samples lexicographically by the labels |
|
|
|
sort!(samples; by = function(x) |
|
|
|
sort!( |
|
|
|
|
|
|
|
samples; by = function(x) |
|
|
|
labels = x.label_values |
|
|
|
labels = x.label_values |
|
|
|
@assert(labels !== nothing) |
|
|
|
@assert(labels !== nothing) |
|
|
|
return labels.label_values |
|
|
|
return labels.label_values |
|
|
|
end) |
|
|
|
end |
|
|
|
|
|
|
|
) |
|
|
|
push!( |
|
|
|
push!( |
|
|
|
metrics, |
|
|
|
metrics, |
|
|
|
Metric(type, family.metric_name, family.help, samples), |
|
|
|
Metric(type, family.metric_name, family.help, samples), |
|
|
|
@ -970,7 +958,7 @@ struct Sample |
|
|
|
label_names::Union{Nothing, LabelNames{N}}, |
|
|
|
label_names::Union{Nothing, LabelNames{N}}, |
|
|
|
label_values::Union{Nothing, LabelValues{N}}, |
|
|
|
label_values::Union{Nothing, LabelValues{N}}, |
|
|
|
value::Real, |
|
|
|
value::Real, |
|
|
|
) where N |
|
|
|
) where {N} |
|
|
|
@assert((label_names === nothing) === (label_values === nothing)) |
|
|
|
@assert((label_names === nothing) === (label_values === nothing)) |
|
|
|
return new(suffix, label_names, label_values, value) |
|
|
|
return new(suffix, label_names, label_values, value) |
|
|
|
end |
|
|
|
end |
|
|
|
@ -1038,6 +1026,7 @@ function expose_metric(io::IO, metric::Metric) |
|
|
|
println(io, " ", isinteger(sample.value) ? Int(sample.value) : sample.value) |
|
|
|
println(io, " ", isinteger(sample.value) ? Int(sample.value) : sample.value) |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
return |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
""" |
|
|
|
""" |
|
|
|
|