6 changed files with 1010 additions and 0 deletions
@ -0,0 +1,22 @@ |
|||||||
|
MIT License |
||||||
|
|
||||||
|
Copyright (c) 2023 Fredrik Ekre and Prometheus.jl contributors |
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||||
|
of this software and associated documentation files (the "Software"), to deal |
||||||
|
in the Software without restriction, including without limitation the rights |
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||||
|
copies of the Software, and to permit persons to whom the Software is |
||||||
|
furnished to do so, subject to the following conditions: |
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all |
||||||
|
copies or substantial portions of the Software. |
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||||
|
SOFTWARE. |
||||||
|
|
||||||
@ -0,0 +1,13 @@ |
|||||||
|
name = "Prometheus" |
||||||
|
uuid = "f25c1797-fe98-4e0c-b252-1b4fe3b6bde6" |
||||||
|
version = "1.0.0" |
||||||
|
|
||||||
|
[deps] |
||||||
|
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" |
||||||
|
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" |
||||||
|
|
||||||
|
[extras] |
||||||
|
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" |
||||||
|
|
||||||
|
[targets] |
||||||
|
test = ["Test"] |
||||||
@ -0,0 +1,90 @@ |
|||||||
|
# Prometheus.jl |
||||||
|
|
||||||
|
*A [Prometheus](https://prometheus.io/) client for Julia* |
||||||
|
|
||||||
|
## Quickstart |
||||||
|
|
||||||
|
1. Install Prometheus.jl and [HTTP.jl](https://github.com/JuliaWeb/HTTP.jl) |
||||||
|
using the package manager: |
||||||
|
``` |
||||||
|
pkg> add Prometheus HTTP |
||||||
|
``` |
||||||
|
|
||||||
|
2. Paste the following code into a Julia REPL. |
||||||
|
```julia |
||||||
|
# Load the packages |
||||||
|
using Prometheus, HTTP |
||||||
|
|
||||||
|
# Create a Counter metric |
||||||
|
const request_counter = Prometheus.Counter("request_count", "Number of handled requests") |
||||||
|
|
||||||
|
# Start a HTTP server on localhost port 8000 to server the metrics |
||||||
|
server = HTTP.listen!(8000) do http |
||||||
|
Prometheus.inc(request_counter) # Increment the request counter |
||||||
|
return Prometheus.expose(http) # Expose the metrics |
||||||
|
end |
||||||
|
``` |
||||||
|
|
||||||
|
3. Visit <http://localhost:8000> in your browser. You will see something like the following |
||||||
|
``` |
||||||
|
# HELP request_count Number of handled requests |
||||||
|
# TYPE request_count counter |
||||||
|
request_count 1 |
||||||
|
``` |
||||||
|
which is how the counter is presented when Prometheus scrapes the metrics. |
||||||
|
Every time you refresh, the counter will increment its value. |
||||||
|
`close(server)` will shutdown the server. |
||||||
|
|
||||||
|
|
||||||
|
## Collectors |
||||||
|
|
||||||
|
### Counter |
||||||
|
|
||||||
|
See <https://prometheus.io/docs/concepts/metric_types/#counter> for details. |
||||||
|
|
||||||
|
Supported methods: |
||||||
|
- `Prometheus.inc(counter)`: increment the counter with 1. |
||||||
|
- `Prometheus.inc(counter, v)`: increment the counter with `v`. |
||||||
|
|
||||||
|
### Gauge |
||||||
|
|
||||||
|
See <https://prometheus.io/docs/concepts/metric_types/#gauge> for details. |
||||||
|
|
||||||
|
Supported methods: |
||||||
|
- `Prometheus.inc(gauge)`: increment the gauges's value with 1. |
||||||
|
- `Prometheus.inc(gauge, v)`: increment the gauge's value with `v`. |
||||||
|
- `Prometheus.dec(gauge)`: decrement the gauges's value with 1. |
||||||
|
- `Prometheus.dec(gauge, v)`: decrement the gauge's value with `v`. |
||||||
|
- `Prometheus.set_to_current_time(gauge)`: set the gauge's value to the current unixtime in |
||||||
|
seconds. |
||||||
|
|
||||||
|
### Summary |
||||||
|
|
||||||
|
See <https://prometheus.io/docs/concepts/metric_types/#summary> for details. |
||||||
|
|
||||||
|
Supported methods: |
||||||
|
- `Prometheus.observe(summary, v)`: record the observed value `v`. |
||||||
|
|
||||||
|
## Labels |
||||||
|
|
||||||
|
See <https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels> for details. |
||||||
|
|
||||||
|
All metrics can be labeled using the special `Prometheus.Family` collector. For example, a |
||||||
|
labeled Counter collector |
||||||
|
```julia |
||||||
|
labelnames = ["endpoint", "status_code"] |
||||||
|
counter_family = Prometheus.Family{Prometheus.Collector}( |
||||||
|
"http_requests", |
||||||
|
"Number of processed requests", |
||||||
|
labelnames, |
||||||
|
) |
||||||
|
``` |
||||||
|
|
||||||
|
Supported methods: |
||||||
|
- `Prometheus.labels(family, ["label 1", "label 2"])`: extract the child collector |
||||||
|
corresponding to the labels `["label 1", "label 2"]`. |
||||||
|
- `Prometheus.remove(family, ["label 1", "label 2"])`: remove the child collector |
||||||
|
corresponding to the labels `["label 1", "label 2"]`. |
||||||
|
- `Prometheus.clear(family)`: clear all child collectors. |
||||||
|
|
||||||
|
## Registries |
||||||
@ -0,0 +1,530 @@ |
|||||||
|
module Prometheus |
||||||
|
|
||||||
|
using HTTP: HTTP |
||||||
|
using Sockets: Sockets |
||||||
|
|
||||||
|
abstract type Collector end |
||||||
|
|
||||||
|
######### |
||||||
|
# Utils # |
||||||
|
######### |
||||||
|
|
||||||
|
abstract type PrometheusException <: Exception end |
||||||
|
|
||||||
|
struct PrometheusUnreachable <: PrometheusException end |
||||||
|
unreachable() = throw(PrometheusUnreachable()) |
||||||
|
|
||||||
|
struct PrometheusAssert <: PrometheusException end |
||||||
|
macro assert(cond) |
||||||
|
return quote |
||||||
|
$(esc(cond)) || throw(PrometheusAssert()) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
########################################### |
||||||
|
# Compat for const fields, @lock, @atomic # |
||||||
|
########################################### |
||||||
|
@eval macro $(Symbol("const"))(field) |
||||||
|
if VERSION >= v"1.8.0-DEV.1148" |
||||||
|
Expr(:const, esc(field)) |
||||||
|
else |
||||||
|
return esc(field) |
||||||
|
end |
||||||
|
end |
||||||
|
if VERSION < v"1.7.0" |
||||||
|
# Defined but not exported |
||||||
|
using Base: @lock |
||||||
|
end |
||||||
|
if !isdefined(Base, Symbol("@atomic")) # v1.7.0 |
||||||
|
const ATOMIC_COMPAT_LOCK = ReentrantLock() |
||||||
|
macro atomic(expr) |
||||||
|
if Meta.isexpr(expr, :(::)) |
||||||
|
return esc(expr) |
||||||
|
else |
||||||
|
return quote |
||||||
|
lock(ATOMIC_COMPAT_LOCK) |
||||||
|
tmp = $(esc(expr)) |
||||||
|
unlock(ATOMIC_COMPAT_LOCK) |
||||||
|
tmp |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
|
||||||
|
##################### |
||||||
|
# CollectorRegistry # |
||||||
|
##################### |
||||||
|
|
||||||
|
struct CollectorRegistry |
||||||
|
lock::ReentrantLock |
||||||
|
collectors::Base.IdSet{Collector} |
||||||
|
function CollectorRegistry() |
||||||
|
return new(ReentrantLock(), Base.IdSet{Collector}()) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
const DEFAULT_REGISTRY = CollectorRegistry() |
||||||
|
|
||||||
|
function register(reg::CollectorRegistry, collector::Collector) |
||||||
|
existing_names = Set{String}() # TODO: Cache existing_names in the registry? |
||||||
|
@lock reg.lock begin |
||||||
|
for c in reg.collectors |
||||||
|
union!(existing_names, metric_names(c)) |
||||||
|
end |
||||||
|
if any(in(existing_names), metric_names(collector)) |
||||||
|
error("not allowed") |
||||||
|
end |
||||||
|
push!(reg.collectors, collector) |
||||||
|
end |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
function unregister(reg::CollectorRegistry, collector::Collector) |
||||||
|
@lock reg.lock delete!(reg.collectors, collector) |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
############## |
||||||
|
# Collectors # |
||||||
|
############## |
||||||
|
|
||||||
|
# abstract type Collector end |
||||||
|
|
||||||
|
function collect(collector::Collector) |
||||||
|
return collect!(Metric[], collector) |
||||||
|
end |
||||||
|
|
||||||
|
######################## |
||||||
|
# Counter <: Collector # |
||||||
|
######################## |
||||||
|
# https://prometheus.io/docs/instrumenting/writing_clientlibs/#counter |
||||||
|
|
||||||
|
# TODO: A counter is ENCOURAGED to have: |
||||||
|
# - A way to count exceptions throw/raised in a given piece of code, and optionally only |
||||||
|
# certain types of exceptions. This is count_exceptions in Python. |
||||||
|
|
||||||
|
mutable struct Counter <: Collector |
||||||
|
@const metric_name::String |
||||||
|
@const help::String |
||||||
|
@atomic value::Float64 |
||||||
|
|
||||||
|
function Counter( |
||||||
|
registry::Union{CollectorRegistry, Nothing}, |
||||||
|
metric_name::String, help::String, |
||||||
|
) |
||||||
|
initial_value = 0.0 |
||||||
|
counter = new(metric_name, help, initial_value) |
||||||
|
if registry !== nothing |
||||||
|
register(registry, counter) |
||||||
|
end |
||||||
|
return counter |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
function Counter(metric_name::String, help::String) |
||||||
|
return Counter(DEFAULT_REGISTRY, metric_name, help) |
||||||
|
end |
||||||
|
|
||||||
|
function metric_names(counter::Counter) |
||||||
|
return (counter.metric_name, ) |
||||||
|
end |
||||||
|
|
||||||
|
function inc(m::Counter, v = 1.0) |
||||||
|
if v < 0 |
||||||
|
error("counting backwards") |
||||||
|
end |
||||||
|
@atomic m.value += v |
||||||
|
return nothing |
||||||
|
end |
||||||
|
|
||||||
|
function collect!(metrics::Vector, counter::Counter) |
||||||
|
push!(metrics, |
||||||
|
Metric( |
||||||
|
"counter", counter.metric_name, counter.help, |
||||||
|
nothing, Sample(nothing, nothing, @atomic(counter.value)), |
||||||
|
), |
||||||
|
) |
||||||
|
return metrics |
||||||
|
end |
||||||
|
|
||||||
|
|
||||||
|
###################### |
||||||
|
# Gauge <: Collector # |
||||||
|
###################### |
||||||
|
# https://prometheus.io/docs/instrumenting/writing_clientlibs/#gauge |
||||||
|
|
||||||
|
# TODO: A gauge is ENCOURAGED to have: |
||||||
|
# - A way to track in-progress requests in some piece of code/function. This is |
||||||
|
# track_inprogress in Python. |
||||||
|
# - A way to time a piece of code and set the gauge to its duration in seconds. This is |
||||||
|
# useful for batch jobs. This is startTimer/setDuration in Java and the time() |
||||||
|
# decorator/context manager in Python. This SHOULD match the pattern in Summary/Histogram |
||||||
|
# (though set() rather than observe()). |
||||||
|
|
||||||
|
mutable struct Gauge <: Collector |
||||||
|
@const metric_name::String |
||||||
|
@const help::String |
||||||
|
@atomic value::Float64 |
||||||
|
|
||||||
|
function Gauge( |
||||||
|
registry::Union{CollectorRegistry, Nothing}, |
||||||
|
metric_name::String, help::String, |
||||||
|
) |
||||||
|
initial_value = 0.0 |
||||||
|
gauge = new(metric_name, help, initial_value) |
||||||
|
if registry !== nothing |
||||||
|
register(registry, gauge) |
||||||
|
end |
||||||
|
return gauge |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
function Gauge(metric_name::String, help::String) |
||||||
|
return Gauge(DEFAULT_REGISTRY, metric_name, help) |
||||||
|
end |
||||||
|
|
||||||
|
function metric_names(gauge::Gauge) |
||||||
|
return (gauge.metric_name, ) |
||||||
|
end |
||||||
|
|
||||||
|
function inc(m::Gauge, v = 1.0) |
||||||
|
if v < 0 |
||||||
|
error("incrementing with negative value, use dec(...)?") |
||||||
|
end |
||||||
|
@atomic m.value += v |
||||||
|
return nothing |
||||||
|
end |
||||||
|
|
||||||
|
function dec(m::Gauge, v = 1.0) |
||||||
|
if v < 0 |
||||||
|
error("decrementing with negative value, use inc(...)?") |
||||||
|
end |
||||||
|
@atomic m.value -= v |
||||||
|
return nothing |
||||||
|
end |
||||||
|
|
||||||
|
function set(m::Gauge, v) |
||||||
|
@atomic m.value = v |
||||||
|
return nothing |
||||||
|
end |
||||||
|
|
||||||
|
function set_to_current_time(m::Gauge) |
||||||
|
@atomic m.value = time() |
||||||
|
return nothing |
||||||
|
end |
||||||
|
|
||||||
|
function collect!(metrics::Vector, gauge::Gauge) |
||||||
|
push!(metrics, |
||||||
|
Metric( |
||||||
|
"gauge", gauge.metric_name, gauge.help, |
||||||
|
nothing, Sample(nothing, nothing, @atomic(gauge.value)), |
||||||
|
), |
||||||
|
) |
||||||
|
return metrics |
||||||
|
end |
||||||
|
|
||||||
|
|
||||||
|
######################## |
||||||
|
# Summary <: Collector # |
||||||
|
######################## |
||||||
|
# https://prometheus.io/docs/instrumenting/writing_clientlibs/#summary |
||||||
|
|
||||||
|
# TODO: A summary SHOULD have the following methods: |
||||||
|
# - Some way to time code for users in seconds. In Python this is the time() |
||||||
|
# decorator/context manager. In Java this is startTimer/observeDuration. Units other than |
||||||
|
# seconds MUST NOT be offered (if a user wants something else, they can do it by hand). |
||||||
|
# This should follow the same pattern as Gauge/Histogram. |
||||||
|
|
||||||
|
mutable struct Summary <: Collector |
||||||
|
@const metric_name::String |
||||||
|
@const help::String |
||||||
|
@atomic _count::Int |
||||||
|
@atomic _sum::Float64 |
||||||
|
|
||||||
|
function Summary( |
||||||
|
registry::Union{CollectorRegistry, Nothing}, |
||||||
|
metric_name::String, help::String, |
||||||
|
) |
||||||
|
initial_count = 0 |
||||||
|
initial_sum = 0.0 |
||||||
|
summary = new(metric_name, help, initial_count, initial_sum) |
||||||
|
if registry !== nothing |
||||||
|
register(registry, summary) |
||||||
|
end |
||||||
|
return summary |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
function Summary(metric_name::String, help::String) |
||||||
|
return Summary(DEFAULT_REGISTRY, metric_name, help) |
||||||
|
end |
||||||
|
|
||||||
|
function metric_names(summary::Summary) |
||||||
|
return (summary.metric_name * "_count", summary.metric_name * "_sum") |
||||||
|
end |
||||||
|
|
||||||
|
function observe(summary::Summary, v) |
||||||
|
@atomic summary._count += 1 |
||||||
|
@atomic summary._sum += v |
||||||
|
return nothing |
||||||
|
end |
||||||
|
|
||||||
|
function collect!(metrics::Vector, summary::Summary) |
||||||
|
push!(metrics, |
||||||
|
Metric( |
||||||
|
"summary", summary.metric_name, summary.help, nothing, |
||||||
|
[ |
||||||
|
Sample("_count", nothing, @atomic(summary._count)), |
||||||
|
Sample("_sum", nothing, @atomic(summary._sum)), |
||||||
|
] |
||||||
|
), |
||||||
|
) |
||||||
|
return metrics |
||||||
|
end |
||||||
|
|
||||||
|
|
||||||
|
#################################### |
||||||
|
# Family{<:Collector} <: Collector # |
||||||
|
#################################### |
||||||
|
|
||||||
|
struct LabelNames |
||||||
|
labelnames::Vector{String} |
||||||
|
end |
||||||
|
|
||||||
|
struct LabelValues |
||||||
|
labelvalues::Vector{String} |
||||||
|
end |
||||||
|
function Base.hash(l::LabelValues, h::UInt) |
||||||
|
h = hash(0x94a2d04ee9e5a55b, h) # hash("Prometheus.LabelValues") on Julia 1.9.3 |
||||||
|
for v in l.labelvalues |
||||||
|
h = hash(v, h) |
||||||
|
end |
||||||
|
return h |
||||||
|
end |
||||||
|
function Base.:(==)(l1::LabelValues, l2::LabelValues) |
||||||
|
return l1.labelvalues == l2.labelvalues |
||||||
|
end |
||||||
|
|
||||||
|
struct Family{C} <: Collector |
||||||
|
metric_name::String |
||||||
|
help::String |
||||||
|
labelnames::LabelNames |
||||||
|
children::Dict{LabelValues, C} |
||||||
|
lock::ReentrantLock |
||||||
|
|
||||||
|
function Family{C}( |
||||||
|
registry::Union{CollectorRegistry, Nothing}, |
||||||
|
metric_name::String, help::String, labelnames::LabelNames, |
||||||
|
) where C |
||||||
|
children = Dict{LabelValues, C}() |
||||||
|
lock = ReentrantLock() |
||||||
|
family = new(metric_name, help, labelnames, children, lock) |
||||||
|
if registry !== nothing |
||||||
|
register(registry, family) |
||||||
|
end |
||||||
|
return family |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
function Family{C}(metric_name::String, help::String, labelnames) where C |
||||||
|
return Family{C}(DEFAULT_REGISTRY, metric_name, help, LabelNames(labelnames)) |
||||||
|
end |
||||||
|
function Family{C}(registry::Union{CollectorRegistry, Nothing}, metric_name::String, help::String, labelnames) where C |
||||||
|
return Family{C}(registry, metric_name, help, LabelNames(labelnames)) |
||||||
|
end |
||||||
|
|
||||||
|
function metric_names(family::Family) |
||||||
|
return (family.metric_name, ) |
||||||
|
end |
||||||
|
|
||||||
|
function labels(family::Family{C}, labelvalues::LabelValues) where C |
||||||
|
collector = @lock family.lock get!(family.children, labelvalues) do |
||||||
|
C(nothing, family.metric_name, family.help) |
||||||
|
end |
||||||
|
return collector |
||||||
|
end |
||||||
|
labels(family::Family, labelvalues) = labels(family, LabelValues(labelvalues)) |
||||||
|
|
||||||
|
function remove(family::Family, labelvalues::LabelValues) |
||||||
|
@lock family.lock delete!(family.children, labelvalues) |
||||||
|
return |
||||||
|
end |
||||||
|
remove(family::Family, labelvalues) = remove(family, LabelValues(labelvalues)) |
||||||
|
|
||||||
|
function clear(family::Family) |
||||||
|
@lock family.lock empty!(family.children) |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
prometheus_type(::Type{Counter}) = "counter" |
||||||
|
prometheus_type(::Type{Gauge}) = "gauge" |
||||||
|
prometheus_type(::Type{Summary}) = "summary" |
||||||
|
prometheus_type(::Type) = unreachable() |
||||||
|
|
||||||
|
function collect!(metrics::Vector, family::Family{C}) where C |
||||||
|
type = prometheus_type(C) |
||||||
|
samples = Sample[] |
||||||
|
buf = Metric[] |
||||||
|
@lock family.lock begin |
||||||
|
for (labels, child) in family.children |
||||||
|
# collect!(...) the child, throw away the metric, but keep the samples |
||||||
|
child_metrics = collect!(resize!(buf, 0), child) |
||||||
|
length(child_metrics) !=1 && error("multiple metrics not supported (yet?)") |
||||||
|
child_metric = child_metrics[1] |
||||||
|
@assert(child_metric.type == type) |
||||||
|
# Unwrap and rewrap samples with the labels |
||||||
|
child_samples = child_metric.samples |
||||||
|
if child_samples isa Sample |
||||||
|
push!(samples, Sample(child_samples.suffix, labels, child_samples.value)) |
||||||
|
else |
||||||
|
@assert(child_samples isa Vector{Sample}) |
||||||
|
for child_sample in child_samples |
||||||
|
@assert(child_sample.labels === nothing) |
||||||
|
push!(samples, Sample(child_sample.suffix, labels, child_sample.value)) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
# Sort samples lexicographically by the labels |
||||||
|
sort!(samples; by = function(x) |
||||||
|
labels = x.labels |
||||||
|
@assert(labels !== nothing) |
||||||
|
return labels.labelvalues |
||||||
|
end) |
||||||
|
push!(metrics, Metric(type, family.metric_name, family.help, family.labelnames, samples)) |
||||||
|
return metrics |
||||||
|
end |
||||||
|
|
||||||
|
|
||||||
|
############## |
||||||
|
# Exposition # |
||||||
|
############## |
||||||
|
|
||||||
|
struct Sample |
||||||
|
suffix::Union{String, Nothing} # e.g. _count or _sum |
||||||
|
labels::Union{LabelValues, Nothing} |
||||||
|
value::Float64 |
||||||
|
end |
||||||
|
|
||||||
|
struct Metric |
||||||
|
type::String |
||||||
|
metric_name::String |
||||||
|
help::String |
||||||
|
labelnames::Union{LabelNames, Nothing} |
||||||
|
# TODO: Union{Tuple{Sample}, Vector{Sample}} would always make this iterable. |
||||||
|
samples::Union{Sample, Vector{Sample}} |
||||||
|
end |
||||||
|
|
||||||
|
function expose_metric(io::IO, metric::Metric) |
||||||
|
println(io, "# HELP ", metric.metric_name, " ", metric.help) |
||||||
|
println(io, "# TYPE ", metric.metric_name, " ", metric.type) |
||||||
|
labelnames = metric.labelnames |
||||||
|
samples = metric.samples |
||||||
|
if samples isa Sample |
||||||
|
# Single sample, no labels |
||||||
|
@assert(labelnames === nothing) |
||||||
|
@assert(samples.labels === nothing) |
||||||
|
@assert(samples.suffix === nothing) |
||||||
|
val = samples.value |
||||||
|
println(io, metric.metric_name, " ", isinteger(val) ? Int(val) : val) |
||||||
|
else |
||||||
|
# Multiple samples, might have labels |
||||||
|
@assert(samples isa Vector{Sample}) |
||||||
|
for sample in samples |
||||||
|
# Print metric name |
||||||
|
print(io, metric.metric_name) |
||||||
|
# Print potential suffix |
||||||
|
if sample.suffix !== nothing |
||||||
|
print(io, sample.suffix) |
||||||
|
end |
||||||
|
# Print potential labels |
||||||
|
labels = sample.labels |
||||||
|
if labelnames !== nothing && labels !== nothing |
||||||
|
first = true |
||||||
|
print(io, "{") |
||||||
|
for (name, value) in zip(labelnames.labelnames, labels.labelvalues) |
||||||
|
first || print(io, ",") |
||||||
|
print(io, name, "=\"", value, "\"") |
||||||
|
first = false |
||||||
|
end |
||||||
|
print(io, "}") |
||||||
|
end |
||||||
|
# Print the value |
||||||
|
println(io, " ", isinteger(sample.value) ? Int(sample.value) : sample.value) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
""" |
||||||
|
Prometheus.expose(file::String, reg::CollectorRegistry = DEFAULT_REGISTRY) |
||||||
|
|
||||||
|
Export all metrics in `reg` by writing them to the file `file`. |
||||||
|
""" |
||||||
|
function expose(path::String, reg::CollectorRegistry = DEFAULT_REGISTRY) |
||||||
|
dir = dirname(path) |
||||||
|
mkpath(dir) |
||||||
|
mktemp(dirname(path)) do tmp_path, tmp_io |
||||||
|
expose_io(tmp_io, reg) |
||||||
|
close(tmp_io) |
||||||
|
mv(tmp_path, path; force=true) |
||||||
|
end |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
""" |
||||||
|
expose(io::IO, reg::CollectorRegistry = DEFAULT_REGISTRY) |
||||||
|
|
||||||
|
Export all metrics in `reg` by writing them to the I/O stream `io`. |
||||||
|
""" |
||||||
|
function expose(io::IO, reg::CollectorRegistry = DEFAULT_REGISTRY) |
||||||
|
return expose_io(io, reg) |
||||||
|
end |
||||||
|
|
||||||
|
function expose_io(io::IO, reg::CollectorRegistry) |
||||||
|
# Collect all metrics |
||||||
|
metrics = Metric[] |
||||||
|
@lock reg.lock for collector in reg.collectors |
||||||
|
collect!(metrics, collector) |
||||||
|
end |
||||||
|
sort!(metrics; by = metric -> metric.metric_name) |
||||||
|
# Write to IO |
||||||
|
buf = IOBuffer(; maxsize=1024^2) # 1 MB |
||||||
|
for metric in metrics |
||||||
|
truncate(buf, 0) |
||||||
|
expose_metric(buf, metric) |
||||||
|
seekstart(buf) |
||||||
|
write(io, buf) |
||||||
|
end |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
####################### |
||||||
|
# HTTP.jl integration # |
||||||
|
####################### |
||||||
|
|
||||||
|
const CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" |
||||||
|
|
||||||
|
""" |
||||||
|
expose(http::HTTP.Stream, reg::CollectorRegistry = DEFAULT_REGISTRY) |
||||||
|
|
||||||
|
Export all metrics in `reg` by writing them to the HTTP stream `http`. |
||||||
|
|
||||||
|
The caller is responsible for checking e.g. the HTTP method and URI target. For |
||||||
|
HEAD requests this method do not write a body, however. |
||||||
|
""" |
||||||
|
function expose(http::HTTP.Stream, reg::CollectorRegistry = DEFAULT_REGISTRY) |
||||||
|
# TODO: Handle Accept request header |
||||||
|
# TODO: Compression if requested/supported |
||||||
|
HTTP.setstatus(http, 200) |
||||||
|
HTTP.setheader(http, "Content-Type" => CONTENT_TYPE_LATEST) |
||||||
|
HTTP.startwrite(http) |
||||||
|
# The user is repsonsible for making sure that e.g. target and method is |
||||||
|
# correct, but at least we skip writing the body for HEAD requests. |
||||||
|
if http.message.method != "HEAD" |
||||||
|
expose_io(http, reg) |
||||||
|
end |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
end # module Prometheus |
||||||
@ -0,0 +1,352 @@ |
|||||||
|
using HTTP: HTTP |
||||||
|
using Prometheus: Prometheus |
||||||
|
using Test: @test, @test_throws, @testset |
||||||
|
|
||||||
|
@testset "Prometheus.CollectorRegistry" begin |
||||||
|
empty!(Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
# Default registry |
||||||
|
c = Prometheus.Counter("metric_name_counter", "A counter.") |
||||||
|
@test c in Prometheus.DEFAULT_REGISTRY.collectors |
||||||
|
@test_throws ErrorException Prometheus.Counter("metric_name_counter", "A counter.") |
||||||
|
Prometheus.unregister(Prometheus.DEFAULT_REGISTRY, c) |
||||||
|
@test !(c in Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
c2 = Prometheus.Counter("metric_name_counter", "A counter.") |
||||||
|
@test c2 in Prometheus.DEFAULT_REGISTRY.collectors |
||||||
|
# Provided registry |
||||||
|
r = Prometheus.CollectorRegistry() |
||||||
|
c = Prometheus.Counter(r, "metric_name_counter", "A counter.") |
||||||
|
@test c in r.collectors |
||||||
|
@test !(c in Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
# No registry on construction, register after |
||||||
|
c = Prometheus.Counter(nothing, "metric_name_counter", "A counter.") |
||||||
|
@test !(c in Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
r = Prometheus.CollectorRegistry() |
||||||
|
Prometheus.register(r, c) |
||||||
|
@test c in r.collectors |
||||||
|
@test_throws ErrorException Prometheus.register(r, c) |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.Counter" begin |
||||||
|
# Constructors and implicit registration |
||||||
|
empty!(Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
c = Prometheus.Counter("metric_name_counter", "A counter.") |
||||||
|
@test c in Prometheus.DEFAULT_REGISTRY.collectors |
||||||
|
r = Prometheus.CollectorRegistry() |
||||||
|
c = Prometheus.Counter(r, "metric_name_counter", "A counter.") |
||||||
|
@test c in r.collectors |
||||||
|
@test c.value == 0 |
||||||
|
# Prometheus.inc(...) |
||||||
|
Prometheus.inc(c) |
||||||
|
@test c.value == 1 |
||||||
|
Prometheus.inc(c, 0) |
||||||
|
@test c.value == 1 |
||||||
|
Prometheus.inc(c, 2) |
||||||
|
@test c.value == 3 |
||||||
|
@test_throws ErrorException Prometheus.inc(c, -1) |
||||||
|
# Prometheus.collect(...) |
||||||
|
metrics = Prometheus.collect(c) |
||||||
|
@test length(metrics) == 1 |
||||||
|
metric = metrics[1] |
||||||
|
@test metric.metric_name == c.metric_name |
||||||
|
@test metric.help == c.help |
||||||
|
@test metric.samples.value == c.value |
||||||
|
# Prometheus.expose_metric(...) |
||||||
|
@test sprint(Prometheus.expose_metric, metric) == |
||||||
|
sprint(Prometheus.expose_io, r) == |
||||||
|
""" |
||||||
|
# HELP metric_name_counter A counter. |
||||||
|
# TYPE metric_name_counter counter |
||||||
|
metric_name_counter 3 |
||||||
|
""" |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.Gauge" begin |
||||||
|
# Constructors and implicit registration |
||||||
|
empty!(Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
c = Prometheus.Gauge("metric_name_gauge", "A gauge.") |
||||||
|
@test c in Prometheus.DEFAULT_REGISTRY.collectors |
||||||
|
r = Prometheus.CollectorRegistry() |
||||||
|
c = Prometheus.Gauge(r, "metric_name_gauge", "A gauge.") |
||||||
|
@test c in r.collectors |
||||||
|
@test c.value == 0 |
||||||
|
# Prometheus.inc(...) |
||||||
|
Prometheus.inc(c) |
||||||
|
@test c.value == 1 |
||||||
|
Prometheus.inc(c, 0) |
||||||
|
@test c.value == 1 |
||||||
|
Prometheus.inc(c, 2) |
||||||
|
@test c.value == 3 |
||||||
|
@test_throws ErrorException Prometheus.inc(c, -1) |
||||||
|
# Prometheus.dec(...) |
||||||
|
Prometheus.dec(c) |
||||||
|
@test c.value == 2 |
||||||
|
Prometheus.dec(c, 1) |
||||||
|
@test c.value == 1 |
||||||
|
@test_throws ErrorException Prometheus.dec(c, -1) |
||||||
|
# Prometheus.set_to_current_time(...) |
||||||
|
t0 = time() |
||||||
|
sleep(0.1) |
||||||
|
Prometheus.set_to_current_time(c) |
||||||
|
sleep(0.1) |
||||||
|
@test t0 < c.value < time() |
||||||
|
# Prometheus.set(...) |
||||||
|
Prometheus.set(c, 42) |
||||||
|
@test c.value == 42 |
||||||
|
# Prometheus.collect(...) |
||||||
|
metrics = Prometheus.collect(c) |
||||||
|
@test length(metrics) == 1 |
||||||
|
metric = metrics[1] |
||||||
|
@test metric.metric_name == c.metric_name |
||||||
|
@test metric.help == c.help |
||||||
|
@test metric.samples.value == c.value |
||||||
|
# Prometheus.expose_metric(...) |
||||||
|
@test sprint(Prometheus.expose_metric, metric) == |
||||||
|
sprint(Prometheus.expose_io, r) == |
||||||
|
""" |
||||||
|
# HELP metric_name_gauge A gauge. |
||||||
|
# TYPE metric_name_gauge gauge |
||||||
|
metric_name_gauge 42 |
||||||
|
""" |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.Summary" begin |
||||||
|
# Constructors and implicit registration |
||||||
|
empty!(Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
c = Prometheus.Summary("metric_name_summary", "A summary.") |
||||||
|
@test c in Prometheus.DEFAULT_REGISTRY.collectors |
||||||
|
r = Prometheus.CollectorRegistry() |
||||||
|
c = Prometheus.Summary(r, "metric_name_summary", "A summary.") |
||||||
|
@test c in r.collectors |
||||||
|
@test c._count == 0 |
||||||
|
@test c._sum == 0 |
||||||
|
# Prometheus.observe(...) |
||||||
|
Prometheus.observe(c, 1) |
||||||
|
@test c._count == 1 |
||||||
|
@test c._sum == 1 |
||||||
|
Prometheus.observe(c, 10) |
||||||
|
@test c._count == 2 |
||||||
|
@test c._sum == 11 |
||||||
|
# Prometheus.collect(...) |
||||||
|
metrics = Prometheus.collect(c) |
||||||
|
@test length(metrics) == 1 |
||||||
|
metric = metrics[1] |
||||||
|
@test metric.metric_name == c.metric_name |
||||||
|
@test metric.help == c.help |
||||||
|
@test length(metric.samples) == 2 |
||||||
|
s1, s2 = metric.samples[1], metric.samples[2] |
||||||
|
@test s1.suffix == "_count" |
||||||
|
@test s2.suffix == "_sum" |
||||||
|
@test s1.labels === nothing |
||||||
|
@test s2.labels === nothing |
||||||
|
@test s1.value == 2 |
||||||
|
@test s2.value == 11 |
||||||
|
# Prometheus.expose_metric(...) |
||||||
|
@test sprint(Prometheus.expose_metric, metric) == |
||||||
|
sprint(Prometheus.expose_io, r) == |
||||||
|
""" |
||||||
|
# HELP metric_name_summary A summary. |
||||||
|
# TYPE metric_name_summary summary |
||||||
|
metric_name_summary_count 2 |
||||||
|
metric_name_summary_sum 11 |
||||||
|
""" |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.LabelNames and Prometheus.LabelValues" begin |
||||||
|
# Custom hashing |
||||||
|
v1 = Prometheus.LabelValues(["foo", "bar"]) |
||||||
|
v2 = Prometheus.LabelValues(["foo", "bar"]) |
||||||
|
v3 = Prometheus.LabelValues(["foo", "baz"]) |
||||||
|
@test hash(v1) == hash(v2) |
||||||
|
@test hash(v1) != hash(v3) |
||||||
|
@test v1 == v2 |
||||||
|
@test v1 != v3 |
||||||
|
@test isequal(v1, v2) |
||||||
|
@test !isequal(v1, v3) |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.Family{$(Collector)}" for Collector in (Prometheus.Counter, Prometheus.Gauge) |
||||||
|
# Constructors and implicit registration |
||||||
|
empty!(Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
c = Prometheus.Family{Collector}( |
||||||
|
"http_requests", "Number of HTTP requests.", |
||||||
|
["endpoint", "status_code"], |
||||||
|
) |
||||||
|
@test c in Prometheus.DEFAULT_REGISTRY.collectors |
||||||
|
r = Prometheus.CollectorRegistry() |
||||||
|
c = Prometheus.Family{Collector}( |
||||||
|
r, "http_requests", "Number of HTTP requests.", |
||||||
|
["endpoint", "status_code"], |
||||||
|
) |
||||||
|
@test c in r.collectors |
||||||
|
@test length(c.children) == 0 |
||||||
|
# Prometheus.labels(...), Prometheus.remove(...), Prometheus.clear() |
||||||
|
l1 = ["/foo/", "200"] |
||||||
|
l2 = ["/bar/", "404"] |
||||||
|
@test Prometheus.labels(c, l1) === Prometheus.labels(c, l1) |
||||||
|
@test Prometheus.labels(c, l2) === Prometheus.labels(c, l2) |
||||||
|
@test length(c.children) == 2 |
||||||
|
@test Prometheus.labels(c, l1).value == 0 |
||||||
|
@test Prometheus.labels(c, l2).value == 0 |
||||||
|
Prometheus.remove(c, l1) |
||||||
|
@test length(c.children) == 1 |
||||||
|
Prometheus.clear(c) |
||||||
|
@test length(c.children) == 0 |
||||||
|
# Prometheus.inc(...) |
||||||
|
Prometheus.inc(Prometheus.labels(c, l1)) |
||||||
|
Prometheus.inc(Prometheus.labels(c, l2)) |
||||||
|
@test Prometheus.labels(c, l1).value == 1 |
||||||
|
@test Prometheus.labels(c, l2).value == 1 |
||||||
|
Prometheus.inc(Prometheus.labels(c, l1), 2) |
||||||
|
Prometheus.inc(Prometheus.labels(c, l2), 2) |
||||||
|
@test Prometheus.labels(c, l1).value == 3 |
||||||
|
@test Prometheus.labels(c, l2).value == 3 |
||||||
|
# Prometheus.collect(...) |
||||||
|
metrics = Prometheus.collect(c) |
||||||
|
@test length(metrics) == 1 |
||||||
|
metric = metrics[1] |
||||||
|
@test metric.metric_name == c.metric_name |
||||||
|
@test metric.help == c.help |
||||||
|
@test length(metric.samples) == 2 |
||||||
|
s1, s2 = metric.samples[1], metric.samples[2] |
||||||
|
@test s1.labels.labelvalues == ["/bar/", "404"] |
||||||
|
@test s2.labels.labelvalues == ["/foo/", "200"] |
||||||
|
@test s1.value == 3 |
||||||
|
@test s2.value == 3 |
||||||
|
# Prometheus.expose_metric(...) |
||||||
|
type = Collector === Prometheus.Counter ? "counter" : "gauge" |
||||||
|
@test sprint(Prometheus.expose_metric, metric) == |
||||||
|
sprint(Prometheus.expose_io, r) == |
||||||
|
""" |
||||||
|
# HELP http_requests Number of HTTP requests. |
||||||
|
# TYPE http_requests $(type) |
||||||
|
http_requests{endpoint="/bar/",status_code="404"} 3 |
||||||
|
http_requests{endpoint="/foo/",status_code="200"} 3 |
||||||
|
""" |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.Family{Summary}" begin |
||||||
|
r = Prometheus.CollectorRegistry() |
||||||
|
c = Prometheus.Family{Prometheus.Summary}( |
||||||
|
r, "http_request_time", "Time to process requests.", |
||||||
|
["endpoint", "status_code"], |
||||||
|
) |
||||||
|
@test c in r.collectors |
||||||
|
@test length(c.children) == 0 |
||||||
|
# Prometheus.inc(...) |
||||||
|
l1 = ["/foo/", "200"] |
||||||
|
l2 = ["/bar/", "404"] |
||||||
|
@test Prometheus.labels(c, l1) === Prometheus.labels(c, l1) |
||||||
|
@test Prometheus.labels(c, l2) === Prometheus.labels(c, l2) |
||||||
|
@test length(c.children) == 2 |
||||||
|
@test Prometheus.labels(c, l1)._count == 0 |
||||||
|
@test Prometheus.labels(c, l1)._sum == 0 |
||||||
|
@test Prometheus.labels(c, l2)._count == 0 |
||||||
|
@test Prometheus.labels(c, l2)._sum == 0 |
||||||
|
Prometheus.observe(Prometheus.labels(c, l1), 1.2) |
||||||
|
Prometheus.observe(Prometheus.labels(c, l2), 2.1) |
||||||
|
@test Prometheus.labels(c, l1)._count == 1 |
||||||
|
@test Prometheus.labels(c, l1)._sum == 1.2 |
||||||
|
@test Prometheus.labels(c, l2)._count == 1 |
||||||
|
@test Prometheus.labels(c, l2)._sum == 2.1 |
||||||
|
Prometheus.observe(Prometheus.labels(c, l1), 3.4) |
||||||
|
Prometheus.observe(Prometheus.labels(c, l2), 4.3) |
||||||
|
@test Prometheus.labels(c, l1)._count == 2 |
||||||
|
@test Prometheus.labels(c, l1)._sum == 4.6 |
||||||
|
@test Prometheus.labels(c, l2)._count == 2 |
||||||
|
@test Prometheus.labels(c, l2)._sum == 6.4 |
||||||
|
# Prometheus.collect(...) |
||||||
|
metrics = Prometheus.collect(c) |
||||||
|
@test length(metrics) == 1 |
||||||
|
metric = metrics[1] |
||||||
|
@test metric.metric_name == c.metric_name |
||||||
|
@test metric.help == c.help |
||||||
|
@test length(metric.samples) == 4 |
||||||
|
s1, s2, s3, s4 = metric.samples |
||||||
|
@test s1.labels.labelvalues == s2.labels.labelvalues == ["/bar/", "404"] |
||||||
|
@test s3.labels.labelvalues == s4.labels.labelvalues == ["/foo/", "200"] |
||||||
|
@test s1.value == 2 # _count |
||||||
|
@test s2.value == 6.4 # _sum |
||||||
|
@test s3.value == 2 # _count |
||||||
|
@test s4.value == 4.6 # _sum |
||||||
|
# Prometheus.expose_metric(...) |
||||||
|
@test sprint(Prometheus.expose_metric, metric) == |
||||||
|
sprint(Prometheus.expose_io, r) == |
||||||
|
""" |
||||||
|
# HELP http_request_time Time to process requests. |
||||||
|
# TYPE http_request_time summary |
||||||
|
http_request_time_count{endpoint="/bar/",status_code="404"} 2 |
||||||
|
http_request_time_sum{endpoint="/bar/",status_code="404"} 6.4 |
||||||
|
http_request_time_count{endpoint="/foo/",status_code="200"} 2 |
||||||
|
http_request_time_sum{endpoint="/foo/",status_code="200"} 4.6 |
||||||
|
""" |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.expose(::Union{String, IO})" begin |
||||||
|
r = Prometheus.DEFAULT_REGISTRY |
||||||
|
empty!(r.collectors) |
||||||
|
Prometheus.inc(Prometheus.Counter(r, "prom_counter", "Counting things")) |
||||||
|
Prometheus.set(Prometheus.Gauge(r, "prom_gauge", "Gauging things"), 1.2) |
||||||
|
mktempdir() do dir |
||||||
|
default = joinpath(dir, "default.prom") |
||||||
|
Prometheus.expose(default) |
||||||
|
reg = joinpath(dir, "reg.prom") |
||||||
|
Prometheus.expose(reg, r) |
||||||
|
default_io = IOBuffer() |
||||||
|
Prometheus.expose(default_io) |
||||||
|
reg_io = IOBuffer() |
||||||
|
Prometheus.expose(reg_io, r) |
||||||
|
@test read(default, String) == |
||||||
|
read(reg, String) == |
||||||
|
String(take!(default_io)) == |
||||||
|
String(take!(reg_io)) == |
||||||
|
""" |
||||||
|
# HELP prom_counter Counting things |
||||||
|
# TYPE prom_counter counter |
||||||
|
prom_counter 1 |
||||||
|
# HELP prom_gauge Gauging things |
||||||
|
# TYPE prom_gauge gauge |
||||||
|
prom_gauge 1.2 |
||||||
|
""" |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Prometheus.expose(::HTTP.Stream)" begin |
||||||
|
empty!(Prometheus.DEFAULT_REGISTRY.collectors) |
||||||
|
Prometheus.inc(Prometheus.Counter("prom_counter", "Counting things")) |
||||||
|
Prometheus.set(Prometheus.Gauge("prom_gauge", "Gauging things"), 1.2) |
||||||
|
iob = IOBuffer() |
||||||
|
Prometheus.expose(iob) |
||||||
|
reference_output = String(take!(iob)) |
||||||
|
# Spin up the server |
||||||
|
server = HTTP.listen!(8123) do http |
||||||
|
if http.message.target == "/metrics/default" |
||||||
|
return Prometheus.expose(http) |
||||||
|
elseif http.message.target == "/metrics/reg" |
||||||
|
return Prometheus.expose(http, Prometheus.DEFAULT_REGISTRY) |
||||||
|
else |
||||||
|
HTTP.setstatus(http, 404) |
||||||
|
HTTP.startwrite(http) |
||||||
|
end |
||||||
|
end |
||||||
|
# Normal requests |
||||||
|
r_default = HTTP.request("GET", "http://localhost:8123/metrics/default") |
||||||
|
r_ref = HTTP.request("GET", "http://localhost:8123/metrics/reg") |
||||||
|
@test String(r_default.body) == String(r_ref.body) == reference_output |
||||||
|
# HEAD |
||||||
|
@test isempty(HTTP.request("HEAD", "http://localhost:8123/metrics/default").body) |
||||||
|
# POST (no filtering in the server above) |
||||||
|
r_post = HTTP.request("POST", "http://localhost:8123/metrics/default") |
||||||
|
@test String(r_post.body) == reference_output |
||||||
|
# Bad URI |
||||||
|
r_bad = HTTP.request("GET", "http://localhost:8123"; status_exception=false) |
||||||
|
@test r_bad.status == 404 |
||||||
|
# Clean up |
||||||
|
close(server) |
||||||
|
wait(server) |
||||||
|
end |
||||||
|
|
||||||
|
@testset "Utilities" begin |
||||||
|
@test_throws Prometheus.PrometheusUnreachable Prometheus.unreachable() |
||||||
|
@test_throws Prometheus.PrometheusAssert Prometheus.@assert false |
||||||
|
@test_throws Prometheus.PrometheusUnreachable Prometheus.prometheus_type(Int) |
||||||
|
end |
||||||
Loading…
Reference in new issue