From 28012d651ec3accac3da4e96507d7fc06dc2ea4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Thu, 4 May 2023 19:40:43 -0300 Subject: [PATCH 1/4] Update losses.jl --- src/losses.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/losses.jl b/src/losses.jl index 0f73ff3..03c90b3 100644 --- a/src/losses.jl +++ b/src/losses.jl @@ -5,7 +5,6 @@ Scalar = Union{Number,CategoricalValue} (loss::DistanceLoss)(output::Number, target::Number) = loss(output - target) deriv(loss::DistanceLoss, output::Number, target::Number) = deriv(loss, output - target) deriv2(loss::DistanceLoss, output::Number, target::Number) = deriv2(loss, output - target) - (loss::MarginLoss)(output::Number, target::Number) = loss(target * output) deriv(loss::MarginLoss, output::Number, target::Number) = target * deriv(loss, target * output) deriv2(loss::MarginLoss, output::Number, target::Number) = deriv2(loss, target * output) @@ -71,4 +70,4 @@ function mean(loss::SupervisedLoss, outputs, targets, weights; normalize=true) m = mean(w * loss(ŷ, y) for (ŷ, y, w) in zip(outputs, targets, weights)) n = normalize ? sum(weights) : one(first(weights)) m / n -end \ No newline at end of file +end From 6741aebd86cb1cf8b2c6612d37b8e3a6de412f59 Mon Sep 17 00:00:00 2001 From: Elias Carvalho <73039601+eliascarv@users.noreply.github.com> Date: Mon, 15 May 2023 09:31:31 -0300 Subject: [PATCH 2/4] Use JuliaFormatter.jl to format source code (#164) --- .JuliaFormatter.toml | 10 ++++++++++ .github/workflows/FormatPR.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 .JuliaFormatter.toml create mode 100644 .github/workflows/FormatPR.yml diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml new file mode 100644 index 0000000..eb77a5b --- /dev/null +++ b/.JuliaFormatter.toml @@ -0,0 +1,10 @@ +indent = 2 +margin = 120 +always_for_in = true +always_use_return = false +whitespace_typedefs = false +whitespace_in_kwargs = false +whitespace_ops_in_indices = true +remove_extra_newlines = true +trailing_comma = false +normalize_line_endings = "unix" diff --git a/.github/workflows/FormatPR.yml b/.github/workflows/FormatPR.yml new file mode 100644 index 0000000..cbf3b18 --- /dev/null +++ b/.github/workflows/FormatPR.yml @@ -0,0 +1,28 @@ +name: FormatPR +on: + push: + branches: + - master +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install JuliaFormatter and format + run: | + julia -e 'import Pkg; Pkg.add("JuliaFormatter")' + julia -e 'using JuliaFormatter; format(".")' + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: ":robot: Format .jl files" + title: '[AUTO] JuliaFormatter.jl run' + branch: auto-juliaformatter-pr + delete-branch: true + labels: formatting, automated pr, no changelog + - name: Check outputs + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" From d52f9160a830d541f6c77d7aadd5dac5285d5334 Mon Sep 17 00:00:00 2001 From: eliascarv Date: Mon, 15 May 2023 12:33:22 +0000 Subject: [PATCH 3/4] :robot: Format .jl files --- docs/make.jl | 57 +- src/LossFunctions.jl | 105 ++-- src/io.jl | 42 +- src/losses.jl | 26 +- src/losses/distance.jl | 190 +++---- src/losses/margin.jl | 96 ++-- src/losses/other.jl | 32 +- src/losses/scaled.jl | 43 +- src/losses/weighted.jl | 64 ++- src/traits.jl | 8 +- test/agg.jl | 100 ++-- test/core.jl | 673 +++++++++++----------- test/props.jl | 1230 ++++++++++++++++++++-------------------- test/runtests.jl | 64 ++- 14 files changed, 1390 insertions(+), 1340 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index aa90555..7c969fd 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -3,42 +3,25 @@ using Documenter, LossFunctions DocMeta.setdocmeta!(LossFunctions, :DocTestSetup, :(using LossFunctions); recursive=true) makedocs( - modules=[LossFunctions], - authors="Christof Stocker, Tom Breloff, Alex Williams", - repo="https://github.com/JuliaML/LossFunctions.jl/blob/{commit}{path}#{line}", - sitename="LossFunctions.jl", - format=Documenter.HTML( - prettyurls=get(ENV, "CI", "false") == "true", - canonical="https://JuliaML.github.io/LossFunctions.jl", - assets=["assets/style.css", "assets/favicon.ico"] - ), - pages=[ - hide("Home" => "index.md"), - "Introduction" => [ - "introduction/gettingstarted.md", - "introduction/motivation.md", - ], - "User's Guide" => [ - "user/interface.md", - "user/aggregate.md", - ], - "Available Losses" => [ - "losses/distance.md", - "losses/margin.md", - "losses/other.md", - ], - "Advances Topics" => [ - "advanced/extend.md", - "advanced/developer.md", - ], - hide("Indices" => "indices.md"), - "acknowledgements.md", - "LICENSE.md", - ] + modules=[LossFunctions], + authors="Christof Stocker, Tom Breloff, Alex Williams", + repo="https://github.com/JuliaML/LossFunctions.jl/blob/{commit}{path}#{line}", + sitename="LossFunctions.jl", + format=Documenter.HTML( + prettyurls=get(ENV, "CI", "false") == "true", + canonical="https://JuliaML.github.io/LossFunctions.jl", + assets=["assets/style.css", "assets/favicon.ico"] + ), + pages=[ + hide("Home" => "index.md"), + "Introduction" => ["introduction/gettingstarted.md", "introduction/motivation.md"], + "User's Guide" => ["user/interface.md", "user/aggregate.md"], + "Available Losses" => ["losses/distance.md", "losses/margin.md", "losses/other.md"], + "Advances Topics" => ["advanced/extend.md", "advanced/developer.md"], + hide("Indices" => "indices.md"), + "acknowledgements.md", + "LICENSE.md" + ] ) -deploydocs( - repo="github.com/JuliaML/LossFunctions.jl.git", - devbranch="master", - push_preview=true -) +deploydocs(repo="github.com/JuliaML/LossFunctions.jl.git", devbranch="master", push_preview=true) diff --git a/src/LossFunctions.jl b/src/LossFunctions.jl index d88e039..98dffb4 100644 --- a/src/LossFunctions.jl +++ b/src/LossFunctions.jl @@ -16,58 +16,67 @@ include("losses.jl") include("io.jl") export - # trait functions - Loss, - SupervisedLoss, - MarginLoss, - DistanceLoss, - deriv, deriv2, - isdistancebased, ismarginbased, - isminimizable, isdifferentiable, - istwicedifferentiable, - isconvex, isstrictlyconvex, - isstronglyconvex, isnemitski, - isunivfishercons, isfishercons, - islipschitzcont, islocallylipschitzcont, - isclipable, isclasscalibrated, issymmetric, + # trait functions + Loss, + SupervisedLoss, + MarginLoss, + DistanceLoss, + deriv, + deriv2, + isdistancebased, + ismarginbased, + isminimizable, + isdifferentiable, + istwicedifferentiable, + isconvex, + isstrictlyconvex, + isstronglyconvex, + isnemitski, + isunivfishercons, + isfishercons, + islipschitzcont, + islocallylipschitzcont, + isclipable, + isclasscalibrated, + issymmetric, - # margin-based losses - ZeroOneLoss, - LogitMarginLoss, - PerceptronLoss, - HingeLoss, - L1HingeLoss, - L2HingeLoss, - SmoothedL1HingeLoss, - ModifiedHuberLoss, - L2MarginLoss, - ExpLoss, - SigmoidLoss, - DWDMarginLoss, + # margin-based losses + ZeroOneLoss, + LogitMarginLoss, + PerceptronLoss, + HingeLoss, + L1HingeLoss, + L2HingeLoss, + SmoothedL1HingeLoss, + ModifiedHuberLoss, + L2MarginLoss, + ExpLoss, + SigmoidLoss, + DWDMarginLoss, - # distance-based losses - LPDistLoss, - L1DistLoss, - L2DistLoss, - PeriodicLoss, - HuberLoss, - EpsilonInsLoss, - L1EpsilonInsLoss, - L2EpsilonInsLoss, - LogitDistLoss, - QuantileLoss, - LogCoshLoss, + # distance-based losses + LPDistLoss, + L1DistLoss, + L2DistLoss, + PeriodicLoss, + HuberLoss, + EpsilonInsLoss, + L1EpsilonInsLoss, + L2EpsilonInsLoss, + LogitDistLoss, + QuantileLoss, + LogCoshLoss, - # other losses - MisclassLoss, - PoissonLoss, - CrossEntropyLoss, + # other losses + MisclassLoss, + PoissonLoss, + CrossEntropyLoss, - # meta losses - ScaledLoss, - WeightedMarginLoss, + # meta losses + ScaledLoss, + WeightedMarginLoss, - # reexport mean - mean + # reexport mean + mean end # module diff --git a/src/io.jl b/src/io.jl index 5d01838..fe80963 100644 --- a/src/io.jl +++ b/src/io.jl @@ -1,26 +1,34 @@ Base.print(io::IO, loss::SupervisedLoss, args...) = print(io, typeof(loss).name.name, args...) Base.print(io::IO, loss::L1DistLoss, args...) = print(io, "L1DistLoss", args...) Base.print(io::IO, loss::L2DistLoss, args...) = print(io, "L2DistLoss", args...) -Base.print(io::IO, loss::LPDistLoss{P}, args...) where {P} = print(io, typeof(loss).name.name, " with P = $(P)", args...) -Base.print(io::IO, loss::L1EpsilonInsLoss, args...) = print(io, typeof(loss).name.name, " with \$\\epsilon\$ = $(loss.ε)", args...) -Base.print(io::IO, loss::L2EpsilonInsLoss, args...) = print(io, typeof(loss).name.name, " with \$\\epsilon\$ = $(loss.ε)", args...) -Base.print(io::IO, loss::QuantileLoss, args...) = print(io, typeof(loss).name.name, " with \$\\tau\$ = $(loss.τ)", args...) -Base.print(io::IO, loss::SmoothedL1HingeLoss, args...) = print(io, typeof(loss).name.name, " with \$\\gamma\$ = $(loss.gamma)", args...) -Base.print(io::IO, loss::HuberLoss, args...) = print(io, typeof(loss).name.name, " with \$\\alpha\$ = $(loss.d)", args...) +Base.print(io::IO, loss::LPDistLoss{P}, args...) where {P} = + print(io, typeof(loss).name.name, " with P = $(P)", args...) +Base.print(io::IO, loss::L1EpsilonInsLoss, args...) = + print(io, typeof(loss).name.name, " with \$\\epsilon\$ = $(loss.ε)", args...) +Base.print(io::IO, loss::L2EpsilonInsLoss, args...) = + print(io, typeof(loss).name.name, " with \$\\epsilon\$ = $(loss.ε)", args...) +Base.print(io::IO, loss::QuantileLoss, args...) = + print(io, typeof(loss).name.name, " with \$\\tau\$ = $(loss.τ)", args...) +Base.print(io::IO, loss::SmoothedL1HingeLoss, args...) = + print(io, typeof(loss).name.name, " with \$\\gamma\$ = $(loss.gamma)", args...) +Base.print(io::IO, loss::HuberLoss, args...) = + print(io, typeof(loss).name.name, " with \$\\alpha\$ = $(loss.d)", args...) Base.print(io::IO, loss::DWDMarginLoss, args...) = print(io, typeof(loss).name.name, " with q = $(loss.q)", args...) -Base.print(io::IO, loss::PeriodicLoss, args...) = print(io, typeof(loss).name.name, " with c = $(round(2π / loss.k, digits=1))", args...) +Base.print(io::IO, loss::PeriodicLoss, args...) = + print(io, typeof(loss).name.name, " with c = $(round(2π / loss.k, digits=1))", args...) Base.print(io::IO, loss::ScaledLoss{T,K}, args...) where {T,K} = print(io, "$(K) * ($(loss.loss))", args...) _round(num) = round(num) == round(num, digits=1) ? round(Int, num) : round(num, digits=1) function _relation(num) - if num <= 0 - "negative only" - elseif num >= 1 - "positive only" - elseif num < 0.5 - "1:$(_round((1-num)/num)) weighted" - else - "$(_round(num/(1-num))):1 weighted" - end + if num <= 0 + "negative only" + elseif num >= 1 + "positive only" + elseif num < 0.5 + "1:$(_round((1-num)/num)) weighted" + else + "$(_round(num/(1-num))):1 weighted" + end end -Base.print(io::IO, loss::WeightedMarginLoss{T,W}, args...) where {T,W} = print(io, "$(_relation(W)) $(loss.loss)", args...) +Base.print(io::IO, loss::WeightedMarginLoss{T,W}, args...) where {T,W} = + print(io, "$(_relation(W)) $(loss.loss)", args...) diff --git a/src/losses.jl b/src/losses.jl index 03c90b3..7e43ce2 100644 --- a/src/losses.jl +++ b/src/losses.jl @@ -2,12 +2,12 @@ Scalar = Union{Number,CategoricalValue} # fallback to unary evaluation -(loss::DistanceLoss)(output::Number, target::Number) = loss(output - target) -deriv(loss::DistanceLoss, output::Number, target::Number) = deriv(loss, output - target) +(loss::DistanceLoss)(output::Number, target::Number) = loss(output - target) +deriv(loss::DistanceLoss, output::Number, target::Number) = deriv(loss, output - target) deriv2(loss::DistanceLoss, output::Number, target::Number) = deriv2(loss, output - target) -(loss::MarginLoss)(output::Number, target::Number) = loss(target * output) -deriv(loss::MarginLoss, output::Number, target::Number) = target * deriv(loss, target * output) -deriv2(loss::MarginLoss, output::Number, target::Number) = deriv2(loss, target * output) +(loss::MarginLoss)(output::Number, target::Number) = loss(target * output) +deriv(loss::MarginLoss, output::Number, target::Number) = target * deriv(loss, target * output) +deriv2(loss::MarginLoss, output::Number, target::Number) = deriv2(loss, target * output) # broadcasting behavior Broadcast.broadcastable(loss::SupervisedLoss) = Ref(loss) @@ -34,7 +34,7 @@ include("losses/weighted.jl") Return sum of `loss` values over the iterables `outputs` and `targets`. """ function sum(loss::SupervisedLoss, outputs, targets) - sum(loss(ŷ, y) for (ŷ, y) in zip(outputs, targets)) + sum(loss(ŷ, y) for (ŷ, y) in zip(outputs, targets)) end """ @@ -45,9 +45,9 @@ The `weights` determine the importance of each observation. The option `normalize` divides the result by the sum of the weights. """ function sum(loss::SupervisedLoss, outputs, targets, weights; normalize=true) - s = sum(w * loss(ŷ, y) for (ŷ, y, w) in zip(outputs, targets, weights)) - n = normalize ? sum(weights) : one(first(weights)) - s / n + s = sum(w * loss(ŷ, y) for (ŷ, y, w) in zip(outputs, targets, weights)) + n = normalize ? sum(weights) : one(first(weights)) + s / n end """ @@ -56,7 +56,7 @@ end Return mean of `loss` values over the iterables `outputs` and `targets`. """ function mean(loss::SupervisedLoss, outputs, targets) - mean(loss(ŷ, y) for (ŷ, y) in zip(outputs, targets)) + mean(loss(ŷ, y) for (ŷ, y) in zip(outputs, targets)) end """ @@ -67,7 +67,7 @@ The `weights` determine the importance of each observation. The option `normalize` divides the result by the sum of the weights. """ function mean(loss::SupervisedLoss, outputs, targets, weights; normalize=true) - m = mean(w * loss(ŷ, y) for (ŷ, y, w) in zip(outputs, targets, weights)) - n = normalize ? sum(weights) : one(first(weights)) - m / n + m = mean(w * loss(ŷ, y) for (ŷ, y, w) in zip(outputs, targets, weights)) + n = normalize ? sum(weights) : one(first(weights)) + m / n end diff --git a/src/losses/distance.jl b/src/losses/distance.jl index 7bb4b43..49bc367 100644 --- a/src/losses/distance.jl +++ b/src/losses/distance.jl @@ -14,19 +14,19 @@ struct LPDistLoss{P} <: DistanceLoss end LPDistLoss(p::Number) = LPDistLoss{p}() (loss::LPDistLoss{P})(difference::Number) where {P} = abs(difference)^P -function deriv(loss::LPDistLoss{P}, difference::T)::promote_type(typeof(P),T) where {P,T<:Number} - if difference == 0 - zero(difference) - else - P * difference * abs(difference)^(P-convert(typeof(P), 2)) - end +function deriv(loss::LPDistLoss{P}, difference::T)::promote_type(typeof(P), T) where {P,T<:Number} + if difference == 0 + zero(difference) + else + P * difference * abs(difference)^(P - convert(typeof(P), 2)) + end end -function deriv2(loss::LPDistLoss{P}, difference::T)::promote_type(typeof(P),T) where {P,T<:Number} - if difference == 0 - zero(difference) - else - (abs2(P)-P) * abs(difference)^P / abs2(difference) - end +function deriv2(loss::LPDistLoss{P}, difference::T)::promote_type(typeof(P), T) where {P,T<:Number} + if difference == 0 + zero(difference) + else + (abs2(P) - P) * abs(difference)^P / abs2(difference) + end end isminimizable(::LPDistLoss{P}) where {P} = true @@ -119,8 +119,8 @@ L(r) = |r|^2 const L2DistLoss = LPDistLoss{2} (loss::L2DistLoss)(difference::Number) = abs2(difference) -deriv(loss::L2DistLoss, difference::T) where {T<:Number} = convert(T,2) * difference -deriv2(loss::L2DistLoss, difference::T) where {T<:Number} = convert(T,2) +deriv(loss::L2DistLoss, difference::T) where {T<:Number} = convert(T, 2) * difference +deriv2(loss::L2DistLoss, difference::T) where {T<:Number} = convert(T, 2) isdifferentiable(::L2DistLoss) = true isdifferentiable(::L2DistLoss, at) = true @@ -143,18 +143,18 @@ L(r) = 1 - \cos \left( \frac{2 r \pi}{c} \right) ``` """ struct PeriodicLoss{T<:AbstractFloat} <: DistanceLoss - k::T # k = 2π/circumference - function PeriodicLoss{T}(circ::T) where T - circ > 0 || error("circumference should be strictly positive") - new{T}(convert(T, 2π/circ)) - end + k::T # k = 2π/circumference + function PeriodicLoss{T}(circ::T) where {T} + circ > 0 || error("circumference should be strictly positive") + new{T}(convert(T, 2π / circ)) + end end PeriodicLoss(circ::T=1.0) where {T<:AbstractFloat} = PeriodicLoss{T}(circ) PeriodicLoss(circ) = PeriodicLoss{Float64}(Float64(circ)) -(loss::PeriodicLoss)(difference::T) where {T<:Number} = 1 - cos(difference*loss.k) -deriv(loss::PeriodicLoss, difference::T) where {T<:Number} = loss.k * sin(difference*loss.k) -deriv2(loss::PeriodicLoss, difference::T) where {T<:Number} = abs2(loss.k) * cos(difference*loss.k) +(loss::PeriodicLoss)(difference::T) where {T<:Number} = 1 - cos(difference * loss.k) +deriv(loss::PeriodicLoss, difference::T) where {T<:Number} = loss.k * sin(difference * loss.k) +deriv2(loss::PeriodicLoss, difference::T) where {T<:Number} = abs2(loss.k) * cos(difference * loss.k) isdifferentiable(::PeriodicLoss) = true isdifferentiable(::PeriodicLoss, at) = true @@ -165,7 +165,6 @@ isconvex(::PeriodicLoss) = false isstrictlyconvex(::PeriodicLoss) = false isstronglyconvex(::PeriodicLoss) = false - # =========================================================== @doc doc""" @@ -198,35 +197,35 @@ L(r) = \begin{cases} \frac{r^2}{2} & \quad \text{if } | r | \le \alpha \\ \alpha ``` """ struct HuberLoss{T<:AbstractFloat} <: DistanceLoss - d::T # boundary between quadratic and linear loss - function HuberLoss{T}(d::T) where T - d > 0 || error("Huber crossover parameter must be strictly positive.") - new{T}(d) - end + d::T # boundary between quadratic and linear loss + function HuberLoss{T}(d::T) where {T} + d > 0 || error("Huber crossover parameter must be strictly positive.") + new{T}(d) + end end HuberLoss(d::T=1.0) where {T<:AbstractFloat} = HuberLoss{T}(d) HuberLoss(d) = HuberLoss{Float64}(Float64(d)) function (loss::HuberLoss{T1})(difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - abs_diff = abs(difference) - if abs_diff <= loss.d - return convert(T,0.5)*abs2(difference) # quadratic - else - return (loss.d*abs_diff) - convert(T,0.5)*abs2(loss.d) # linear - end + T = promote_type(T1, T2) + abs_diff = abs(difference) + if abs_diff <= loss.d + return convert(T, 0.5) * abs2(difference) # quadratic + else + return (loss.d * abs_diff) - convert(T, 0.5) * abs2(loss.d) # linear + end end function deriv(loss::HuberLoss{T1}, difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - if abs(difference) <= loss.d - return convert(T,difference) # quadratic - else - return loss.d*convert(T,sign(difference)) # linear - end + T = promote_type(T1, T2) + if abs(difference) <= loss.d + return convert(T, difference) # quadratic + else + return loss.d * convert(T, sign(difference)) # linear + end end function deriv2(loss::HuberLoss{T1}, difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - abs(difference) <= loss.d ? one(T) : zero(T) + T = promote_type(T1, T2) + abs(difference) <= loss.d ? one(T) : zero(T) end isdifferentiable(::HuberLoss) = true @@ -271,26 +270,26 @@ L(r) = \max \{ 0, | r | - \epsilon \} ``` """ struct L1EpsilonInsLoss{T<:AbstractFloat} <: DistanceLoss - ε::T + ε::T - function L1EpsilonInsLoss{T}(ɛ::T) where T - ɛ > 0 || error("ɛ must be strictly positive") - new{T}(ɛ) - end + function L1EpsilonInsLoss{T}(ɛ::T) where {T} + ɛ > 0 || error("ɛ must be strictly positive") + new{T}(ɛ) + end end const EpsilonInsLoss = L1EpsilonInsLoss @inline L1EpsilonInsLoss(ε::T) where {T<:AbstractFloat} = L1EpsilonInsLoss{T}(ε) @inline L1EpsilonInsLoss(ε::Number) = L1EpsilonInsLoss{Float64}(Float64(ε)) function (loss::L1EpsilonInsLoss{T1})(difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - max(zero(T), abs(difference) - loss.ε) + T = promote_type(T1, T2) + max(zero(T), abs(difference) - loss.ε) end function deriv(loss::L1EpsilonInsLoss{T1}, difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - abs(difference) <= loss.ε ? zero(T) : convert(T,sign(difference)) + T = promote_type(T1, T2) + abs(difference) <= loss.ε ? zero(T) : convert(T, sign(difference)) end -deriv2(loss::L1EpsilonInsLoss{T1}, difference::T2) where {T1,T2<:Number} = zero(promote_type(T1,T2)) +deriv2(loss::L1EpsilonInsLoss{T1}, difference::T2) where {T1,T2<:Number} = zero(promote_type(T1, T2)) issymmetric(::L1EpsilonInsLoss) = true isdifferentiable(::L1EpsilonInsLoss) = false @@ -334,28 +333,28 @@ L(r) = \max \{ 0, | r | - \epsilon \}^2 ``` """ struct L2EpsilonInsLoss{T<:AbstractFloat} <: DistanceLoss - ε::T + ε::T - function L2EpsilonInsLoss{T}(ɛ::T) where T - ɛ > 0 || error("ɛ must be strictly positive") - new{T}(ɛ) - end + function L2EpsilonInsLoss{T}(ɛ::T) where {T} + ɛ > 0 || error("ɛ must be strictly positive") + new{T}(ɛ) + end end L2EpsilonInsLoss(ε::T) where {T<:AbstractFloat} = L2EpsilonInsLoss{T}(ε) L2EpsilonInsLoss(ε) = L2EpsilonInsLoss{Float64}(Float64(ε)) function (loss::L2EpsilonInsLoss{T1})(difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - abs2(max(zero(T), abs(difference) - loss.ε)) + T = promote_type(T1, T2) + abs2(max(zero(T), abs(difference) - loss.ε)) end function deriv(loss::L2EpsilonInsLoss{T1}, difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - absr = abs(difference) - absr <= loss.ε ? zero(T) : convert(T,2)*sign(difference)*(absr - loss.ε) + T = promote_type(T1, T2) + absr = abs(difference) + absr <= loss.ε ? zero(T) : convert(T, 2) * sign(difference) * (absr - loss.ε) end function deriv2(loss::L2EpsilonInsLoss{T1}, difference::T2) where {T1,T2<:Number} - T = promote_type(T1,T2) - abs(difference) <= loss.ε ? zero(T) : convert(T,2) + T = promote_type(T1, T2) + abs(difference) <= loss.ε ? zero(T) : convert(T, 2) end issymmetric(::L2EpsilonInsLoss) = true @@ -400,17 +399,17 @@ L(r) = - \ln \frac{4 e^r}{(1 + e^r)^2} struct LogitDistLoss <: DistanceLoss end function (loss::LogitDistLoss)(difference::Number) - er = exp(difference) - T = typeof(er) - -log(convert(T,4)) - difference + 2log(one(T) + er) + er = exp(difference) + T = typeof(er) + -log(convert(T, 4)) - difference + 2log(one(T) + er) end -function deriv(loss::LogitDistLoss, difference::T) where T<:Number - tanh(difference / convert(T,2)) +function deriv(loss::LogitDistLoss, difference::T) where {T<:Number} + tanh(difference / convert(T, 2)) end function deriv2(loss::LogitDistLoss, difference::Number) - er = exp(difference) - T = typeof(er) - convert(T,2)*er / abs2(one(T) + er) + er = exp(difference) + T = typeof(er) + convert(T, 2) * er / abs2(one(T) + er) end issymmetric(::LogitDistLoss) = true @@ -423,7 +422,6 @@ isconvex(::LogitDistLoss) = true isstrictlyconvex(::LogitDistLoss) = true isstronglyconvex(::LogitDistLoss) = false - # =========================================================== @doc doc""" QuantileLoss <: DistanceLoss @@ -454,19 +452,19 @@ L(r) = \begin{cases} -\left( 1 - \tau \right) r & \quad \text{if } r < 0 \\ \ta ŷ - y ŷ - y ``` """ -struct QuantileLoss{T <: AbstractFloat} <: DistanceLoss - τ::T +struct QuantileLoss{T<:AbstractFloat} <: DistanceLoss + τ::T end -function (loss::QuantileLoss{T1})(diff::T2) where {T1, T2 <: Number} - T = promote_type(T1, T2) - diff * (convert(T,diff > 0) - loss.τ) +function (loss::QuantileLoss{T1})(diff::T2) where {T1,T2<:Number} + T = promote_type(T1, T2) + diff * (convert(T, diff > 0) - loss.τ) end -function deriv(loss::QuantileLoss{T1}, diff::T2) where {T1, T2 <: Number} - T = promote_type(T1, T2) - convert(T,diff > 0) - loss.τ +function deriv(loss::QuantileLoss{T1}, diff::T2) where {T1,T2<:Number} + T = promote_type(T1, T2) + convert(T, diff > 0) - loss.τ end -deriv2(::QuantileLoss{T1}, diff::T2) where {T1, T2 <: Number} = zero(promote_type(T1, T2)) +deriv2(::QuantileLoss{T1}, diff::T2) where {T1,T2<:Number} = zero(promote_type(T1, T2)) issymmetric(loss::QuantileLoss) = loss.τ == 0.5 isdifferentiable(::QuantileLoss) = false @@ -509,27 +507,27 @@ L(r) = log ( cosh ( x )) """ struct LogCoshLoss <: DistanceLoss end -_softplus(x::T) where T<:Number = x > zero(T) ? x + log1p(exp(-x)) : log1p(exp(x)) -_log_cosh(x::T) where T<:Number = x + _softplus(-2x) - log(convert(T, 2)) +_softplus(x::T) where {T<:Number} = x > zero(T) ? x + log1p(exp(-x)) : log1p(exp(x)) +_log_cosh(x::T) where {T<:Number} = x + _softplus(-2x) - log(convert(T, 2)) -function (loss::LogCoshLoss)(diff::T) where {T <: Number} +function (loss::LogCoshLoss)(diff::T) where {T<:Number} _log_cosh(diff) end -function deriv(loss::LogCoshLoss, diff::T) where {T <: Number} +function deriv(loss::LogCoshLoss, diff::T) where {T<:Number} tanh.(diff) end -function deriv2(::LogCoshLoss, diff::T) where {T <: Number} +function deriv2(::LogCoshLoss, diff::T) where {T<:Number} (sech.(diff))^2 end -issymmetric(loss::LogCoshLoss) = true -isdifferentiable(::LogCoshLoss) = true -isdifferentiable(::LogCoshLoss, at) = true +issymmetric(loss::LogCoshLoss) = true +isdifferentiable(::LogCoshLoss) = true +isdifferentiable(::LogCoshLoss, at) = true istwicedifferentiable(::LogCoshLoss, at) = true -istwicedifferentiable(::LogCoshLoss) = true -islipschitzcont(::LogCoshLoss) = true -isconvex(::LogCoshLoss) = true -isstrictlyconvex(::LogCoshLoss) = true -isstronglyconvex(::LogCoshLoss) = true +istwicedifferentiable(::LogCoshLoss) = true +islipschitzcont(::LogCoshLoss) = true +isconvex(::LogCoshLoss) = true +isstrictlyconvex(::LogCoshLoss) = true +isstronglyconvex(::LogCoshLoss) = true diff --git a/src/losses/margin.jl b/src/losses/margin.jl index 661e6a7..96c5680 100644 --- a/src/losses/margin.jl +++ b/src/losses/margin.jl @@ -127,7 +127,8 @@ L(a) = \ln (1 + e^{-a}) struct LogitMarginLoss <: MarginLoss end (loss::LogitMarginLoss)(agreement::Number) = log1p(exp(-agreement)) deriv(loss::LogitMarginLoss, agreement::Number) = -one(agreement) / (one(agreement) + exp(agreement)) -deriv2(loss::LogitMarginLoss, agreement::Number) = (eᵗ = exp(agreement); eᵗ / abs2(one(eᵗ) + eᵗ)) +deriv2(loss::LogitMarginLoss, agreement::Number) = (eᵗ = exp(agreement); +eᵗ / abs2(one(eᵗ) + eᵗ)) isunivfishercons(::LogitMarginLoss) = true isdifferentiable(::LogitMarginLoss) = true @@ -222,8 +223,9 @@ L(a) = \max \{ 0, 1 - a \}^2 struct L2HingeLoss <: MarginLoss end (loss::L2HingeLoss)(agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : abs2(one(T) - agreement) -deriv(loss::L2HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : convert(T,2) * (agreement - one(T)) -deriv2(loss::L2HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : convert(T,2) +deriv(loss::L2HingeLoss, agreement::T) where {T<:Number} = + agreement >= 1 ? zero(T) : convert(T, 2) * (agreement - one(T)) +deriv2(loss::L2HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : convert(T, 2) isunivfishercons(::L2HingeLoss) = true isdifferentiable(::L2HingeLoss) = true @@ -267,32 +269,32 @@ L(a) = \begin{cases} \frac{0.5}{\gamma} \cdot \max \{ 0, 1 - a \} ^2 & \quad \te ``` """ struct SmoothedL1HingeLoss{T<:AbstractFloat} <: MarginLoss - gamma::T + gamma::T - function SmoothedL1HingeLoss{T}(γ::T) where T - γ > 0 || error("γ must be strictly positive") - new{T}(γ) - end + function SmoothedL1HingeLoss{T}(γ::T) where {T} + γ > 0 || error("γ must be strictly positive") + new{T}(γ) + end end SmoothedL1HingeLoss(γ::T) where {T<:AbstractFloat} = SmoothedL1HingeLoss{T}(γ) SmoothedL1HingeLoss(γ) = SmoothedL1HingeLoss(Float64(γ)) -function (loss::SmoothedL1HingeLoss{R})(agreement::T)::promote_type(R,T) where {R,T<:Number} - if agreement >= 1 - loss.gamma - R(0.5) / loss.gamma * abs2(max(zero(T), one(T) - agreement)) - else - one(T) - loss.gamma / R(2) - agreement - end +function (loss::SmoothedL1HingeLoss{R})(agreement::T)::promote_type(R, T) where {R,T<:Number} + if agreement >= 1 - loss.gamma + R(0.5) / loss.gamma * abs2(max(zero(T), one(T) - agreement)) + else + one(T) - loss.gamma / R(2) - agreement + end end -function deriv(loss::SmoothedL1HingeLoss{R}, agreement::T)::promote_type(R,T) where {R,T<:Number} - if agreement >= 1 - loss.gamma - agreement >= 1 ? zero(T) : (agreement - one(T)) / loss.gamma - else - -one(T) - end +function deriv(loss::SmoothedL1HingeLoss{R}, agreement::T)::promote_type(R, T) where {R,T<:Number} + if agreement >= 1 - loss.gamma + agreement >= 1 ? zero(T) : (agreement - one(T)) / loss.gamma + else + -one(T) + end end -function deriv2(loss::SmoothedL1HingeLoss{R}, agreement::T)::promote_type(R,T) where {R,T<:Number} - agreement < 1 - loss.gamma || agreement > 1 ? zero(T) : one(T) / loss.gamma +function deriv2(loss::SmoothedL1HingeLoss{R}, agreement::T)::promote_type(R, T) where {R,T<:Number} + agreement < 1 - loss.gamma || agreement > 1 ? zero(T) : one(T) / loss.gamma end isdifferentiable(::SmoothedL1HingeLoss) = true @@ -339,17 +341,17 @@ L(a) = \begin{cases} \max \{ 0, 1 - a \} ^2 & \quad \text{if } a \ge -1 \\ - 4 a struct ModifiedHuberLoss <: MarginLoss end function (loss::ModifiedHuberLoss)(agreement::T) where {T<:Number} - agreement >= -1 ? abs2(max(zero(T), one(agreement) - agreement)) : -convert(T,4) * agreement + agreement >= -1 ? abs2(max(zero(T), one(agreement) - agreement)) : -convert(T, 4) * agreement end function deriv(loss::ModifiedHuberLoss, agreement::T) where {T<:Number} - if agreement >= -1 - agreement > 1 ? zero(T) : convert(T,2)*agreement - convert(T,2) - else - -convert(T,4) - end + if agreement >= -1 + agreement > 1 ? zero(T) : convert(T, 2) * agreement - convert(T, 2) + else + -convert(T, 4) + end end function deriv2(loss::ModifiedHuberLoss, agreement::T) where {T<:Number} - agreement < -1 || agreement > 1 ? zero(T) : convert(T,2) + agreement < -1 || agreement > 1 ? zero(T) : convert(T, 2) end isdifferentiable(::ModifiedHuberLoss) = true @@ -396,8 +398,8 @@ L(a) = {\left( 1 - a \right)}^2 struct L2MarginLoss <: MarginLoss end (loss::L2MarginLoss)(agreement::T) where {T<:Number} = abs2(one(T) - agreement) -deriv(loss::L2MarginLoss, agreement::T) where {T<:Number} = convert(T,2) * (agreement - one(T)) -deriv2(loss::L2MarginLoss, agreement::T) where {T<:Number} = convert(T,2) +deriv(loss::L2MarginLoss, agreement::T) where {T<:Number} = convert(T, 2) * (agreement - one(T)) +deriv2(loss::L2MarginLoss, agreement::T) where {T<:Number} = convert(T, 2) isunivfishercons(::L2MarginLoss) = true isdifferentiable(::L2MarginLoss) = true @@ -494,7 +496,7 @@ struct SigmoidLoss <: MarginLoss end (loss::SigmoidLoss)(agreement::Number) = one(agreement) - tanh(agreement) deriv(loss::SigmoidLoss, agreement::Number) = -abs2(sech(agreement)) -deriv2(loss::SigmoidLoss, agreement::T) where {T<:Number} = convert(T,2) * tanh(agreement) * abs2(sech(agreement)) +deriv2(loss::SigmoidLoss, agreement::T) where {T<:Number} = convert(T, 2) * tanh(agreement) * abs2(sech(agreement)) isunivfishercons(::SigmoidLoss) = true isdifferentiable(::SigmoidLoss) = true @@ -541,32 +543,32 @@ L(a) = \begin{cases} 1 - a & \quad \text{if } a \ge \frac{q}{q+1} \\ \frac{1}{a^ ``` """ struct DWDMarginLoss{T<:AbstractFloat} <: MarginLoss - q::T - function DWDMarginLoss{T}(q::T) where T - q > 0 || error("q must be strictly positive") - new{T}(q) - end + q::T + function DWDMarginLoss{T}(q::T) where {T} + q > 0 || error("q must be strictly positive") + new{T}(q) + end end DWDMarginLoss(q::T) where {T<:AbstractFloat} = DWDMarginLoss{T}(q) DWDMarginLoss(q) = DWDMarginLoss(Float64(q)) function (loss::DWDMarginLoss{R})(agreement::T)::promote_type(R, T) where {R,T<:Number} - q = loss.q - if agreement <= q/(q+1) - R(1) - agreement - else - (q^q/(q+1)^(q+1)) / agreement^q - end + q = loss.q + if agreement <= q / (q + 1) + R(1) - agreement + else + (q^q / (q + 1)^(q + 1)) / agreement^q + end end function deriv(loss::DWDMarginLoss{R}, agreement::T)::promote_type(R, T) where {R,T<:Number} - q = loss.q - agreement <= q/(q+1) ? -one(T) : -(q/(q+1))^(q+1) / agreement^(q+1) + q = loss.q + agreement <= q / (q + 1) ? -one(T) : -(q / (q + 1))^(q + 1) / agreement^(q + 1) end function deriv2(loss::DWDMarginLoss{R}, agreement::T)::promote_type(R, T) where {R,T<:Number} - q = loss.q - agreement <= q/(q+1) ? zero(T) : ( (q^(q+1))/((q+1)^q) ) / agreement^(q+2) + q = loss.q + agreement <= q / (q + 1) ? zero(T) : ((q^(q + 1)) / ((q + 1)^q)) / agreement^(q + 2) end isdifferentiable(::DWDMarginLoss) = true diff --git a/src/losses/other.jl b/src/losses/other.jl index 2982d22..63b80f1 100644 --- a/src/losses/other.jl +++ b/src/losses/other.jl @@ -13,9 +13,9 @@ struct MisclassLoss{R<:AbstractFloat} <: SupervisedLoss end MisclassLoss() = MisclassLoss{Float64}() # return floating point to avoid big integers in aggregations -(::MisclassLoss{R})(agreement::Bool) where R = ifelse(agreement, zero(R), one(R)) -deriv(::MisclassLoss{R}, agreement::Bool) where R = zero(R) -deriv2(::MisclassLoss{R}, agreement::Bool) where R = zero(R) +(::MisclassLoss{R})(agreement::Bool) where {R} = ifelse(agreement, zero(R), one(R)) +deriv(::MisclassLoss{R}, agreement::Bool) where {R} = zero(R) +deriv2(::MisclassLoss{R}, agreement::Bool) where {R} = zero(R) (loss::MisclassLoss)(output::Scalar, target::Scalar) = loss(target == output) deriv(loss::MisclassLoss, output::Scalar, target::Scalar) = deriv(loss, target == output) @@ -43,7 +43,7 @@ Loss under a Poisson noise distribution (KL-divergence) """ struct PoissonLoss <: SupervisedLoss end -(loss::PoissonLoss)(output::Number, target::Number) = exp(output) - target*output +(loss::PoissonLoss)(output::Number, target::Number) = exp(output) - target * output deriv(loss::PoissonLoss, output::Number, target::Number) = exp(output) - target deriv2(loss::PoissonLoss, output::Number, target::Number) = exp(output) @@ -68,21 +68,21 @@ The cross-entropy loss is defined as: struct CrossEntropyLoss <: SupervisedLoss end function (loss::CrossEntropyLoss)(output::Number, target::Number) - target >= 0 && target <=1 || error("target must be in [0,1]") - output >= 0 && output <=1 || error("output must be in [0,1]") - if target == 0 - -log(1 - output) - elseif target == 1 - -log(output) - else - -(target * log(output) + (1-target) * log(1-output)) - end + target >= 0 && target <= 1 || error("target must be in [0,1]") + output >= 0 && output <= 1 || error("output must be in [0,1]") + if target == 0 + -log(1 - output) + elseif target == 1 + -log(output) + else + -(target * log(output) + (1 - target) * log(1 - output)) + end end -deriv(loss::CrossEntropyLoss, output::Number, target::Number) = (1-target) / (1-output) - target / output -deriv2(loss::CrossEntropyLoss, output::Number, target::Number) = (1-target) / (1-output)^2 + target / output^2 +deriv(loss::CrossEntropyLoss, output::Number, target::Number) = (1 - target) / (1 - output) - target / output +deriv2(loss::CrossEntropyLoss, output::Number, target::Number) = (1 - target) / (1 - output)^2 + target / output^2 isdifferentiable(::CrossEntropyLoss) = true isdifferentiable(::CrossEntropyLoss, y, t) = t != 0 && t != 1 istwicedifferentiable(::CrossEntropyLoss) = true istwicedifferentiable(::CrossEntropyLoss, y, t) = t != 0 && t != 1 -isconvex(::CrossEntropyLoss) = true \ No newline at end of file +isconvex(::CrossEntropyLoss) = true diff --git a/src/losses/scaled.jl b/src/losses/scaled.jl index a0faf81..c085bd6 100644 --- a/src/losses/scaled.jl +++ b/src/losses/scaled.jl @@ -5,39 +5,48 @@ Can be used to represent a `K` times scaled version of a given type of loss `L`. """ struct ScaledLoss{L<:SupervisedLoss,K} <: SupervisedLoss - loss::L + loss::L end @generated function (::Type{ScaledLoss{L,K}})(args...) where {L<:SupervisedLoss,K} - ScaledLoss{L,K}(L(args...)) + ScaledLoss{L,K}(L(args...)) end ScaledLoss(loss::T, ::Val{K}) where {T,K} = ScaledLoss{T,K}(loss) ScaledLoss(loss::T, k::Number) where {T} = ScaledLoss(loss, Val(k)) Base.:*(::Val{K}, loss::SupervisedLoss) where {K} = ScaledLoss(loss, Val(K)) Base.:*(k::Number, loss::SupervisedLoss) = ScaledLoss(loss, Val(k)) -@generated ScaledLoss(s::ScaledLoss{T,K1}, ::Val{K2}) where {T,K1,K2} = :(ScaledLoss(s.loss, Val($(K1*K2)))) +@generated ScaledLoss(s::ScaledLoss{T,K1}, ::Val{K2}) where {T,K1,K2} = :(ScaledLoss(s.loss, Val($(K1 * K2)))) (l::ScaledLoss{T,K})(output::Number, target::Number) where {T,K} = K * l.loss(output, target) for FUN in (:deriv, :deriv2) - @eval function ($FUN)( - l::ScaledLoss{T,K}, - output::Number, - target::Number) where {T,K} - K * ($FUN)(l.loss, output, target) - end + @eval function ($FUN)(l::ScaledLoss{T,K}, output::Number, target::Number) where {T,K} + K * ($FUN)(l.loss, output, target) + end end -for FUN in (:isminimizable, :isdifferentiable, :istwicedifferentiable, - :isconvex, :isstrictlyconvex, :isstronglyconvex, - :isnemitski, :isunivfishercons, :isfishercons, - :islipschitzcont, :islocallylipschitzcont, - :isclipable, :ismarginbased, :isclasscalibrated, - :isdistancebased, :issymmetric) - @eval ($FUN)(l::ScaledLoss) = ($FUN)(l.loss) +for FUN in ( + :isminimizable, + :isdifferentiable, + :istwicedifferentiable, + :isconvex, + :isstrictlyconvex, + :isstronglyconvex, + :isnemitski, + :isunivfishercons, + :isfishercons, + :islipschitzcont, + :islocallylipschitzcont, + :isclipable, + :ismarginbased, + :isclasscalibrated, + :isdistancebased, + :issymmetric +) + @eval ($FUN)(l::ScaledLoss) = ($FUN)(l.loss) end for FUN in (:isdifferentiable, :istwicedifferentiable) - @eval ($FUN)(l::ScaledLoss, at) = ($FUN)(l.loss, at) + @eval ($FUN)(l::ScaledLoss, at) = ($FUN)(l.loss, at) end diff --git a/src/losses/weighted.jl b/src/losses/weighted.jl index d7843ee..bf563c0 100644 --- a/src/losses/weighted.jl +++ b/src/losses/weighted.jl @@ -7,46 +7,46 @@ denotes the relative weight of the positive class, while the relative weight of the negative class will be `1 - W`. """ struct WeightedMarginLoss{L<:MarginLoss,W} <: SupervisedLoss - loss::L + loss::L end _werror() = throw(ArgumentError("The given \"weight\" has to be a number in the interval [0, 1]")) @generated function WeightedMarginLoss(loss::L, ::Val{W}) where {L<:MarginLoss,W} - typeof(W) <: Number && 0 <= W <= 1 || _werror() - :(WeightedMarginLoss{L,W}(loss)) + typeof(W) <: Number && 0 <= W <= 1 || _werror() + :(WeightedMarginLoss{L,W}(loss)) end function WeightedMarginLoss(loss::MarginLoss, w::Number) - WeightedMarginLoss(loss, Val(w)) + WeightedMarginLoss(loss, Val(w)) end @generated function WeightedMarginLoss(s::WeightedMarginLoss{T,W1}, ::Val{W2}) where {T,W1,W2} - :(WeightedMarginLoss(s.loss, Val($(W1*W2)))) + :(WeightedMarginLoss(s.loss, Val($(W1 * W2)))) end function WeightedMarginLoss(s::WeightedMarginLoss{L,W}, w::Number) where {L<:MarginLoss,W} - WeightedMarginLoss(s.loss, Val(W*w)) + WeightedMarginLoss(s.loss, Val(W * w)) end function (l::WeightedMarginLoss{L,W})(output::Number, target::Number) where {L,W} - # We interpret the W to be the weight of the positive class - if target > 0 - W * l.loss(output, target) - else - (1-W) * l.loss(output, target) - end + # We interpret the W to be the weight of the positive class + if target > 0 + W * l.loss(output, target) + else + (1 - W) * l.loss(output, target) + end end for FUN in (:deriv, :deriv2) - @eval function ($FUN)(l::WeightedMarginLoss{L,W}, output::Number, target::Number) where {L,W} - # We interpret the W to be the weight of the positive class - if target > 0 - W * ($FUN)(l.loss, output, target) - else - (1-W) * ($FUN)(l.loss, output, target) - end + @eval function ($FUN)(l::WeightedMarginLoss{L,W}, output::Number, target::Number) where {L,W} + # We interpret the W to be the weight of the positive class + if target > 0 + W * ($FUN)(l.loss, output, target) + else + (1 - W) * ($FUN)(l.loss, output, target) end + end end # An α-weighted version of a classification callibrated margin loss is @@ -56,15 +56,25 @@ isclasscalibrated(l::WeightedMarginLoss{T,W}) where {T,W} = W == 0.5 && isclassc # TODO: Think about this semantic issymmetric(::WeightedMarginLoss) = false -for FUN in (:isminimizable, :isdifferentiable, :istwicedifferentiable, - :isconvex, :isstrictlyconvex, :isstronglyconvex, - :isnemitski, :isunivfishercons, :isfishercons, - :islipschitzcont, :islocallylipschitzcont, - :isclipable, :ismarginbased, - :isdistancebased) - @eval ($FUN)(l::WeightedMarginLoss) = ($FUN)(l.loss) +for FUN in ( + :isminimizable, + :isdifferentiable, + :istwicedifferentiable, + :isconvex, + :isstrictlyconvex, + :isstronglyconvex, + :isnemitski, + :isunivfishercons, + :isfishercons, + :islipschitzcont, + :islocallylipschitzcont, + :isclipable, + :ismarginbased, + :isdistancebased +) + @eval ($FUN)(l::WeightedMarginLoss) = ($FUN)(l.loss) end for FUN in (:isdifferentiable, :istwicedifferentiable) - @eval ($FUN)(l::WeightedMarginLoss, at) = ($FUN)(l.loss, at) + @eval ($FUN)(l::WeightedMarginLoss, at) = ($FUN)(l.loss, at) end diff --git a/src/traits.jl b/src/traits.jl index 45d306f..88588ff 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -134,8 +134,7 @@ such that Every convex function is locally lipschitz continuous. """ -islocallylipschitzcont(loss::SupervisedLoss) = - isconvex(loss) || islipschitzcont(loss) +islocallylipschitzcont(loss::SupervisedLoss) = isconvex(loss) || islipschitzcont(loss) """ islipschitzcont(loss) -> Bool @@ -220,8 +219,7 @@ ismarginbased(loss::MarginLoss) = true isclasscalibrated(loss) -> Bool """ isclasscalibrated(loss::SupervisedLoss) = false -isclasscalibrated(loss::MarginLoss) = - isconvex(loss) && isdifferentiable(loss, 0) && deriv(loss, 0) < 0 +isclasscalibrated(loss::MarginLoss) = isconvex(loss) && isdifferentiable(loss, 0) && deriv(loss, 0) < 0 """ issymmetric(loss) -> Bool @@ -241,4 +239,4 @@ issymmetric(loss::SupervisedLoss) = false Return `true` if the given `loss` is a minimizable loss. """ -isminimizable(loss::SupervisedLoss) = isconvex(loss) \ No newline at end of file +isminimizable(loss::SupervisedLoss) = isconvex(loss) diff --git a/test/agg.jl b/test/agg.jl index 484feca..8ad87e9 100644 --- a/test/agg.jl +++ b/test/agg.jl @@ -1,70 +1,64 @@ function test_vector_value(l, o, t) - ref = [l(o[i], t[i]) for i in 1:length(o)] - v(l, o, t) = l.(o, t) - @test @inferred(v(l, o, t)) == ref - n = length(ref) - s = sum(ref) - @test @inferred(sum(l, o, t)) ≈ s - @test @inferred(mean(l, o, t)) ≈ s / n - @test @inferred(sum(l, o, t, ones(n), normalize=false)) ≈ s - @test @inferred(sum(l, o, t, ones(n), normalize=true)) ≈ s / n - @test @inferred(mean(l, o, t, ones(n), normalize=false)) ≈ s / n - @test @inferred(mean(l, o, t, ones(n), normalize=true)) ≈ (s / n) / n + ref = [l(o[i], t[i]) for i in 1:length(o)] + v(l, o, t) = l.(o, t) + @test @inferred(v(l, o, t)) == ref + n = length(ref) + s = sum(ref) + @test @inferred(sum(l, o, t)) ≈ s + @test @inferred(mean(l, o, t)) ≈ s / n + @test @inferred(sum(l, o, t, ones(n), normalize=false)) ≈ s + @test @inferred(sum(l, o, t, ones(n), normalize=true)) ≈ s / n + @test @inferred(mean(l, o, t, ones(n), normalize=false)) ≈ s / n + @test @inferred(mean(l, o, t, ones(n), normalize=true)) ≈ (s / n) / n end function test_vector_deriv(l, o, t) - ref = [deriv(l, o[i], t[i]) for i in 1:length(o)] - d(l, o, t) = deriv.(l, o, t) - @test @inferred(d(l, o, t)) == ref + ref = [deriv(l, o[i], t[i]) for i in 1:length(o)] + d(l, o, t) = deriv.(l, o, t) + @test @inferred(d(l, o, t)) == ref end function test_vector_deriv2(l, o, t) - ref = [deriv2(l, o[i], t[i]) for i in 1:length(o)] - d(l, o, t) = deriv2.(l, o, t) - @test @inferred(d(l, o, t)) == ref + ref = [deriv2(l, o[i], t[i]) for i in 1:length(o)] + d(l, o, t) = deriv2.(l, o, t) + @test @inferred(d(l, o, t)) == ref end @testset "Vectorized API" begin - for T in (Float32, Float64) - for O in (Float32, Float64) - @testset "Margin-based $T -> $O" begin - for (targets,outputs) in ( - (rand(T[-1, 1], 4), (rand(O, 4) .- O(.5)) .* O(20)), - ) - for loss in (LogitMarginLoss(),ModifiedHuberLoss(), - L1HingeLoss(),SigmoidLoss()) - @testset "$(loss): " begin - test_vector_value(loss, outputs, targets) - test_vector_deriv(loss, outputs, targets) - test_vector_deriv2(loss, outputs, targets) - end - end - end + for T in (Float32, Float64) + for O in (Float32, Float64) + @testset "Margin-based $T -> $O" begin + for (targets, outputs) in ((rand(T[-1, 1], 4), (rand(O, 4) .- O(0.5)) .* O(20)),) + for loss in (LogitMarginLoss(), ModifiedHuberLoss(), L1HingeLoss(), SigmoidLoss()) + @testset "$(loss): " begin + test_vector_value(loss, outputs, targets) + test_vector_deriv(loss, outputs, targets) + test_vector_deriv2(loss, outputs, targets) end - @testset "Distance-based $T -> $O" begin - for (targets,outputs) in ( - ((rand(T, 4) .- T(.5)) .* T(20), (rand(O, 4) .- O(.5)) .* O(20)), - ) - for loss in (QuantileLoss(0.75),L2DistLoss(), - EpsilonInsLoss(1)) - @testset "$(loss): " begin - test_vector_value(loss, outputs, targets) - test_vector_deriv(loss, outputs, targets) - test_vector_deriv2(loss, outputs, targets) - end - end - end + end + end + end + @testset "Distance-based $T -> $O" begin + for (targets, outputs) in (((rand(T, 4) .- T(0.5)) .* T(20), (rand(O, 4) .- O(0.5)) .* O(20)),) + for loss in (QuantileLoss(0.75), L2DistLoss(), EpsilonInsLoss(1)) + @testset "$(loss): " begin + test_vector_value(loss, outputs, targets) + test_vector_deriv(loss, outputs, targets) + test_vector_deriv2(loss, outputs, targets) end + end end + end end + end end @testset "Aggregation with categorical values" begin - c = categorical(["Foo","Bar","Baz","Foo"]) - l = MisclassLoss() - @test sum(l, c, reverse(c)) == 2.0 - @test mean(l, c, reverse(c)) == 0.5 - @test sum(l, c, reverse(c), 2*ones(4), normalize=false) == 4.0 - @test mean(l, c, reverse(c), 2*ones(4), normalize=false) == 1.0 - @test mean(l, c, reverse(c), 2*ones(4), normalize=true) == 0.125 -end \ No newline at end of file + c = categorical(["Foo", "Bar", "Baz", "Foo"]) + l = MisclassLoss() + @test sum(l, c, reverse(c)) == 2.0 + @test mean(l, c, reverse(c)) == 0.5 + @test sum(l, c, reverse(c), 2 * ones(4), normalize=false) == 4.0 + @test mean(l, c, reverse(c), 2 * ones(4), normalize=false) == 1.0 + @test mean(l, c, reverse(c), 2 * ones(4), normalize=true) == 0.125 +end diff --git a/test/core.jl b/test/core.jl index 3aed376..54891f7 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1,435 +1,462 @@ function test_value_typestable(l::SupervisedLoss) - @testset "$(l): " begin - for o in (-1, 1, Int32(-1), Int32(1), -1.5, 1.5, Float32(-.5), Float32(.5)) - for t in (-2, 2, Int32(-1), Int32(1), -.5, .5, Float32(-1), Float32(1)) - # check inference - @inferred deriv(l, o, t) - @inferred deriv2(l, o, t) - - # get expected return type - T = promote_type(typeof(o), typeof(t)) - - # test basic loss - val = @inferred l(o, t) - @test typeof(val) <: T - - # test scaled version of loss - @test typeof((T(2)*l)(o, t)) <: T - end - end + @testset "$(l): " begin + for o in (-1, 1, Int32(-1), Int32(1), -1.5, 1.5, Float32(-0.5), Float32(0.5)) + for t in (-2, 2, Int32(-1), Int32(1), -0.5, 0.5, Float32(-1), Float32(1)) + # check inference + @inferred deriv(l, o, t) + @inferred deriv2(l, o, t) + + # get expected return type + T = promote_type(typeof(o), typeof(t)) + + # test basic loss + val = @inferred l(o, t) + @test typeof(val) <: T + + # test scaled version of loss + @test typeof((T(2) * l)(o, t)) <: T + end end + end end function test_value_float32_preserving(l::SupervisedLoss) - @testset "$(l): " begin - for o in (-1, 1, Int32(-1), Int32(1), -1.5, 1.5, Float32(-.5), Float32(.5)) - for t in (-2, 2, Int32(-1), Int32(1), -.5, .5, Float32(-1), Float32(1)) - # check inference - @inferred deriv(l, o, t) - @inferred deriv2(l, o, t) - - val = @inferred l(o, t) - T = promote_type(typeof(o),typeof(t)) - if !(T <: AbstractFloat) - # cast Integers to a float - # (whether its Float32 or Float64 depends on the loss...) - @test (typeof(val) <: AbstractFloat) - elseif T <: Float32 - # preserve Float32 - @test (typeof(val) <: Float32) - else - @test (typeof(val) <: Float64) - end - end + @testset "$(l): " begin + for o in (-1, 1, Int32(-1), Int32(1), -1.5, 1.5, Float32(-0.5), Float32(0.5)) + for t in (-2, 2, Int32(-1), Int32(1), -0.5, 0.5, Float32(-1), Float32(1)) + # check inference + @inferred deriv(l, o, t) + @inferred deriv2(l, o, t) + + val = @inferred l(o, t) + T = promote_type(typeof(o), typeof(t)) + if !(T <: AbstractFloat) + # cast Integers to a float + # (whether its Float32 or Float64 depends on the loss...) + @test (typeof(val) <: AbstractFloat) + elseif T <: Float32 + # preserve Float32 + @test (typeof(val) <: Float32) + else + @test (typeof(val) <: Float64) end + end end + end end function test_value_float64_forcing(l::SupervisedLoss) - @testset "$(l): " begin - for o in (-1, 1, Int32(-1), Int32(1), -1.5, 1.5, Float32(-.5), Float32(.5)) - for t in (-2, 2, Int32(-1), Int32(1), -.5, .5, Float32(-1), Float32(1)) - # check inference - @inferred deriv(l, o, t) - @inferred deriv2(l, o, t) - - val = @inferred l(o, t) - @test (typeof(val) <: Float64) - end - end + @testset "$(l): " begin + for o in (-1, 1, Int32(-1), Int32(1), -1.5, 1.5, Float32(-0.5), Float32(0.5)) + for t in (-2, 2, Int32(-1), Int32(1), -0.5, 0.5, Float32(-1), Float32(1)) + # check inference + @inferred deriv(l, o, t) + @inferred deriv2(l, o, t) + + val = @inferred l(o, t) + @test (typeof(val) <: Float64) + end end + end end function test_value(l::SupervisedLoss, f::Function, o_vec, t_vec) - @testset "$(l): " begin - for o in o_vec, t in t_vec - @test abs(l(o, t) - f(o, t)) < 1e-10 - end + @testset "$(l): " begin + for o in o_vec, t in t_vec + @test abs(l(o, t) - f(o, t)) < 1e-10 end + end end function test_deriv(l::MarginLoss, o_vec) - @testset "$(l): " begin - for o in o_vec, t in [-1., 1.] - if isdifferentiable(l, o*t) - d_dual = epsilon(l(dual(o, one(o)), dual(t, zero(t)))) - d_comp = @inferred deriv(l, o, t) - @test abs(d_dual - d_comp) < 1e-10 - val = @inferred l(o, t) - @test val ≈ l(o, t) - @test val ≈ l(o*t) - @test d_comp ≈ t*deriv(l, o*t) - end - end + @testset "$(l): " begin + for o in o_vec, t in [-1.0, 1.0] + if isdifferentiable(l, o * t) + d_dual = epsilon(l(dual(o, one(o)), dual(t, zero(t)))) + d_comp = @inferred deriv(l, o, t) + @test abs(d_dual - d_comp) < 1e-10 + val = @inferred l(o, t) + @test val ≈ l(o, t) + @test val ≈ l(o * t) + @test d_comp ≈ t * deriv(l, o * t) + end end + end end function test_deriv(l::DistanceLoss, o_vec) - @testset "$(l): " begin - for o in o_vec, t in -10:.2:10 - if isdifferentiable(l, o-t) - d_dual = epsilon(l(dual(o-t, one(o-t)))) - d_comp = @inferred deriv(l, o, t) - @test abs(d_dual - d_comp) < 1e-10 - val = @inferred l(o, t) - @test val ≈ l(o, t) - @test val ≈ l(o-t) - @test d_comp ≈ deriv(l, o-t) - end - end + @testset "$(l): " begin + for o in o_vec, t in -10:0.2:10 + if isdifferentiable(l, o - t) + d_dual = epsilon(l(dual(o - t, one(o - t)))) + d_comp = @inferred deriv(l, o, t) + @test abs(d_dual - d_comp) < 1e-10 + val = @inferred l(o, t) + @test val ≈ l(o, t) + @test val ≈ l(o - t) + @test d_comp ≈ deriv(l, o - t) + end end + end end function test_deriv(l::SupervisedLoss, o_vec, t_vec) - @testset "$(l): " begin - for o in o_vec, t in t_vec - if isdifferentiable(l, o, t) - d_dual = epsilon(l(dual(o, one(o)), dual(t, zero(t)))) - d_comp = @inferred deriv(l, o, t) - @test abs(d_dual - d_comp) < 1e-10 - val = @inferred l(o, t) - @test val ≈ l(o, t) - @test d_comp ≈ deriv(l, o, t) - end - end + @testset "$(l): " begin + for o in o_vec, t in t_vec + if isdifferentiable(l, o, t) + d_dual = epsilon(l(dual(o, one(o)), dual(t, zero(t)))) + d_comp = @inferred deriv(l, o, t) + @test abs(d_dual - d_comp) < 1e-10 + val = @inferred l(o, t) + @test val ≈ l(o, t) + @test d_comp ≈ deriv(l, o, t) + end end + end end function test_deriv2(l::MarginLoss, o_vec) - @testset "$(l): " begin - for o in o_vec, t in [-1., 1] - if istwicedifferentiable(l, o*t) && isdifferentiable(l, o*t) - d2_dual = epsilon(deriv(l, dual(o, one(o)), dual(t, zero(t)))) - d2_comp = @inferred deriv2(l, o, t) - @test abs(d2_dual - d2_comp) < 1e-10 - @test d2_comp ≈ @inferred deriv2(l, o, t) - @test d2_comp ≈ @inferred deriv2(l, o*t) - end - end + @testset "$(l): " begin + for o in o_vec, t in [-1.0, 1] + if istwicedifferentiable(l, o * t) && isdifferentiable(l, o * t) + d2_dual = epsilon(deriv(l, dual(o, one(o)), dual(t, zero(t)))) + d2_comp = @inferred deriv2(l, o, t) + @test abs(d2_dual - d2_comp) < 1e-10 + @test d2_comp ≈ @inferred deriv2(l, o, t) + @test d2_comp ≈ @inferred deriv2(l, o * t) + end end + end end function test_deriv2(l::DistanceLoss, o_vec) - @testset "$(l): " begin - for o in o_vec, t in -10:.2:10 - if istwicedifferentiable(l, o-t) && isdifferentiable(l, o-t) - d2_dual = epsilon(deriv(l, dual(o-t, one(o-t)))) - d2_comp = @inferred deriv2(l, o, t) - @test abs(d2_dual - d2_comp) < 1e-10 - @test d2_comp ≈ @inferred deriv2(l, o, t) - @test d2_comp ≈ @inferred deriv2(l, o-t) - end - end + @testset "$(l): " begin + for o in o_vec, t in -10:0.2:10 + if istwicedifferentiable(l, o - t) && isdifferentiable(l, o - t) + d2_dual = epsilon(deriv(l, dual(o - t, one(o - t)))) + d2_comp = @inferred deriv2(l, o, t) + @test abs(d2_dual - d2_comp) < 1e-10 + @test d2_comp ≈ @inferred deriv2(l, o, t) + @test d2_comp ≈ @inferred deriv2(l, o - t) + end end + end end function test_deriv2(l::SupervisedLoss, o_vec, t_vec) - @testset "$(l): " begin - for o in o_vec, t in t_vec - if istwicedifferentiable(l, o, t) && isdifferentiable(l, o, t) - d2_dual = epsilon(deriv(l, dual(o, one(o)), dual(t, zero(t)))) - d2_comp = @inferred deriv2(l, o, t) - @test abs(d2_dual - d2_comp) < 1e-10 - @test d2_comp ≈ @inferred deriv2(l, o, t) - end - end + @testset "$(l): " begin + for o in o_vec, t in t_vec + if istwicedifferentiable(l, o, t) && isdifferentiable(l, o, t) + d2_dual = epsilon(deriv(l, dual(o, one(o)), dual(t, zero(t)))) + d2_comp = @inferred deriv2(l, o, t) + @test abs(d2_dual - d2_comp) < 1e-10 + @test d2_comp ≈ @inferred deriv2(l, o, t) + end end + end end function test_scaledloss(l::SupervisedLoss, o_vec, t_vec) - @testset "Scaling for $(l): " begin - for λ = (2.0, 2) - sl = ScaledLoss(l,λ) - @test typeof(sl) <: ScaledLoss{typeof(l),λ} - @test 3 * sl == @inferred(ScaledLoss(sl,Val(3))) - @test (λ*3) * l == @inferred(ScaledLoss(sl,Val(3))) - @test sl == @inferred(ScaledLoss(l,Val(λ))) - @test sl == λ * l - @test sl == @inferred(Val(λ) * l) - for o in o_vec, t in t_vec - @test @inferred(sl(o, t)) == λ * l(o, t) - @test @inferred(deriv(sl, o, t)) == λ * deriv(l, o, t) - @test @inferred(deriv2(sl, o, t)) == λ * deriv2(l, o, t) - end - end + @testset "Scaling for $(l): " begin + for λ in (2.0, 2) + sl = ScaledLoss(l, λ) + @test typeof(sl) <: ScaledLoss{typeof(l),λ} + @test 3 * sl == @inferred(ScaledLoss(sl, Val(3))) + @test (λ * 3) * l == @inferred(ScaledLoss(sl, Val(3))) + @test sl == @inferred(ScaledLoss(l, Val(λ))) + @test sl == λ * l + @test sl == @inferred(Val(λ) * l) + for o in o_vec, t in t_vec + @test @inferred(sl(o, t)) == λ * l(o, t) + @test @inferred(deriv(sl, o, t)) == λ * deriv(l, o, t) + @test @inferred(deriv2(sl, o, t)) == λ * deriv2(l, o, t) + end end + end end function test_weightedloss(l::MarginLoss, o_vec, t_vec) - @testset "Weighted version for $(l): " begin - for w in (0., 0.2, 0.7, 1.) - wl = WeightedMarginLoss(l, w) - @test typeof(wl) <: WeightedMarginLoss{typeof(l),w} - @test WeightedMarginLoss(l, w * 0.1) == WeightedMarginLoss(wl, 0.1) - for o in o_vec, t in t_vec - if t == 1 - @test wl(o, t) == w * l(o, t) - @test deriv(wl, o, t) == w * deriv(l, o, t) - @test deriv2(wl, o, t) == w * deriv2(l, o, t) - else - @test wl(o, t) == (1-w) * l(o, t) - @test deriv(wl, o, t) == (1-w) * deriv(l, o, t) - @test deriv2(wl, o, t) == (1-w) * deriv2(l, o, t) - end - end + @testset "Weighted version for $(l): " begin + for w in (0.0, 0.2, 0.7, 1.0) + wl = WeightedMarginLoss(l, w) + @test typeof(wl) <: WeightedMarginLoss{typeof(l),w} + @test WeightedMarginLoss(l, w * 0.1) == WeightedMarginLoss(wl, 0.1) + for o in o_vec, t in t_vec + if t == 1 + @test wl(o, t) == w * l(o, t) + @test deriv(wl, o, t) == w * deriv(l, o, t) + @test deriv2(wl, o, t) == w * deriv2(l, o, t) + else + @test wl(o, t) == (1 - w) * l(o, t) + @test deriv(wl, o, t) == (1 - w) * deriv(l, o, t) + @test deriv2(wl, o, t) == (1 - w) * deriv2(l, o, t) end + end end + end end # ==================================================================== @testset "Test typealias" begin - @test L1DistLoss === LPDistLoss{1} - @test L2DistLoss === LPDistLoss{2} - @test HingeLoss === L1HingeLoss - @test EpsilonInsLoss === L1EpsilonInsLoss + @test L1DistLoss === LPDistLoss{1} + @test L2DistLoss === LPDistLoss{2} + @test HingeLoss === L1HingeLoss + @test EpsilonInsLoss === L1EpsilonInsLoss end @testset "Test typestable supervised loss for type stability" begin - for loss in [L1HingeLoss(), L2HingeLoss(), ModifiedHuberLoss(), - PerceptronLoss(), LPDistLoss(1), LPDistLoss(2), - LPDistLoss(3), L2MarginLoss()] - test_value_typestable(loss) - # TODO: add ZeroOneLoss after scaling works... - end + for loss in [ + L1HingeLoss(), + L2HingeLoss(), + ModifiedHuberLoss(), + PerceptronLoss(), + LPDistLoss(1), + LPDistLoss(2), + LPDistLoss(3), + L2MarginLoss() + ] + test_value_typestable(loss) + # TODO: add ZeroOneLoss after scaling works... + end end @testset "Test float-forcing supervised loss for type stability" begin - # Losses that should always return Float64 - for loss in [SmoothedL1HingeLoss(0.5), SmoothedL1HingeLoss(1), - L1EpsilonInsLoss(0.5), L1EpsilonInsLoss(1), - L2EpsilonInsLoss(0.5), L2EpsilonInsLoss(1), - PeriodicLoss(1), PeriodicLoss(1.5), - HuberLoss(1.0), QuantileLoss(.8), - DWDMarginLoss(0.5), DWDMarginLoss(1), DWDMarginLoss(2)] - test_value_float64_forcing(loss) - test_value_float64_forcing(2.0 * loss) - end - test_value_float64_forcing(2.0 * LogitDistLoss()) - test_value_float64_forcing(2.0 * LogitMarginLoss()) - test_value_float64_forcing(2.0 * ExpLoss()) - test_value_float64_forcing(2.0 * SigmoidLoss()) - - # Losses that should return an AbstractFloat, preserving type if possible - for loss in [SmoothedL1HingeLoss(0.5f0), SmoothedL1HingeLoss(1f0), - PeriodicLoss(1f0), PeriodicLoss(0.5f0), - LogitDistLoss(), LogitMarginLoss(), ExpLoss(), SigmoidLoss(), - L1EpsilonInsLoss(1f0), L1EpsilonInsLoss(0.5f0), - L2EpsilonInsLoss(1f0), L2EpsilonInsLoss(0.5f0), - HuberLoss(1.0f0), QuantileLoss(.8f0), DWDMarginLoss(0.5f0)] - test_value_float32_preserving(loss) - test_value_float32_preserving(2f0 * loss) - end + # Losses that should always return Float64 + for loss in [ + SmoothedL1HingeLoss(0.5), + SmoothedL1HingeLoss(1), + L1EpsilonInsLoss(0.5), + L1EpsilonInsLoss(1), + L2EpsilonInsLoss(0.5), + L2EpsilonInsLoss(1), + PeriodicLoss(1), + PeriodicLoss(1.5), + HuberLoss(1.0), + QuantileLoss(0.8), + DWDMarginLoss(0.5), + DWDMarginLoss(1), + DWDMarginLoss(2) + ] + test_value_float64_forcing(loss) + test_value_float64_forcing(2.0 * loss) + end + test_value_float64_forcing(2.0 * LogitDistLoss()) + test_value_float64_forcing(2.0 * LogitMarginLoss()) + test_value_float64_forcing(2.0 * ExpLoss()) + test_value_float64_forcing(2.0 * SigmoidLoss()) + + # Losses that should return an AbstractFloat, preserving type if possible + for loss in [ + SmoothedL1HingeLoss(0.5f0), + SmoothedL1HingeLoss(1.0f0), + PeriodicLoss(1.0f0), + PeriodicLoss(0.5f0), + LogitDistLoss(), + LogitMarginLoss(), + ExpLoss(), + SigmoidLoss(), + L1EpsilonInsLoss(1.0f0), + L1EpsilonInsLoss(0.5f0), + L2EpsilonInsLoss(1.0f0), + L2EpsilonInsLoss(0.5f0), + HuberLoss(1.0f0), + QuantileLoss(0.8f0), + DWDMarginLoss(0.5f0) + ] + test_value_float32_preserving(loss) + test_value_float32_preserving(2.0f0 * loss) + end end @testset "Test margin-based loss against reference function" begin - _zerooneloss(o, t) = sign(o*t) < 0 ? 1 : 0 - test_value(ZeroOneLoss(), _zerooneloss, -10:0.2:10, [-1.,1.]) + _zerooneloss(o, t) = sign(o * t) < 0 ? 1 : 0 + test_value(ZeroOneLoss(), _zerooneloss, -10:0.2:10, [-1.0, 1.0]) - _hingeloss(o, t) = max(0, 1 - o.*t) - test_value(HingeLoss(), _hingeloss, -10:0.2:10, [-1.,1.]) + _hingeloss(o, t) = max(0, 1 - o .* t) + test_value(HingeLoss(), _hingeloss, -10:0.2:10, [-1.0, 1.0]) - _l2hingeloss(o, t) = max(0, 1 - o.*t)^2 - test_value(L2HingeLoss(), _l2hingeloss, -10:0.2:10, [-1.,1.]) + _l2hingeloss(o, t) = max(0, 1 - o .* t)^2 + test_value(L2HingeLoss(), _l2hingeloss, -10:0.2:10, [-1.0, 1.0]) - _perceptronloss(o, t) = max(0, -o.*t) - test_value(PerceptronLoss(), _perceptronloss, -10:0.2:10, [-1.,1.]) + _perceptronloss(o, t) = max(0, -o .* t) + test_value(PerceptronLoss(), _perceptronloss, -10:0.2:10, [-1.0, 1.0]) - _logitmarginloss(o, t) = log(1 + exp(-o.*t)) - test_value(LogitMarginLoss(), _logitmarginloss, -10:0.2:10, [-1.,1.]) + _logitmarginloss(o, t) = log(1 + exp(-o .* t)) + test_value(LogitMarginLoss(), _logitmarginloss, -10:0.2:10, [-1.0, 1.0]) - function _smoothedl1hingeloss(γ) - function _value(o, t) - if o.*t >= 1 - γ - 1/(2γ) * max(0, 1- o.*t)^2 - else - 1 - γ / 2 - o.*t - end - end - _value + function _smoothedl1hingeloss(γ) + function _value(o, t) + if o .* t >= 1 - γ + 1 / (2γ) * max(0, 1 - o .* t)^2 + else + 1 - γ / 2 - o .* t + end end - test_value(SmoothedL1HingeLoss(.5), _smoothedl1hingeloss(.5), -10:0.2:10, [-1.,1.]) - test_value(SmoothedL1HingeLoss(1), _smoothedl1hingeloss(1), -10:0.2:10, [-1.,1.]) - test_value(SmoothedL1HingeLoss(2), _smoothedl1hingeloss(2), -10:0.2:10, [-1.,1.]) - - function _modhuberloss(o, t) - if o .* t >= -1 - max(0, 1 - o .* t)^2 - else - -4 .* o .* t - end + _value + end + test_value(SmoothedL1HingeLoss(0.5), _smoothedl1hingeloss(0.5), -10:0.2:10, [-1.0, 1.0]) + test_value(SmoothedL1HingeLoss(1), _smoothedl1hingeloss(1), -10:0.2:10, [-1.0, 1.0]) + test_value(SmoothedL1HingeLoss(2), _smoothedl1hingeloss(2), -10:0.2:10, [-1.0, 1.0]) + + function _modhuberloss(o, t) + if o .* t >= -1 + max(0, 1 - o .* t)^2 + else + -4 .* o .* t end - test_value(ModifiedHuberLoss(), _modhuberloss, -10:0.2:10, [-1.,1.]) + end + test_value(ModifiedHuberLoss(), _modhuberloss, -10:0.2:10, [-1.0, 1.0]) - _l2marginloss(o, t) = (1 - o.*t)^2 - test_value(L2MarginLoss(), _l2marginloss, -10:0.2:10, [-1.,1.]) + _l2marginloss(o, t) = (1 - o .* t)^2 + test_value(L2MarginLoss(), _l2marginloss, -10:0.2:10, [-1.0, 1.0]) - _exploss(o, t) = exp(-o.*t) - test_value(ExpLoss(), _exploss, -10:0.2:10, [-1.,1.]) + _exploss(o, t) = exp(-o .* t) + test_value(ExpLoss(), _exploss, -10:0.2:10, [-1.0, 1.0]) - _sigmoidloss(o, t) = (1-tanh(o.*t)) - test_value(SigmoidLoss(), _sigmoidloss, -10:0.2:10, [-1.,1.]) + _sigmoidloss(o, t) = (1 - tanh(o .* t)) + test_value(SigmoidLoss(), _sigmoidloss, -10:0.2:10, [-1.0, 1.0]) - function _dwdmarginloss(q) - function _value(o, t) - if o.*t <= q/(q+1) - convert(Float64, 1 - o.*t) - else - ((q^q)/(q+1)^(q+1)) / (o.*t)^q - end - end - _value + function _dwdmarginloss(q) + function _value(o, t) + if o .* t <= q / (q + 1) + convert(Float64, 1 - o .* t) + else + ((q^q) / (q + 1)^(q + 1)) / (o .* t)^q + end end - test_value(DWDMarginLoss(0.5), _dwdmarginloss(0.5), -10:0.2:10, [-1.,1.]) - test_value(DWDMarginLoss(1), _dwdmarginloss(1), -10:0.2:10, [-1.,1.]) - test_value(DWDMarginLoss(2), _dwdmarginloss(2), -10:0.2:10, [-1.,1.]) + _value + end + test_value(DWDMarginLoss(0.5), _dwdmarginloss(0.5), -10:0.2:10, [-1.0, 1.0]) + test_value(DWDMarginLoss(1), _dwdmarginloss(1), -10:0.2:10, [-1.0, 1.0]) + test_value(DWDMarginLoss(2), _dwdmarginloss(2), -10:0.2:10, [-1.0, 1.0]) end @testset "Test distance-based loss against reference function" begin - or = range(-10, stop=20, length=10) - tr = range(-30, stop=30, length=10) - - _l1distloss(o, t) = abs(t - o) - test_value(L1DistLoss(), _l1distloss, or, tr) - - _l2distloss(o, t) = (t - o)^2 - test_value(L2DistLoss(), _l2distloss, or, tr) - - _lp15distloss(o, t) = abs(t - o)^(1.5) - test_value(LPDistLoss(1.5), _lp15distloss, or, tr) - - function _periodicloss(c) - (o, t) -> 1 - cos((o-t)*2π/c) - end - test_value(PeriodicLoss(0.5), _periodicloss(0.5), or, tr) - test_value(PeriodicLoss(1), _periodicloss(1), or, tr) - test_value(PeriodicLoss(1.5), _periodicloss(1.5), or, tr) - - function _huberloss(d) - (o, t) -> abs(o-t) max(0, abs(t - o) - ɛ) - end - test_value(EpsilonInsLoss(0.5), _l1epsinsloss(0.5), or, tr) - test_value(EpsilonInsLoss(1), _l1epsinsloss(1), or, tr) - test_value(EpsilonInsLoss(1.5), _l1epsinsloss(1.5), or, tr) - - function _l2epsinsloss(ɛ) - (o, t) -> max(0, abs(t - o) - ɛ)^2 - end - test_value(L2EpsilonInsLoss(0.5), _l2epsinsloss(0.5), or, tr) - test_value(L2EpsilonInsLoss(1), _l2epsinsloss(1), or, tr) - test_value(L2EpsilonInsLoss(1.5), _l2epsinsloss(1.5), or, tr) - - _logitdistloss(o, t) = -log((4*exp(t-o))/(1+exp(t-o))^2) - test_value(LogitDistLoss(), _logitdistloss, or, tr) - - function _quantileloss(τ) - (o, t) -> (o - t) * ((o - t > 0) - τ) - end - test_value(QuantileLoss(.7), _quantileloss(.7), or, tr) - - function _logcoshloss(o, t) - log(cosh(o - t)) - end - test_value(LogCoshLoss(), _logcoshloss, or, tr) + or = range(-10, stop=20, length=10) + tr = range(-30, stop=30, length=10) + + _l1distloss(o, t) = abs(t - o) + test_value(L1DistLoss(), _l1distloss, or, tr) + + _l2distloss(o, t) = (t - o)^2 + test_value(L2DistLoss(), _l2distloss, or, tr) + + _lp15distloss(o, t) = abs(t - o)^(1.5) + test_value(LPDistLoss(1.5), _lp15distloss, or, tr) + + function _periodicloss(c) + (o, t) -> 1 - cos((o - t) * 2π / c) + end + test_value(PeriodicLoss(0.5), _periodicloss(0.5), or, tr) + test_value(PeriodicLoss(1), _periodicloss(1), or, tr) + test_value(PeriodicLoss(1.5), _periodicloss(1.5), or, tr) + + function _huberloss(d) + (o, t) -> abs(o - t) < d ? (abs2(o - t) / 2) : (d * (abs(o - t) - (d / 2))) + end + test_value(HuberLoss(0.5), _huberloss(0.5), or, tr) + test_value(HuberLoss(1), _huberloss(1), or, tr) + test_value(HuberLoss(1.5), _huberloss(1.5), or, tr) + + function _l1epsinsloss(ɛ) + (o, t) -> max(0, abs(t - o) - ɛ) + end + test_value(EpsilonInsLoss(0.5), _l1epsinsloss(0.5), or, tr) + test_value(EpsilonInsLoss(1), _l1epsinsloss(1), or, tr) + test_value(EpsilonInsLoss(1.5), _l1epsinsloss(1.5), or, tr) + + function _l2epsinsloss(ɛ) + (o, t) -> max(0, abs(t - o) - ɛ)^2 + end + test_value(L2EpsilonInsLoss(0.5), _l2epsinsloss(0.5), or, tr) + test_value(L2EpsilonInsLoss(1), _l2epsinsloss(1), or, tr) + test_value(L2EpsilonInsLoss(1.5), _l2epsinsloss(1.5), or, tr) + + _logitdistloss(o, t) = -log((4 * exp(t - o)) / (1 + exp(t - o))^2) + test_value(LogitDistLoss(), _logitdistloss, or, tr) + + function _quantileloss(τ) + (o, t) -> (o - t) * ((o - t > 0) - τ) + end + test_value(QuantileLoss(0.7), _quantileloss(0.7), or, tr) + + function _logcoshloss(o, t) + log(cosh(o - t)) + end + test_value(LogCoshLoss(), _logcoshloss, or, tr) end @testset "Test other loss" begin - _misclassloss(o, t) = o == t ? 0 : 1 - test_value(MisclassLoss(), _misclassloss, vcat(1:5,7:11), 1:10) + _misclassloss(o, t) = o == t ? 0 : 1 + test_value(MisclassLoss(), _misclassloss, vcat(1:5, 7:11), 1:10) - _crossentropyloss(o, t) = -t*log(o) - (1-t)*log(1-o) - test_value(CrossEntropyLoss(), _crossentropyloss, 0.01:0.01:0.99, 0:0.01:1) + _crossentropyloss(o, t) = -t * log(o) - (1 - t) * log(1 - o) + test_value(CrossEntropyLoss(), _crossentropyloss, 0.01:0.01:0.99, 0:0.01:1) - _poissonloss(o, t) = exp(o) - t*o - test_value(PoissonLoss(), _poissonloss, range(0,stop=10,length=11), 0:10) + _poissonloss(o, t) = exp(o) - t * o + test_value(PoissonLoss(), _poissonloss, range(0, stop=10, length=11), 0:10) end @testset "Test scaled loss" begin - for loss in distance_losses - test_scaledloss(loss, -10:0.5:10, -10:.2:10) - end + for loss in distance_losses + test_scaledloss(loss, -10:0.5:10, -10:0.2:10) + end - for loss in margin_losses - test_scaledloss(loss, -10:0.2:10, [-1.,1.]) - end + for loss in margin_losses + test_scaledloss(loss, -10:0.2:10, [-1.0, 1.0]) + end - test_scaledloss(PoissonLoss(), range(0,stop=10,length=11), 0:10) + test_scaledloss(PoissonLoss(), range(0, stop=10, length=11), 0:10) end @testset "Test weighted loss" begin - for loss in margin_losses - test_weightedloss(loss, -10:0.2:10, [-1.,1.]) - end + for loss in margin_losses + test_weightedloss(loss, -10:0.2:10, [-1.0, 1.0]) + end end # -------------------------------------------------------------- @testset "Test first derivatives" begin - for loss in distance_losses - test_deriv(loss, -10:0.5:10) - end + for loss in distance_losses + test_deriv(loss, -10:0.5:10) + end - for loss in margin_losses - test_deriv(loss, -10:0.2:10) - end + for loss in margin_losses + test_deriv(loss, -10:0.2:10) + end - test_deriv(PoissonLoss(), -10:.2:10, 0:30) - test_deriv(CrossEntropyLoss(), 0.01:0.01:0.99, 0:0.01:1) + test_deriv(PoissonLoss(), -10:0.2:10, 0:30) + test_deriv(CrossEntropyLoss(), 0.01:0.01:0.99, 0:0.01:1) end # -------------------------------------------------------------- @testset "Test second derivatives" begin - for loss in distance_losses - test_deriv2(loss, -10:0.5:10) - end + for loss in distance_losses + test_deriv2(loss, -10:0.5:10) + end - for loss in margin_losses - test_deriv2(loss, -10:0.2:10) - end + for loss in margin_losses + test_deriv2(loss, -10:0.2:10) + end - test_deriv2(PoissonLoss(), -10:.2:10, 0:30) - test_deriv2(CrossEntropyLoss(), 0.01:0.01:0.99, 0:0.01:1) + test_deriv2(PoissonLoss(), -10:0.2:10, 0:30) + test_deriv2(CrossEntropyLoss(), 0.01:0.01:0.99, 0:0.01:1) end # -------------------------------------------------------------- @testset "Test losses with categorical values" begin - c = categorical(["Foo","Bar","Baz","Foo"]) + c = categorical(["Foo", "Bar", "Baz", "Foo"]) - l = MisclassLoss() - @test l(c[1], c[1]) == 0.0 - @test l(c[1], c[2]) == 1.0 - @test l.(c, reverse(c)) == [0.0, 1.0, 1.0, 0.0] + l = MisclassLoss() + @test l(c[1], c[1]) == 0.0 + @test l(c[1], c[2]) == 1.0 + @test l.(c, reverse(c)) == [0.0, 1.0, 1.0, 0.0] - l = MisclassLoss{Float32}() - @test l(c[1], c[1]) isa Float32 - @test l.(c, c) isa Vector{Float32} + l = MisclassLoss{Float32}() + @test l(c[1], c[1]) isa Float32 + @test l.(c, c) isa Vector{Float32} end diff --git a/test/props.jl b/test/props.jl index a2ad5ba..d9d17c0 100644 --- a/test/props.jl +++ b/test/props.jl @@ -1,723 +1,719 @@ struct TstVanillaLoss <: SupervisedLoss end @testset "Fallback implementations; not prior knowledge" begin - @test isminimizable(TstVanillaLoss()) == false - @test isdifferentiable(TstVanillaLoss()) == false - @test isdifferentiable(TstVanillaLoss(), 0) == false - @test istwicedifferentiable(TstVanillaLoss()) == false - @test istwicedifferentiable(TstVanillaLoss(), 0) == false - - @test isstronglyconvex(TstVanillaLoss()) == false - @test isstrictlyconvex(TstVanillaLoss()) == false - @test isconvex(TstVanillaLoss()) == false - - @test isnemitski(TstVanillaLoss()) == false - @test islipschitzcont(TstVanillaLoss()) == false - @test islocallylipschitzcont(TstVanillaLoss()) == false - @test isclipable(TstVanillaLoss()) == false - @test ismarginbased(TstVanillaLoss()) == false - @test isdistancebased(TstVanillaLoss()) == false - @test issymmetric(TstVanillaLoss()) == false + @test isminimizable(TstVanillaLoss()) == false + @test isdifferentiable(TstVanillaLoss()) == false + @test isdifferentiable(TstVanillaLoss(), 0) == false + @test istwicedifferentiable(TstVanillaLoss()) == false + @test istwicedifferentiable(TstVanillaLoss(), 0) == false + + @test isstronglyconvex(TstVanillaLoss()) == false + @test isstrictlyconvex(TstVanillaLoss()) == false + @test isconvex(TstVanillaLoss()) == false + + @test isnemitski(TstVanillaLoss()) == false + @test islipschitzcont(TstVanillaLoss()) == false + @test islocallylipschitzcont(TstVanillaLoss()) == false + @test isclipable(TstVanillaLoss()) == false + @test ismarginbased(TstVanillaLoss()) == false + @test isdistancebased(TstVanillaLoss()) == false + @test issymmetric(TstVanillaLoss()) == false end - struct TstStronglyConvexLoss <: SupervisedLoss end LossFunctions.isstronglyconvex(::TstStronglyConvexLoss) = true @testset "Fallback implementations; strongly convex" begin - @test isminimizable(TstStronglyConvexLoss()) == true - - @test isdifferentiable(TstStronglyConvexLoss()) == false - @test isdifferentiable(TstStronglyConvexLoss(), 0) == false - @test istwicedifferentiable(TstStronglyConvexLoss()) == false - @test istwicedifferentiable(TstStronglyConvexLoss(), 0) == false - - @test isstronglyconvex(TstStronglyConvexLoss()) == true - @test isstrictlyconvex(TstStronglyConvexLoss()) == true - @test isconvex(TstStronglyConvexLoss()) == true - - @test isnemitski(TstStronglyConvexLoss()) == true - @test islipschitzcont(TstStronglyConvexLoss()) == false - @test islocallylipschitzcont(TstStronglyConvexLoss()) == true - @test isclipable(TstStronglyConvexLoss()) == false - @test ismarginbased(TstStronglyConvexLoss()) == false - @test isdistancebased(TstStronglyConvexLoss()) == false - @test issymmetric(TstStronglyConvexLoss()) == false + @test isminimizable(TstStronglyConvexLoss()) == true + + @test isdifferentiable(TstStronglyConvexLoss()) == false + @test isdifferentiable(TstStronglyConvexLoss(), 0) == false + @test istwicedifferentiable(TstStronglyConvexLoss()) == false + @test istwicedifferentiable(TstStronglyConvexLoss(), 0) == false + + @test isstronglyconvex(TstStronglyConvexLoss()) == true + @test isstrictlyconvex(TstStronglyConvexLoss()) == true + @test isconvex(TstStronglyConvexLoss()) == true + + @test isnemitski(TstStronglyConvexLoss()) == true + @test islipschitzcont(TstStronglyConvexLoss()) == false + @test islocallylipschitzcont(TstStronglyConvexLoss()) == true + @test isclipable(TstStronglyConvexLoss()) == false + @test ismarginbased(TstStronglyConvexLoss()) == false + @test isdistancebased(TstStronglyConvexLoss()) == false + @test issymmetric(TstStronglyConvexLoss()) == false end - struct TstTwiceDiffLoss <: SupervisedLoss end LossFunctions.istwicedifferentiable(::TstTwiceDiffLoss) = true @testset "Fallback implementations; twice differentiable" begin - @test isminimizable(TstTwiceDiffLoss()) == false - - @test isdifferentiable(TstTwiceDiffLoss()) == true - @test isdifferentiable(TstTwiceDiffLoss(), 0) == true - @test istwicedifferentiable(TstTwiceDiffLoss()) == true - @test istwicedifferentiable(TstTwiceDiffLoss(), 0) == true - - @test isstronglyconvex(TstTwiceDiffLoss()) == false - @test isstrictlyconvex(TstTwiceDiffLoss()) == false - @test isconvex(TstTwiceDiffLoss()) == false - - @test isnemitski(TstTwiceDiffLoss()) == false - @test islipschitzcont(TstTwiceDiffLoss()) == false - @test islocallylipschitzcont(TstTwiceDiffLoss()) == false - @test isclipable(TstTwiceDiffLoss()) == false - @test ismarginbased(TstTwiceDiffLoss()) == false - @test isdistancebased(TstTwiceDiffLoss()) == false - @test issymmetric(TstTwiceDiffLoss()) == false + @test isminimizable(TstTwiceDiffLoss()) == false + + @test isdifferentiable(TstTwiceDiffLoss()) == true + @test isdifferentiable(TstTwiceDiffLoss(), 0) == true + @test istwicedifferentiable(TstTwiceDiffLoss()) == true + @test istwicedifferentiable(TstTwiceDiffLoss(), 0) == true + + @test isstronglyconvex(TstTwiceDiffLoss()) == false + @test isstrictlyconvex(TstTwiceDiffLoss()) == false + @test isconvex(TstTwiceDiffLoss()) == false + + @test isnemitski(TstTwiceDiffLoss()) == false + @test islipschitzcont(TstTwiceDiffLoss()) == false + @test islocallylipschitzcont(TstTwiceDiffLoss()) == false + @test isclipable(TstTwiceDiffLoss()) == false + @test ismarginbased(TstTwiceDiffLoss()) == false + @test isdistancebased(TstTwiceDiffLoss()) == false + @test issymmetric(TstTwiceDiffLoss()) == false end - struct TstMarginLoss <: MarginLoss end @testset "Fallback implementations; margin-based" begin - @test isminimizable(TstMarginLoss()) == false - - @test isdifferentiable(TstMarginLoss()) == false - @test isdifferentiable(TstMarginLoss(), 0) == false - @test istwicedifferentiable(TstMarginLoss()) == false - @test istwicedifferentiable(TstMarginLoss(), 0) == false - - @test isstronglyconvex(TstMarginLoss()) == false - @test isstrictlyconvex(TstMarginLoss()) == false - @test isconvex(TstMarginLoss()) == false - - @test isnemitski(TstMarginLoss()) == true - @test islipschitzcont(TstMarginLoss()) == false - @test islocallylipschitzcont(TstMarginLoss()) == false - @test isclipable(TstMarginLoss()) == false - @test ismarginbased(TstMarginLoss()) == true - @test isdistancebased(TstMarginLoss()) == false - @test issymmetric(TstMarginLoss()) == false - - @test isclasscalibrated(TstMarginLoss()) == false - @test isfishercons(TstMarginLoss()) == false - @test isunivfishercons(TstMarginLoss()) == false + @test isminimizable(TstMarginLoss()) == false + + @test isdifferentiable(TstMarginLoss()) == false + @test isdifferentiable(TstMarginLoss(), 0) == false + @test istwicedifferentiable(TstMarginLoss()) == false + @test istwicedifferentiable(TstMarginLoss(), 0) == false + + @test isstronglyconvex(TstMarginLoss()) == false + @test isstrictlyconvex(TstMarginLoss()) == false + @test isconvex(TstMarginLoss()) == false + + @test isnemitski(TstMarginLoss()) == true + @test islipschitzcont(TstMarginLoss()) == false + @test islocallylipschitzcont(TstMarginLoss()) == false + @test isclipable(TstMarginLoss()) == false + @test ismarginbased(TstMarginLoss()) == true + @test isdistancebased(TstMarginLoss()) == false + @test issymmetric(TstMarginLoss()) == false + + @test isclasscalibrated(TstMarginLoss()) == false + @test isfishercons(TstMarginLoss()) == false + @test isunivfishercons(TstMarginLoss()) == false end - struct TstDistanceLoss <: DistanceLoss end @testset "Fallback implementations; distance-based" begin - @test isminimizable(TstDistanceLoss()) == false - - @test isdifferentiable(TstDistanceLoss()) == false - @test isdifferentiable(TstDistanceLoss(), 0) == false - @test istwicedifferentiable(TstDistanceLoss()) == false - @test istwicedifferentiable(TstDistanceLoss(), 0) == false - - @test isstronglyconvex(TstDistanceLoss()) == false - @test isstrictlyconvex(TstDistanceLoss()) == false - @test isconvex(TstDistanceLoss()) == false - - @test isnemitski(TstDistanceLoss()) == false - @test islipschitzcont(TstDistanceLoss()) == false - @test islocallylipschitzcont(TstDistanceLoss()) == false - @test isclipable(TstDistanceLoss()) == true - @test ismarginbased(TstDistanceLoss()) == false - @test isdistancebased(TstDistanceLoss()) == true - @test issymmetric(TstDistanceLoss()) == false + @test isminimizable(TstDistanceLoss()) == false + + @test isdifferentiable(TstDistanceLoss()) == false + @test isdifferentiable(TstDistanceLoss(), 0) == false + @test istwicedifferentiable(TstDistanceLoss()) == false + @test istwicedifferentiable(TstDistanceLoss(), 0) == false + + @test isstronglyconvex(TstDistanceLoss()) == false + @test isstrictlyconvex(TstDistanceLoss()) == false + @test isconvex(TstDistanceLoss()) == false + + @test isnemitski(TstDistanceLoss()) == false + @test islipschitzcont(TstDistanceLoss()) == false + @test islocallylipschitzcont(TstDistanceLoss()) == false + @test isclipable(TstDistanceLoss()) == true + @test ismarginbased(TstDistanceLoss()) == false + @test isdistancebased(TstDistanceLoss()) == true + @test issymmetric(TstDistanceLoss()) == false end # -------------------------------------------------------------- @testset "LPDistLoss{0.5}" begin - loss = LPDistLoss(0.5) - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == false - @test isdifferentiable(loss, 0) == false - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == false - - @test isnemitski(loss) == false - @test islipschitzcont(loss) == false - @test islocallylipschitzcont(loss) == false - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = LPDistLoss(0.5) + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == false + @test isdifferentiable(loss, 0) == false + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == false + + @test isnemitski(loss) == false + @test islipschitzcont(loss) == false + @test islocallylipschitzcont(loss) == false + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "L1DistLoss" begin - loss = L1DistLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == false - @test isdifferentiable(loss, 0) == false - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = L1DistLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == false + @test isdifferentiable(loss, 0) == false + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "L2DistLoss" begin - loss = L2DistLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - - @test isstronglyconvex(loss) == true - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == false - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = L2DistLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + + @test isstronglyconvex(loss) == true + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == false + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "LPDistLoss{3}" begin - loss = LPDistLoss(3) - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - - @test isstronglyconvex(loss) == true - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == false - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = LPDistLoss(3) + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + + @test isstronglyconvex(loss) == true + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == false + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "HuberLoss(1)" begin - loss = HuberLoss(1) - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == false - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = HuberLoss(1) + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == false + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "L1EpsilonInsLoss(1)" begin - loss = L1EpsilonInsLoss(1) - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == false - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == false - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 0) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = L1EpsilonInsLoss(1) + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == false + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == false + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 0) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "L2EpsilonInsLoss(1)" begin - loss = L2EpsilonInsLoss(1) - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == false - - @test isstronglyconvex(loss) == true - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == false - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = L2EpsilonInsLoss(1) + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == false + + @test isstronglyconvex(loss) == true + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == false + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "LogitDistLoss()" begin - loss = LogitDistLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = LogitDistLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end @testset "QuantileLoss" begin - l1 = QuantileLoss(.5) - l2 = QuantileLoss(.7) - - @test issymmetric(l1) == true - @test issymmetric(l2) == false - - @test isminimizable(l2) == true - - @test isdifferentiable(l2) == false - @test isdifferentiable(l2, 0) == false - @test isdifferentiable(l2, 1) == true - @test istwicedifferentiable(l2) == false - @test istwicedifferentiable(l2, 0) == false - @test istwicedifferentiable(l2, 1) == true - - @test isstronglyconvex(l2) == false - @test isstrictlyconvex(l2) == false - @test isconvex(l2) == true - - # @test isnemitski(l2) == ? - @test islipschitzcont(l2) == true - @test islocallylipschitzcont(l2) == true - # @test isclipable(l2) == ? - @test ismarginbased(l2) == false - @test isdistancebased(l2) == true + l1 = QuantileLoss(0.5) + l2 = QuantileLoss(0.7) + + @test issymmetric(l1) == true + @test issymmetric(l2) == false + + @test isminimizable(l2) == true + + @test isdifferentiable(l2) == false + @test isdifferentiable(l2, 0) == false + @test isdifferentiable(l2, 1) == true + @test istwicedifferentiable(l2) == false + @test istwicedifferentiable(l2, 0) == false + @test istwicedifferentiable(l2, 1) == true + + @test isstronglyconvex(l2) == false + @test isstrictlyconvex(l2) == false + @test isconvex(l2) == true + + # @test isnemitski(l2) == ? + @test islipschitzcont(l2) == true + @test islocallylipschitzcont(l2) == true + # @test isclipable(l2) == ? + @test ismarginbased(l2) == false + @test isdistancebased(l2) == true end @testset "LogCoshLoss" begin - loss = LogCoshLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - - @test isstronglyconvex(loss) == true - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - #@test isnemitski(loss) == ? - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == false - @test isdistancebased(loss) == true - @test issymmetric(loss) == true + loss = LogCoshLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + + @test isstronglyconvex(loss) == true + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + #@test isnemitski(loss) == ? + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == false + @test isdistancebased(loss) == true + @test issymmetric(loss) == true end # -------------------------------------------------------------- @testset "ZeroOneLoss" begin - loss = ZeroOneLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == false - @test isdifferentiable(loss, 0) == false - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == false - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true + loss = ZeroOneLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == false + @test isdifferentiable(loss, 0) == false + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == false + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true end @testset "PerceptronLoss" begin - loss = PerceptronLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == false - @test isdifferentiable(loss, 0) == false - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 0) == false - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == false + loss = PerceptronLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == false + @test isdifferentiable(loss, 0) == false + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 0) == false + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == false end @testset "LogitMarginLoss" begin - loss = LogitMarginLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == false - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true - @test isfishercons(loss) == true - @test isunivfishercons(loss) == true + loss = LogitMarginLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == false + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true + @test isfishercons(loss) == true + @test isunivfishercons(loss) == true end @testset "L1HingeLoss" begin - loss = L1HingeLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == false - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == false - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == false - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true - @test isfishercons(loss) == true - @test isunivfishercons(loss) == false + loss = L1HingeLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == false + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == false + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == false + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true + @test isfishercons(loss) == true + @test isunivfishercons(loss) == false end @testset "L2HingeLoss" begin - loss = L2HingeLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == false - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == false - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true - @test isfishercons(loss) == true - @test isunivfishercons(loss) == true + loss = L2HingeLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == false + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == false + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true + @test isfishercons(loss) == true + @test isunivfishercons(loss) == true end @testset "SmoothedL1HingeLoss" begin - loss = SmoothedL1HingeLoss(2) - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, -1) == false - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == false - @test istwicedifferentiable(loss, 2) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true + loss = SmoothedL1HingeLoss(2) + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, -1) == false + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == false + @test istwicedifferentiable(loss, 2) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true end @testset "ModifiedHuberLoss" begin - loss = ModifiedHuberLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == false - @test istwicedifferentiable(loss, -1) == false - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == false - @test istwicedifferentiable(loss, 2) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true + loss = ModifiedHuberLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == false + @test istwicedifferentiable(loss, -1) == false + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == false + @test istwicedifferentiable(loss, 2) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true end @testset "L2MarginLoss" begin - loss = L2MarginLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == true - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == false - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == true - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true - @test isfishercons(loss) == true - @test isunivfishercons(loss) == true + loss = L2MarginLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == true + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == false + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == true + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true + @test isfishercons(loss) == true + @test isunivfishercons(loss) == true end @testset "ExpLoss" begin - loss = ExpLoss() - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == true - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == false - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == false - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true - @test isfishercons(loss) == true - @test isunivfishercons(loss) == true + loss = ExpLoss() + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == true + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == false + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == false + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true + @test isfishercons(loss) == true + @test isunivfishercons(loss) == true end @testset "SigmoidLoss" begin - loss = SigmoidLoss() - - @test isminimizable(loss) == false - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == false - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == false - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true - @test isfishercons(loss) == true - @test isunivfishercons(loss) == true + loss = SigmoidLoss() + + @test isminimizable(loss) == false + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == false + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == false + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true + @test isfishercons(loss) == true + @test isunivfishercons(loss) == true end @testset "DWDMarginLoss" begin - loss = DWDMarginLoss(2) - - @test isminimizable(loss) == true - - @test isdifferentiable(loss) == true - @test isdifferentiable(loss, 0) == true - @test isdifferentiable(loss, 1) == true - @test istwicedifferentiable(loss) == true - @test istwicedifferentiable(loss, 0) == true - @test istwicedifferentiable(loss, 1) == true - - @test isstronglyconvex(loss) == false - @test isstrictlyconvex(loss) == false - @test isconvex(loss) == true - - @test isnemitski(loss) == true - @test islipschitzcont(loss) == true - @test islocallylipschitzcont(loss) == true - @test isclipable(loss) == false - @test ismarginbased(loss) == true - @test isdistancebased(loss) == false - @test issymmetric(loss) == false - @test isclasscalibrated(loss) == true - @test isfishercons(loss) == true - @test isunivfishercons(loss) == true + loss = DWDMarginLoss(2) + + @test isminimizable(loss) == true + + @test isdifferentiable(loss) == true + @test isdifferentiable(loss, 0) == true + @test isdifferentiable(loss, 1) == true + @test istwicedifferentiable(loss) == true + @test istwicedifferentiable(loss, 0) == true + @test istwicedifferentiable(loss, 1) == true + + @test isstronglyconvex(loss) == false + @test isstrictlyconvex(loss) == false + @test isconvex(loss) == true + + @test isnemitski(loss) == true + @test islipschitzcont(loss) == true + @test islocallylipschitzcont(loss) == true + @test isclipable(loss) == false + @test ismarginbased(loss) == true + @test isdistancebased(loss) == false + @test issymmetric(loss) == false + @test isclasscalibrated(loss) == true + @test isfishercons(loss) == true + @test isunivfishercons(loss) == true end # -------------------------------------------------------------- -function compare_losses(l1, l2, ccal = true) - @test isminimizable(l1) == isminimizable(l2) - - @test isdifferentiable(l1) == isdifferentiable(l2) - @test isdifferentiable(l1, 0) == isdifferentiable(l2, 0) - @test isdifferentiable(l1, 1) == isdifferentiable(l2, 1) - @test istwicedifferentiable(l1) == istwicedifferentiable(l2) - @test istwicedifferentiable(l1, 0) == istwicedifferentiable(l2, 0) - @test istwicedifferentiable(l1, 1) == istwicedifferentiable(l2, 1) - @test istwicedifferentiable(l1, 2) == istwicedifferentiable(l2, 2) - - @test isstronglyconvex(l1) == isstronglyconvex(l2) - @test isstrictlyconvex(l1) == isstrictlyconvex(l2) - @test isconvex(l1) == isconvex(l2) - - @test isnemitski(l1) == isnemitski(l2) - @test islipschitzcont(l1) == islipschitzcont(l2) - @test islocallylipschitzcont(l1) == islocallylipschitzcont(l2) - @test isclipable(l1) == isclipable(l2) - @test ismarginbased(l1) == ismarginbased(l2) - @test isdistancebased(l1) == isdistancebased(l2) - @test issymmetric(l1) == issymmetric(l2) - @test (ccal && isclasscalibrated(l1)) == isclasscalibrated(l2) +function compare_losses(l1, l2, ccal=true) + @test isminimizable(l1) == isminimizable(l2) + + @test isdifferentiable(l1) == isdifferentiable(l2) + @test isdifferentiable(l1, 0) == isdifferentiable(l2, 0) + @test isdifferentiable(l1, 1) == isdifferentiable(l2, 1) + @test istwicedifferentiable(l1) == istwicedifferentiable(l2) + @test istwicedifferentiable(l1, 0) == istwicedifferentiable(l2, 0) + @test istwicedifferentiable(l1, 1) == istwicedifferentiable(l2, 1) + @test istwicedifferentiable(l1, 2) == istwicedifferentiable(l2, 2) + + @test isstronglyconvex(l1) == isstronglyconvex(l2) + @test isstrictlyconvex(l1) == isstrictlyconvex(l2) + @test isconvex(l1) == isconvex(l2) + + @test isnemitski(l1) == isnemitski(l2) + @test islipschitzcont(l1) == islipschitzcont(l2) + @test islocallylipschitzcont(l1) == islocallylipschitzcont(l2) + @test isclipable(l1) == isclipable(l2) + @test ismarginbased(l1) == ismarginbased(l2) + @test isdistancebased(l1) == isdistancebased(l2) + @test issymmetric(l1) == issymmetric(l2) + @test (ccal && isclasscalibrated(l1)) == isclasscalibrated(l2) end -compare_losses(PoissonLoss(), 2*PoissonLoss()) -compare_losses(PoissonLoss(), 0.5*PoissonLoss()) +compare_losses(PoissonLoss(), 2 * PoissonLoss()) +compare_losses(PoissonLoss(), 0.5 * PoissonLoss()) @testset "Scaled losses" begin - for loss in distance_losses ∪ margin_losses ∪ other_losses - @testset "$loss" begin - compare_losses(loss, 2*loss) - compare_losses(loss, 0.5*loss) - end + for loss in distance_losses ∪ margin_losses ∪ other_losses + @testset "$loss" begin + compare_losses(loss, 2 * loss) + compare_losses(loss, 0.5 * loss) end + end end @testset "Weighted Margin-based" begin - for loss in margin_losses - @testset "$loss" begin - compare_losses(loss, WeightedMarginLoss(loss,0.2), false) - compare_losses(loss, WeightedMarginLoss(loss,0.5), true) - compare_losses(loss, WeightedMarginLoss(loss,0.7), false) - end + for loss in margin_losses + @testset "$loss" begin + compare_losses(loss, WeightedMarginLoss(loss, 0.2), false) + compare_losses(loss, WeightedMarginLoss(loss, 0.5), true) + compare_losses(loss, WeightedMarginLoss(loss, 0.7), false) end + end end diff --git a/test/runtests.jl b/test/runtests.jl index 3e07aa6..8dd5b82 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,42 +7,58 @@ using Statistics using Random using Test -tests = [ - "core.jl", - "props.jl", - "agg.jl" -] +tests = ["core.jl", "props.jl", "agg.jl"] # for deterministic testing Random.seed!(1234) distance_losses = [ - L2DistLoss(), LPDistLoss(2.0), L1DistLoss(), LPDistLoss(1.0), - LPDistLoss(0.5), LPDistLoss(1.5), LPDistLoss(3), - LogitDistLoss(), L1EpsilonInsLoss(0.5), EpsilonInsLoss(1.5), - L2EpsilonInsLoss(0.5), L2EpsilonInsLoss(1.5), - PeriodicLoss(1), PeriodicLoss(1.5), HuberLoss(1), - HuberLoss(1.5), QuantileLoss(.2), QuantileLoss(.5), - QuantileLoss(.8), LogCoshLoss() + L2DistLoss(), + LPDistLoss(2.0), + L1DistLoss(), + LPDistLoss(1.0), + LPDistLoss(0.5), + LPDistLoss(1.5), + LPDistLoss(3), + LogitDistLoss(), + L1EpsilonInsLoss(0.5), + EpsilonInsLoss(1.5), + L2EpsilonInsLoss(0.5), + L2EpsilonInsLoss(1.5), + PeriodicLoss(1), + PeriodicLoss(1.5), + HuberLoss(1), + HuberLoss(1.5), + QuantileLoss(0.2), + QuantileLoss(0.5), + QuantileLoss(0.8), + LogCoshLoss() ] margin_losses = [ - LogitMarginLoss(), L1HingeLoss(), L2HingeLoss(), - PerceptronLoss(), SmoothedL1HingeLoss(.5), - SmoothedL1HingeLoss(1), SmoothedL1HingeLoss(2), - ModifiedHuberLoss(), ZeroOneLoss(), L2MarginLoss(), - ExpLoss(), SigmoidLoss(), DWDMarginLoss(.5), - DWDMarginLoss(1), DWDMarginLoss(2) + LogitMarginLoss(), + L1HingeLoss(), + L2HingeLoss(), + PerceptronLoss(), + SmoothedL1HingeLoss(0.5), + SmoothedL1HingeLoss(1), + SmoothedL1HingeLoss(2), + ModifiedHuberLoss(), + ZeroOneLoss(), + L2MarginLoss(), + ExpLoss(), + SigmoidLoss(), + DWDMarginLoss(0.5), + DWDMarginLoss(1), + DWDMarginLoss(2) ] -other_losses = [ - MisclassLoss(), PoissonLoss(), CrossEntropyLoss() -] +other_losses = [MisclassLoss(), PoissonLoss(), CrossEntropyLoss()] for t in tests - @testset "$t" begin - include(t) - end + @testset "$t" begin + include(t) + end end end # module From 1fff2f962ff769282107f66bbcb15207ff74e644 Mon Sep 17 00:00:00 2001 From: Miles Cranmer Date: Fri, 21 Jul 2023 18:05:53 -0400 Subject: [PATCH 4/4] Move CategoricalArrays.jl into extension (#168) * Move CategoricalArrays.jl into extension * Add [extras] section for backwards compatibility --- Project.toml | 14 ++++++++++++-- ext/LossFunctionsCategoricalArraysExt.jl | 18 ++++++++++++++++++ src/LossFunctions.jl | 7 ++++++- src/losses.jl | 3 --- src/losses/other.jl | 6 +++--- 5 files changed, 39 insertions(+), 9 deletions(-) create mode 100644 ext/LossFunctionsCategoricalArraysExt.jl diff --git a/Project.toml b/Project.toml index d2ee3bb..b7a29e3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,12 +1,22 @@ name = "LossFunctions" uuid = "30fc2ffe-d236-52d8-8643-a9d8f7c094a7" -version = "0.10.0" +version = "0.10.1" [deps] -CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +Requires = "ae029012-a4dd-5104-9daa-d747884805df" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +[weakdeps] +CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" + +[extensions] +LossFunctionsCategoricalArraysExt = "CategoricalArrays" + [compat] CategoricalArrays = "0.10" +Requires = "1" julia = "1.6" + +[extras] +CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/ext/LossFunctionsCategoricalArraysExt.jl b/ext/LossFunctionsCategoricalArraysExt.jl new file mode 100644 index 0000000..a2993c8 --- /dev/null +++ b/ext/LossFunctionsCategoricalArraysExt.jl @@ -0,0 +1,18 @@ +module LossFunctionsCategoricalArraysExt + +if isdefined(Base, :get_extension) + import LossFunctions: MisclassLoss, deriv, deriv2 + import CategoricalArrays: CategoricalValue +else + import ..LossFunctions: MisclassLoss, deriv, deriv2 + import ..CategoricalArrays: CategoricalValue +end + +# type alias to make code more readable +const Scalar = Union{Number,CategoricalValue} + +(loss::MisclassLoss)(output::Scalar, target::Scalar) = loss(target == output) +deriv(loss::MisclassLoss, output::Scalar, target::Scalar) = deriv(loss, target == output) +deriv2(loss::MisclassLoss, output::Scalar, target::Scalar) = deriv2(loss, target == output) + +end diff --git a/src/LossFunctions.jl b/src/LossFunctions.jl index 98dffb4..d832628 100644 --- a/src/LossFunctions.jl +++ b/src/LossFunctions.jl @@ -1,10 +1,10 @@ module LossFunctions using Markdown -using CategoricalArrays: CategoricalValue import Base: sum import Statistics: mean +import Requires: @init, @require # trait functions include("traits.jl") @@ -15,6 +15,11 @@ include("losses.jl") # IO methods include("io.jl") +# Extensions +if !isdefined(Base, :get_extension) + @init @require CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" include("../ext/LossFunctionsCategoricalArraysExt.jl") +end + export # trait functions Loss, diff --git a/src/losses.jl b/src/losses.jl index 7e43ce2..79eac96 100644 --- a/src/losses.jl +++ b/src/losses.jl @@ -1,6 +1,3 @@ -# type alias to make code more readable -Scalar = Union{Number,CategoricalValue} - # fallback to unary evaluation (loss::DistanceLoss)(output::Number, target::Number) = loss(output - target) deriv(loss::DistanceLoss, output::Number, target::Number) = deriv(loss, output - target) diff --git a/src/losses/other.jl b/src/losses/other.jl index 63b80f1..ca93a0b 100644 --- a/src/losses/other.jl +++ b/src/losses/other.jl @@ -17,9 +17,9 @@ MisclassLoss() = MisclassLoss{Float64}() deriv(::MisclassLoss{R}, agreement::Bool) where {R} = zero(R) deriv2(::MisclassLoss{R}, agreement::Bool) where {R} = zero(R) -(loss::MisclassLoss)(output::Scalar, target::Scalar) = loss(target == output) -deriv(loss::MisclassLoss, output::Scalar, target::Scalar) = deriv(loss, target == output) -deriv2(loss::MisclassLoss, output::Scalar, target::Scalar) = deriv2(loss, target == output) +(loss::MisclassLoss)(output::Number, target::Number) = loss(target == output) +deriv(loss::MisclassLoss, output::Number, target::Number) = deriv(loss, target == output) +deriv2(loss::MisclassLoss, output::Number, target::Number) = deriv2(loss, target == output) isminimizable(::MisclassLoss) = false isdifferentiable(::MisclassLoss) = false