diff --git a/Project.toml b/Project.toml index c17d178..362f016 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ authors = ["jeremiedb "] name = "NeuroTreeModels" uuid = "1db4e0a5-a364-4b0c-897c-2bd5a4a3a1f2" -version = "1.5.0" +version = "1.5.1" [deps] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" diff --git a/benchmarks/Higgs-logloss.jl b/benchmarks/Higgs-logloss.jl index e1a1b15..c20a928 100644 --- a/benchmarks/Higgs-logloss.jl +++ b/benchmarks/Higgs-logloss.jl @@ -10,11 +10,11 @@ using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") +aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") bucket = "jeremiedb" path = "share/data/higgs/HIGGS.arrow" -df_tot = Connectors.read_arrow_aws(path; bucket="jeremiedb", aws_config) +df_tot = Connectors.read_arrow_aws(path; bucket = "jeremiedb", aws_config) rename!(df_tot, "Column1" => "y") feature_names = setdiff(names(df_tot), ["y"]) @@ -26,21 +26,21 @@ target_name = "y" # transform!(df_tot, feature_names .=> percent_rank .=> feature_names) -dtrain = df_tot[1:end-1_000_000, :]; -deval = df_tot[end-1_000_000+1:end-500_000, :]; -dtest = df_tot[end-500_000+1:end, :]; +dtrain = df_tot[1:(end-1_000_000), :]; +deval = df_tot[(end-1_000_000+1):(end-500_000), :]; +dtest = df_tot[(end-500_000+1):end, :]; config = NeuroTreeRegressor( - loss=:logloss, - nrounds=200, - depth=4, - lr=2e-3, - ntrees=128, - stack_size=2, - hidden_size=16, - batchsize=8092, - early_stopping_rounds=2, - device=:gpu, + loss = :logloss, + nrounds = 200, + depth = 4, + lr = 2e-3, + ntrees = 128, + stack_size = 2, + hidden_size = 16, + batchsize = 8092, + early_stopping_rounds = 2, + device = :gpu, ) @time m = NeuroTreeModels.fit( @@ -49,15 +49,25 @@ config = NeuroTreeRegressor( deval, target_name, feature_names, - print_every_n=1 + print_every_n = 1, ); -dinfer_eval = NeuroTreeModels.get_df_loader_infer(deval; feature_names, batchsize=config.batchsize, device=config.device); +dinfer_eval = NeuroTreeModels.get_df_loader_infer( + deval; + feature_names, + batchsize = config.batchsize, + device = config.device, +); p_eval = m(dinfer_eval); error_eval = 1 - mean(round.(Int, p_eval) .== deval.y) @info "ERROR - deval" error_eval -dinfer_test = NeuroTreeModels.get_df_loader_infer(dtest; feature_names, batchsize=config.batchsize, device=config.device); +dinfer_test = NeuroTreeModels.get_df_loader_infer( + dtest; + feature_names, + batchsize = config.batchsize, + device = config.device, +); p_test = m(dinfer_test); error_test = 1 - mean(round.(Int, p_test) .== dtest.y) @info "ERROR - dtest" error_test @@ -82,4 +92,3 @@ error_test = 1 - mean(round.(Int, p_test) .== dtest.y) # 34568.885039 seconds (7.51 G allocations: 1.109 TiB, 1.01% gc time) # ┌ Info: ERROR - dtest # └ error_test = 0.22153599999999996 - diff --git a/benchmarks/MSRank.jl b/benchmarks/MSRank.jl index 242490e..74a39a7 100644 --- a/benchmarks/MSRank.jl +++ b/benchmarks/MSRank.jl @@ -13,18 +13,24 @@ using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") +aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") bucket = "jeremiedb" # initial prep -function read_libsvm_aws(file::String; has_query=false, aws_config=AWSConfig()) - raw = S3.get_object("jeremiedb", file, Dict("response-content-type" => "application/octet-stream"); aws_config) +function read_libsvm_aws(file::String; has_query = false, aws_config = AWSConfig()) + raw = S3.get_object( + "jeremiedb", + file, + Dict("response-content-type" => "application/octet-stream"); + aws_config, + ) return read_libsvm(raw; has_query) end -@time train_raw = read_libsvm_aws("share/data/msrank/train.txt"; has_query=true, aws_config); -@time eval_raw = read_libsvm_aws("share/data/msrank/vali.txt"; has_query=true, aws_config); -@time test_raw = read_libsvm_aws("share/data/msrank/test.txt"; has_query=true, aws_config); +@time train_raw = + read_libsvm_aws("share/data/msrank/train.txt"; has_query = true, aws_config); +@time eval_raw = read_libsvm_aws("share/data/msrank/vali.txt"; has_query = true, aws_config); +@time test_raw = read_libsvm_aws("share/data/msrank/test.txt"; has_query = true, aws_config); dtrain = DataFrame(train_raw[:x], :auto) dtrain.y_raw = train_raw[:y] @@ -53,17 +59,17 @@ transform!(deval, feature_names .=> percent_rank .=> feature_names) transform!(dtest, feature_names .=> percent_rank .=> feature_names) config = NeuroTreeRegressor( - loss=:mse, - nrounds=2, - actA=:tanh, - depth=4, - ntrees=64, - stack_size=2, - hidden_size=16, - batchsize=4096, - lr=3e-4, - early_stopping_rounds=3, - device=:gpu, + loss = :mse, + nrounds = 2, + actA = :tanh, + depth = 4, + ntrees = 64, + stack_size = 2, + hidden_size = 16, + batchsize = 4096, + lr = 3e-4, + early_stopping_rounds = 3, + device = :gpu, ) @time m = NeuroTreeModels.fit( @@ -72,7 +78,7 @@ config = NeuroTreeRegressor( deval, target_name, feature_names, - print_every_n=1, + print_every_n = 1, ); p_eval = m(deval); diff --git a/benchmarks/YEAR-gaussian.jl b/benchmarks/YEAR-gaussian.jl index 08224e3..736776a 100644 --- a/benchmarks/YEAR-gaussian.jl +++ b/benchmarks/YEAR-gaussian.jl @@ -14,20 +14,35 @@ using NeuroTreeModels using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") +aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") path = "share/data/year/year.csv" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -df = DataFrame(CSV.File(raw, header=false)) +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +df = DataFrame(CSV.File(raw, header = false)) df_tot = copy(df) path = "share/data/year/year-train-idx.txt" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -train_idx = DataFrame(CSV.File(raw, header=false))[:, 1] .+ 1 +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +train_idx = DataFrame(CSV.File(raw, header = false))[:, 1] .+ 1 path = "share/data/year/year-eval-idx.txt" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -eval_idx = DataFrame(CSV.File(raw, header=false))[:, 1] .+ 1 +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +eval_idx = DataFrame(CSV.File(raw, header = false))[:, 1] .+ 1 transform!(df_tot, "Column1" => identity => "y_raw") transform!(df_tot, "y_raw" => (x -> (x .- mean(x)) ./ std(x)) => "y_norm") @@ -43,19 +58,19 @@ dtest = df_tot[(end-51630+1):end, :]; device = :gpu config = NeuroTreeRegressor(; - loss=:gaussian_mle, - actA=:identity, - init_scale=0.0, - nrounds=200, - depth=4, - ntrees=32, - hidden_size=8, - stack_size=1, - MLE_tree_split=true, - batchsize=2048, - lr=1e-3, - early_stopping_rounds=2, - device + loss = :gaussian_mle, + actA = :identity, + init_scale = 0.0, + nrounds = 200, + depth = 4, + ntrees = 32, + hidden_size = 8, + stack_size = 1, + MLE_tree_split = true, + batchsize = 2048, + lr = 1e-3, + early_stopping_rounds = 2, + device, ) @time m = NeuroTreeModels.fit( @@ -64,7 +79,7 @@ config = NeuroTreeRegressor(; deval, target_name, feature_names, - print_every_n=5 + print_every_n = 5, ); @time p_eval = m(deval; device); diff --git a/benchmarks/YEAR-mse.jl b/benchmarks/YEAR-mse.jl index 087f35b..7e9ead1 100644 --- a/benchmarks/YEAR-mse.jl +++ b/benchmarks/YEAR-mse.jl @@ -9,20 +9,35 @@ using NeuroTreeModels using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") +aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") path = "share/data/year/year.csv" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -df = DataFrame(CSV.File(raw, header=false)) +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +df = DataFrame(CSV.File(raw, header = false)) df_tot = copy(df) path = "share/data/year/year-train-idx.txt" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -train_idx = DataFrame(CSV.File(raw, header=false))[:, 1] .+ 1 +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +train_idx = DataFrame(CSV.File(raw, header = false))[:, 1] .+ 1 path = "share/data/year/year-eval-idx.txt" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -eval_idx = DataFrame(CSV.File(raw, header=false))[:, 1] .+ 1 +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +eval_idx = DataFrame(CSV.File(raw, header = false))[:, 1] .+ 1 transform!(df_tot, "Column1" => identity => "y_raw") transform!(df_tot, "y_raw" => (x -> (x .- mean(x)) ./ std(x)) => "y_norm") @@ -38,19 +53,19 @@ dtest = df_tot[(end-51630+1):end, :]; device = :gpu config = NeuroTreeRegressor(; - loss=:mse, - actA=:identity, - scaler=true, - init_scale=0.1, - nrounds=200, - depth=4, - ntrees=32, - stack_size=1, - hidden_size=8, - batchsize=2048, - lr=1e-3, - early_stopping_rounds=2, - device + loss = :mse, + actA = :identity, + scaler = true, + init_scale = 0.1, + nrounds = 200, + depth = 4, + ntrees = 32, + stack_size = 1, + hidden_size = 8, + batchsize = 2048, + lr = 1e-3, + early_stopping_rounds = 2, + device, ) @time m = NeuroTreeModels.fit( @@ -59,7 +74,7 @@ config = NeuroTreeRegressor(; deval, target_name, feature_names, - print_every_n=5 + print_every_n = 5, ); @time p_eval = m(deval; device); diff --git a/benchmarks/YEAR-tweedie.jl b/benchmarks/YEAR-tweedie.jl index b9be569..23055e3 100644 --- a/benchmarks/YEAR-tweedie.jl +++ b/benchmarks/YEAR-tweedie.jl @@ -14,20 +14,35 @@ using NeuroTreeModels using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") +aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") path = "share/data/year/year.csv" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -df = DataFrame(CSV.File(raw, header=false)) +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +df = DataFrame(CSV.File(raw, header = false)) df_tot = copy(df) path = "share/data/year/year-train-idx.txt" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -train_idx = DataFrame(CSV.File(raw, header=false))[:, 1] .+ 1 +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +train_idx = DataFrame(CSV.File(raw, header = false))[:, 1] .+ 1 path = "share/data/year/year-eval-idx.txt" -raw = S3.get_object("jeremiedb", path, Dict("response-content-type" => "application/octet-stream"); aws_config) -eval_idx = DataFrame(CSV.File(raw, header=false))[:, 1] .+ 1 +raw = S3.get_object( + "jeremiedb", + path, + Dict("response-content-type" => "application/octet-stream"); + aws_config, +) +eval_idx = DataFrame(CSV.File(raw, header = false))[:, 1] .+ 1 transform!(df_tot, "Column1" => identity => "y_raw") transform!(df_tot, "y_raw" => (x -> (x .- minimum(x)) ./ std(x)) => "y_norm") @@ -40,18 +55,18 @@ deval = df_tot[eval_idx, :]; dtest = df_tot[(end-51630+1):end, :]; config = NeuroTreeRegressor( - loss=:tweedie, - actA=:identity, - init_scale=0.1, - nrounds=200, - depth=5, - ntrees=32, - stack_size=1, - hidden_size=8, - batchsize=2048, - lr=1e-3, - early_stopping_rounds=2, - device=:gpu + loss = :tweedie, + actA = :identity, + init_scale = 0.1, + nrounds = 200, + depth = 5, + ntrees = 32, + stack_size = 1, + hidden_size = 8, + batchsize = 2048, + lr = 1e-3, + early_stopping_rounds = 2, + device = :gpu, ) @time m = NeuroTreeModels.fit( @@ -60,7 +75,7 @@ config = NeuroTreeRegressor( deval, target_name, feature_names, - print_every_n=5, + print_every_n = 5, ); p_eval = m(deval); diff --git a/benchmarks/Yahoo-LTRC.jl b/benchmarks/Yahoo-LTRC.jl index ba206dc..7c33c12 100644 --- a/benchmarks/Yahoo-LTRC.jl +++ b/benchmarks/Yahoo-LTRC.jl @@ -15,23 +15,28 @@ using NeuroTreeModels using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") - -function read_libsvm_aws(file::String; has_query=false, aws_config=AWSConfig()) - raw = S3.get_object("jeremiedb", file, Dict("response-content-type" => "application/octet-stream"); aws_config) +aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") + +function read_libsvm_aws(file::String; has_query = false, aws_config = AWSConfig()) + raw = S3.get_object( + "jeremiedb", + file, + Dict("response-content-type" => "application/octet-stream"); + aws_config, + ) return read_libsvm(raw; has_query) end -function ndcg(p, y, k=10) +function ndcg(p, y, k = 10) k = min(k, length(p)) - p_order = partialsortperm(p, 1:k, rev=true) - y_order = partialsortperm(y, 1:k, rev=true) + p_order = partialsortperm(p, 1:k, rev = true) + y_order = partialsortperm(y, 1:k, rev = true) _y = y[p_order] gains = 2 .^ _y .- 1 discounts = log2.((1:k) .+ 1) ndcg = sum(gains ./ discounts) - y_order = partialsortperm(y, 1:k, rev=true) + y_order = partialsortperm(y, 1:k, rev = true) _y = y[y_order] gains = 2 .^ _y .- 1 discounts = log2.((1:k) .+ 1) @@ -40,9 +45,12 @@ function ndcg(p, y, k=10) return idcg == 0 ? 1.0 : ndcg / idcg end -@time train_raw = read_libsvm_aws("share/data/yahoo-ltrc/set1.train.txt"; has_query=true, aws_config); -@time eval_raw = read_libsvm_aws("share/data/yahoo-ltrc/set1.valid.txt"; has_query=true, aws_config); -@time test_raw = read_libsvm_aws("share/data/yahoo-ltrc/set1.test.txt"; has_query=true, aws_config); +@time train_raw = + read_libsvm_aws("share/data/yahoo-ltrc/set1.train.txt"; has_query = true, aws_config); +@time eval_raw = + read_libsvm_aws("share/data/yahoo-ltrc/set1.valid.txt"; has_query = true, aws_config); +@time test_raw = + read_libsvm_aws("share/data/yahoo-ltrc/set1.test.txt"; has_query = true, aws_config); colsums_train = map(sum, eachcol(train_raw[:x])) # colsums_eval = map(sum, eachcol(eval_raw[:x])) @@ -61,9 +69,9 @@ x_train_miss = x_train .== 0 x_eval_miss = x_eval .== 0 x_test_miss = x_test .== 0 -x_train[x_train.==0] .= 0.5 -x_eval[x_eval.==0] .= 0.5 -x_test[x_test.==0] .= 0.5 +x_train[x_train .== 0] .= 0.5 +x_eval[x_eval .== 0] .= 0.5 +x_test[x_test .== 0] .= 0.5 x_train = hcat(x_train, x_train_miss) x_eval = hcat(x_eval, x_eval_miss) @@ -94,20 +102,20 @@ target_name = "y" # training ##################################### config = NeuroTreeRegressor( - loss=:logloss, - nrounds=400, - actA=:identity, - init_scale=1.0, - scaler=true, - depth=4, - ntrees=256, - hidden_size=1, - stack_size=1, - batchsize=1024, - shuffle=true, - lr=3e-4, - metric=:logloss, - device=:gpu + loss = :logloss, + nrounds = 400, + actA = :identity, + init_scale = 1.0, + scaler = true, + depth = 4, + ntrees = 256, + hidden_size = 1, + stack_size = 1, + batchsize = 1024, + shuffle = true, + lr = 3e-4, + metric = :logloss, + device = :gpu, ) @time m = NeuroTreeModels.fit( @@ -116,13 +124,18 @@ config = NeuroTreeRegressor( deval, target_name, feature_names, - print_every_n=5 + print_every_n = 5, ); -dinfer = NeuroTreeModels.get_df_loader_infer(dtest; feature_names, batchsize=config.batchsize, device=config.device); +dinfer = NeuroTreeModels.get_df_loader_infer( + dtest; + feature_names, + batchsize = config.batchsize, + device = config.device, +); p_test = m(dinfer) .* 4; -test_df = DataFrame(p=p_test, y=dtest.y_raw, q=dtest.q) +test_df = DataFrame(p = p_test, y = dtest.y_raw, q = dtest.q) test_df_agg = combine(groupby(test_df, "q"), ["p", "y"] => ndcg => "ndcg") ndcg_test = mean(test_df_agg.ndcg) mse_test = mean((p_test .- dtest.y_raw) .^ 2) diff --git a/benchmarks/aicrowd-test.jl b/benchmarks/aicrowd-test.jl index 38f7cc7..3f29c25 100644 --- a/benchmarks/aicrowd-test.jl +++ b/benchmarks/aicrowd-test.jl @@ -9,7 +9,7 @@ using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") +aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") path = "share/data/aicrowd/insurance-aicrowd.csv" raw = S3.get_object( @@ -42,7 +42,7 @@ setdiff(feature_names, names(df)) seed!(123) nobs = nrow(df) -id_train = sample(1:nobs, Int(round(0.8 * nobs)), replace=false) +id_train = sample(1:nobs, Int(round(0.8 * nobs)), replace = false) dtrain = dropmissing(df[id_train, [feature_names..., target_name]]) deval = dropmissing(df[Not(id_train), [feature_names..., target_name]]) @@ -51,16 +51,16 @@ deval = dropmissing(df[Not(id_train), [feature_names..., target_name]]) # NeuroTrees ############################## config = NeuroTreeRegressor( - loss=:logloss, - nrounds=400, - actA=:tanh, - depth=4, - ntrees=32, - batchsize=4096, - rng=123, - lr=3e-3, - early_stopping_rounds=5, - device=:gpu + loss = :logloss, + nrounds = 400, + actA = :tanh, + depth = 4, + ntrees = 32, + batchsize = 4096, + rng = 123, + lr = 3e-3, + early_stopping_rounds = 5, + device = :gpu, ) @time m = NeuroTreeModels.fit( @@ -69,27 +69,29 @@ config = NeuroTreeRegressor( deval, feature_names, target_name, - print_every_n=5, + print_every_n = 5, ); pred_eval_neuro = m(deval) ############################## # EvoTrees ############################## -config = EvoTreeRegressor(T=Float32, - loss=:logistic, - nrounds=1000, - eta=0.02, - L2=1, - lambda=0.02, - nbins=32, - max_depth=5, - rowsample=0.5, - colsample=0.8, - early_stopping_rounds=50 +config = EvoTreeRegressor( + T = Float32, + loss = :logistic, + nrounds = 1000, + eta = 0.02, + L2 = 1, + lambda = 0.02, + nbins = 32, + max_depth = 5, + rowsample = 0.5, + colsample = 0.8, + early_stopping_rounds = 50, ) -@time m = EvoTrees.fit(config, dtrain; deval, feature_names, target_name, print_every_n=25); +@time m = + EvoTrees.fit(config, dtrain; deval, feature_names, target_name, print_every_n = 25); pred_eval_evo = m(deval) function logloss(p::Vector{T}, y::Vector{T}) where {T<:AbstractFloat} diff --git a/benchmarks/benchmark_gauss.jl b/benchmarks/benchmark_gauss.jl index 1021dc9..5cbf433 100644 --- a/benchmarks/benchmark_gauss.jl +++ b/benchmarks/benchmark_gauss.jl @@ -35,13 +35,10 @@ config = Dict( :batch_size => 2048, :shuffle => true, :early_stopping_rounds => 0, - :opt => Dict( - :type => "nadam", - :lr => 1e-3, - :rho => 0.9) + :opt => Dict(:type => "nadam", :lr => 1e-3, :rho => 0.9), ) -@time NeuroTreeModels.fit!(loss, θ, dtrain, opt, nrounds=1, cb=cb); +@time NeuroTreeModels.fit!(loss, θ, dtrain, opt, nrounds = 1, cb = cb); ####### # mse ####### @@ -98,8 +95,8 @@ function gauss1(μ, σ, y) return gauss / length(y) end -@btime NeuroTreeModels.gauss_𝑙(preds[1,:], preds[1,:], y) -@btime gauss1(preds[1,:], preds[1,:], y) +@btime NeuroTreeModels.gauss_𝑙(preds[1, :], preds[1, :], y) +@btime gauss1(preds[1, :], preds[1, :], y) # gauss_𝑙(μ, σ, y) = mean(-σ .- (y .- μ) .^ 2 ./ (2 .* max.(Float32(2e-7), exp.(2 .* σ)))) diff --git a/benchmarks/benchmark_mse.jl b/benchmarks/benchmark_mse.jl index 15e0644..cce8719 100644 --- a/benchmarks/benchmark_mse.jl +++ b/benchmarks/benchmark_mse.jl @@ -19,28 +19,22 @@ X = rand(Float32, nobs, num_feat) Y = randn(Float32, size(X, 1)) config = NeuroTreeRegressor( - loss=:mse, - nrounds=1, - actA=:identity, - depth=5, - ntrees=64, - batchsize=2048, - rng=123, - lr=1e-3, - device=:gpu, + loss = :mse, + nrounds = 1, + actA = :identity, + depth = 5, + ntrees = 64, + batchsize = 2048, + rng = 123, + lr = 1e-3, + device = :gpu, ) -CUDA.@time m, cache = NeuroTreeModels.init(config, x_train=X, y_train=Y); +CUDA.@time m, cache = NeuroTreeModels.init(config, x_train = X, y_train = Y); CUDA.@time NeuroTreeModels.fit!(m, cache); -CUDA.@time NeuroTreeModels.fit(config, x_train=X, y_train=Y); -@time NeuroTrees.fit(config, x_train=X, y_train=Y); -CUDA.@time NeuroTreeModels.fit( - config, - x_train=X, - y_train=Y, - x_eval=X, - y_eval=Y, -); +CUDA.@time NeuroTreeModels.fit(config, x_train = X, y_train = Y); +@time NeuroTrees.fit(config, x_train = X, y_train = Y); +CUDA.@time NeuroTreeModels.fit(config, x_train = X, y_train = Y, x_eval = X, y_eval = Y); ####### # mse @@ -53,9 +47,9 @@ CUDA.@time NeuroTreeModels.fit( _device = config.device == "cpu" ? NeuroTrees.cpu : NeuroTrees.gpu dinfer = NeuroTreeModels.DataLoader( Matrix{Float32}(X') |> _device, - batchsize=config.batchsize, - shuffle=false, - partial=true, + batchsize = config.batchsize, + shuffle = false, + partial = true, ) @time pred = NeuroTreeModels.infer(m, dinfer); @btime pred = NeuroTreeModels.infer($m, $dinfer); diff --git a/benchmarks/titanic-logloss.jl b/benchmarks/titanic-logloss.jl index 14e06dc..7ee67df 100644 --- a/benchmarks/titanic-logloss.jl +++ b/benchmarks/titanic-logloss.jl @@ -24,7 +24,7 @@ transform!(df, :Age => (x -> coalesce.(x, median(skipmissing(x)))) => :Age); df = df[:, Not([:PassengerId, :Name, :Embarked, :Cabin, :Ticket])] train_ratio = 0.8 -train_indices = randperm(nrow(df))[1:Int(round(train_ratio * nrow(df)))] +train_indices = randperm(nrow(df))[1:Int(round(train_ratio*nrow(df)))] dtrain = df[train_indices, :] deval = df[setdiff(1:nrow(df), train_indices), :] @@ -33,13 +33,13 @@ target_name = "Survived" feature_names = setdiff(names(df), ["Survived"]) config = NeuroTreeRegressor(; - loss=:logloss, - actA=:identity, - nrounds=400, - depth=4, - lr=5e-2, - early_stopping_rounds=3, - device=:cpu + loss = :logloss, + actA = :identity, + nrounds = 400, + depth = 4, + lr = 5e-2, + early_stopping_rounds = 3, + device = :cpu, ) m = NeuroTreeModels.fit( @@ -48,11 +48,11 @@ m = NeuroTreeModels.fit( deval, target_name, feature_names, - print_every_n=10 + print_every_n = 10, ) -p_train = m(dtrain; device=:cpu) -p_eval = m(deval; device=:cpu) +p_train = m(dtrain; device = :cpu) +p_eval = m(deval; device = :cpu) @info mean((p_train .> 0.5) .== (dtrain[!, target_name] .> 0.5)) @info mean((p_eval .> 0.5) .== (deval[!, target_name] .> 0.5)) @@ -61,12 +61,12 @@ p_eval = m(deval; device=:cpu) # MLJ ################################### using MLJBase, NeuroTreeModels -m = NeuroTreeRegressor(depth=5, nrounds=40, batchsize=1024, device=:cpu) +m = NeuroTreeRegressor(depth = 5, nrounds = 40, batchsize = 1024, device = :cpu) mach = machine(m, dtrain[:, feature_names], Float32.(dtrain[!, target_name])) |> fit! p = predict(mach, dtrain[:, feature_names]) @info mean((p .> 0.5) .== (dtrain[!, target_name] .> 0.5)) -m = NeuroTreeRegressor(depth=5, nrounds=40, batchsize=1024, device=:gpu) +m = NeuroTreeRegressor(depth = 5, nrounds = 40, batchsize = 1024, device = :gpu) mach = machine(m, dtrain[:, feature_names], Float32.(dtrain[!, target_name])) |> fit! p = predict(mach, dtrain[:, feature_names]) @info mean((p .> 0.5) .== (dtrain[!, target_name] .> 0.5)) diff --git a/benchmarks/titanic-mlogloss.jl b/benchmarks/titanic-mlogloss.jl index fb3a292..acc82f3 100644 --- a/benchmarks/titanic-mlogloss.jl +++ b/benchmarks/titanic-mlogloss.jl @@ -27,7 +27,7 @@ transform!(df, :Age => (x -> coalesce.(x, median(skipmissing(x)))) => :Age); df = df[:, Not([:PassengerId, :Name, :Embarked, :Cabin, :Ticket])] train_ratio = 0.8 -train_indices = randperm(nrow(df))[1:Int(round(train_ratio * nrow(df)))] +train_indices = randperm(nrow(df))[1:Int(round(train_ratio*nrow(df)))] dtrain = df[train_indices, :] deval = df[setdiff(1:nrow(df), train_indices), :] @@ -37,11 +37,11 @@ feature_names = setdiff(names(df), ["y_cat", "Survived"]) eltype(dtrain[:, "y_cat"]) config = NeuroTreeClassifier( - nrounds=400, - depth=4, - lr=3e-2, - early_stopping_rounds=3, - device=:cpu + nrounds = 400, + depth = 4, + lr = 3e-2, + early_stopping_rounds = 3, + device = :cpu, ) m = NeuroTreeModels.fit( @@ -50,7 +50,7 @@ m = NeuroTreeModels.fit( deval, target_name, feature_names, - print_every_n=10, + print_every_n = 10, ) p_train = m(dtrain) diff --git a/docs/make.jl b/docs/make.jl index 55ce05f..775131a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -12,29 +12,29 @@ pages = [ "Regression - Boston" => "tutorials/regression-boston.md", "Logistic - Titanic" => "tutorials/logistic-titanic.md", "Classification - IRIS" => "tutorials/classification-iris.md", - ] + ], ] makedocs(; - sitename="NeuroTreeModels", - authors="Evovest and contributors.", - format=DocumenterVitepress.MarkdownVitepress( - repo="github.com/Evovest/NeuroTreeModels.jl", # this must be the full URL! - devbranch="main", - devurl="dev"; + sitename = "NeuroTreeModels", + authors = "Evovest and contributors.", + format = DocumenterVitepress.MarkdownVitepress( + repo = "github.com/Evovest/NeuroTreeModels.jl", # this must be the full URL! + devbranch = "main", + devurl = "dev"; ), - modules=[NeuroTreeModels], - warnonly=true, - checkdocs=:all, - pages=pages, + modules = [NeuroTreeModels], + warnonly = true, + checkdocs = :all, + pages = pages, ) deploydocs(; - repo="github.com/Evovest/NeuroTreeModels.jl", # this must be the full URL! - target="build", # this is where Vitepress stores its output - branch="gh-pages", - devbranch="main", - push_preview=true + repo = "github.com/Evovest/NeuroTreeModels.jl", # this must be the full URL! + target = "build", # this is where Vitepress stores its output + branch = "gh-pages", + devbranch = "main", + push_preview = true, ) diff --git a/experiments/dataloader.jl b/experiments/dataloader.jl index feedeb8..239350b 100644 --- a/experiments/dataloader.jl +++ b/experiments/dataloader.jl @@ -19,14 +19,15 @@ batchsize = 32 # CPU ################################### device = :cpu -dtrain = NeuroTreeModels.get_df_loader_train(df; feature_names, target_name, batchsize, device) +dtrain = + NeuroTreeModels.get_df_loader_train(df; feature_names, target_name, batchsize, device) for d in dtrain @info length(d) @info size(d[1]) end -deval = NeuroTreeModels.get_df_loader_infer(df; feature_names, batchsize=32) +deval = NeuroTreeModels.get_df_loader_infer(df; feature_names, batchsize = 32) for d in deval @info size(d) end @@ -35,14 +36,15 @@ end # GPU ################################### device = :gpu -dtrain = NeuroTreeModels.get_df_loader_train(df; feature_names, target_name, batchsize, device) +dtrain = + NeuroTreeModels.get_df_loader_train(df; feature_names, target_name, batchsize, device) for d in dtrain @info length(d) @info size(d[1]) end -deval = NeuroTreeModels.get_df_loader_infer(df; feature_names, batchsize=32) +deval = NeuroTreeModels.get_df_loader_infer(df; feature_names, batchsize = 32) for d in deval @info size(d) end @@ -59,7 +61,8 @@ x = rand(nobs, nfeats); df = DataFrame(x, :auto); df.y = categorical(rand(1:2, nobs)); -dtrain = NeuroTreeModels.get_df_loader_train(df; feature_names, target_name, batchsize, device) +dtrain = + NeuroTreeModels.get_df_loader_train(df; feature_names, target_name, batchsize, device) for d in dtrain @info length(d) @info size(d[1]) diff --git a/experiments/diagnosis-iris.jl b/experiments/diagnosis-iris.jl index aa7cb6a..b20bcb1 100644 --- a/experiments/diagnosis-iris.jl +++ b/experiments/diagnosis-iris.jl @@ -14,18 +14,13 @@ target_name = "class" feature_names = setdiff(names(df), [target_name]) train_ratio = 0.8 -train_indices = randperm(nrow(df))[1:Int(train_ratio * nrow(df))] +train_indices = randperm(nrow(df))[1:Int(train_ratio*nrow(df))] dtrain = df[train_indices, :] deval = df[setdiff(1:nrow(df), train_indices), :] -config = NeuroTreeClassifier( - nrounds=400, - depth=3, - ntrees=4, - lr=5e-2, - batchsize=60, -) +config = + NeuroTreeClassifier(nrounds = 400, depth = 3, ntrees = 4, lr = 5e-2, batchsize = 60) m = NeuroTreeModels.fit( config, @@ -33,9 +28,9 @@ m = NeuroTreeModels.fit( deval, target_name, feature_names, - metric=:mlogloss, - print_every_n=10, - early_stopping_rounds=2, + metric = :mlogloss, + print_every_n = 10, + early_stopping_rounds = 2, ) p_train = m(dtrain) @@ -48,8 +43,8 @@ nt = nts.trees[1] w = nt.w xnames = "feat" .* string.(1:4) -ynames = ["T$j/N$i" for i in 1:(2^config.depth-1), j in 1:config.ntrees] +ynames = ["T$j/N$i" for i = 1:(2^config.depth-1), j = 1:config.ntrees] ynames = vec(ynames) # p = plot(z=w, type="heatmap") # Make plot -p = plot(z=w, x=xnames, y=ynames, type="heatmap") # Make plot +p = plot(z = w, x = xnames, y = ynames, type = "heatmap") # Make plot diff --git a/experiments/gradients.jl b/experiments/gradients.jl index 2b0343e..c12495c 100644 --- a/experiments/gradients.jl +++ b/experiments/gradients.jl @@ -12,21 +12,12 @@ df = DataFrame(x, :auto); df.y = y; feature_names = Symbol("x" .* string.(1:nfeats)) -config = NeuroTreeRegressor(; - actA=:identity, - depth=3, - ntrees=32, - init_scale=0.1, -) +config = NeuroTreeRegressor(; actA = :identity, depth = 3, ntrees = 32, init_scale = 0.1) loss = NeuroTreeModels.get_loss_fn(config) L = NeuroTreeModels.get_loss_type(config) -chain = NeuroTreeModels.get_model_chain(L; config, nfeats, outsize=1) -info = Dict( - :device => :cpu, - :nrounds => 0, - :feature_names => feature_names -) +chain = NeuroTreeModels.get_model_chain(L; config, nfeats, outsize = 1) +info = Dict(:device => :cpu, :nrounds => 0, :feature_names => feature_names) m = NeuroTreeModel(L, chain, info) xb = x' yb = y @@ -46,11 +37,11 @@ db = grad_neuro[:b] dp = grad_neuro[:p] # fig = plot(x=vec(w); type=:scatter, mode="markers") -fig = plot(x=vec(w); type=:histogram) -fig = plot(x=vec(dw); type=:histogram) +fig = plot(x = vec(w); type = :histogram) +fig = plot(x = vec(dw); type = :histogram) -fig = plot(x=vec(b); type=:histogram) -fig = plot(x=vec(db); type=:histogram) +fig = plot(x = vec(b); type = :histogram) +fig = plot(x = vec(db); type = :histogram) -fig = plot(x=vec(p); type=:histogram) -fig = plot(x=vec(dp); type=:histogram) +fig = plot(x = vec(p); type = :histogram) +fig = plot(x = vec(dp); type = :histogram) diff --git a/experiments/issue-xai.jl b/experiments/issue-xai.jl index e8eef66..a12f592 100644 --- a/experiments/issue-xai.jl +++ b/experiments/issue-xai.jl @@ -14,18 +14,18 @@ target_name = "class" feature_names = setdiff(names(df), [target_name]) train_ratio = 0.8 -train_indices = randperm(nrow(df))[1:Int(train_ratio * nrow(df))] +train_indices = randperm(nrow(df))[1:Int(train_ratio*nrow(df))] dtrain = df[train_indices, :] deval = df[setdiff(1:nrow(df), train_indices), :] config = NeuroTreeRegressor( - device=:cpu, - loss=:mlogloss, - nrounds=400, - outsize=3, - depth=4, - lr=2e-2, + device = :cpu, + loss = :mlogloss, + nrounds = 400, + outsize = 3, + depth = 4, + lr = 2e-2, ) m = NeuroTreeModels.fit( @@ -34,13 +34,13 @@ m = NeuroTreeModels.fit( deval, target_name, feature_names, - metric=:mlogloss, - print_every_n=10, - early_stopping_rounds=2, + metric = :mlogloss, + print_every_n = 10, + early_stopping_rounds = 2, ) # Predictions depend on the number of samples in the dataset -m(dtrain[1:1, :])[1:1,:] -m(dtrain[1:2, :])[1:1,:] -m(dtrain[1:3, :])[1:1,:] -m(dtrain[1:10, :])[1:1,:] +m(dtrain[1:1, :])[1:1, :] +m(dtrain[1:2, :])[1:1, :] +m(dtrain[1:3, :])[1:1, :] +m(dtrain[1:10, :])[1:1, :] diff --git a/src/MLJ.jl b/src/MLJ.jl index f34734b..8d96c25 100644 --- a/src/MLJ.jl +++ b/src/MLJ.jl @@ -1,99 +1,100 @@ -function MMI.fit( - model::NeuroTypes, - verbosity::Int, - A, - y, - w=nothing) +function MMI.fit(model::NeuroTypes, verbosity::Int, A, y, w = nothing) - Tables.istable(A) ? dtrain = DataFrame(A) : error("`A` must be a Table") - feature_names = string.(collect(Tables.schema(dtrain).names)) - @assert "_target" ∉ feature_names - dtrain._target = y - target_name = "_target" + Tables.istable(A) ? dtrain = DataFrame(A) : error("`A` must be a Table") + feature_names = string.(collect(Tables.schema(dtrain).names)) + @assert "_target" ∉ feature_names + dtrain._target = y + target_name = "_target" - if !isnothing(w) - @assert "_weight" ∉ feature_names - dtrain._weight = w - weight_name = "_weight" - else - weight_name = nothing - end - offset_name = nothing + if !isnothing(w) + @assert "_weight" ∉ feature_names + dtrain._weight = w + weight_name = "_weight" + else + weight_name = nothing + end + offset_name = nothing - fitresult, cache = init(model, dtrain; feature_names, target_name, weight_name, offset_name) + fitresult, cache = + init(model, dtrain; feature_names, target_name, weight_name, offset_name) - while fitresult.info[:nrounds] < model.nrounds - fit_iter!(fitresult, cache) - end + while fitresult.info[:nrounds] < model.nrounds + fit_iter!(fitresult, cache) + end - report = (features=fitresult.info[:feature_names],) - return fitresult, cache, report + report = (features = fitresult.info[:feature_names],) + return fitresult, cache, report end function okay_to_continue(model, fitresult, cache) - return model.nrounds - fitresult.info[:nrounds] >= 0 + return model.nrounds - fitresult.info[:nrounds] >= 0 end # For EarlyStopping.jl support MMI.iteration_parameter(::Type{<:NeuroTypes}) = :nrounds function MMI.update( - model::NeuroTypes, - verbosity::Integer, - fitresult, - cache, - A, - y, - w=nothing, + model::NeuroTypes, + verbosity::Integer, + fitresult, + cache, + A, + y, + w = nothing, ) - if okay_to_continue(model, fitresult, cache) - while fitresult.info[:nrounds] < model.nrounds - fit_iter!(fitresult, cache) + if okay_to_continue(model, fitresult, cache) + while fitresult.info[:nrounds] < model.nrounds + fit_iter!(fitresult, cache) + end + report = (features = fitresult.info[:feature_names],) + else + fitresult, cache, report = fit(model, verbosity, A, y, w) end - report = (features=fitresult.info[:feature_names],) - else - fitresult, cache, report = fit(model, verbosity, A, y, w) - end - return fitresult, cache, report + return fitresult, cache, report end function predict(::NeuroTreeRegressor, fitresult, A) - df = DataFrame(A) - Tables.istable(A) ? df = DataFrame(A) : error("`A` must be a Table") - pred = fitresult(df) - return pred + df = DataFrame(A) + Tables.istable(A) ? df = DataFrame(A) : error("`A` must be a Table") + pred = fitresult(df) + return pred end function predict(::NeuroTreeClassifier, fitresult, A) - df = DataFrame(A) - Tables.istable(A) ? df = DataFrame(A) : error("`A` must be a Table") - pred = fitresult(df) - return MMI.UnivariateFinite(fitresult.info[:target_levels], pred, pool=missing, ordered=fitresult.info[:target_isordered]) + df = DataFrame(A) + Tables.istable(A) ? df = DataFrame(A) : error("`A` must be a Table") + pred = fitresult(df) + return MMI.UnivariateFinite( + fitresult.info[:target_levels], + pred, + pool = missing, + ordered = fitresult.info[:target_isordered], + ) end # Metadata MMI.metadata_pkg.( - (NeuroTreeRegressor, NeuroTreeClassifier), - name="NeuroTreeModels", - uuid="1db4e0a5-a364-4b0c-897c-2bd5a4a3a1f2", - url="https://github.com/Evovest/NeuroTreeModels.jl", - julia=true, - license="Apache", - is_wrapper=false, + (NeuroTreeRegressor, NeuroTreeClassifier), + name = "NeuroTreeModels", + uuid = "1db4e0a5-a364-4b0c-897c-2bd5a4a3a1f2", + url = "https://github.com/Evovest/NeuroTreeModels.jl", + julia = true, + license = "Apache", + is_wrapper = false, ) MMI.metadata_model( - NeuroTreeRegressor, - input_scitype=MMI.Table(MMI.Continuous, MMI.Count, MMI.OrderedFactor), - target_scitype=AbstractVector{<:MMI.Continuous}, - weights=true, - path="NeuroTreeModels.NeuroTreeRegressor", + NeuroTreeRegressor, + input_scitype = MMI.Table(MMI.Continuous, MMI.Count, MMI.OrderedFactor), + target_scitype = AbstractVector{<:MMI.Continuous}, + weights = true, + path = "NeuroTreeModels.NeuroTreeRegressor", ) MMI.metadata_model( - NeuroTreeClassifier, - input_scitype=MMI.Table(MMI.Continuous, MMI.Count, MMI.OrderedFactor), - target_scitype=AbstractVector{<:MMI.Finite}, - weights=true, - path="NeuroTreeModels.NeuroTreeClassifier", + NeuroTreeClassifier, + input_scitype = MMI.Table(MMI.Continuous, MMI.Count, MMI.OrderedFactor), + target_scitype = AbstractVector{<:MMI.Finite}, + weights = true, + path = "NeuroTreeModels.NeuroTreeClassifier", ) diff --git a/src/NeuroTreeModels.jl b/src/NeuroTreeModels.jl index 0dd8fdd..0388613 100644 --- a/src/NeuroTreeModels.jl +++ b/src/NeuroTreeModels.jl @@ -11,7 +11,8 @@ using CUDA using CUDA: CuIterator import Optimisers -import Optimisers: OptimiserChain, WeightDecay, Adam, NAdam, Nesterov, Descent, Momentum, AdaDelta +import Optimisers: + OptimiserChain, WeightDecay, Adam, NAdam, Nesterov, Descent, Momentum, AdaDelta import Flux import Flux: @layer, trainmode!, gradient, Chain, DataLoader, cpu, gpu diff --git a/src/callback.jl b/src/callback.jl index 7ed91dd..1f8ac31 100644 --- a/src/callback.jl +++ b/src/callback.jl @@ -16,7 +16,7 @@ struct CallBack{F,D} end function (cb::CallBack)(logger, iter, m) - metric = Metrics.eval(m, cb.feval, cb.deval) + metric = Metrics.evaluate(m, cb.feval, cb.deval) update_logger!(logger; iter, metric) return nothing end @@ -26,14 +26,22 @@ function CallBack( deval::AbstractDataFrame; feature_names, target_name, - weight_name=nothing, - offset_name=nothing + weight_name = nothing, + offset_name = nothing, ) device = config.device batchsize = config.batchsize feval = metric_dict[config.metric] - deval = get_df_loader_train(deval; feature_names, target_name, weight_name, offset_name, batchsize, device) + deval = get_df_loader_train( + deval; + feature_names, + target_name, + weight_name, + offset_name, + batchsize, + device, + ) return CallBack(feval, deval) end @@ -43,7 +51,7 @@ function init_logger(config::NeuroTypes) :maximise => is_maximise(metric_dict[config.metric]), :early_stopping_rounds => config.early_stopping_rounds, :nrounds => 0, - :metrics => (iter=Int[], metric=Float64[]), + :metrics => (iter = Int[], metric = Float64[]), :iter_since_best => 0, :best_iter => 0, :best_metric => 0.0, @@ -64,7 +72,8 @@ function update_logger!(logger; iter, metric) logger[:best_iter] = iter logger[:iter_since_best] = 0 else - logger[:iter_since_best] += logger[:metrics][:iter][end] - logger[:metrics][:iter][end-1] + logger[:iter_since_best] += + logger[:metrics][:iter][end] - logger[:metrics][:iter][end-1] end end end @@ -78,7 +87,7 @@ function agg_logger(logger_raw::Vector{Dict}) best_metrics = [d[:best_metric] for d in logger_raw] best_metric = last(best_metrics) - metrics = (layer=Int[], iter=Int[], metric=Float64[]) + metrics = (layer = Int[], iter = Int[], metric = Float64[]) for i in eachindex(logger_raw) _l = logger_raw[i] append!(metrics[:layer], zeros(Int, length(_l[:metrics][:iter])) .+ i) @@ -100,4 +109,4 @@ function agg_logger(logger_raw::Vector{Dict}) return logger end -end \ No newline at end of file +end diff --git a/src/data.jl b/src/data.jl index 247554d..cd00666 100644 --- a/src/data.jl +++ b/src/data.jl @@ -13,25 +13,37 @@ end length(data::ContainerTrain) = size(data.x, 2) -function getindex(data::ContainerTrain{A,B,C,D}, idx::AbstractVector) where {A,B,C<:Nothing,D<:Nothing} +function getindex( + data::ContainerTrain{A,B,C,D}, + idx::AbstractVector, +) where {A,B,C<:Nothing,D<:Nothing} x = data.x[:, idx] y = data.y[idx] return (x, y) end -function getindex(data::ContainerTrain{A,B,C,D}, idx::AbstractVector) where {A,B,C<:AbstractVector,D<:Nothing} +function getindex( + data::ContainerTrain{A,B,C,D}, + idx::AbstractVector, +) where {A,B,C<:AbstractVector,D<:Nothing} x = data.x[:, idx] y = data.y[idx] w = data.w[idx] return (x, y, w) end -function getindex(data::ContainerTrain{A,B,C,D}, idx::AbstractVector) where {A,B,C<:AbstractVector,D<:AbstractVector} +function getindex( + data::ContainerTrain{A,B,C,D}, + idx::AbstractVector, +) where {A,B,C<:AbstractVector,D<:AbstractVector} x = data.x[:, idx] y = data.y[idx] w = data.w[idx] offset = data.offset[idx] return (x, y, w, offset) end -function getindex(data::ContainerTrain{A,B,C,D}, idx::AbstractVector) where {A,B,C<:AbstractVector,D<:AbstractMatrix} +function getindex( + data::ContainerTrain{A,B,C,D}, + idx::AbstractVector, +) where {A,B,C<:AbstractVector,D<:AbstractMatrix} x = data.x[:, idx] y = data.y[idx] w = data.w[idx] @@ -44,11 +56,12 @@ function get_df_loader_train( df::AbstractDataFrame; feature_names, target_name, - weight_name=nothing, - offset_name=nothing, + weight_name = nothing, + offset_name = nothing, batchsize, - shuffle=true, - device=:cpu) + shuffle = true, + device = :cpu, +) feature_names = Symbol.(feature_names) x = Matrix{Float32}(Matrix{Float32}(select(df, feature_names))') @@ -64,12 +77,13 @@ function get_df_loader_train( offset = if isnothing(offset_name) nothing else - isa(offset_name, String) ? Float32.(df[!, offset_name]) : offset = Matrix{Float32}(Matrix{Float32}(df[!, data.offset_name])') + isa(offset_name, String) ? Float32.(df[!, offset_name]) : + offset = Matrix{Float32}(Matrix{Float32}(df[!, data.offset_name])') end container = ContainerTrain(x, y, w, offset) batchsize = min(batchsize, length(container)) - dtrain = DataLoader(container; shuffle, batchsize, partial=true, parallel=false) + dtrain = DataLoader(container; shuffle, batchsize, partial = true, parallel = false) if device == :gpu return CuIterator(dtrain) else @@ -93,12 +107,18 @@ function getindex(data::ContainerInfer{A,D}, idx::AbstractVector) where {A,D<:No x = data.x[:, idx] return x end -function getindex(data::ContainerTrain{A,D}, idx::AbstractVector) where {A,D<:AbstractVector} +function getindex( + data::ContainerTrain{A,D}, + idx::AbstractVector, +) where {A,D<:AbstractVector} x = data.x[:, idx] offset = data.offset[idx] return (x, offset) end -function getindex(data::ContainerTrain{A,D}, idx::AbstractVector) where {A,D<:AbstractMatrix} +function getindex( + data::ContainerTrain{A,D}, + idx::AbstractVector, +) where {A,D<:AbstractMatrix} x = data.x[:, idx] offset = data.offset[:, idx] return (x, offset) @@ -107,9 +127,10 @@ end function get_df_loader_infer( df::AbstractDataFrame; feature_names, - offset_name=nothing, + offset_name = nothing, batchsize, - device=:cpu) + device = :cpu, +) feature_names = Symbol.(feature_names) x = Matrix{Float32}(Matrix{Float32}(select(df, feature_names))') @@ -117,12 +138,14 @@ function get_df_loader_infer( offset = if isnothing(offset_name) nothing else - isa(offset_name, String) ? Float32.(df[!, offset_name]) : offset = Matrix{Float32}(Matrix{Float32}(df[!, data.offset_name])') + isa(offset_name, String) ? Float32.(df[!, offset_name]) : + offset = Matrix{Float32}(Matrix{Float32}(df[!, data.offset_name])') end container = ContainerInfer(x, offset) batchsize = min(batchsize, length(container)) - dinfer = DataLoader(container; shuffle=false, batchsize, partial=true, parallel=false) + dinfer = + DataLoader(container; shuffle = false, batchsize, partial = true, parallel = false) if device == :gpu return CuIterator(dinfer) else diff --git a/src/entmax.jl b/src/entmax.jl index 4f4bc16..b7d1eda 100644 --- a/src/entmax.jl +++ b/src/entmax.jl @@ -5,16 +5,16 @@ one_to_vec(x::AnyCuArray) = reshape(CUDA.CuArray(1:size(x, 2)), 1, :) function entmax_threshold_and_support(x) rho = one_to_vec(x) - x_sort = sort(x, dims=2, rev=true) - x_μ = cumsum(x_sort, dims=2) ./ rho - x_μ² = cumsum(x_sort .^ 2, dims=2) ./ rho + x_sort = sort(x, dims = 2, rev = true) + x_μ = cumsum(x_sort, dims = 2) ./ rho + x_μ² = cumsum(x_sort .^ 2, dims = 2) ./ rho ss = rho .* (x_μ² .- x_μ .^ 2) delta = (1 .- ss) ./ rho delta_nz = max.(delta, 0) τ = x_μ .- sqrt.(delta_nz) - support = dropdims(sum(x_sort .>= τ, dims=2), dims=2) + support = dropdims(sum(x_sort .>= τ, dims = 2), dims = 2) idx = collect(zip(1:size(x, 1), support)) τ′ = Flux.NNlib.gather(τ, idx) # !!! TO DO return τ′ @@ -28,7 +28,7 @@ WIP: entmax15 activation function for sparse feature selection. Doesn't improve performanceon YEAR dataset as stated in NODE paper. """ function entmax15(x) - x_norm = (x .- maximum(x, dims=2)) ./ 2 + x_norm = (x .- maximum(x, dims = 2)) ./ 2 τ′ = entmax_threshold_and_support(x_norm) out = max.(x_norm .- reshape(τ′, :, 1), 0) .^ 2 return out @@ -45,7 +45,7 @@ end function Δ_entmax15(ȳ, out) gppr = sqrt.(out) # = 1 / g'' (Y) Δ = ȳ .* gppr - q = sum(Δ, dims=2) ./ sum(gppr, dims=2) + q = sum(Δ, dims = 2) ./ sum(gppr, dims = 2) Δ = Δ .- q .* gppr return NoTangent(), Δ end diff --git a/src/fit.jl b/src/fit.jl index fcdf68f..7ea2fa1 100644 --- a/src/fit.jl +++ b/src/fit.jl @@ -3,8 +3,8 @@ function init( df::AbstractDataFrame; feature_names, target_name, - weight_name=nothing, - offset_name=nothing, + weight_name = nothing, + offset_name = nothing, ) device = config.device @@ -17,19 +17,30 @@ function init( target_isordered = false outsize = 1 if L <: MLogLoss - eltype(df[!, target_name]) <: CategoricalValue || error("Target variable `$target_name` must have its elements `<: CategoricalValue`") + eltype(df[!, target_name]) <: CategoricalValue || error( + "Target variable `$target_name` must have its elements `<: CategoricalValue`", + ) target_levels = CategoricalArrays.levels(df[!, target_name]) target_isordered = isordered(df[!, target_name]) outsize = length(target_levels) end - dtrain = NeuroTreeModels.get_df_loader_train(df; feature_names, target_name, weight_name, offset_name, batchsize, device) + dtrain = NeuroTreeModels.get_df_loader_train( + df; + feature_names, + target_name, + weight_name, + offset_name, + batchsize, + device, + ) chain = get_model_chain(L; config, nfeats, outsize) info = Dict( :nrounds => 0, :feature_names => feature_names, :target_levels => target_levels, - :target_isordered => target_isordered) + :target_isordered => target_isordered, + ) m = NeuroTreeModel(L, chain, info) if device == :gpu m = m |> gpu @@ -38,7 +49,7 @@ function init( optim = OptimiserChain(NAdam(config.lr), WeightDecay(config.wd)) opts = Optimisers.setup(optim, m) - cache = (dtrain=dtrain, loss=loss, opts=opts, info=info) + cache = (dtrain = dtrain, loss = loss, opts = opts, info = info) return m, cache end @@ -78,11 +89,11 @@ function fit( dtrain; feature_names, target_name, - weight_name=nothing, - offset_name=nothing, - deval=nothing, - print_every_n=9999, - verbosity=1, + weight_name = nothing, + offset_name = nothing, + deval = nothing, + print_every_n = 9999, + verbosity = 1, ) device = Symbol(config.device) diff --git a/src/infer.jl b/src/infer.jl index d302cce..d7386df 100644 --- a/src/infer.jl +++ b/src/infer.jl @@ -11,7 +11,12 @@ infer(m::NeuroTreeModel, data) Return the inference of a `NeuroTreeModel` over `data`, where `data` is `AbstractDataFrame`. """ function infer(m::NeuroTreeModel, data::AbstractDataFrame) - dinfer = get_df_loader_infer(data; feature_names=m.info[:feature_names], batchsize=2048, device=m.info[:device]) + dinfer = get_df_loader_infer( + data; + feature_names = m.info[:feature_names], + batchsize = 2048, + device = m.info[:device], + ) p = infer(m, dinfer) return p end @@ -41,7 +46,7 @@ function infer(m::NeuroTreeModel{<:MLogLoss}, data::DL) push!(preds, Matrix(m(x)')) end p = vcat(preds...) - softmax!(p; dims=2) + softmax!(p; dims = 2) return p end diff --git a/src/leaf_weights.jl b/src/leaf_weights.jl index cbe588a..17cfff7 100644 --- a/src/leaf_weights.jl +++ b/src/leaf_weights.jl @@ -14,7 +14,7 @@ function leaf_weights!(nw) end end end - @views lw = cw[size(nw, 1)+1:size(cw, 1), :, 1:size(nw, 3)] + @views lw = cw[(size(nw, 1)+1):size(cw, 1), :, 1:size(nw, 3)] return (cw, lw) end @@ -29,7 +29,7 @@ function leaf_weights!(nw::AnyCuArray) threads = size(nw, 2) @cuda threads = threads blocks = blocks leaf_weights!_kernel!(cw, nw) CUDA.synchronize() - @views lw = cw[size(nw, 1)+1:size(cw, 1), :, 1:size(nw, 3)] + @views lw = cw[(size(nw, 1)+1):size(cw, 1), :, 1:size(nw, 3)] return (cw, lw) end @@ -38,7 +38,7 @@ function leaf_weights!_kernel!(cw::CuDeviceArray, nw::CuDeviceArray) tree = threadIdx().x # one thread per tree batch = blockIdx().x # one block per batch - @inbounds for i in 2:2:size(cw, 1) + @inbounds for i = 2:2:size(cw, 1) # child cumulative probability is obtained from the product of the parent cumulative prob (cw[parent]) and the parent probability (np[parent]) parent = cw[i>>1, tree, batch] child = nw[i>>1, tree, batch] @@ -139,7 +139,7 @@ function Δ_leaf_weights_kernel!( leaf_offset = step * (i - 2^depth) # offset on the leaf row child = nw[i, tree, batch] # loop on node weight leaf dependencies - half positive + half negative - @inbounds for j in (1+leaf_offset:step÷2+leaf_offset) + @inbounds for j in ((1+leaf_offset):(step÷2+leaf_offset)) k = j + node_offset # move from leaf position to full tree position Δnw[i, tree, batch] += ȳ[j, tree, batch] * cw[k, tree, batch] / max(1.0f-8, child) diff --git a/src/learners.jl b/src/learners.jl index 01d2ceb..c94da22 100644 --- a/src/learners.jl +++ b/src/learners.jl @@ -15,33 +15,33 @@ mk_rng(rng::Random.AbstractRNG) = rng mk_rng(rng::T) where {T<:Integer} = Random.MersenneTwister(rng) const _loss_type_dict = Dict( - :mse => MSE, - :mae => MAE, - :logloss => LogLoss, - :tweedie => Tweedie, - :gaussian_mle => GaussianMLE, - :mlogloss => MLogLoss + :mse => MSE, + :mae => MAE, + :logloss => LogLoss, + :tweedie => Tweedie, + :gaussian_mle => GaussianMLE, + :mlogloss => MLogLoss, ) mutable struct NeuroTreeRegressor <: MMI.Deterministic - loss::Symbol - metric::Symbol - nrounds::Int - early_stopping_rounds::Int - lr::Float32 - wd::Float32 - batchsize::Int - actA::Symbol - depth::Int - ntrees::Int - hidden_size::Int - stack_size::Int - scaler::Bool - init_scale::Float32 - MLE_tree_split::Bool - rng::AbstractRNG - device::Symbol - gpuID::Int + loss::Symbol + metric::Symbol + nrounds::Int + early_stopping_rounds::Int + lr::Float32 + wd::Float32 + batchsize::Int + actA::Symbol + depth::Int + ntrees::Int + hidden_size::Int + stack_size::Int + scaler::Bool + init_scale::Float32 + MLE_tree_split::Bool + rng::AbstractRNG + device::Symbol + gpuID::Int end """ @@ -167,103 +167,104 @@ p = predict(mach, X) """ function NeuroTreeRegressor(; kwargs...) - # defaults arguments - args = Dict{Symbol,Any}( - :loss => :mse, - :metric => nothing, - :nrounds => 100, - :early_stopping_rounds => typemax(Int), - :lr => 1.0f-2, - :wd => 0.0f0, - :batchsize => 2048, - :actA => :tanh, - :depth => 4, - :ntrees => 64, - :hidden_size => 1, - :stack_size => 1, - :scaler => true, - :init_scale => 0.1, - :MLE_tree_split => false, - :rng => 123, - :device => :cpu, - :gpuID => 0 - ) - - args_ignored = setdiff(keys(kwargs), keys(args)) - args_ignored_str = join(args_ignored, ", ") - length(args_ignored) > 0 && - @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)." - - args_default = setdiff(keys(args), keys(kwargs)) - args_default_str = join(args_default, ", ") - length(args_default) > 0 && - @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)." - - args_override = intersect(keys(args), keys(kwargs)) - for arg in args_override - args[arg] = kwargs[arg] - end - - loss = Symbol(args[:loss]) - loss ∉ [:mse, :mae, :logloss, :tweedie, :gaussian_mle] && error("The provided kwarg `loss`: $loss is not supported.") - - _metric_list = [:mse, :mae, :logloss, :tweedie, :gaussian_mle] - if isnothing(args[:metric]) - metric = loss - else - metric = Symbol(args[:metric]) - end - if metric ∉ _metric_list - error("Invalid metric. Must be one of: $_metric_list") - end - - rng = mk_rng(args[:rng]) - device = Symbol(args[:device]) - - config = NeuroTreeRegressor( - loss, - metric, - args[:nrounds], - args[:early_stopping_rounds], - Float32(args[:lr]), - Float32(args[:wd]), - args[:batchsize], - Symbol(args[:actA]), - args[:depth], - args[:ntrees], - args[:hidden_size], - args[:stack_size], - args[:scaler], - args[:init_scale], - args[:MLE_tree_split], - rng, - device, - args[:gpuID] - ) - - return config + # defaults arguments + args = Dict{Symbol,Any}( + :loss => :mse, + :metric => nothing, + :nrounds => 100, + :early_stopping_rounds => typemax(Int), + :lr => 1.0f-2, + :wd => 0.0f0, + :batchsize => 2048, + :actA => :tanh, + :depth => 4, + :ntrees => 64, + :hidden_size => 1, + :stack_size => 1, + :scaler => true, + :init_scale => 0.1, + :MLE_tree_split => false, + :rng => 123, + :device => :cpu, + :gpuID => 0, + ) + + args_ignored = setdiff(keys(kwargs), keys(args)) + args_ignored_str = join(args_ignored, ", ") + length(args_ignored) > 0 && + @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)." + + args_default = setdiff(keys(args), keys(kwargs)) + args_default_str = join(args_default, ", ") + length(args_default) > 0 && + @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)." + + args_override = intersect(keys(args), keys(kwargs)) + for arg in args_override + args[arg] = kwargs[arg] + end + + loss = Symbol(args[:loss]) + loss ∉ [:mse, :mae, :logloss, :tweedie, :gaussian_mle] && + error("The provided kwarg `loss`: $loss is not supported.") + + _metric_list = [:mse, :mae, :logloss, :tweedie, :gaussian_mle] + if isnothing(args[:metric]) + metric = loss + else + metric = Symbol(args[:metric]) + end + if metric ∉ _metric_list + error("Invalid metric. Must be one of: $_metric_list") + end + + rng = mk_rng(args[:rng]) + device = Symbol(args[:device]) + + config = NeuroTreeRegressor( + loss, + metric, + args[:nrounds], + args[:early_stopping_rounds], + Float32(args[:lr]), + Float32(args[:wd]), + args[:batchsize], + Symbol(args[:actA]), + args[:depth], + args[:ntrees], + args[:hidden_size], + args[:stack_size], + args[:scaler], + args[:init_scale], + args[:MLE_tree_split], + rng, + device, + args[:gpuID], + ) + + return config end mutable struct NeuroTreeClassifier <: MMI.Probabilistic - loss::Symbol - metric::Symbol - nrounds::Int - early_stopping_rounds::Int - lr::Float32 - wd::Float32 - batchsize::Int - actA::Symbol - depth::Int - ntrees::Int - hidden_size::Int - stack_size::Int - scaler::Bool - init_scale::Float32 - MLE_tree_split::Bool - rng::AbstractRNG - device::Symbol - gpuID::Int + loss::Symbol + metric::Symbol + nrounds::Int + early_stopping_rounds::Int + lr::Float32 + wd::Float32 + batchsize::Int + actA::Symbol + depth::Int + ntrees::Int + hidden_size::Int + stack_size::Int + scaler::Bool + init_scale::Float32 + MLE_tree_split::Bool + rng::AbstractRNG + device::Symbol + gpuID::Int end """ @@ -379,69 +380,69 @@ p = predict(mach, X) """ function NeuroTreeClassifier(; kwargs...) - # defaults arguments - args = Dict{Symbol,Any}( - :nrounds => 100, - :early_stopping_rounds => typemax(Int), - :lr => 1.0f-2, - :wd => 0.0f0, - :batchsize => 2048, - :actA => :tanh, - :depth => 4, - :ntrees => 64, - :hidden_size => 1, - :stack_size => 1, - :scaler => true, - :init_scale => 0.1, - :MLE_tree_split => false, - :rng => 123, - :device => :cpu, - :gpuID => 0 - ) - - args_ignored = setdiff(keys(kwargs), keys(args)) - args_ignored_str = join(args_ignored, ", ") - length(args_ignored) > 0 && - @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)." - - args_default = setdiff(keys(args), keys(kwargs)) - args_default_str = join(args_default, ", ") - length(args_default) > 0 && - @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)." - - args_override = intersect(keys(args), keys(kwargs)) - for arg in args_override - args[arg] = kwargs[arg] - end - - loss = :mlogloss - metric = :mlogloss - - rng = mk_rng(args[:rng]) - device = Symbol(args[:device]) - - config = NeuroTreeClassifier( - loss, - metric, - args[:nrounds], - args[:early_stopping_rounds], - Float32(args[:lr]), - Float32(args[:wd]), - args[:batchsize], - Symbol(args[:actA]), - args[:depth], - args[:ntrees], - args[:hidden_size], - args[:stack_size], - args[:scaler], - args[:init_scale], - args[:MLE_tree_split], - rng, - device, - args[:gpuID], - ) - - return config + # defaults arguments + args = Dict{Symbol,Any}( + :nrounds => 100, + :early_stopping_rounds => typemax(Int), + :lr => 1.0f-2, + :wd => 0.0f0, + :batchsize => 2048, + :actA => :tanh, + :depth => 4, + :ntrees => 64, + :hidden_size => 1, + :stack_size => 1, + :scaler => true, + :init_scale => 0.1, + :MLE_tree_split => false, + :rng => 123, + :device => :cpu, + :gpuID => 0, + ) + + args_ignored = setdiff(keys(kwargs), keys(args)) + args_ignored_str = join(args_ignored, ", ") + length(args_ignored) > 0 && + @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)." + + args_default = setdiff(keys(args), keys(kwargs)) + args_default_str = join(args_default, ", ") + length(args_default) > 0 && + @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)." + + args_override = intersect(keys(args), keys(kwargs)) + for arg in args_override + args[arg] = kwargs[arg] + end + + loss = :mlogloss + metric = :mlogloss + + rng = mk_rng(args[:rng]) + device = Symbol(args[:device]) + + config = NeuroTreeClassifier( + loss, + metric, + args[:nrounds], + args[:early_stopping_rounds], + Float32(args[:lr]), + Float32(args[:wd]), + args[:batchsize], + Symbol(args[:actA]), + args[:depth], + args[:ntrees], + args[:hidden_size], + args[:stack_size], + args[:scaler], + args[:init_scale], + args[:MLE_tree_split], + rng, + device, + args[:gpuID], + ) + + return config end const NeuroTypes = Union{NeuroTreeRegressor,NeuroTreeClassifier} diff --git a/src/loss.jl b/src/loss.jl index 74d7c14..2220472 100644 --- a/src/loss.jl +++ b/src/loss.jl @@ -34,44 +34,61 @@ end function tweedie(m, x, y) rho = eltype(x)(1.5) p = exp.(m(x)) - mean(2 .* (y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + - p .^ (2 - rho) / (2 - rho)) + mean( + 2 .* ( + y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + + p .^ (2 - rho) / (2 - rho) + ), ) end function tweedie(m, x, y, w) rho = eltype(x)(1.5) p = exp.(m(x)) - sum(w .* 2 .* (y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + - p .^ (2 - rho) / (2 - rho)) + sum( + w .* 2 .* ( + y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + + p .^ (2 - rho) / (2 - rho) + ), ) / sum(w) end function tweedie(m, x, y, w, offset) rho = eltype(x)(1.5) p = exp.(m(x) .+ offset) - sum(w .* 2 .* (y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + - p .^ (2 - rho) / (2 - rho)) + sum( + w .* 2 .* ( + y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + + p .^ (2 - rho) / (2 - rho) + ), ) / sum(w) end function mlogloss(m, x, y) - p = logsoftmax(m(x); dims=1) + p = logsoftmax(m(x); dims = 1) k = size(p, 1) - mean(-sum(onehotbatch(y, 1:k) .* p; dims=1)) + mean(-sum(onehotbatch(y, 1:k) .* p; dims = 1)) end function mlogloss(m, x, y, w) - p = logsoftmax(m(x); dims=1) + p = logsoftmax(m(x); dims = 1) k = size(p, 1) - sum(-sum(onehotbatch(y, 1:k) .* p; dims=1) .* w) / sum(w) + sum(-sum(onehotbatch(y, 1:k) .* p; dims = 1) .* w) / sum(w) end function mlogloss(m, x, y, w, offset) - p = logsoftmax(m(x) .+ offset; dims=1) + p = logsoftmax(m(x) .+ offset; dims = 1) k = size(p, 1) - sum(-sum(onehotbatch(y, 1:k) .* p; dims=1) .* w) / sum(w) + sum(-sum(onehotbatch(y, 1:k) .* p; dims = 1) .* w) / sum(w) end -gaussian_mle_loss(μ::AbstractVector{T}, σ::AbstractVector{T}, y::AbstractVector{T}) where {T} = - -sum(-σ .- (y .- μ) .^ 2 ./ (2 .* max.(T(2e-7), exp.(2 .* σ)))) -gaussian_mle_loss(μ::AbstractVector{T}, σ::AbstractVector{T}, y::AbstractVector{T}, w::AbstractVector{T}) where {T} = +gaussian_mle_loss( + μ::AbstractVector{T}, + σ::AbstractVector{T}, + y::AbstractVector{T}, +) where {T} = -sum(-σ .- (y .- μ) .^ 2 ./ (2 .* max.(T(2e-7), exp.(2 .* σ)))) +gaussian_mle_loss( + μ::AbstractVector{T}, + σ::AbstractVector{T}, + y::AbstractVector{T}, + w::AbstractVector{T}, +) where {T} = -sum((-σ .- (y .- μ) .^ 2 ./ (2 .* max.(T(2e-7), exp.(2 .* σ)))) .* w) / sum(w) function gaussian_mle(m, x, y) diff --git a/src/metrics.jl b/src/metrics.jl index 3545e99..273bdef 100644 --- a/src/metrics.jl +++ b/src/metrics.jl @@ -10,15 +10,15 @@ import Flux: Chain, logσ, logsoftmax, onehotbatch mse(x, y, w; agg=mean) mse(x, y, w, offset; agg=mean) """ -function mse(m, x, y; agg=mean) +function mse(m, x, y; agg = mean) metric = agg((m(x) .- y) .^ 2) return metric end -function mse(m, x, y, w; agg=mean) +function mse(m, x, y, w; agg = mean) metric = agg((m(x) .- y) .^ 2 .* w) return metric end -function mse(m, x, y, w, offset; agg=mean) +function mse(m, x, y, w, offset; agg = mean) metric = agg((m(x) .+ offset .- y) .^ 2 .* w) return metric end @@ -28,15 +28,15 @@ end mae(x, y, w; agg=mean) mae(x, y, w, offset; agg=mean) """ -function mae(m, x, y; agg=mean) +function mae(m, x, y; agg = mean) metric = agg(abs.(m(x) .- y)) return metric end -function mae(m, x, y, w; agg=mean) +function mae(m, x, y, w; agg = mean) metric = agg(abs.(m(x) .- y) .* w) return metric end -function mae(m, x, y, w, offset; agg=mean) +function mae(m, x, y, w, offset; agg = mean) metric = agg(abs.(m(x) .+ offset .- y) .* w) return metric end @@ -47,17 +47,17 @@ end logloss(x, y, w; agg=mean) logloss(x, y, w, offset; agg=mean) """ -function logloss(m, x, y; agg=mean) +function logloss(m, x, y; agg = mean) p = m(x) metric = agg((1 .- y) .* p .- logσ.(p)) return metric end -function logloss(m, x, y, w; agg=mean) +function logloss(m, x, y, w; agg = mean) p = m(x) metric = agg(((1 .- y) .* p .- logσ.(p)) .* w) return metric end -function logloss(m, x, y, w, offset; agg=mean) +function logloss(m, x, y, w, offset; agg = mean) p = m(x) .+ offset metric = agg(((1 .- y) .* p .- logσ.(p)) .* w) return metric @@ -69,26 +69,35 @@ end tweedie(x, y, w; agg=mean) tweedie(x, y, w, offset; agg=mean) """ -function tweedie(m, x, y; agg=mean) +function tweedie(m, x, y; agg = mean) rho = eltype(x)(1.5) p = exp.(m(x)) - agg(2 .* (y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + - p .^ (2 - rho) / (2 - rho)) + agg( + 2 .* ( + y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + + p .^ (2 - rho) / (2 - rho) + ), ) end function tweedie(m, x, y, w) agg = mean rho = eltype(x)(1.5) p = exp.(m(x)) - agg(w .* 2 .* (y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + - p .^ (2 - rho) / (2 - rho)) + agg( + w .* 2 .* ( + y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + + p .^ (2 - rho) / (2 - rho) + ), ) end -function tweedie(m, x, y, w, offset; agg=mean) +function tweedie(m, x, y, w, offset; agg = mean) rho = eltype(x)(1.5) p = exp.(m(x) .+ offset) - agg(w .* 2 .* (y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + - p .^ (2 - rho) / (2 - rho)) + agg( + w .* 2 .* ( + y .^ (2 - rho) / (1 - rho) / (2 - rho) - y .* p .^ (1 - rho) / (1 - rho) + + p .^ (2 - rho) / (2 - rho) + ), ) end @@ -97,24 +106,24 @@ end mlogloss(x, y, w; agg=mean) mlogloss(x, y, w, offset; agg=mean) """ -function mlogloss(m, x, y; agg=mean) - p = logsoftmax(m(x); dims=1) +function mlogloss(m, x, y; agg = mean) + p = logsoftmax(m(x); dims = 1) k = size(p, 1) - raw = dropdims(-sum(onehotbatch(y, 1:k) .* p; dims=1); dims=1) + raw = dropdims(-sum(onehotbatch(y, 1:k) .* p; dims = 1); dims = 1) metric = agg(raw) return metric end -function mlogloss(m, x, y, w; agg=mean) - p = logsoftmax(m(x); dims=1) +function mlogloss(m, x, y, w; agg = mean) + p = logsoftmax(m(x); dims = 1) k = size(p, 1) - raw = dropdims(-sum(onehotbatch(y, 1:k) .* p; dims=1); dims=1) + raw = dropdims(-sum(onehotbatch(y, 1:k) .* p; dims = 1); dims = 1) metric = agg(raw .* w) return metric end -function mlogloss(m, x, y, w, offset; agg=mean) - p = logsoftmax(m(x) .+ offset; dims=1) +function mlogloss(m, x, y, w, offset; agg = mean) + p = logsoftmax(m(x) .+ offset; dims = 1) k = size(p, 1) - raw = dropdims(-sum(onehotbatch(y, 1:k) .* p; dims=1); dims=1) + raw = dropdims(-sum(onehotbatch(y, 1:k) .* p; dims = 1); dims = 1) metric = agg(raw .* w) return metric end @@ -134,27 +143,27 @@ gaussian_mle(μ::T, σ::T, y::T, w::T) where {T<:AbstractFloat} = gaussian_mle(x, y, w; agg=mean) gaussian_mle(x, y, w, offset; agg=mean) """ -function gaussian_mle(m, x, y; agg=mean) +function gaussian_mle(m, x, y; agg = mean) p = m(x) metric = agg(gaussian_mle.(view(p, 1, :), view(p, 2, :), y)) return metric end -function gaussian_mle(m, x, y, w; agg=mean) +function gaussian_mle(m, x, y, w; agg = mean) p = m(x) metric = agg(gaussian_mle.(view(p, 1, :), view(p, 2, :), y, w)) return metric end -function gaussian_mle(m, x, y, w, offset; agg=mean) +function gaussian_mle(m, x, y, w, offset; agg = mean) p = m(x) .+ offset metric = agg(gaussian_mle.(view(p, 1, :), view(p, 2, :), y, w)) return metric end -function eval(m, f::Function, data) +function evaluate(m, f::Function, data) metric = 0.0f0 ws = 0.0f0 for d in data - metric += f(m, d...; agg=sum) + metric += f(m, d...; agg = sum) if length(d) >= 3 ws += sum(d[3]) else diff --git a/src/model.jl b/src/model.jl index 259e6ea..e97582a 100644 --- a/src/model.jl +++ b/src/model.jl @@ -32,7 +32,8 @@ function (m::NeuroTree{W,B,P,F})(x::W) where {W,B,P,F} return pred end -dot_prod_agg(lw, p) = dropdims(sum(reshape(lw, 1, size(lw)...) .* p, dims=(2, 3)), dims=(2, 3)) +dot_prod_agg(lw, p) = + dropdims(sum(reshape(lw, 1, size(lw)...) .* p, dims = (2, 3)), dims = (2, 3)) """ NeuroTree(; ins, outs, depth=4, ntrees=64, actA=identity, init_scale=1e-1) @@ -40,7 +41,15 @@ dot_prod_agg(lw, p) = dropdims(sum(reshape(lw, 1, size(lw)...) .* p, dims=(2, 3) Initialization of a NeuroTree. """ -function NeuroTree(; ins, outs, depth=4, ntrees=64, actA=identity, scaler=true, init_scale=1e-1) +function NeuroTree(; + ins, + outs, + depth = 4, + ntrees = 64, + actA = identity, + scaler = true, + init_scale = 1e-1, +) nnodes = 2^depth - 1 nleaves = 2^depth nt = NeuroTree( @@ -49,11 +58,18 @@ function NeuroTree(; ins, outs, depth=4, ntrees=64, actA=identity, scaler=true, Float32.((rand(nnodes * ntrees) .- 0.5) ./ 4), # b Float32.(randn(outs, nleaves, ntrees) .* init_scale), # p actA, - scaler + scaler, ) return nt end -function NeuroTree((ins, outs)::Pair{<:Integer,<:Integer}; depth=4, ntrees=64, actA=identity, scaler=true, init_scale=1e-1) +function NeuroTree( + (ins, outs)::Pair{<:Integer,<:Integer}; + depth = 4, + ntrees = 64, + actA = identity, + scaler = true, + init_scale = 1e-1, +) nnodes = 2^depth - 1 nleaves = 2^depth nt = NeuroTree( @@ -62,7 +78,7 @@ function NeuroTree((ins, outs)::Pair{<:Integer,<:Integer}; depth=4, ntrees=64, a Float32.((rand(nnodes * ntrees) .- 0.5) ./ 4), # b Float32.(randn(outs, nleaves, ntrees) .* init_scale), # p actA, - scaler + scaler, ) return nt end @@ -76,20 +92,37 @@ struct StackTree end @layer StackTree -function StackTree((ins, outs)::Pair{<:Integer,<:Integer}; depth=4, ntrees=64, stack_size=2, hidden_size=8, actA=identity, scaler=true, init_scale=1e-1) +function StackTree( + (ins, outs)::Pair{<:Integer,<:Integer}; + depth = 4, + ntrees = 64, + stack_size = 2, + hidden_size = 8, + actA = identity, + scaler = true, + init_scale = 1e-1, +) @assert stack_size == 1 || hidden_size >= outs trees = [] - for i in 1:stack_size + for i = 1:stack_size if i == 1 if i < stack_size - tree = NeuroTree(ins => hidden_size; depth, ntrees, actA, scaler, init_scale) + tree = + NeuroTree(ins => hidden_size; depth, ntrees, actA, scaler, init_scale) push!(trees, tree) else tree = NeuroTree(ins => outs; depth, ntrees, actA, scaler, init_scale) push!(trees, tree) end elseif i < stack_size - tree = NeuroTree(hidden_size => hidden_size; depth, ntrees, actA, scaler, init_scale) + tree = NeuroTree( + hidden_size => hidden_size; + depth, + ntrees, + actA, + scaler, + init_scale, + ) push!(trees, tree) else tree = NeuroTree(hidden_size => outs; depth, ntrees, actA, scaler, init_scale) @@ -102,7 +135,7 @@ end function (m::StackTree)(x::AbstractMatrix) p = m.trees[1](x) - for i in 2:length(m.trees) + for i = 2:length(m.trees) if i < length(m.trees) p = p .+ m.trees[i](p) else @@ -134,31 +167,36 @@ Inference for NeuroTreeModel function (m::NeuroTreeModel)(x::AbstractMatrix) p = m.chain(x) if size(p, 1) == 1 - p = dropdims(p; dims=1) + p = dropdims(p; dims = 1) end return p end -function (m::NeuroTreeModel)(data::AbstractDataFrame; device=:cpu, gpuID=0) +function (m::NeuroTreeModel)(data::AbstractDataFrame; device = :cpu, gpuID = 0) if device == :gpu CUDA.device!(gpuID) end m = device == :cpu ? m |> cpu : m |> gpu - dinfer = get_df_loader_infer(data; feature_names=m.info[:feature_names], batchsize=2048, device) + dinfer = get_df_loader_infer( + data; + feature_names = m.info[:feature_names], + batchsize = 2048, + device, + ) p = infer(m, dinfer) return p end function _identity_act(x) - return x ./ sum(abs.(x), dims=2) + return x ./ sum(abs.(x), dims = 2) end function _tanh_act(x) x = Flux.tanh_fast.(x) - return x ./ sum(abs.(x), dims=2) + return x ./ sum(abs.(x), dims = 2) end function _hardtanh_act(x) x = Flux.hardtanh.(x) - return x ./ sum(abs.(x), dims=2) + return x ./ sum(abs.(x), dims = 2) end """ @@ -170,11 +208,8 @@ end Dictionary mapping features activation name to their function. """ -const act_dict = Dict( - :identity => _identity_act, - :tanh => _tanh_act, - :hardtanh => _hardtanh_act, -) +const act_dict = + Dict(:identity => _identity_act, :tanh => _tanh_act, :hardtanh => _hardtanh_act) function get_model_chain(L; config, nfeats, outsize) @@ -183,36 +218,42 @@ function get_model_chain(L; config, nfeats, outsize) BatchNorm(nfeats), Parallel( vcat, - StackTree(nfeats => outsize; - depth=config.depth, - ntrees=config.ntrees, - stack_size=config.stack_size, - hidden_size=config.hidden_size, - actA=act_dict[config.actA], - scaler=config.scaler, - init_scale=config.init_scale), - StackTree(nfeats => outsize; - depth=config.depth, - ntrees=config.ntrees, - stack_size=config.stack_size, - hidden_size=config.hidden_size, - actA=act_dict[config.actA], - scaler=config.scaler, - init_scale=config.init_scale) - ) + StackTree( + nfeats => outsize; + depth = config.depth, + ntrees = config.ntrees, + stack_size = config.stack_size, + hidden_size = config.hidden_size, + actA = act_dict[config.actA], + scaler = config.scaler, + init_scale = config.init_scale, + ), + StackTree( + nfeats => outsize; + depth = config.depth, + ntrees = config.ntrees, + stack_size = config.stack_size, + hidden_size = config.hidden_size, + actA = act_dict[config.actA], + scaler = config.scaler, + init_scale = config.init_scale, + ), + ), ) else outsize = L <: GaussianMLE ? 2 * outsize : outsize chain = Chain( BatchNorm(nfeats), - StackTree(nfeats => outsize; - depth=config.depth, - ntrees=config.ntrees, - stack_size=config.stack_size, - hidden_size=config.hidden_size, - actA=act_dict[config.actA], - scaler=config.scaler, - init_scale=config.init_scale) + StackTree( + nfeats => outsize; + depth = config.depth, + ntrees = config.ntrees, + stack_size = config.stack_size, + hidden_size = config.hidden_size, + actA = act_dict[config.actA], + scaler = config.scaler, + init_scale = config.init_scale, + ), ) end diff --git a/test/MLJ.jl b/test/MLJ.jl index 78d01ca..1f6c958 100644 --- a/test/MLJ.jl +++ b/test/MLJ.jl @@ -8,9 +8,9 @@ sigmoid(x::AbstractVector) = sigmoid.(x) failures, summary = MLJTestInterface.test( [NeuroTreeRegressor], MLJTestInterface.make_regression()...; - mod=@__MODULE__, - verbosity=0, # bump to debug - throw=true # set to true to debug + mod = @__MODULE__, + verbosity = 0, # bump to debug + throw = true, # set to true to debug ) @test isempty(failures) end @@ -19,18 +19,18 @@ sigmoid(x::AbstractVector) = sigmoid.(x) failures, summary = MLJTestInterface.test( [NeuroTreeClassifier], MLJTestInterface.make_binary()...; - mod=@__MODULE__, - verbosity=0, # bump to debug - throw=true # set to true to debug + mod = @__MODULE__, + verbosity = 0, # bump to debug + throw = true, # set to true to debug ) @test isempty(failures) failures, summary = MLJTestInterface.test( [NeuroTreeClassifier], MLJTestInterface.make_multiclass()...; - mod=@__MODULE__, - verbosity=0, # bump to debug - throw=true # set to true to debug + mod = @__MODULE__, + verbosity = 0, # bump to debug + throw = true, # set to true to debug ) @test isempty(failures) @@ -50,13 +50,13 @@ end y = Y X = MLJBase.table(X) - tree_model = NeuroTreeRegressor(max_depth=5, eta=0.05, nrounds=10) + tree_model = NeuroTreeRegressor(max_depth = 5, eta = 0.05, nrounds = 10) mach = machine(tree_model, X, y) - train, test = partition(eachindex(y), 0.7, shuffle=true) # 70:30 split - fit!(mach, rows=train, verbosity=1) + train, test = partition(eachindex(y), 0.7, shuffle = true) # 70:30 split + fit!(mach, rows = train, verbosity = 1) mach.model.nrounds += 10 - fit!(mach, rows=train, verbosity=1) + fit!(mach, rows = train, verbosity = 1) _report = report(mach) # predict on train data @@ -82,7 +82,7 @@ end end @testset "MLJ - named tuples - NeuroTreeRegressor" begin - X, y = (x1=rand(100), x2=rand(100)), rand(100) + X, y = (x1 = rand(100), x2 = rand(100)), rand(100) booster = NeuroTreeRegressor() # smoke tests: mach = machine(booster, X, y) |> fit! @@ -94,20 +94,15 @@ end @testset "MLJ - classification" begin X, y = @load_crabs - tree_model = NeuroTreeClassifier( - depth=4, - lr=0.1, - nrounds=20, - batchsize=64 - ) + tree_model = NeuroTreeClassifier(depth = 4, lr = 0.1, nrounds = 20, batchsize = 64) # @load EvoTreeRegressor mach = machine(tree_model, X, y) - train, test = partition(eachindex(y), 0.7, shuffle=true) # 70:30 split - fit!(mach, rows=train, verbosity=1) + train, test = partition(eachindex(y), 0.7, shuffle = true) # 70:30 split + fit!(mach, rows = train, verbosity = 1) mach.model.nrounds += 50 - fit!(mach, rows=train, verbosity=1) + fit!(mach, rows = train, verbosity = 1) pred_train = predict(mach, selectrows(X, train)) pred_train_mode = predict_mode(mach, selectrows(X, train)) @@ -120,7 +115,7 @@ end end @testset "MLJ - support for ordered factor predictions" begin - X = (; x=rand(10)) + X = (; x = rand(10)) y = coerce(rand("ab", 10), OrderedFactor) model = NeuroTreeClassifier() mach = machine(model, X, y) |> fit! diff --git a/test/core.jl b/test/core.jl index 42e94ff..23225ae 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1,16 +1,16 @@ @testset "Core - internals test" begin config = NeuroTreeRegressor( - loss=:mse, - actA=:identity, - init_scale=1.0, - nrounds=200, - depth=4, - ntrees=32, - stack_size=1, - hidden_size=1, - batchsize=2048, - lr=1e-3, + loss = :mse, + actA = :identity, + init_scale = 1.0, + nrounds = 200, + depth = 4, + ntrees = 32, + stack_size = 1, + hidden_size = 1, + batchsize = 2048, + lr = 1e-3, ) # stack tree @@ -23,11 +23,7 @@ loss = NeuroTreeModels.get_loss_fn(config) L = NeuroTreeModels.get_loss_type(config) chain = NeuroTreeModels.get_model_chain(L; config, nfeats, outsize) - info = Dict( - :device => :cpu, - :nrounds => 0, - :feature_names => feature_names - ) + info = Dict(:device => :cpu, :nrounds => 0, :feature_names => feature_names) m = NeuroTreeModel(L, chain, info) end @@ -42,32 +38,16 @@ end feature_names = setdiff(names(df), [target_name]) train_ratio = 0.8 - train_indices = randperm(nrow(df))[1:Int(train_ratio * nrow(df))] + train_indices = randperm(nrow(df))[1:Int(train_ratio*nrow(df))] dtrain = df[train_indices, :] deval = df[setdiff(1:nrow(df), train_indices), :] - config = NeuroTreeRegressor( - loss=:mse, - nrounds=20, - depth=3, - lr=1e-1, - ) + config = NeuroTreeRegressor(loss = :mse, nrounds = 20, depth = 3, lr = 1e-1) - m = NeuroTreeModels.fit( - config, - dtrain; - target_name, - feature_names - ) + m = NeuroTreeModels.fit(config, dtrain; target_name, feature_names) - m = NeuroTreeModels.fit( - config, - dtrain; - target_name, - feature_names, - deval - ) + m = NeuroTreeModels.fit(config, dtrain; target_name, feature_names, deval) end @@ -81,27 +61,22 @@ end feature_names = setdiff(names(df), [target_name]) train_ratio = 0.8 - train_indices = randperm(nrow(df))[1:Int(train_ratio * nrow(df))] + train_indices = randperm(nrow(df))[1:Int(train_ratio*nrow(df))] dtrain = df[train_indices, :] deval = df[setdiff(1:nrow(df), train_indices), :] config = NeuroTreeClassifier( - nrounds=100, - depth=4, - lr=3e-2, - batchsize=64, - early_stopping_rounds=10, - device=:cpu) - - m = NeuroTreeModels.fit( - config, - dtrain; - deval, - target_name, - feature_names + nrounds = 100, + depth = 4, + lr = 3e-2, + batchsize = 64, + early_stopping_rounds = 10, + device = :cpu, ) + m = NeuroTreeModels.fit(config, dtrain; deval, target_name, feature_names) + # Predictions depend on the number of samples in the dataset ptrain = [argmax(x) for x in eachrow(m(dtrain))] peval = [argmax(x) for x in eachrow(m(deval))]