diff --git a/Project.toml b/Project.toml index b9f516a1..c8baa160 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GeometricMachineLearning" uuid = "194d25b2-d3f5-49f0-af24-c124f4aa80cc" -authors = ["Michael Kraus "] version = "0.4.6" +authors = ["Michael Kraus "] [deps] AbstractNeuralNetworks = "60874f82-5ada-4c70-bd1c-fa6be7711c8a" @@ -29,7 +29,6 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" SymbolicNeuralNetworks = "aed23131-dcd0-47ca-8090-d21e605652e3" Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -UpdateJulia = "770da0de-323d-4d28-9202-0e205c1e0aff" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" @@ -54,7 +53,6 @@ StatsBase = "0.33, 0.34" SymbolicNeuralNetworks = "0.3" Symbolics = "6.22" TimerOutputs = "0.5" -UpdateJulia = "0.4" Zygote = "0.6" ZygoteRules = "0.2.7" julia = "1.8" diff --git a/src/GeometricMachineLearning.jl b/src/GeometricMachineLearning.jl index bdc2d8e1..7c7e6564 100644 --- a/src/GeometricMachineLearning.jl +++ b/src/GeometricMachineLearning.jl @@ -5,7 +5,7 @@ module GeometricMachineLearning using ChainRulesCore using Distances using GeometricBase - using GeometricSolutions: GeometricSolution, EnsembleSolution, DataSeries, StateVariable + using GeometricSolutions: GeometricSolution, EnsembleSolution, DataSeries, StateVariable, TimeSeries using GeometricEquations: EnsembleProblem, ODEProblem, HODEProblem, ODEEnsemble, HODEEnsemble using KernelAbstractions using LinearAlgebra @@ -46,7 +46,7 @@ module GeometricMachineLearning export initialparameters export parameterlength export NeuralNetworkParameters - + export σ, sigmoid, softmax # from GeometricBase to print docs @@ -106,7 +106,7 @@ module GeometricMachineLearning # this defines empty retraction type structs (doesn't rely on anything) include("optimizers/manifold_related/retraction_types.jl") - + export MatrixSoftmax, VectorSoftmax include("activations/softmax.jl") @@ -115,7 +115,7 @@ module GeometricMachineLearning # + operation has been overloaded to work with NamedTuples! export _add, apply_toNT, split_and_flatten, add! - + # GPU specific operations export convert_to_dev, Device, CPUDevice @@ -139,7 +139,7 @@ module GeometricMachineLearning include("optimizers/optimizer_caches.jl") include("optimizers/optimizer.jl") include("optimizers/gradient_optimizer.jl") - include("optimizers/momentum_optimizer.jl") + include("optimizers/momentum_optimizer.jl") include("optimizers/adam_optimizer.jl") include("optimizers/adam_optimizer_with_learning_rate_decay.jl") include("optimizers/bfgs_cache.jl") @@ -202,11 +202,11 @@ module GeometricMachineLearning export AbstractTrainingMethod export loss_single #, loss - + export HnnTrainingMethod export LnnTrainingMethod export SympNetTrainingMethod - + include("training_method/abstract_training_method.jl") # INCLUDE DATA TRAINING STRUCTURE @@ -227,8 +227,8 @@ module GeometricMachineLearning export TrainingMethod export symbol, shape export min_length_batch - - + + include("training_method/training_method.jl") # INCLUDE DATA TRAINING STRUCTURE @@ -237,11 +237,11 @@ module GeometricMachineLearning export shape, symbols, dim, noisemaker, data_symbols # , problem export reduce_symbols, reshape_intoSampledData export aresame - + include("data/data_training.jl") export get_batch, complete_batch_size, check_batch_size - + include("data/batch.jl") # INCLUDE BACKENDS @@ -329,7 +329,7 @@ module GeometricMachineLearning export NeuralNetSolution export problem, timestep, history, size_history export set_sizemax_history - + include("nnsolution/neural_net_solution.jl") export EnsembleNeuralNetSolution @@ -386,7 +386,7 @@ module GeometricMachineLearning include("training_method/sympnet_basic_method.jl") export default_method - + include("training/default_method.jl") @@ -409,4 +409,4 @@ module GeometricMachineLearning export SymplecticTransformer include("map_to_cpu.jl") -end \ No newline at end of file +end diff --git a/src/data_loader/data_loader.jl b/src/data_loader/data_loader.jl index 70e39ea7..774faf02 100644 --- a/src/data_loader/data_loader.jl +++ b/src/data_loader/data_loader.jl @@ -7,7 +7,7 @@ This is designed to make training convenient. # Fields of `DataLoader` -The fields of the `DataLoader` struct are the following: +The fields of the `DataLoader` struct are the following: - `input`: The input data with axes (i) system dimension, (ii) number of time steps and (iii) number of parameters. - `output`: The tensor that contains the output (supervised learning) - this may be of type `Nothing` if the constructor is only called with one tensor (unsupervised learning). - `input_dim`: The *dimension* of the system, i.e. what is taken as input by a regular neural network. @@ -59,7 +59,7 @@ end Make an instance of DataLoader for data that are in tensor format. -# Arguments +# Arguments There are two optional keyword arguments: - `autoencoder = false` and @@ -78,9 +78,9 @@ By default we have: ```jldoctest using GeometricMachineLearning -data = [ 1; 2; 3;; +data = [ 1; 2; 3;; 4; 5; 6;;; - 7; 8; 9;; + 7; 8; 9;; 10; 11; 12] DataLoader(data) @@ -97,9 +97,9 @@ But if we write ```jldoctest using GeometricMachineLearning -data = [ 1; 2; 3;; +data = [ 1; 2; 3;; 4; 5; 6;;; - 7; 8; 9;; + 7; 8; 9;; 10; 11; 12] DataLoader(data; suppress_info = true) @@ -128,7 +128,7 @@ end Make an instance of [`DataLoader`](@ref) based on a matrix. -# Arguments +# Arguments See [`DataLoader(::AbstractArray{<:Number, 3})`](@ref) for details. @@ -145,7 +145,7 @@ function DataLoader(data::AbstractMatrix{T}; autoencoder=true, suppress_info = f input_dim, time_steps = size(data) reshaped_data = reshape(data, input_dim, time_steps, 1) return DataLoader{T, typeof(reshaped_data), Nothing, :TimeSeries}(reshaped_data, nothing, input_dim, time_steps, 1, nothing, nothing) - elseif autoencoder == true + elseif autoencoder == true input_dim, n_params = size(data) reshaped_data = reshape(data, input_dim, 1, n_params) return DataLoader{T, typeof(reshaped_data), Nothing, :RegularData}(reshaped_data, nothing, input_dim, 1, n_params, nothing, nothing) @@ -189,7 +189,7 @@ end @doc raw""" DataLoader(data::AbstractArray{T, 3}, target::AbstractVector) -Make an instance of DataLoader for a classification problem. +Make an instance of DataLoader for a classification problem. Target here is a vector of labels. This is tailored towards being used with the package [`MLDatasets.jl`](https://github.com/JuliaML/MLDatasets.jl). @@ -202,15 +202,15 @@ There are two keyword arguments: For the example of the MNIST data set all images are of size ``49\times49``. For `patch_length = 7` the image is therefore split into 16 ``7\times7`` patches [brantner2023generalizing](@cite). """ -function DataLoader(data::AbstractArray{T, 3}, target::AbstractVector{T1}; +function DataLoader(data::AbstractArray{T, 3}, target::AbstractVector{T1}; patch_length=7, suppress_info = false) where {T, T1} if !suppress_info @info "You provided a tensor and a vector as input. This will be treated as a classification problem (MNIST). Tensor axes: (i) & (ii) image axes and (iii) parameter dimension." end im_dim₁, im_dim₂, n_params = size(data) - @assert length(target) == n_params - number_of_patches = (im_dim₁ ÷ patch_length) * (im_dim₂ ÷ patch_length) + @assert length(target) == n_params + number_of_patches = (im_dim₁ ÷ patch_length) * (im_dim₂ ÷ patch_length) target = onehotbatch(target) data_preprocessed = split_and_flatten(data, patch_length = patch_length, number_of_patches = number_of_patches) DataLoader{T, typeof(data_preprocessed), typeof(target), :TimeSeries}( @@ -230,8 +230,8 @@ In this case the field `input_dim` of `DataLoader` is interpreted as the sum of Apart from this the input is treated similarly as if it were an `Array`, i.e. everything is converted to tensors internally. See e.g. [`DataLoader{::AbstractArray{<:Number, 3}}`](@ref). """ -function DataLoader(data::NamedTuple{(:q, :p), Tuple{AT, AT}}; - autoencoder=false, +function DataLoader(data::NamedTuple{(:q, :p), Tuple{AT, AT}}; + autoencoder=false, suppress_info = false) where {T, AT<:AbstractMatrix{T}} if !suppress_info @info "You have provided a NamedTuple with keys q and p; the data are matrices. This is interpreted as *symplectic data*." @@ -248,13 +248,13 @@ function DataLoader(data::NamedTuple{(:q, :p), Tuple{AT, AT}}; end end -function DataLoader(data::NamedTuple{(:q, :p), Tuple{AT, AT}}; +function DataLoader(data::NamedTuple{(:q, :p), Tuple{AT, AT}}; autoencoder = false, suppress_info = false) where {T, AT<:AbstractArray{T, 3}} if !suppress_info @info "You have provided a NamedTuple with keys q and p; the data are tensors. This is interpreted as *symplectic data*." end - + dim2, time_steps, n_params = size(data.q) if autoencoder == false @@ -264,18 +264,18 @@ function DataLoader(data::NamedTuple{(:q, :p), Tuple{AT, AT}}; end end -function DataLoader(data::NamedTuple{(:q, :p), Tuple{VT, VT}}; +function DataLoader(data::NamedTuple{(:q, :p), Tuple{VT, VT}}; suppress_info = false) where {VT <: AbstractVector} DataLoader((q = reshape(data.q, 1, length(data.q)), p = reshape(data.p, 1, length(data.p))); suppress_info = suppress_info) end -function data_tensors_from_geometric_solution(solution::GeometricSolution{T, <:Number, NT}) where {T <: Number, DT <: DataSeries{T}, NT<:NamedTuple{(:q, :p), Tuple{DT, DT}}} +function data_tensors_from_geometric_solution(solution::GeometricSolution{T, T2, TT, NT}) where {T <: Number, T2 <: Number, TT <: TimeSeries{T2}, DT <: DataSeries{T}, NT<:NamedTuple{(:q, :p), Tuple{DT, DT}}} sys_dim, input_time_steps = length(solution.s.q[0]), length(solution.t) data = (q = zeros(T, sys_dim, input_time_steps, 1), p = zeros(T, sys_dim, input_time_steps, 1)) - for dim in 1:sys_dim + for dim in 1:sys_dim data.q[dim, :, 1] = solution.q[:, dim] data.p[dim, :, 1] = solution.p[:, dim] end @@ -283,11 +283,11 @@ function data_tensors_from_geometric_solution(solution::GeometricSolution{T, <:N data end -function data_tensors_from_geometric_solution(solution::GeometricSolution{T, <:Number, NT}) where {T <: Number, DT <: DataSeries{T}, NT<:NamedTuple{(:q, ), Tuple{DT}}} +function data_tensors_from_geometric_solution(solution::GeometricSolution{T, T2, TT, NT}) where {T <: Number, T2 <: Number, TT <: TimeSeries{T2}, DT <: DataSeries{T}, NT<:NamedTuple{(:q, ), Tuple{DT}}} sys_dim, input_time_steps = length(solution.s.q[0]), length(solution.t) data = zeros(T, sys_dim, input_time_steps, 1) - for dim in 1:sys_dim + for dim in 1:sys_dim data[dim, :, 1] = solution.q[:, dim] end @@ -303,21 +303,23 @@ Make an instance of `DataLoader` for a `GeometricSolution`. # Arguments -This functor for [`DataLoader`](@ref) also has the keyword arguments +This functor for [`DataLoader`](@ref) also has the keyword arguments - `autoencoder = false` and -- `suppress_info = false`. +- `suppress_info = false`. See the docstring for [`DataLoader(::AbstractArray{<:Number, 3})`](@ref). -# Implementation +# Implementation Internally this stores the data as a tensor where the third axis has length 1. """ -function DataLoader(solution::GeometricSolution{T, <:Number, NT}, suppress_info = false; kwargs...) where - {T <: Number, +function DataLoader(solution::GeometricSolution{T, T2, TT, NT}, suppress_info = false; kwargs...) where + {T <: Number, + T2 <: Number, + TT <: TimeSeries{T2}, DT <: DataSeries{T}, NT <: Union{ - NamedTuple{(:q, :p), - Tuple{DT, DT}}, + NamedTuple{(:q, :p), + Tuple{DT, DT}}, NamedTuple{(:q, ), Tuple{DT}}}} data = data_tensors_from_geometric_solution(solution) @@ -326,9 +328,9 @@ function DataLoader(solution::GeometricSolution{T, <:Number, NT}, suppress_info end function DataLoader(ensemble_solution::EnsembleSolution{T, T1, Vector{ST}}; - suppress_info = false) where {T, - T1, - DT, + suppress_info = false) where {T, + T1, + DT, ST <: GeometricSolution{T, T1, NamedTuple{(:q, ), Tuple{DT}}}} sys_dim = length(ensemble_solution.s[1].q[0]) @@ -338,9 +340,9 @@ function DataLoader(ensemble_solution::EnsembleSolution{T, T1, Vector{ST}}; data = zeros(sys_dim, input_time_steps, n_params) for (solution, i) in zip(ensemble_solution.s, axes(ensemble_solution.s, 1)) - for dim in 1:sys_dim + for dim in 1:sys_dim data[dim, :, i] = solution.q[:, dim] - end + end end DataLoader(data; suppress_info = suppress_info) @@ -355,9 +357,9 @@ Make an instance of `DataLoader` for a `EnsembleSolution`. # Arguments -This functor for `DataLoader` also has the keyword arguments +This functor for `DataLoader` also has the keyword arguments - `autoencoder = false` and -- `suppress_info = false`. +- `suppress_info = false`. See the docstring for [`DataLoader(::AbstractArray{<:Number, 3})`](@ref). @@ -365,12 +367,12 @@ See the docstring for [`DataLoader(::AbstractArray{<:Number, 3})`](@ref). Internally this stores the data as a tensor where the third axis has length equal to the number of solutions in the ensemble. """ -function DataLoader(ensemble_solution::EnsembleSolution{T, T1, Vector{ST}}; +function DataLoader(ensemble_solution::EnsembleSolution{T, T1, Vector{ST}}; autoencoder = false, suppress_info = false - ) where {T, - T1, - DT <: DataSeries{T}, + ) where {T, + T1, + DT <: DataSeries{T}, ST <: GeometricSolution{T, T1, NamedTuple{(:q, :p), Tuple{DT, DT}}} } @@ -381,10 +383,10 @@ function DataLoader(ensemble_solution::EnsembleSolution{T, T1, Vector{ST}}; data = (q = zeros(T, sys_dim, input_time_steps, n_params), p = zeros(T, sys_dim, input_time_steps, n_params)) for (solution, i) in zip(ensemble_solution.s, axes(ensemble_solution.s, 1)) - for dim in 1:sys_dim + for dim in 1:sys_dim data.q[dim, :, i] = solution.q[:, dim] data.p[dim, :, i] = solution.p[:, dim] - end + end end DataLoader(data; autoencoder = autoencoder, suppress_info = suppress_info) @@ -420,7 +422,7 @@ This allocates new memory of the same size as is used for the original `dl` and By default the data type remains unchanged, i.e. `eltype(DataLoader(dl, backend)) == eltype(dl)` is `true`. -If you want to change the data type write e.g. +If you want to change the data type write e.g. ```julia DataLoader(dl, backend, Float32) @@ -428,17 +430,17 @@ If you want to change the data type write e.g. # Arguments -There is an optional keyword argument -- `autoencoder = nothing`. +There is an optional keyword argument +- `autoencoder = nothing`. By default this inherits the autoencoder property form `dl`. See the docstring for [`DataLoader(data::AbstractArray{<:Number, 3})`](@ref). """ -function DataLoader(dl::DataLoader{T1, <:QPTOAT, Nothing, Type}, - backend::KernelAbstractions.Backend=networkbackend(dl), - T::DataType=T1; - autoencoder = nothing +function DataLoader(dl::DataLoader{T1, <:QPTOAT, Nothing, Type}, + backend::KernelAbstractions.Backend=networkbackend(dl), + T::DataType=T1; + autoencoder = nothing ) where {T1, Type} DT = if isnothing(autoencoder) Type @@ -448,16 +450,16 @@ function DataLoader(dl::DataLoader{T1, <:QPTOAT, Nothing, Type}, :TimeSeries end - input = + input = if T == T1 dl.input else map_to_type(dl.input, T) end - new_input = + new_input = if backend == networkbackend(dl) - input + input else map_to_new_backend(input, backend) end @@ -508,4 +510,4 @@ Base.eltype(::DataLoader{T}) where T = T networkbackend(dl::DataLoader) = networkbackend(dl.input) function networkbackend(dl::DataLoader{T, <:QPT{T}}) where T networkbackend(dl.input.q) -end \ No newline at end of file +end